diff --git a/.gitignore b/.gitignore index f53598b4b4c..7acee3ad016 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,8 @@ /docs/publish /docs/edit /docs/website -/docs/venv/ -/docs/tools/venv/ +/docs/venv +/docs/tools/venv /docs/tools/translate/venv /docs/tools/translate/output.md /docs/en/single.md diff --git a/.gitmodules b/.gitmodules index 4a5b21ce30a..14661e8a32d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,7 +13,7 @@ url = https://github.com/edenhill/librdkafka.git [submodule "contrib/cctz"] path = contrib/cctz - url = https://github.com/google/cctz.git + url = https://github.com/ClickHouse-Extras/cctz.git [submodule "contrib/zlib-ng"] path = contrib/zlib-ng url = https://github.com/ClickHouse-Extras/zlib-ng.git @@ -151,3 +151,6 @@ [submodule "website/images/feathericons"] path = website/images/feathericons url = https://github.com/feathericons/feather +[submodule "contrib/msgpack-c"] + path = contrib/msgpack-c + url = https://github.com/msgpack/msgpack-c diff --git a/CHANGELOG.md b/CHANGELOG.md index a0ea7f62b6c..8375f4ba152 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,86 @@ +--- +toc_folder_title: Changelog +toc_priority: 74 +toc_title: '2020' +--- + ## ClickHouse release v20.3 +### ClickHouse release v20.3.7.46, 2020-04-17 + +#### Bug Fix + +* Fix `Logical error: CROSS JOIN has expressions` error for queries with comma and names joins mix. [#10311](https://github.com/ClickHouse/ClickHouse/pull/10311) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix queries with `max_bytes_before_external_group_by`. [#10302](https://github.com/ClickHouse/ClickHouse/pull/10302) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix move-to-prewhere optimization in presense of arrayJoin functions (in certain cases). This fixes [#10092](https://github.com/ClickHouse/ClickHouse/issues/10092). [#10195](https://github.com/ClickHouse/ClickHouse/pull/10195) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add the ability to relax the restriction on non-deterministic functions usage in mutations with `allow_nondeterministic_mutations` setting. [#10186](https://github.com/ClickHouse/ClickHouse/pull/10186) ([filimonov](https://github.com/filimonov)). + +### ClickHouse release v20.3.6.40, 2020-04-16 + +#### New Feature + +* Added function `isConstant`. This function checks whether its argument is constant expression and returns 1 or 0. It is intended for development, debugging and demonstration purposes. [#10198](https://github.com/ClickHouse/ClickHouse/pull/10198) ([alexey-milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix + +* Fix error `Pipeline stuck` with `max_rows_to_group_by` and `group_by_overflow_mode = 'break'`. [#10279](https://github.com/ClickHouse/ClickHouse/pull/10279) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix rare possible exception `Cannot drain connections: cancel first`. [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed bug where ClickHouse would throw "Unknown function lambda." error message when user tries to run ALTER UPDATE/DELETE on tables with ENGINE = Replicated*. Check for nondeterministic functions now handles lambda expressions correctly. [#10237](https://github.com/ClickHouse/ClickHouse/pull/10237) ([Alexander Kazakov](https://github.com/Akazz)). +* Fixed "generateRandom" function for Date type. This fixes [#9973](https://github.com/ClickHouse/ClickHouse/issues/9973). Fix an edge case when dates with year 2106 are inserted to MergeTree tables with old-style partitioning but partitions are named with year 1970. [#10218](https://github.com/ClickHouse/ClickHouse/pull/10218) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Convert types if the table definition of a View does not correspond to the SELECT query. This fixes [#10180](https://github.com/ClickHouse/ClickHouse/issues/10180) and [#10022](https://github.com/ClickHouse/ClickHouse/issues/10022). [#10217](https://github.com/ClickHouse/ClickHouse/pull/10217) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix `parseDateTimeBestEffort` for strings in RFC-2822 when day of week is Tuesday or Thursday. This fixes [#10082](https://github.com/ClickHouse/ClickHouse/issues/10082). [#10214](https://github.com/ClickHouse/ClickHouse/pull/10214) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix column names of constants inside JOIN that may clash with names of constants outside of JOIN. [#10207](https://github.com/ClickHouse/ClickHouse/pull/10207) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix possible inifinite query execution when the query actually should stop on LIMIT, while reading from infinite source like `system.numbers` or `system.zeros`. [#10206](https://github.com/ClickHouse/ClickHouse/pull/10206) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix using the current database for access checking when the database isn't specified. [#10192](https://github.com/ClickHouse/ClickHouse/pull/10192) ([Vitaly Baranov](https://github.com/vitlibar)). +* Convert blocks if structure does not match on INSERT into Distributed(). [#10135](https://github.com/ClickHouse/ClickHouse/pull/10135) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible incorrect result for extremes in processors pipeline. [#10131](https://github.com/ClickHouse/ClickHouse/pull/10131) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix some kinds of alters with compact parts. [#10130](https://github.com/ClickHouse/ClickHouse/pull/10130) ([Anton Popov](https://github.com/CurtizJ)). +* Fix incorrect `index_granularity_bytes` check while creating new replica. Fixes [#10098](https://github.com/ClickHouse/ClickHouse/issues/10098). [#10121](https://github.com/ClickHouse/ClickHouse/pull/10121) ([alesapin](https://github.com/alesapin)). +* Fix SIGSEGV on INSERT into Distributed table when its structure differs from the underlying tables. [#10105](https://github.com/ClickHouse/ClickHouse/pull/10105) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node doesn't exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)). +* Add some arguments check and support identifier arguments for MySQL Database Engine. [#10077](https://github.com/ClickHouse/ClickHouse/pull/10077) ([Winter Zhang](https://github.com/zhang2014)). +* Fix bug in clickhouse dictionary source from localhost clickhouse server. The bug may lead to memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)). +* Fix bug in `CHECK TABLE` query when table contain skip indices. [#10068](https://github.com/ClickHouse/ClickHouse/pull/10068) ([alesapin](https://github.com/alesapin)). +* Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with different level from different shards (mixed single and two level aggregation). [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix a segmentation fault that could occur in GROUP BY over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix parallel distributed INSERT SELECT for remote table. This PR fixes the solution provided in [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759). [#9999](https://github.com/ClickHouse/ClickHouse/pull/9999) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix the number of threads used for remote query execution (performance regression, since 20.3). This happened when query from `Distributed` table was executed simultaneously on local and remote shards. Fixes [#9965](https://github.com/ClickHouse/ClickHouse/issues/9965). [#9971](https://github.com/ClickHouse/ClickHouse/pull/9971) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix bug in which the necessary tables weren't retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)). +* Fix 'Not found column in block' error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839). [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)). +* Fix parsing multiple hosts set in the CREATE USER command, e.g. `CREATE USER user6 HOST NAME REGEXP 'lo.?*host', NAME REGEXP 'lo*host'`. [#9924](https://github.com/ClickHouse/ClickHouse/pull/9924) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)). +* Fix "scalar doesn't exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)). +* Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)). +* Fix error with qualified names in `distributed_product_mode='local'`. Fixes [#4756](https://github.com/ClickHouse/ClickHouse/issues/4756). [#9891](https://github.com/ClickHouse/ClickHouse/pull/9891) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix calculating grants for introspection functions from the setting 'allow_introspection_functions'. [#9840](https://github.com/ClickHouse/ClickHouse/pull/9840) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Build/Testing/Packaging Improvement + +* Fix integration test `test_settings_constraints`. [#9962](https://github.com/ClickHouse/ClickHouse/pull/9962) ([Vitaly Baranov](https://github.com/vitlibar)). +* Removed dependency on `clock_getres`. [#9833](https://github.com/ClickHouse/ClickHouse/pull/9833) ([alexey-milovidov](https://github.com/alexey-milovidov)). + + +### ClickHouse release v20.3.5.21, 2020-03-27 + +#### Bug Fix + +* Fix 'Different expressions with the same alias' error when query has PREWHERE and WHERE on distributed table and `SET distributed_product_mode = 'local'`. [#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix mutations excessive memory consumption for tables with a composite primary key. This fixes [#9850](https://github.com/ClickHouse/ClickHouse/issues/9850). [#9860](https://github.com/ClickHouse/ClickHouse/pull/9860) ([alesapin](https://github.com/alesapin)). +* For INSERT queries shard now clamps the settings got from the initiator to the shard's constaints instead of throwing an exception. This fix allows to send INSERT queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix 'COMMA to CROSS JOIN rewriter is not enabled or cannot rewrite query' error in case of subqueries with COMMA JOIN out of tables lists (i.e. in WHERE). Fixes [#9782](https://github.com/ClickHouse/ClickHouse/issues/9782). [#9830](https://github.com/ClickHouse/ClickHouse/pull/9830) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix possible exception `Got 0 in totals chunk, expected 1` on client. It happened for queries with `JOIN` in case if right joined table had zero rows. Example: `select * from system.one t1 join system.one t2 on t1.dummy = t2.dummy limit 0 FORMAT TabSeparated;`. Fixes [#9777](https://github.com/ClickHouse/ClickHouse/issues/9777). [#9823](https://github.com/ClickHouse/ClickHouse/pull/9823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix SIGSEGV with optimize_skip_unused_shards when type cannot be converted. [#9804](https://github.com/ClickHouse/ClickHouse/pull/9804) ([Azat Khuzhin](https://github.com/azat)). +* Fix broken `ALTER TABLE DELETE COLUMN` query for compact parts. [#9779](https://github.com/ClickHouse/ClickHouse/pull/9779) ([alesapin](https://github.com/alesapin)). +* Fix max_distributed_connections (w/ and w/o Processors). [#9673](https://github.com/ClickHouse/ClickHouse/pull/9673) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a few cases when timezone of the function argument wasn't used properly. [#9574](https://github.com/ClickHouse/ClickHouse/pull/9574) ([Vasily Nemkov](https://github.com/Enmk)). + +#### Improvement + +* Remove order by stage from mutations because we read from a single ordered part in a single thread. Also add check that the order of rows in mutation is ordered in sorting key order and this order is not violated. [#9886](https://github.com/ClickHouse/ClickHouse/pull/9886) ([alesapin](https://github.com/alesapin)). + + ### ClickHouse release v20.3.4.10, 2020-03-20 #### Bug Fix @@ -255,6 +336,55 @@ ## ClickHouse release v20.1 +### ClickHouse release v20.1.10.70, 2020-04-17 + +#### Bug Fix + +* Fix rare possible exception `Cannot drain connections: cancel first`. [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed bug where ClickHouse would throw `'Unknown function lambda.'` error message when user tries to run `ALTER UPDATE/DELETE` on tables with `ENGINE = Replicated*`. Check for nondeterministic functions now handles lambda expressions correctly. [#10237](https://github.com/ClickHouse/ClickHouse/pull/10237) ([Alexander Kazakov](https://github.com/Akazz)). +* Fix `parseDateTimeBestEffort` for strings in RFC-2822 when day of week is Tuesday or Thursday. This fixes [#10082](https://github.com/ClickHouse/ClickHouse/issues/10082). [#10214](https://github.com/ClickHouse/ClickHouse/pull/10214) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix column names of constants inside `JOIN` that may clash with names of constants outside of `JOIN`. [#10207](https://github.com/ClickHouse/ClickHouse/pull/10207) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix possible inifinite query execution when the query actually should stop on LIMIT, while reading from infinite source like `system.numbers` or `system.zeros`. [#10206](https://github.com/ClickHouse/ClickHouse/pull/10206) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix move-to-prewhere optimization in presense of `arrayJoin` functions (in certain cases). This fixes [#10092](https://github.com/ClickHouse/ClickHouse/issues/10092). [#10195](https://github.com/ClickHouse/ClickHouse/pull/10195) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add the ability to relax the restriction on non-deterministic functions usage in mutations with `allow_nondeterministic_mutations` setting. [#10186](https://github.com/ClickHouse/ClickHouse/pull/10186) ([filimonov](https://github.com/filimonov)). +* Convert blocks if structure does not match on `INSERT` into table with `Distributed` engine. [#10135](https://github.com/ClickHouse/ClickHouse/pull/10135) ([Azat Khuzhin](https://github.com/azat)). +* Fix `SIGSEGV` on `INSERT` into `Distributed` table when its structure differs from the underlying tables. [#10105](https://github.com/ClickHouse/ClickHouse/pull/10105) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add arguments check and support identifier arguments for MySQL Database Engine. [#10077](https://github.com/ClickHouse/ClickHouse/pull/10077) ([Winter Zhang](https://github.com/zhang2014)). +* Fix bug in clickhouse dictionary source from localhost clickhouse server. The bug may lead to memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)). +* Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with different level from different shards (mixed single and two level aggregation). [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix a segmentation fault that could occur in `GROUP BY` over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix bug in which the necessary tables weren't retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)). +* Fix `'Not found column in block'` error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839). [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)). +* Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)). +* Fix `'scalar doesn't exist'` error in ALTER queries ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)). +* Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)). +* Fixed `DeleteOnDestroy` logic in `ATTACH PART` which could lead to automatic removal of attached part and added few tests. [#9410](https://github.com/ClickHouse/ClickHouse/pull/9410) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Build/Testing/Packaging Improvement + +* Fix unit test `collapsing_sorted_stream`. [#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([Deleted user](https://github.com/ghost)). + +### ClickHouse release v20.1.9.54, 2020-03-28 + +#### Bug Fix + +* Fix `'Different expressions with the same alias'` error when query has `PREWHERE` and `WHERE` on distributed table and `SET distributed_product_mode = 'local'`. [#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix mutations excessive memory consumption for tables with a composite primary key. This fixes [#9850](https://github.com/ClickHouse/ClickHouse/issues/9850). [#9860](https://github.com/ClickHouse/ClickHouse/pull/9860) ([alesapin](https://github.com/alesapin)). +* For INSERT queries shard now clamps the settings got from the initiator to the shard's constaints instead of throwing an exception. This fix allows to send `INSERT` queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix possible exception `Got 0 in totals chunk, expected 1` on client. It happened for queries with `JOIN` in case if right joined table had zero rows. Example: `select * from system.one t1 join system.one t2 on t1.dummy = t2.dummy limit 0 FORMAT TabSeparated;`. Fixes [#9777](https://github.com/ClickHouse/ClickHouse/issues/9777). [#9823](https://github.com/ClickHouse/ClickHouse/pull/9823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `SIGSEGV` with `optimize_skip_unused_shards` when type cannot be converted. [#9804](https://github.com/ClickHouse/ClickHouse/pull/9804) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a few cases when timezone of the function argument wasn't used properly. [#9574](https://github.com/ClickHouse/ClickHouse/pull/9574) ([Vasily Nemkov](https://github.com/Enmk)). + +#### Improvement + +* Remove `ORDER BY` stage from mutations because we read from a single ordered part in a single thread. Also add check that the order of rows in mutation is ordered in sorting key order and this order is not violated. [#9886](https://github.com/ClickHouse/ClickHouse/pull/9886) ([alesapin](https://github.com/alesapin)). + +#### Build/Testing/Packaging Improvement + +* Clean up duplicated linker flags. Make sure the linker won't look up an unexpected symbol. [#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([Amos Bird](https://github.com/amosbird)). + ### ClickHouse release v20.1.8.41, 2020-03-20 #### Bug Fix @@ -641,4 +771,4 @@ #### Security Fix * Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) -## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) +## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/whats_new/changelog/2019.md) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92433dcbe34..a05989fbb6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -213,10 +213,14 @@ if (COMPILER_CLANG) # TODO investigate that set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer") + if (OS_DARWIN) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -Wl,-U,_inside_main") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-U,_inside_main") endif() + + # Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. + set(COMPILER_FLAGS "${COMPILER_FLAGS} -fdiagnostics-absolute-paths") endif () option (ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) @@ -343,6 +347,7 @@ include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) include (cmake/find/orc.cmake) include (cmake/find/avro.cmake) +include (cmake/find/msgpack.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) diff --git a/README.md b/README.md index 84af1e30a6b..a7a4cb97b2c 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,11 @@ ClickHouse is an open-source column-oriented database management system that all * [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events. * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. -* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. +* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person. ## Upcoming Events -* [ClickHouse in Avito (online in Russian)](https://avitotech.timepad.ru/event/1290051/) on April 9, 2020. +* [ClickHouse Online Meetup West (in English)](https://www.eventbrite.com/e/clickhouse-online-meetup-registration-102886791162) on April 24, 2020. +* [ClickHouse Online Meetup East (in English)](https://www.eventbrite.com/e/clickhouse-online-meetup-east-registration-102989325846) on April 28, 2020. * [ClickHouse Workshop in Novosibirsk](https://2020.codefest.ru/lecture/1628) on TBD date. * [Yandex C++ Open-Source Sprints in Moscow](https://events.yandex.ru/events/otkrytyj-kod-v-yandek-28-03-2020) on TBD date. diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index e202021fe48..cfa54fe2ca4 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -3,8 +3,10 @@ if (USE_CLANG_TIDY) endif () add_subdirectory (common) -add_subdirectory (loggers) add_subdirectory (daemon) +add_subdirectory (loggers) +add_subdirectory (pcg-random) +add_subdirectory (widechar_width) if (USE_MYSQL) add_subdirectory (mysqlxx) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index d7ab0046992..71501a815d3 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -2,16 +2,19 @@ #include #include +#include +#include #include +#include + +#include #include #include #include #include #include -#define DATE_LUT_MIN 0 - namespace { @@ -47,7 +50,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) assert(inside_main); size_t i = 0; - time_t start_of_day = DATE_LUT_MIN; + time_t start_of_day = 0; cctz::time_zone cctz_time_zone; if (!cctz::load_time_zone(time_zone, &cctz_time_zone)) @@ -133,7 +136,10 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) } /// Fill lookup table for years and months. - for (size_t day = 0; day < DATE_LUT_SIZE && lut[day].year <= DATE_LUT_MAX_YEAR; ++day) + size_t year_months_lut_index = 0; + size_t first_day_of_last_month = 0; + + for (size_t day = 0; day < DATE_LUT_SIZE; ++day) { const Values & values = lut[day]; @@ -141,7 +147,93 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) { if (values.month == 1) years_lut[values.year - DATE_LUT_MIN_YEAR] = day; - years_months_lut[(values.year - DATE_LUT_MIN_YEAR) * 12 + values.month - 1] = day; + + year_months_lut_index = (values.year - DATE_LUT_MIN_YEAR) * 12 + values.month - 1; + years_months_lut[year_months_lut_index] = day; + first_day_of_last_month = day; } } + + /// Fill the rest of lookup table with the same last month (2106-02-01). + for (; year_months_lut_index < DATE_LUT_YEARS * 12; ++year_months_lut_index) + { + years_months_lut[year_months_lut_index] = first_day_of_last_month; + } } + + +#if !defined(ARCADIA_BUILD) /// Arcadia's variant of CCTZ already has the same implementation. + +/// Prefer to load timezones from blobs linked to the binary. +/// The blobs are provided by "tzdata" library. +/// This allows to avoid dependency on system tzdata. +namespace cctz_extension +{ + namespace + { + class Source : public cctz::ZoneInfoSource + { + public: + Source(const char * data_, size_t size_) : data(data_), size(size_) {} + + size_t Read(void * buf, size_t bytes) override + { + if (bytes > size) + bytes = size; + memcpy(buf, data, bytes); + data += bytes; + size -= bytes; + return bytes; + } + + int Skip(size_t offset) override + { + if (offset <= size) + { + data += offset; + size -= offset; + return 0; + } + else + { + errno = EINVAL; + return -1; + } + } + private: + const char * data; + size_t size; + }; + + std::unique_ptr custom_factory( + const std::string & name, + const std::function(const std::string & name)> & fallback) + { + std::string name_replaced = name; + std::replace(name_replaced.begin(), name_replaced.end(), '/', '_'); + std::replace(name_replaced.begin(), name_replaced.end(), '-', '_'); + + /// These are the names that are generated by "ld -r -b binary" + std::string symbol_name_data = "_binary_" + name_replaced + "_start"; + std::string symbol_name_size = "_binary_" + name_replaced + "_size"; + + const void * sym_data = dlsym(RTLD_DEFAULT, symbol_name_data.c_str()); + const void * sym_size = dlsym(RTLD_DEFAULT, symbol_name_size.c_str()); + + if (sym_data && sym_size) + return std::make_unique(static_cast(sym_data), unalignedLoad(&sym_size)); + +#if defined(NDEBUG) + return fallback(name); +#else + /// In debug builds, ensure that only embedded timezones can be loaded - this is intended for tests. + (void)fallback; + return nullptr; +#endif + } + } + + ZoneInfoSourceFactory zone_info_source_factory = custom_factory; +} + +#endif diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index d9d27c56ee3..cb41d2ef1c1 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -12,7 +12,7 @@ /// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check. #define DATE_LUT_SIZE 0x10000 #define DATE_LUT_MIN_YEAR 1970 -#define DATE_LUT_MAX_YEAR 2105 /// Last supported year +#define DATE_LUT_MAX_YEAR 2106 /// Last supported year (incomplete) #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table #if defined(__PPC__) @@ -99,7 +99,7 @@ private: return guess; /// Time zones that have offset 0 from UTC do daylight saving time change (if any) towards increasing UTC offset (example: British Standard Time). - if (offset_at_start_of_epoch >= 0) + if (t >= lut[DayNum(guess + 1)].date) return DayNum(guess + 1); return DayNum(guess - 1); @@ -579,7 +579,7 @@ public: return t / 3600; /// Assume that if offset was fractional, then the fraction is the same as at the beginning of epoch. - /// NOTE This assumption is false for "Pacific/Pitcairn" time zone. + /// NOTE This assumption is false for "Pacific/Pitcairn" and "Pacific/Kiritimati" time zones. return (t + 86400 - offset_at_start_of_epoch) / 3600; } diff --git a/base/common/LineReader.cpp b/base/common/LineReader.cpp index d0aedabb02b..0d06e5ef225 100644 --- a/base/common/LineReader.cpp +++ b/base/common/LineReader.cpp @@ -127,7 +127,7 @@ LineReader::InputStatus LineReader::readOneLine(const String & prompt) #ifdef OS_LINUX if (!readline_ptr) { - for (auto name : {"libreadline.so", "libreadline.so.0", "libeditline.so", "libeditline.so.0"}) + for (const auto * name : {"libreadline.so", "libreadline.so.0", "libeditline.so", "libeditline.so.0"}) { void * dl_handle = dlopen(name, RTLD_LAZY); if (dl_handle) diff --git a/base/common/ReplxxLineReader.cpp b/base/common/ReplxxLineReader.cpp index 40fcc04dedc..52c42235f1b 100644 --- a/base/common/ReplxxLineReader.cpp +++ b/base/common/ReplxxLineReader.cpp @@ -37,13 +37,13 @@ ReplxxLineReader::ReplxxLineReader(const Suggest & suggest, const String & histo /// By default C-p/C-n binded to COMPLETE_NEXT/COMPLETE_PREV, /// bind C-p/C-n to history-previous/history-next like readline. - rx.bind_key(Replxx::KEY::control('N'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::HISTORY_NEXT, _1)); - rx.bind_key(Replxx::KEY::control('P'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::HISTORY_PREVIOUS, _1)); + rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); }); + rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); }); /// By default COMPLETE_NEXT/COMPLETE_PREV was binded to C-p/C-n, re-bind /// to M-P/M-N (that was used for HISTORY_COMMON_PREFIX_SEARCH before, but /// it also binded to M-p/M-n). - rx.bind_key(Replxx::KEY::meta('N'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::COMPLETE_NEXT, _1)); - rx.bind_key(Replxx::KEY::meta('P'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::COMPLETE_PREVIOUS, _1)); + rx.bind_key(Replxx::KEY::meta('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_NEXT, code); }); + rx.bind_key(Replxx::KEY::meta('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_PREVIOUS, code); }); } ReplxxLineReader::~ReplxxLineReader() diff --git a/base/common/argsToConfig.cpp b/base/common/argsToConfig.cpp index e6b65c7bb01..36801bea7a8 100644 --- a/base/common/argsToConfig.cpp +++ b/base/common/argsToConfig.cpp @@ -11,7 +11,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4 Poco::AutoPtr map_config = new Poco::Util::MapConfiguration; std::string key; - for (auto & arg : argv) + for (const auto & arg : argv) { auto key_start = arg.find_first_not_of('-'); auto pos_minus = arg.find('-'); diff --git a/base/common/phdr_cache.cpp b/base/common/phdr_cache.cpp index fc81c20e8dd..cef386a85b8 100644 --- a/base/common/phdr_cache.cpp +++ b/base/common/phdr_cache.cpp @@ -70,7 +70,7 @@ extern "C" #endif int dl_iterate_phdr(int (*callback) (dl_phdr_info * info, size_t size, void * data), void * data) { - auto current_phdr_cache = phdr_cache.load(); + auto * current_phdr_cache = phdr_cache.load(); if (!current_phdr_cache) { // Cache is not yet populated, pass through to the original function. diff --git a/base/common/strong_typedef.h b/base/common/strong_typedef.h index ae502b4ef97..a46eb415e15 100644 --- a/base/common/strong_typedef.h +++ b/base/common/strong_typedef.h @@ -1,15 +1,9 @@ #pragma once -#include #include -/** https://svn.boost.org/trac/boost/ticket/5182 - */ - template struct StrongTypedef - : boost::totally_ordered1< StrongTypedef - , boost::totally_ordered2< StrongTypedef, T> > { private: using Self = StrongTypedef; diff --git a/base/common/types.h b/base/common/types.h index 8f125146add..238e4e3956b 100644 --- a/base/common/types.h +++ b/base/common/types.h @@ -11,6 +11,10 @@ using Int16 = int16_t; using Int32 = int32_t; using Int64 = int64_t; +#if __cplusplus <= 201703L +using char8_t = unsigned char; +#endif + using UInt8 = char8_t; using UInt16 = uint16_t; using UInt32 = uint32_t; diff --git a/base/common/ya.make b/base/common/ya.make index a41c8f4c583..31a66893330 100644 --- a/base/common/ya.make +++ b/base/common/ya.make @@ -1,12 +1,47 @@ LIBRARY() +ADDINCL( + GLOBAL clickhouse/base + contrib/libs/cctz/include +) + +CFLAGS (GLOBAL -DARCADIA_BUILD) + +IF (OS_DARWIN) + CFLAGS (GLOBAL -DOS_DARWIN) +ELSEIF (OS_FREEBSD) + CFLAGS (GLOBAL -DOS_FREEBSD) +ELSEIF (OS_LINUX) + CFLAGS (GLOBAL -DOS_LINUX) +ENDIF () + PEERDIR( + contrib/libs/cctz/src + contrib/libs/cxxsupp/libcxx-filesystem + contrib/libs/poco/Net contrib/libs/poco/Util + contrib/restricted/boost + contrib/restricted/cityhash-1.0.2 ) SRCS( argsToConfig.cpp coverage.cpp + DateLUT.cpp + DateLUTImpl.cpp + demangle.cpp + getFQDNOrHostName.cpp + getMemoryAmount.cpp + getThreadId.cpp + JSON.cpp + LineReader.cpp + mremap.cpp + phdr_cache.cpp + preciseExp10.c + setTerminalEcho.cpp + shift10.cpp + sleep.cpp + terminalColors.cpp ) END() diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index c150dc03014..c7012a3ba04 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -50,11 +50,13 @@ #include #include #include -#include -#ifdef __APPLE__ -// ucontext is not available without _XOPEN_SOURCE -#define _XOPEN_SOURCE 700 +#if !defined(ARCADIA_BUILD) +# include +#endif + +#if defined(OS_DARWIN) +# define _XOPEN_SOURCE 700 // ucontext is not available without _XOPEN_SOURCE #endif #include @@ -231,7 +233,6 @@ private: Logger * log; BaseDaemon & daemon; -private: void onTerminate(const std::string & message, UInt32 thread_num) const { LOG_FATAL(log, "(version " << VERSION_STRING << VERSION_OFFICIAL << ") (from thread " << thread_num << ") " << message); @@ -410,7 +411,7 @@ std::string BaseDaemon::getDefaultCorePath() const void BaseDaemon::closeFDs() { -#if defined(__FreeBSD__) || (defined(__APPLE__) && defined(__MACH__)) +#if defined(OS_FREEBSD) || defined(OS_DARWIN) Poco::File proc_path{"/dev/fd"}; #else Poco::File proc_path{"/proc/self/fd"}; @@ -430,7 +431,7 @@ void BaseDaemon::closeFDs() else { int max_fd = -1; -#ifdef _SC_OPEN_MAX +#if defined(_SC_OPEN_MAX) max_fd = sysconf(_SC_OPEN_MAX); if (max_fd == -1) #endif @@ -448,7 +449,7 @@ namespace /// the maximum is 1000, and chromium uses 300 for its tab processes. Ignore /// whatever errors that occur, because it's just a debugging aid and we don't /// care if it breaks. -#if defined(__linux__) && !defined(NDEBUG) +#if defined(OS_LINUX) && !defined(NDEBUG) void debugIncreaseOOMScore() { const std::string new_score = "555"; diff --git a/base/daemon/ya.make b/base/daemon/ya.make new file mode 100644 index 00000000000..1c72af3ed53 --- /dev/null +++ b/base/daemon/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +NO_COMPILER_WARNINGS() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + BaseDaemon.cpp + GraphiteWriter.cpp +) + +END() diff --git a/base/loggers/OwnPatternFormatter.cpp b/base/loggers/OwnPatternFormatter.cpp index 1f918f01697..029d06ff949 100644 --- a/base/loggers/OwnPatternFormatter.cpp +++ b/base/loggers/OwnPatternFormatter.cpp @@ -75,7 +75,11 @@ void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext, if (color) writeCString(resetColor(), wb); writeCString("> ", wb); + if (color) + writeString(setColor(std::hash()(msg.getSource())), wb); DB::writeString(msg.getSource(), wb); + if (color) + writeCString(resetColor(), wb); writeCString(": ", wb); DB::writeString(msg.getText(), wb); } diff --git a/base/loggers/OwnSplitChannel.cpp b/base/loggers/OwnSplitChannel.cpp index eda61e1105e..3f77b594fbb 100644 --- a/base/loggers/OwnSplitChannel.cpp +++ b/base/loggers/OwnSplitChannel.cpp @@ -20,7 +20,7 @@ void OwnSplitChannel::log(const Poco::Message & msg) if (channels.empty() && (logs_queue == nullptr || msg.getPriority() > logs_queue->max_priority)) return; - if (auto masker = SensitiveDataMasker::getInstance()) + if (auto * masker = SensitiveDataMasker::getInstance()) { auto message_text = msg.getText(); auto matches = masker->wipeSensitiveData(message_text); diff --git a/base/loggers/ya.make b/base/loggers/ya.make new file mode 100644 index 00000000000..b1c84042eee --- /dev/null +++ b/base/loggers/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + ExtendedLogChannel.cpp + Loggers.cpp + OwnFormattingChannel.cpp + OwnPatternFormatter.cpp + OwnSplitChannel.cpp +) + +END() diff --git a/base/pcg-random/CMakeLists.txt b/base/pcg-random/CMakeLists.txt new file mode 100644 index 00000000000..88acabba6a7 --- /dev/null +++ b/base/pcg-random/CMakeLists.txt @@ -0,0 +1,2 @@ +add_library(pcg_random INTERFACE) +target_include_directories(pcg_random INTERFACE .) diff --git a/contrib/libpcg-random/LICENSE-APACHE.txt b/base/pcg-random/LICENSE similarity index 100% rename from contrib/libpcg-random/LICENSE-APACHE.txt rename to base/pcg-random/LICENSE diff --git a/contrib/libpcg-random/README b/base/pcg-random/README similarity index 100% rename from contrib/libpcg-random/README rename to base/pcg-random/README diff --git a/contrib/libpcg-random/include/pcg_extras.hpp b/base/pcg-random/pcg_extras.hpp similarity index 98% rename from contrib/libpcg-random/include/pcg_extras.hpp rename to base/pcg-random/pcg_extras.hpp index 929c756b151..118b726dd57 100644 --- a/contrib/libpcg-random/include/pcg_extras.hpp +++ b/base/pcg-random/pcg_extras.hpp @@ -292,7 +292,7 @@ inline itype rotl(itype value, bitcount_t rot) { constexpr bitcount_t bits = sizeof(itype) * 8; constexpr bitcount_t mask = bits - 1; -#if PCG_USE_ZEROCHECK_ROTATE_IDIOM +#if defined(PCG_USE_ZEROCHECK_ROTATE_IDIOM) return rot ? (value << rot) | (value >> (bits - rot)) : value; #else return (value << rot) | (value >> ((- rot) & mask)); @@ -304,7 +304,7 @@ inline itype rotr(itype value, bitcount_t rot) { constexpr bitcount_t bits = sizeof(itype) * 8; constexpr bitcount_t mask = bits - 1; -#if PCG_USE_ZEROCHECK_ROTATE_IDIOM +#if defined(PCG_USE_ZEROCHECK_ROTATE_IDIOM) return rot ? (value >> rot) | (value << (bits - rot)) : value; #else return (value >> rot) | (value << ((- rot) & mask)); @@ -318,7 +318,7 @@ inline itype rotr(itype value, bitcount_t rot) * * These overloads will be preferred over the general template code above. */ -#if PCG_USE_INLINE_ASM && __GNUC__ && (__x86_64__ || __i386__) +#if defined(PCG_USE_INLINE_ASM) && __GNUC__ && (__x86_64__ || __i386__) inline uint8_t rotr(uint8_t value, bitcount_t rot) { @@ -600,7 +600,7 @@ std::ostream& operator<<(std::ostream& out, printable_typename) { #ifdef __GNUC__ int status; char* pretty_name = - abi::__cxa_demangle(implementation_typename, NULL, NULL, &status); + abi::__cxa_demangle(implementation_typename, nullptr, nullptr, &status); if (status == 0) out << pretty_name; free(static_cast(pretty_name)); diff --git a/contrib/libpcg-random/include/pcg_random.hpp b/base/pcg-random/pcg_random.hpp similarity index 100% rename from contrib/libpcg-random/include/pcg_random.hpp rename to base/pcg-random/pcg_random.hpp diff --git a/contrib/libpcg-random/include/pcg_uint128.hpp b/base/pcg-random/pcg_uint128.hpp similarity index 100% rename from contrib/libpcg-random/include/pcg_uint128.hpp rename to base/pcg-random/pcg_uint128.hpp diff --git a/base/pcg-random/ya.make b/base/pcg-random/ya.make new file mode 100644 index 00000000000..c6a50887178 --- /dev/null +++ b/base/pcg-random/ya.make @@ -0,0 +1,5 @@ +LIBRARY() + +ADDINCL (GLOBAL clickhouse/base/pcg-random) + +END() diff --git a/contrib/widecharwidth/CMakeLists.txt b/base/widechar_width/CMakeLists.txt similarity index 100% rename from contrib/widecharwidth/CMakeLists.txt rename to base/widechar_width/CMakeLists.txt diff --git a/contrib/widecharwidth/LICENSE b/base/widechar_width/LICENSE similarity index 100% rename from contrib/widecharwidth/LICENSE rename to base/widechar_width/LICENSE diff --git a/contrib/widecharwidth/README.md b/base/widechar_width/README.md similarity index 100% rename from contrib/widecharwidth/README.md rename to base/widechar_width/README.md diff --git a/contrib/widecharwidth/widechar_width.cpp b/base/widechar_width/widechar_width.cpp similarity index 100% rename from contrib/widecharwidth/widechar_width.cpp rename to base/widechar_width/widechar_width.cpp diff --git a/contrib/widecharwidth/widechar_width.h b/base/widechar_width/widechar_width.h similarity index 100% rename from contrib/widecharwidth/widechar_width.h rename to base/widechar_width/widechar_width.h diff --git a/base/widechar_width/ya.make b/base/widechar_width/ya.make new file mode 100644 index 00000000000..fa0b4f705db --- /dev/null +++ b/base/widechar_width/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +ADDINCL(GLOBAL clickhouse/base/widechar_width) + +SRCS( + widechar_width.cpp +) + +END() diff --git a/base/ya.make b/base/ya.make index 25ab5886b2c..004da9af2ae 100644 --- a/base/ya.make +++ b/base/ya.make @@ -1,3 +1,7 @@ RECURSE( common + daemon + loggers + pcg-random + widechar_width ) diff --git a/cmake/find/msgpack.cmake b/cmake/find/msgpack.cmake new file mode 100644 index 00000000000..46344fc162f --- /dev/null +++ b/cmake/find/msgpack.cmake @@ -0,0 +1,17 @@ +option (USE_INTERNAL_MSGPACK_LIBRARY "Set to FALSE to use system msgpack library instead of bundled" ${NOT_UNBUNDLED}) + +if (USE_INTERNAL_MSGPACK_LIBRARY) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include/msgpack.hpp") + message(WARNING "Submodule contrib/msgpack-c is missing. To fix try run: \n git submodule update --init --recursive") + set(USE_INTERNAL_MSGPACK_LIBRARY 0) + set(MISSING_INTERNAL_MSGPACK_LIBRARY 1) + endif() +endif() + +if (USE_INTERNAL_MSGPACK_LIBRARY) + set(MSGPACK_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include) +else() + find_path(MSGPACK_INCLUDE_DIR NAMES msgpack.hpp PATHS ${MSGPACK_INCLUDE_PATHS}) +endif() + +message(STATUS "Using msgpack: ${MSGPACK_INCLUDE_DIR}") diff --git a/cmake/freebsd/default_libs.cmake b/cmake/freebsd/default_libs.cmake index 2bb76c6a761..d60df52bc6d 100644 --- a/cmake/freebsd/default_libs.cmake +++ b/cmake/freebsd/default_libs.cmake @@ -4,7 +4,11 @@ if (NOT COMPILER_CLANG) message (FATAL_ERROR "FreeBSD build is supported only for Clang") endif () -execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) +if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64") + execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) +else () + execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) +endif () set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread") diff --git a/cmake/lib_name.cmake b/cmake/lib_name.cmake index 8f5bebf4abe..f18b2e52576 100644 --- a/cmake/lib_name.cmake +++ b/cmake/lib_name.cmake @@ -2,4 +2,3 @@ set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide) set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/src ${ClickHouse_BINARY_DIR}/src) set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion) set(METROHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src) -set(PCG_RANDOM_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libpcg-random/include) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 8eccbfd9f83..5eeec3b2f0a 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -6,18 +6,18 @@ endif () if (COMPILER_GCC) # Require minimum version of gcc - set (GCC_MINIMUM_VERSION 8) + set (GCC_MINIMUM_VERSION 9) if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9) message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.") endif () elseif (COMPILER_CLANG) # Require minimum version of clang - set (CLANG_MINIMUM_VERSION 7) + set (CLANG_MINIMUM_VERSION 8) if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.") endif () else () - message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.") + message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang and GCC.") endif () STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION}) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9c0a0aac0dd..1d438e29dae 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -333,6 +333,5 @@ add_subdirectory(grpc-cmake) add_subdirectory(replxx-cmake) add_subdirectory(FastMemcpy) -add_subdirectory(widecharwidth) add_subdirectory(consistent-hashing) add_subdirectory(consistent-hashing-sumbur) diff --git a/contrib/cctz b/contrib/cctz index 4f9776a310f..5a3f785329c 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit 4f9776a310f4952454636363def82c2bf6641d5f +Subproject commit 5a3f785329cecdd2b68cd950e0647e9246774ef2 diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt index 5770b1ef09c..434b8ffb4ae 100644 --- a/contrib/cctz-cmake/CMakeLists.txt +++ b/contrib/cctz-cmake/CMakeLists.txt @@ -23,6 +23,600 @@ if (USE_INTERNAL_CCTZ) # yes, need linux, because bsd check inside linux in time_zone_libc.cc:24 target_compile_definitions (cctz PRIVATE __USE_BSD linux _XOPEN_SOURCE=600) endif () + + # Build a libray with embedded tzdata + + # We invoke 'ld' and 'objcopy' directly because lld linker has no option to generate object file with binary data. + # Note: we can invoke specific ld from toolchain and relax condition on ARCH_AMD64. + + if (OS_LINUX AND ARCH_AMD64) + + set (TIMEZONES + Africa/Abidjan + Africa/Accra + Africa/Addis_Ababa + Africa/Algiers + Africa/Asmara + Africa/Asmera + Africa/Bamako + Africa/Bangui + Africa/Banjul + Africa/Bissau + Africa/Blantyre + Africa/Brazzaville + Africa/Bujumbura + Africa/Cairo + Africa/Casablanca + Africa/Ceuta + Africa/Conakry + Africa/Dakar + Africa/Dar_es_Salaam + Africa/Djibouti + Africa/Douala + Africa/El_Aaiun + Africa/Freetown + Africa/Gaborone + Africa/Harare + Africa/Johannesburg + Africa/Juba + Africa/Kampala + Africa/Khartoum + Africa/Kigali + Africa/Kinshasa + Africa/Lagos + Africa/Libreville + Africa/Lome + Africa/Luanda + Africa/Lubumbashi + Africa/Lusaka + Africa/Malabo + Africa/Maputo + Africa/Maseru + Africa/Mbabane + Africa/Mogadishu + Africa/Monrovia + Africa/Nairobi + Africa/Ndjamena + Africa/Niamey + Africa/Nouakchott + Africa/Ouagadougou + Africa/Porto-Novo + Africa/Sao_Tome + Africa/Timbuktu + Africa/Tripoli + Africa/Tunis + Africa/Windhoek + America/Adak + America/Anchorage + America/Anguilla + America/Antigua + America/Araguaina + America/Argentina/Buenos_Aires + America/Argentina/Catamarca + America/Argentina/ComodRivadavia + America/Argentina/Cordoba + America/Argentina/Jujuy + America/Argentina/La_Rioja + America/Argentina/Mendoza + America/Argentina/Rio_Gallegos + America/Argentina/Salta + America/Argentina/San_Juan + America/Argentina/San_Luis + America/Argentina/Tucuman + America/Argentina/Ushuaia + America/Aruba + America/Asuncion + America/Atikokan + America/Atka + America/Bahia + America/Bahia_Banderas + America/Barbados + America/Belem + America/Belize + America/Blanc-Sablon + America/Boa_Vista + America/Bogota + America/Boise + America/Buenos_Aires + America/Cambridge_Bay + America/Campo_Grande + America/Cancun + America/Caracas + America/Catamarca + America/Cayenne + America/Cayman + America/Chicago + America/Chihuahua + America/Coral_Harbour + America/Cordoba + America/Costa_Rica + America/Creston + America/Cuiaba + America/Curacao + America/Danmarkshavn + America/Dawson + America/Dawson_Creek + America/Denver + America/Detroit + America/Dominica + America/Edmonton + America/Eirunepe + America/El_Salvador + America/Ensenada + America/Fortaleza + America/Fort_Nelson + America/Fort_Wayne + America/Glace_Bay + America/Godthab + America/Goose_Bay + America/Grand_Turk + America/Grenada + America/Guadeloupe + America/Guatemala + America/Guayaquil + America/Guyana + America/Halifax + America/Havana + America/Hermosillo + America/Indiana/Indianapolis + America/Indiana/Knox + America/Indiana/Marengo + America/Indiana/Petersburg + America/Indianapolis + America/Indiana/Tell_City + America/Indiana/Vevay + America/Indiana/Vincennes + America/Indiana/Winamac + America/Inuvik + America/Iqaluit + America/Jamaica + America/Jujuy + America/Juneau + America/Kentucky/Louisville + America/Kentucky/Monticello + America/Knox_IN + America/Kralendijk + America/La_Paz + America/Lima + America/Los_Angeles + America/Louisville + America/Lower_Princes + America/Maceio + America/Managua + America/Manaus + America/Marigot + America/Martinique + America/Matamoros + America/Mazatlan + America/Mendoza + America/Menominee + America/Merida + America/Metlakatla + America/Mexico_City + America/Miquelon + America/Moncton + America/Monterrey + America/Montevideo + America/Montreal + America/Montserrat + America/Nassau + America/New_York + America/Nipigon + America/Nome + America/Noronha + America/North_Dakota/Beulah + America/North_Dakota/Center + America/North_Dakota/New_Salem + America/Ojinaga + America/Panama + America/Pangnirtung + America/Paramaribo + America/Phoenix + America/Port-au-Prince + America/Porto_Acre + America/Port_of_Spain + America/Porto_Velho + America/Puerto_Rico + America/Punta_Arenas + America/Rainy_River + America/Rankin_Inlet + America/Recife + America/Regina + America/Resolute + America/Rio_Branco + America/Rosario + America/Santa_Isabel + America/Santarem + America/Santiago + America/Santo_Domingo + America/Sao_Paulo + America/Scoresbysund + America/Shiprock + America/Sitka + America/St_Barthelemy + America/St_Johns + America/St_Kitts + America/St_Lucia + America/St_Thomas + America/St_Vincent + America/Swift_Current + America/Tegucigalpa + America/Thule + America/Thunder_Bay + America/Tijuana + America/Toronto + America/Tortola + America/Vancouver + America/Virgin + America/Whitehorse + America/Winnipeg + America/Yakutat + America/Yellowknife + Antarctica/Casey + Antarctica/Davis + Antarctica/DumontDUrville + Antarctica/Macquarie + Antarctica/Mawson + Antarctica/McMurdo + Antarctica/Palmer + Antarctica/Rothera + Antarctica/South_Pole + Antarctica/Syowa + Antarctica/Troll + Antarctica/Vostok + Arctic/Longyearbyen + Asia/Aden + Asia/Almaty + Asia/Amman + Asia/Anadyr + Asia/Aqtau + Asia/Aqtobe + Asia/Ashgabat + Asia/Ashkhabad + Asia/Atyrau + Asia/Baghdad + Asia/Bahrain + Asia/Baku + Asia/Bangkok + Asia/Barnaul + Asia/Beirut + Asia/Bishkek + Asia/Brunei + Asia/Calcutta + Asia/Chita + Asia/Choibalsan + Asia/Chongqing + Asia/Chungking + Asia/Colombo + Asia/Dacca + Asia/Damascus + Asia/Dhaka + Asia/Dili + Asia/Dubai + Asia/Dushanbe + Asia/Famagusta + Asia/Gaza + Asia/Harbin + Asia/Hebron + Asia/Ho_Chi_Minh + Asia/Hong_Kong + Asia/Hovd + Asia/Irkutsk + Asia/Istanbul + Asia/Jakarta + Asia/Jayapura + Asia/Jerusalem + Asia/Kabul + Asia/Kamchatka + Asia/Karachi + Asia/Kashgar + Asia/Kathmandu + Asia/Katmandu + Asia/Khandyga + Asia/Kolkata + Asia/Krasnoyarsk + Asia/Kuala_Lumpur + Asia/Kuching + Asia/Kuwait + Asia/Macao + Asia/Macau + Asia/Magadan + Asia/Makassar + Asia/Manila + Asia/Muscat + Asia/Nicosia + Asia/Novokuznetsk + Asia/Novosibirsk + Asia/Omsk + Asia/Oral + Asia/Phnom_Penh + Asia/Pontianak + Asia/Pyongyang + Asia/Qatar + Asia/Qostanay + Asia/Qyzylorda + Asia/Rangoon + Asia/Riyadh + Asia/Saigon + Asia/Sakhalin + Asia/Samarkand + Asia/Seoul + Asia/Shanghai + Asia/Singapore + Asia/Srednekolymsk + Asia/Taipei + Asia/Tashkent + Asia/Tbilisi + Asia/Tehran + Asia/Tel_Aviv + Asia/Thimbu + Asia/Thimphu + Asia/Tokyo + Asia/Tomsk + Asia/Ujung_Pandang + Asia/Ulaanbaatar + Asia/Ulan_Bator + Asia/Urumqi + Asia/Ust-Nera + Asia/Vientiane + Asia/Vladivostok + Asia/Yakutsk + Asia/Yangon + Asia/Yekaterinburg + Asia/Yerevan + Atlantic/Azores + Atlantic/Bermuda + Atlantic/Canary + Atlantic/Cape_Verde + Atlantic/Faeroe + Atlantic/Faroe + Atlantic/Jan_Mayen + Atlantic/Madeira + Atlantic/Reykjavik + Atlantic/South_Georgia + Atlantic/Stanley + Atlantic/St_Helena + Australia/ACT + Australia/Adelaide + Australia/Brisbane + Australia/Broken_Hill + Australia/Canberra + Australia/Currie + Australia/Darwin + Australia/Eucla + Australia/Hobart + Australia/LHI + Australia/Lindeman + Australia/Lord_Howe + Australia/Melbourne + Australia/North + Australia/NSW + Australia/Perth + Australia/Queensland + Australia/South + Australia/Sydney + Australia/Tasmania + Australia/Victoria + Australia/West + Australia/Yancowinna + Brazil/Acre + Brazil/DeNoronha + Brazil/East + Brazil/West + Canada/Atlantic + Canada/Central + Canada/Eastern + Canada/Mountain + Canada/Newfoundland + Canada/Pacific + Canada/Saskatchewan + Canada/Yukon + CET + Chile/Continental + Chile/EasterIsland + CST6CDT + Cuba + EET + Egypt + Eire + EST + EST5EDT + Etc/GMT + Etc/Greenwich + Etc/UCT + Etc/Universal + Etc/UTC + Etc/Zulu + Europe/Amsterdam + Europe/Andorra + Europe/Astrakhan + Europe/Athens + Europe/Belfast + Europe/Belgrade + Europe/Berlin + Europe/Bratislava + Europe/Brussels + Europe/Bucharest + Europe/Budapest + Europe/Busingen + Europe/Chisinau + Europe/Copenhagen + Europe/Dublin + Europe/Gibraltar + Europe/Guernsey + Europe/Helsinki + Europe/Isle_of_Man + Europe/Istanbul + Europe/Jersey + Europe/Kaliningrad + Europe/Kiev + Europe/Kirov + Europe/Lisbon + Europe/Ljubljana + Europe/London + Europe/Luxembourg + Europe/Madrid + Europe/Malta + Europe/Mariehamn + Europe/Minsk + Europe/Monaco + Europe/Moscow + Europe/Nicosia + Europe/Oslo + Europe/Paris + Europe/Podgorica + Europe/Prague + Europe/Riga + Europe/Rome + Europe/Samara + Europe/San_Marino + Europe/Sarajevo + Europe/Saratov + Europe/Simferopol + Europe/Skopje + Europe/Sofia + Europe/Stockholm + Europe/Tallinn + Europe/Tirane + Europe/Tiraspol + Europe/Ulyanovsk + Europe/Uzhgorod + Europe/Vaduz + Europe/Vatican + Europe/Vienna + Europe/Vilnius + Europe/Volgograd + Europe/Warsaw + Europe/Zagreb + Europe/Zaporozhye + Europe/Zurich + Factory + GB + GB-Eire + GMT + GMT0 + Greenwich + Hongkong + HST + Iceland + Indian/Antananarivo + Indian/Chagos + Indian/Christmas + Indian/Cocos + Indian/Comoro + Indian/Kerguelen + Indian/Mahe + Indian/Maldives + Indian/Mauritius + Indian/Mayotte + Indian/Reunion + Iran + Israel + Jamaica + Japan + Kwajalein + Libya + MET + Mexico/BajaNorte + Mexico/BajaSur + Mexico/General + MST + MST7MDT + Navajo + NZ + NZ-CHAT + Pacific/Apia + Pacific/Auckland + Pacific/Bougainville + Pacific/Chatham + Pacific/Chuuk + Pacific/Easter + Pacific/Efate + Pacific/Enderbury + Pacific/Fakaofo + Pacific/Fiji + Pacific/Funafuti + Pacific/Galapagos + Pacific/Gambier + Pacific/Guadalcanal + Pacific/Guam + Pacific/Honolulu + Pacific/Johnston + Pacific/Kiritimati + Pacific/Kosrae + Pacific/Kwajalein + Pacific/Majuro + Pacific/Marquesas + Pacific/Midway + Pacific/Nauru + Pacific/Niue + Pacific/Norfolk + Pacific/Noumea + Pacific/Pago_Pago + Pacific/Palau + Pacific/Pitcairn + Pacific/Pohnpei + Pacific/Ponape + Pacific/Port_Moresby + Pacific/Rarotonga + Pacific/Saipan + Pacific/Samoa + Pacific/Tahiti + Pacific/Tarawa + Pacific/Tongatapu + Pacific/Truk + Pacific/Wake + Pacific/Wallis + Pacific/Yap + Poland + Portugal + PRC + PST8PDT + ROC + ROK + Singapore + Turkey + UCT + Universal + US/Alaska + US/Aleutian + US/Arizona + US/Central + US/Eastern + US/East-Indiana + US/Hawaii + US/Indiana-Starke + US/Michigan + US/Mountain + US/Pacific + US/Samoa + UTC + WET + W-SU + Zulu) + + set(TZDIR ${LIBRARY_DIR}/testdata/zoneinfo) + set(TZ_OBJS) + + foreach(TIMEZONE ${TIMEZONES}) + string(REPLACE "/" "_" TIMEZONE_ID ${TIMEZONE}) + set(TZ_OBJ ${TIMEZONE_ID}.o) + set(TZ_OBJS ${TZ_OBJS} ${TZ_OBJ}) + + # https://stackoverflow.com/questions/14776463/compile-and-add-an-object-file-from-a-binary-with-cmake + add_custom_command(OUTPUT ${TZ_OBJ} + COMMAND cd ${TZDIR} && ld -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/${TZ_OBJ} ${TIMEZONE} + COMMAND objcopy --rename-section .data=.rodata,alloc,load,readonly,data,contents + ${CMAKE_CURRENT_BINARY_DIR}/${TZ_OBJ} ${CMAKE_CURRENT_BINARY_DIR}/${TZ_OBJ}) + + set_source_files_properties(${TZ_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true) + endforeach(TIMEZONE) + + add_library(tzdata STATIC ${TZ_OBJS}) + set_target_properties(tzdata PROPERTIES LINKER_LANGUAGE C) + target_link_libraries(cctz -Wl,--whole-archive tzdata -Wl,--no-whole-archive) # whole-archive prevents symbols from being discarded + endif () + else () find_library (LIBRARY_CCTZ cctz) find_path (INCLUDE_CCTZ NAMES cctz/civil_time.h) diff --git a/contrib/libc-headers b/contrib/libc-headers index 9676d2645a7..92c74f938cf 160000 --- a/contrib/libc-headers +++ b/contrib/libc-headers @@ -1 +1 @@ -Subproject commit 9676d2645a713e679dc981ffd84dee99fcd68b8e +Subproject commit 92c74f938cf2c4dd529cae4f3d2923d153b029a7 diff --git a/contrib/libdivide/libdivide.h b/contrib/libdivide/libdivide.h index eaeaec7db6b..a153e7f9c5e 100644 --- a/contrib/libdivide/libdivide.h +++ b/contrib/libdivide/libdivide.h @@ -1,117 +1,106 @@ -/* libdivide.h - Copyright 2010 ridiculous_fish -*/ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" +// libdivide.h - Optimized integer division +// https://libdivide.com +// +// Copyright (C) 2010 - 2019 ridiculous_fish, +// Copyright (C) 2016 - 2019 Kim Walisch, +// +// libdivide is dual-licensed under the Boost or zlib licenses. +// You may use libdivide under the terms of either of these. +// See LICENSE.txt for more details. -#if defined(_WIN32) || defined(WIN32) -#define LIBDIVIDE_WINDOWS 1 -#endif +#ifndef LIBDIVIDE_H +#define LIBDIVIDE_H -#if defined(_MSC_VER) -#define LIBDIVIDE_VC 1 -#endif +#define LIBDIVIDE_VERSION "3.0" +#define LIBDIVIDE_VERSION_MAJOR 3 +#define LIBDIVIDE_VERSION_MINOR 0 -#ifdef __cplusplus -#include -#include -#include -#else -#include -#include -#include -#endif - -#if ! LIBDIVIDE_HAS_STDINT_TYPES && (! LIBDIVIDE_VC || _MSC_VER >= 1600) -/* Only Visual C++ 2010 and later include stdint.h */ #include -#define LIBDIVIDE_HAS_STDINT_TYPES 1 + +#if defined(__cplusplus) + #include + #include + #include +#else + #include + #include #endif -#if ! LIBDIVIDE_HAS_STDINT_TYPES -typedef __int32 int32_t; -typedef unsigned __int32 uint32_t; -typedef __int64 int64_t; -typedef unsigned __int64 uint64_t; -typedef __int8 int8_t; -typedef unsigned __int8 uint8_t; -#endif - -#if LIBDIVIDE_USE_SSE2 +#if defined(LIBDIVIDE_AVX512) + #include +#elif defined(LIBDIVIDE_AVX2) + #include +#elif defined(LIBDIVIDE_SSE2) #include #endif -#if LIBDIVIDE_VC +#if defined(_MSC_VER) #include + // disable warning C4146: unary minus operator applied + // to unsigned type, result still unsigned + #pragma warning(disable: 4146) + #define LIBDIVIDE_VC #endif -#ifndef __has_builtin -#define __has_builtin(x) 0 // Compatibility with non-clang compilers. +#if !defined(__has_builtin) + #define __has_builtin(x) 0 #endif -#ifdef __ICC -#define HAS_INT128_T 0 -#else -#define HAS_INT128_T __LP64__ +#if defined(__SIZEOF_INT128__) + #define HAS_INT128_T + // clang-cl on Windows does not yet support 128-bit division + #if !(defined(__clang__) && defined(LIBDIVIDE_VC)) + #define HAS_INT128_DIV + #endif #endif -#if defined(__x86_64__) || defined(_WIN64) || defined(_M_64) -#define LIBDIVIDE_IS_X86_64 1 +#if defined(__x86_64__) || defined(_M_X64) + #define LIBDIVIDE_X86_64 #endif #if defined(__i386__) -#define LIBDIVIDE_IS_i386 1 + #define LIBDIVIDE_i386 #endif -#if __GNUC__ || __clang__ -#define LIBDIVIDE_GCC_STYLE_ASM 1 +#if defined(__GNUC__) || defined(__clang__) + #define LIBDIVIDE_GCC_STYLE_ASM #endif +#if defined(__cplusplus) || defined(LIBDIVIDE_VC) + #define LIBDIVIDE_FUNCTION __FUNCTION__ +#else + #define LIBDIVIDE_FUNCTION __func__ +#endif -/* libdivide may use the pmuldq (vector signed 32x32->64 mult instruction) which is in SSE 4.1. However, signed multiplication can be emulated efficiently with unsigned multiplication, and SSE 4.1 is currently rare, so it is OK to not turn this on */ -#ifdef LIBDIVIDE_USE_SSE4_1 -#include +#define LIBDIVIDE_ERROR(msg) \ + do { \ + fprintf(stderr, "libdivide.h:%d: %s(): Error: %s\n", \ + __LINE__, LIBDIVIDE_FUNCTION, msg); \ + exit(-1); \ + } while (0) + +#if defined(LIBDIVIDE_ASSERTIONS_ON) + #define LIBDIVIDE_ASSERT(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "libdivide.h:%d: %s(): Assertion failed: %s\n", \ + __LINE__, LIBDIVIDE_FUNCTION, #x); \ + exit(-1); \ + } \ + } while (0) +#else + #define LIBDIVIDE_ASSERT(x) #endif #ifdef __cplusplus -/* We place libdivide within the libdivide namespace, and that goes in an anonymous namespace so that the functions are only visible to files that #include this header and don't get external linkage. At least that's the theory. */ -namespace { namespace libdivide { #endif -/* Explanation of "more" field: bit 6 is whether to use shift path. If we are using the shift path, bit 7 is whether the divisor is negative in the signed case; in the unsigned case it is 0. Bits 0-4 is shift value (for shift path or mult path). In 32 bit case, bit 5 is always 0. We use bit 7 as the "negative divisor indicator" so that we can use sign extension to efficiently go to a full-width -1. - - -u32: [0-4] shift value - [5] ignored - [6] add indicator - [7] shift path - -s32: [0-4] shift value - [5] shift path - [6] add indicator - [7] indicates negative divisor - -u64: [0-5] shift value - [6] add indicator - [7] shift path - -s64: [0-5] shift value - [6] add indicator - [7] indicates negative divisor - magic number of 0 indicates shift path (we ran out of bits!) -*/ - -enum { - LIBDIVIDE_32_SHIFT_MASK = 0x1F, - LIBDIVIDE_64_SHIFT_MASK = 0x3F, - LIBDIVIDE_ADD_MARKER = 0x40, - LIBDIVIDE_U32_SHIFT_PATH = 0x80, - LIBDIVIDE_U64_SHIFT_PATH = 0x80, - LIBDIVIDE_S32_SHIFT_PATH = 0x20, - LIBDIVIDE_NEGATIVE_DIVISOR = 0x80 -}; - +// pack divider structs to prevent compilers from padding. +// This reduces memory usage by up to 43% when using a large +// array of libdivide dividers and improves performance +// by up to 10% because of reduced memory bandwidth. +#pragma pack(push, 1) struct libdivide_u32_t { uint32_t magic; @@ -133,497 +122,446 @@ struct libdivide_s64_t { uint8_t more; }; +struct libdivide_u32_branchfree_t { + uint32_t magic; + uint8_t more; +}; +struct libdivide_s32_branchfree_t { + int32_t magic; + uint8_t more; +}; -#ifndef LIBDIVIDE_API - #ifdef __cplusplus - /* In C++, we don't want our public functions to be static, because they are arguments to templates and static functions can't do that. They get internal linkage through virtue of the anonymous namespace. In C, they should be static. */ - #define LIBDIVIDE_API - #else - #define LIBDIVIDE_API static - #endif -#endif +struct libdivide_u64_branchfree_t { + uint64_t magic; + uint8_t more; +}; -#ifdef __APPLE__ -typedef signed long Int64; -typedef unsigned long UInt64; -#endif +struct libdivide_s64_branchfree_t { + int64_t magic; + uint8_t more; +}; -LIBDIVIDE_API struct libdivide_s32_t libdivide_s32_gen(int32_t y); -LIBDIVIDE_API struct libdivide_u32_t libdivide_u32_gen(uint32_t y); -LIBDIVIDE_API struct libdivide_s64_t libdivide_s64_gen(int64_t y); -LIBDIVIDE_API struct libdivide_u64_t libdivide_u64_gen(uint64_t y); -#if defined(__APPLE__) && defined(__cplusplus) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -LIBDIVIDE_API struct libdivide_s64_t libdivide_s64_gen(Int64 y) { return libdivide_s64_gen(int64_t(y)); }; -LIBDIVIDE_API struct libdivide_u64_t libdivide_u64_gen(UInt64 y) { return libdivide_u64_gen(uint64_t(y)); }; -#pragma GCC diagnostic pop -#endif +#pragma pack(pop) -LIBDIVIDE_API int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom); -LIBDIVIDE_API uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom); -LIBDIVIDE_API int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom); -LIBDIVIDE_API uint64_t libdivide_u64_do(uint64_t y, const struct libdivide_u64_t *denom); -#if defined(__APPLE__) && defined(__cplusplus) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -LIBDIVIDE_API Int64 libdivide_s64_do(Int64 numer, const struct libdivide_s64_t *denom) { return Int64(libdivide_s64_do(int64_t(numer), denom)); }; -LIBDIVIDE_API UInt64 libdivide_u64_do(UInt64 y, const struct libdivide_u64_t *denom) { return UInt64(libdivide_u64_do(uint64_t(y), denom)); }; -#pragma GCC diagnostic pop -#endif +// Explanation of the "more" field: +// +// * Bits 0-5 is the shift value (for shift path or mult path). +// * Bit 6 is the add indicator for mult path. +// * Bit 7 is set if the divisor is negative. We use bit 7 as the negative +// divisor indicator so that we can efficiently use sign extension to +// create a bitmask with all bits set to 1 (if the divisor is negative) +// or 0 (if the divisor is positive). +// +// u32: [0-4] shift value +// [5] ignored +// [6] add indicator +// magic number of 0 indicates shift path +// +// s32: [0-4] shift value +// [5] ignored +// [6] add indicator +// [7] indicates negative divisor +// magic number of 0 indicates shift path +// +// u64: [0-5] shift value +// [6] add indicator +// magic number of 0 indicates shift path +// +// s64: [0-5] shift value +// [6] add indicator +// [7] indicates negative divisor +// magic number of 0 indicates shift path +// +// In s32 and s64 branchfree modes, the magic number is negated according to +// whether the divisor is negated. In branchfree strategy, it is not negated. -LIBDIVIDE_API int libdivide_u32_get_algorithm(const struct libdivide_u32_t *denom); -LIBDIVIDE_API uint32_t libdivide_u32_do_alg0(uint32_t numer, const struct libdivide_u32_t *denom); -LIBDIVIDE_API uint32_t libdivide_u32_do_alg1(uint32_t numer, const struct libdivide_u32_t *denom); -LIBDIVIDE_API uint32_t libdivide_u32_do_alg2(uint32_t numer, const struct libdivide_u32_t *denom); +enum { + LIBDIVIDE_32_SHIFT_MASK = 0x1F, + LIBDIVIDE_64_SHIFT_MASK = 0x3F, + LIBDIVIDE_ADD_MARKER = 0x40, + LIBDIVIDE_NEGATIVE_DIVISOR = 0x80 +}; -LIBDIVIDE_API int libdivide_u64_get_algorithm(const struct libdivide_u64_t *denom); -LIBDIVIDE_API uint64_t libdivide_u64_do_alg0(uint64_t numer, const struct libdivide_u64_t *denom); -LIBDIVIDE_API uint64_t libdivide_u64_do_alg1(uint64_t numer, const struct libdivide_u64_t *denom); -LIBDIVIDE_API uint64_t libdivide_u64_do_alg2(uint64_t numer, const struct libdivide_u64_t *denom); -#if defined(__APPLE__) && defined(__cplusplus) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -LIBDIVIDE_API UInt64 libdivide_u64_do_alg0(UInt64 numer, const struct libdivide_u64_t *denom) { return UInt64(libdivide_u64_do_alg0(uint64_t(numer), denom)); } -LIBDIVIDE_API UInt64 libdivide_u64_do_alg1(UInt64 numer, const struct libdivide_u64_t *denom) { return UInt64(libdivide_u64_do_alg1(uint64_t(numer), denom)); } -LIBDIVIDE_API UInt64 libdivide_u64_do_alg2(UInt64 numer, const struct libdivide_u64_t *denom) { return UInt64(libdivide_u64_do_alg2(uint64_t(numer), denom)); } -#pragma GCC diagnostic pop -#endif +static inline struct libdivide_s32_t libdivide_s32_gen(int32_t d); +static inline struct libdivide_u32_t libdivide_u32_gen(uint32_t d); +static inline struct libdivide_s64_t libdivide_s64_gen(int64_t d); +static inline struct libdivide_u64_t libdivide_u64_gen(uint64_t d); -LIBDIVIDE_API int libdivide_s32_get_algorithm(const struct libdivide_s32_t *denom); -LIBDIVIDE_API int32_t libdivide_s32_do_alg0(int32_t numer, const struct libdivide_s32_t *denom); -LIBDIVIDE_API int32_t libdivide_s32_do_alg1(int32_t numer, const struct libdivide_s32_t *denom); -LIBDIVIDE_API int32_t libdivide_s32_do_alg2(int32_t numer, const struct libdivide_s32_t *denom); -LIBDIVIDE_API int32_t libdivide_s32_do_alg3(int32_t numer, const struct libdivide_s32_t *denom); -LIBDIVIDE_API int32_t libdivide_s32_do_alg4(int32_t numer, const struct libdivide_s32_t *denom); +static inline struct libdivide_s32_branchfree_t libdivide_s32_branchfree_gen(int32_t d); +static inline struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d); +static inline struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d); +static inline struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d); -LIBDIVIDE_API int libdivide_s64_get_algorithm(const struct libdivide_s64_t *denom); -LIBDIVIDE_API int64_t libdivide_s64_do_alg0(int64_t numer, const struct libdivide_s64_t *denom); -LIBDIVIDE_API int64_t libdivide_s64_do_alg1(int64_t numer, const struct libdivide_s64_t *denom); -LIBDIVIDE_API int64_t libdivide_s64_do_alg2(int64_t numer, const struct libdivide_s64_t *denom); -LIBDIVIDE_API int64_t libdivide_s64_do_alg3(int64_t numer, const struct libdivide_s64_t *denom); -LIBDIVIDE_API int64_t libdivide_s64_do_alg4(int64_t numer, const struct libdivide_s64_t *denom); -#if defined(__APPLE__) && defined(__cplusplus) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -LIBDIVIDE_API Int64 libdivide_s64_do_alg0(Int64 numer, const struct libdivide_s64_t *denom) { return Int64(libdivide_s64_do_alg0(int64_t(numer), denom)); } -LIBDIVIDE_API Int64 libdivide_s64_do_alg1(Int64 numer, const struct libdivide_s64_t *denom) { return Int64(libdivide_s64_do_alg1(int64_t(numer), denom)); } -LIBDIVIDE_API Int64 libdivide_s64_do_alg2(Int64 numer, const struct libdivide_s64_t *denom) { return Int64(libdivide_s64_do_alg2(int64_t(numer), denom)); } -LIBDIVIDE_API Int64 libdivide_s64_do_alg3(Int64 numer, const struct libdivide_s64_t *denom) { return Int64(libdivide_s64_do_alg3(int64_t(numer), denom)); } -LIBDIVIDE_API Int64 libdivide_s64_do_alg4(Int64 numer, const struct libdivide_s64_t *denom) { return Int64(libdivide_s64_do_alg4(int64_t(numer), denom)); } -#pragma GCC diagnostic pop -#endif +static inline int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom); +static inline uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom); +static inline int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom); +static inline uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom); +static inline int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom); +static inline uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom); +static inline int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom); +static inline uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom); -#if LIBDIVIDE_USE_SSE2 -LIBDIVIDE_API __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t * denom); - -LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg0(__m128i numers, const struct libdivide_u32_t * denom); -LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg1(__m128i numers, const struct libdivide_u32_t * denom); -LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg2(__m128i numers, const struct libdivide_u32_t * denom); - -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg0(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg1(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg2(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg3(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg4(__m128i numers, const struct libdivide_s32_t * denom); - -LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg0(__m128i numers, const struct libdivide_u64_t * denom); -LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg1(__m128i numers, const struct libdivide_u64_t * denom); -LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg2(__m128i numers, const struct libdivide_u64_t * denom); - -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg0(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg1(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg2(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg3(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg4(__m128i numers, const struct libdivide_s64_t * denom); -#endif - +static inline int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom); +static inline uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom); +static inline int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom); +static inline uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom); +static inline int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom); +static inline uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom); +static inline int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom); +static inline uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom); //////// Internal Utility Functions -static inline uint32_t libdivide__mullhi_u32(uint32_t x, uint32_t y) { +static inline uint32_t libdivide_mullhi_u32(uint32_t x, uint32_t y) { uint64_t xl = x, yl = y; uint64_t rl = xl * yl; return (uint32_t)(rl >> 32); } -static uint64_t libdivide__mullhi_u64(uint64_t x, uint64_t y) { -#if HAS_INT128_T +static inline int32_t libdivide_mullhi_s32(int32_t x, int32_t y) { + int64_t xl = x, yl = y; + int64_t rl = xl * yl; + // needs to be arithmetic shift + return (int32_t)(rl >> 32); +} + +static inline uint64_t libdivide_mullhi_u64(uint64_t x, uint64_t y) { +#if defined(LIBDIVIDE_VC) && \ + defined(LIBDIVIDE_X86_64) + return __umulh(x, y); +#elif defined(HAS_INT128_T) __uint128_t xl = x, yl = y; __uint128_t rl = xl * yl; return (uint64_t)(rl >> 64); #else - //full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) - const uint32_t mask = 0xFFFFFFFF; - const uint32_t x0 = (uint32_t)(x & mask), x1 = (uint32_t)(x >> 32); - const uint32_t y0 = (uint32_t)(y & mask), y1 = (uint32_t)(y >> 32); - const uint32_t x0y0_hi = libdivide__mullhi_u32(x0, y0); - const uint64_t x0y1 = x0 * (uint64_t)y1; - const uint64_t x1y0 = x1 * (uint64_t)y0; - const uint64_t x1y1 = x1 * (uint64_t)y1; - + // full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) + uint32_t mask = 0xFFFFFFFF; + uint32_t x0 = (uint32_t)(x & mask); + uint32_t x1 = (uint32_t)(x >> 32); + uint32_t y0 = (uint32_t)(y & mask); + uint32_t y1 = (uint32_t)(y >> 32); + uint32_t x0y0_hi = libdivide_mullhi_u32(x0, y0); + uint64_t x0y1 = x0 * (uint64_t)y1; + uint64_t x1y0 = x1 * (uint64_t)y0; + uint64_t x1y1 = x1 * (uint64_t)y1; uint64_t temp = x1y0 + x0y0_hi; - uint64_t temp_lo = temp & mask, temp_hi = temp >> 32; + uint64_t temp_lo = temp & mask; + uint64_t temp_hi = temp >> 32; + return x1y1 + temp_hi + ((temp_lo + x0y1) >> 32); #endif } -static inline int64_t libdivide__mullhi_s64(int64_t x, int64_t y) { -#if HAS_INT128_T +static inline int64_t libdivide_mullhi_s64(int64_t x, int64_t y) { +#if defined(LIBDIVIDE_VC) && \ + defined(LIBDIVIDE_X86_64) + return __mulh(x, y); +#elif defined(HAS_INT128_T) __int128_t xl = x, yl = y; __int128_t rl = xl * yl; return (int64_t)(rl >> 64); #else - //full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) - const uint32_t mask = 0xFFFFFFFF; - const uint32_t x0 = (uint32_t)(x & mask), y0 = (uint32_t)(y & mask); - const int32_t x1 = (int32_t)(x >> 32), y1 = (int32_t)(y >> 32); - const uint32_t x0y0_hi = libdivide__mullhi_u32(x0, y0); - const int64_t t = x1*(int64_t)y0 + x0y0_hi; - const int64_t w1 = x0*(int64_t)y1 + (t & mask); - return x1*(int64_t)y1 + (t >> 32) + (w1 >> 32); + // full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) + uint32_t mask = 0xFFFFFFFF; + uint32_t x0 = (uint32_t)(x & mask); + uint32_t y0 = (uint32_t)(y & mask); + int32_t x1 = (int32_t)(x >> 32); + int32_t y1 = (int32_t)(y >> 32); + uint32_t x0y0_hi = libdivide_mullhi_u32(x0, y0); + int64_t t = x1 * (int64_t)y0 + x0y0_hi; + int64_t w1 = x0 * (int64_t)y1 + (t & mask); + + return x1 * (int64_t)y1 + (t >> 32) + (w1 >> 32); #endif } -#if LIBDIVIDE_USE_SSE2 - -static inline __m128i libdivide__u64_to_m128(uint64_t x) { -#if LIBDIVIDE_VC && ! _WIN64 - //64 bit windows doesn't seem to have an implementation of any of these load intrinsics, and 32 bit Visual C++ crashes - _declspec(align(16)) uint64_t temp[2] = {x, x}; - return _mm_load_si128((const __m128i*)temp); -#elif defined(__ICC) - uint64_t __attribute__((aligned(16))) temp[2] = {x,x}; - return _mm_load_si128((const __m128i*)temp); -#elif __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wc++11-narrowing" // narrowing from uint64_t (aka 'unsigned long') to 'long long' - // clang does not provide this intrinsic either - return (__m128i){x, x}; -#pragma clang diagnostic pop -#else - // everyone else gets it right - return _mm_set1_epi64x(x); -#endif -} - -static inline __m128i libdivide_get_FFFFFFFF00000000(void) { - //returns the same as _mm_set1_epi64(0xFFFFFFFF00000000ULL) without touching memory - __m128i result = _mm_set1_epi8(-1); //optimizes to pcmpeqd on OS X - return _mm_slli_epi64(result, 32); -} - -static inline __m128i libdivide_get_00000000FFFFFFFF(void) { - //returns the same as _mm_set1_epi64(0x00000000FFFFFFFFULL) without touching memory - __m128i result = _mm_set1_epi8(-1); //optimizes to pcmpeqd on OS X - result = _mm_srli_epi64(result, 32); - return result; -} - -#if __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wuninitialized" -#endif -static inline __m128i libdivide_get_0000FFFF(void) { - //returns the same as _mm_set1_epi32(0x0000FFFFULL) without touching memory - __m128i result; //we don't care what its contents are - result = _mm_cmpeq_epi8(result, result); //all 1s - result = _mm_srli_epi32(result, 16); - return result; -} -#if __clang__ -#pragma clang diagnostic pop -#endif - -/// This is a bug in gcc-8, _MM_SHUFFLE was forgotten, though in trunk it is ok https://github.com/gcc-mirror/gcc/blob/master/gcc/config/rs6000/xmmintrin.h#L61 -#if defined(__PPC__) -#ifndef _MM_SHUFFLE -#define _MM_SHUFFLE(w,x,y,z) (((w) << 6) | ((x) << 4) | ((y) << 2) | (z)) -#endif -#endif - -static inline __m128i libdivide_s64_signbits(__m128i v) { - //we want to compute v >> 63, that is, _mm_srai_epi64(v, 63). But there is no 64 bit shift right arithmetic instruction in SSE2. So we have to fake it by first duplicating the high 32 bit values, and then using a 32 bit shift. Another option would be to use _mm_srli_epi64(v, 63) and then subtract that from 0, but that approach appears to be substantially slower for unknown reasons - __m128i hiBitsDuped = _mm_shuffle_epi32(v, _MM_SHUFFLE(3, 3, 1, 1)); - __m128i signBits = _mm_srai_epi32(hiBitsDuped, 31); - return signBits; -} - -/* Returns an __m128i whose low 32 bits are equal to amt and has zero elsewhere. */ -static inline __m128i libdivide_u32_to_m128i(uint32_t amt) { - return _mm_set_epi32(0, 0, 0, amt); -} - -static inline __m128i libdivide_s64_shift_right_vector(__m128i v, int amt) { - //implementation of _mm_sra_epi64. Here we have two 64 bit values which are shifted right to logically become (64 - amt) values, and are then sign extended from a (64 - amt) bit number. - const int b = 64 - amt; - __m128i m = libdivide__u64_to_m128(1ULL << (b - 1)); - __m128i x = _mm_srl_epi64(v, libdivide_u32_to_m128i(amt)); - __m128i result = _mm_sub_epi64(_mm_xor_si128(x, m), m); //result = x^m - m - return result; -} - -/* Here, b is assumed to contain one 32 bit value repeated four times. If it did not, the function would not work. */ -static inline __m128i libdivide__mullhi_u32_flat_vector(__m128i a, __m128i b) { - __m128i hi_product_0Z2Z = _mm_srli_epi64(_mm_mul_epu32(a, b), 32); - __m128i a1X3X = _mm_srli_epi64(a, 32); - __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epu32(a1X3X, b), libdivide_get_FFFFFFFF00000000()); - return _mm_or_si128(hi_product_0Z2Z, hi_product_Z1Z3); // = hi_product_0123 -} - - -/* Here, y is assumed to contain one 64 bit value repeated twice. */ -static inline __m128i libdivide_mullhi_u64_flat_vector(__m128i x, __m128i y) { - //full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) - const __m128i mask = libdivide_get_00000000FFFFFFFF(); - const __m128i x0 = _mm_and_si128(x, mask), x1 = _mm_srli_epi64(x, 32); //x0 is low half of 2 64 bit values, x1 is high half in low slots - const __m128i y0 = _mm_and_si128(y, mask), y1 = _mm_srli_epi64(y, 32); - const __m128i x0y0_hi = _mm_srli_epi64(_mm_mul_epu32(x0, y0), 32); //x0 happens to have the low half of the two 64 bit values in 32 bit slots 0 and 2, so _mm_mul_epu32 computes their full product, and then we shift right by 32 to get just the high values - const __m128i x0y1 = _mm_mul_epu32(x0, y1); - const __m128i x1y0 = _mm_mul_epu32(x1, y0); - const __m128i x1y1 = _mm_mul_epu32(x1, y1); - - const __m128i temp = _mm_add_epi64(x1y0, x0y0_hi); - __m128i temp_lo = _mm_and_si128(temp, mask), temp_hi = _mm_srli_epi64(temp, 32); - temp_lo = _mm_srli_epi64(_mm_add_epi64(temp_lo, x0y1), 32); - temp_hi = _mm_add_epi64(x1y1, temp_hi); - - return _mm_add_epi64(temp_lo, temp_hi); -} - -/* y is one 64 bit value repeated twice */ -static inline __m128i libdivide_mullhi_s64_flat_vector(__m128i x, __m128i y) { - __m128i p = libdivide_mullhi_u64_flat_vector(x, y); - __m128i t1 = _mm_and_si128(libdivide_s64_signbits(x), y); - p = _mm_sub_epi64(p, t1); - __m128i t2 = _mm_and_si128(libdivide_s64_signbits(y), x); - p = _mm_sub_epi64(p, t2); - return p; -} - -#ifdef LIBDIVIDE_USE_SSE4_1 - -/* b is one 32 bit value repeated four times. */ -static inline __m128i libdivide_mullhi_s32_flat_vector(__m128i a, __m128i b) { - __m128i hi_product_0Z2Z = _mm_srli_epi64(_mm_mul_epi32(a, b), 32); - __m128i a1X3X = _mm_srli_epi64(a, 32); - __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epi32(a1X3X, b), libdivide_get_FFFFFFFF00000000()); - return _mm_or_si128(hi_product_0Z2Z, hi_product_Z1Z3); // = hi_product_0123 -} - -#else - -/* SSE2 does not have a signed multiplication instruction, but we can convert unsigned to signed pretty efficiently. Again, b is just a 32 bit value repeated four times. */ -static inline __m128i libdivide_mullhi_s32_flat_vector(__m128i a, __m128i b) { - __m128i p = libdivide__mullhi_u32_flat_vector(a, b); - __m128i t1 = _mm_and_si128(_mm_srai_epi32(a, 31), b); //t1 = (a >> 31) & y, arithmetic shift - __m128i t2 = _mm_and_si128(_mm_srai_epi32(b, 31), a); - p = _mm_sub_epi32(p, t1); - p = _mm_sub_epi32(p, t2); - return p; -} -#endif -#endif - -static inline int32_t libdivide__count_trailing_zeros32(uint32_t val) { -#if __GNUC__ || __has_builtin(__builtin_ctz) - /* Fast way to count trailing zeros */ - return __builtin_ctz(val); -#elif LIBDIVIDE_VC - unsigned long result; - if (_BitScanForward(&result, val)) { - return result; - } - return 0; -#else - /* Dorky way to count trailing zeros. Note that this hangs for val = 0! */ - int32_t result = 0; - val = (val ^ (val - 1)) >> 1; // Set v's trailing 0s to 1s and zero rest - while (val) { - val >>= 1; - result++; - } - return result; -#endif -} - -static inline int32_t libdivide__count_trailing_zeros64(uint64_t val) { -#if __LP64__ && (__GNUC__ || __has_builtin(__builtin_ctzll)) - /* Fast way to count trailing zeros. Note that we disable this in 32 bit because gcc does something horrible - it calls through to a dynamically bound function. */ - return __builtin_ctzll(val); -#elif LIBDIVIDE_VC && _WIN64 - unsigned long result; - if (_BitScanForward64(&result, val)) { - return result; - } - return 0; -#else - /* Pretty good way to count trailing zeros. Note that this hangs for val = 0! */ - uint32_t lo = val & 0xFFFFFFFF; - if (lo != 0) return libdivide__count_trailing_zeros32(lo); - return 32 + libdivide__count_trailing_zeros32(val >> 32); -#endif -} - -static inline int32_t libdivide__count_leading_zeros32(uint32_t val) { -#if __GNUC__ || __has_builtin(__builtin_clzll) - /* Fast way to count leading zeros */ +static inline int32_t libdivide_count_leading_zeros32(uint32_t val) { +#if defined(__GNUC__) || \ + __has_builtin(__builtin_clz) + // Fast way to count leading zeros return __builtin_clz(val); -#elif LIBDIVIDE_VC +#elif defined(LIBDIVIDE_VC) unsigned long result; if (_BitScanReverse(&result, val)) { return 31 - result; } return 0; #else - /* Dorky way to count leading zeros. Note that this hangs for val = 0! */ int32_t result = 0; - while (! (val & (1U << 31))) { - val <<= 1; + uint32_t hi = 1U << 31; + for (; ~val & hi; hi >>= 1) { result++; } return result; #endif } -static inline int32_t libdivide__count_leading_zeros64(uint64_t val) { -#if __GNUC__ || __has_builtin(__builtin_clzll) - /* Fast way to count leading zeros */ +static inline int32_t libdivide_count_leading_zeros64(uint64_t val) { +#if defined(__GNUC__) || \ + __has_builtin(__builtin_clzll) + // Fast way to count leading zeros return __builtin_clzll(val); -#elif LIBDIVIDE_VC && _WIN64 +#elif defined(LIBDIVIDE_VC) && defined(_WIN64) unsigned long result; if (_BitScanReverse64(&result, val)) { return 63 - result; } return 0; #else - /* Dorky way to count leading zeros. Note that this hangs for val = 0! */ - int32_t result = 0; - while (! (val & (1ULL << 63))) { - val <<= 1; - result++; - } - return result; + uint32_t hi = val >> 32; + uint32_t lo = val & 0xFFFFFFFF; + if (hi != 0) return libdivide_count_leading_zeros32(hi); + return 32 + libdivide_count_leading_zeros32(lo); #endif } -//libdivide_64_div_32_to_32: divides a 64 bit uint {u1, u0} by a 32 bit uint {v}. The result must fit in 32 bits. Returns the quotient directly and the remainder in *r -#if (LIBDIVIDE_IS_i386 || LIBDIVIDE_IS_X86_64) && LIBDIVIDE_GCC_STYLE_ASM -static uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { +// libdivide_64_div_32_to_32: divides a 64-bit uint {u1, u0} by a 32-bit +// uint {v}. The result must fit in 32 bits. +// Returns the quotient directly and the remainder in *r +static inline uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { +#if (defined(LIBDIVIDE_i386) || defined(LIBDIVIDE_X86_64)) && \ + defined(LIBDIVIDE_GCC_STYLE_ASM) uint32_t result; __asm__("divl %[v]" : "=a"(result), "=d"(*r) : [v] "r"(v), "a"(u0), "d"(u1) ); return result; -} #else -static uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { - uint64_t n = (((uint64_t)u1) << 32) | u0; + uint64_t n = ((uint64_t)u1 << 32) | u0; uint32_t result = (uint32_t)(n / v); *r = (uint32_t)(n - result * (uint64_t)v); return result; -} #endif +} -#if LIBDIVIDE_IS_X86_64 && LIBDIVIDE_GCC_STYLE_ASM +// libdivide_128_div_64_to_64: divides a 128-bit uint {u1, u0} by a 64-bit +// uint {v}. The result must fit in 64 bits. +// Returns the quotient directly and the remainder in *r static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, uint64_t *r) { - //u0 -> rax - //u1 -> rdx - //divq +#if defined(LIBDIVIDE_X86_64) && \ + defined(LIBDIVIDE_GCC_STYLE_ASM) uint64_t result; __asm__("divq %[v]" : "=a"(result), "=d"(*r) : [v] "r"(v), "a"(u0), "d"(u1) ); return result; - -} +#elif defined(HAS_INT128_T) && \ + defined(HAS_INT128_DIV) + __uint128_t n = ((__uint128_t)u1 << 64) | u0; + uint64_t result = (uint64_t)(n / v); + *r = (uint64_t)(n - result * (__uint128_t)v); + return result; #else + // Code taken from Hacker's Delight: + // http://www.hackersdelight.org/HDcode/divlu.c. + // License permits inclusion here per: + // http://www.hackersdelight.org/permissions.htm -/* Code taken from Hacker's Delight, http://www.hackersdelight.org/HDcode/divlu.c . License permits inclusion here per http://www.hackersdelight.org/permissions.htm - */ -static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, uint64_t *r) { - const uint64_t b = (1ULL << 32); // Number base (16 bits). - uint64_t un1, un0, // Norm. dividend LSD's. - vn1, vn0, // Norm. divisor digits. - q1, q0, // Quotient digits. - un64, un21, un10,// Dividend digit pairs. - rhat; // A remainder. - int s; // Shift amount for norm. + const uint64_t b = (1ULL << 32); // Number base (32 bits) + uint64_t un1, un0; // Norm. dividend LSD's + uint64_t vn1, vn0; // Norm. divisor digits + uint64_t q1, q0; // Quotient digits + uint64_t un64, un21, un10; // Dividend digit pairs + uint64_t rhat; // A remainder + int32_t s; // Shift amount for norm - if (u1 >= v) { // If overflow, set rem. - if (r != NULL) // to an impossible value, - *r = (uint64_t)(-1); // and return the largest - return (uint64_t)(-1);} // possible quotient. + // If overflow, set rem. to an impossible value, + // and return the largest possible quotient + if (u1 >= v) { + *r = (uint64_t) -1; + return (uint64_t) -1; + } - /* count leading zeros */ - s = libdivide__count_leading_zeros64(v); // 0 <= s <= 63. + // count leading zeros + s = libdivide_count_leading_zeros64(v); if (s > 0) { - v = v << s; // Normalize divisor. - un64 = (u1 << s) | ((u0 >> (64 - s)) & (-s >> 31)); - un10 = u0 << s; // Shift dividend left. + // Normalize divisor + v = v << s; + un64 = (u1 << s) | (u0 >> (64 - s)); + un10 = u0 << s; // Shift dividend left } else { - // Avoid undefined behavior. - un64 = u1 | u0; + // Avoid undefined behavior of (u0 >> 64). + // The behavior is undefined if the right operand is + // negative, or greater than or equal to the length + // in bits of the promoted left operand. + un64 = u1; un10 = u0; } - vn1 = v >> 32; // Break divisor up into - vn0 = v & 0xFFFFFFFF; // two 32-bit digits. + // Break divisor up into two 32-bit digits + vn1 = v >> 32; + vn0 = v & 0xFFFFFFFF; - un1 = un10 >> 32; // Break right half of - un0 = un10 & 0xFFFFFFFF; // dividend into two digits. + // Break right half of dividend into two digits + un1 = un10 >> 32; + un0 = un10 & 0xFFFFFFFF; - q1 = un64/vn1; // Compute the first - rhat = un64 - q1*vn1; // quotient digit, q1. -again1: - if (q1 >= b || q1*vn0 > b*rhat + un1) { + // Compute the first quotient digit, q1 + q1 = un64 / vn1; + rhat = un64 - q1 * vn1; + + while (q1 >= b || q1 * vn0 > b * rhat + un1) { q1 = q1 - 1; rhat = rhat + vn1; - if (rhat < b) goto again1;} + if (rhat >= b) + break; + } - un21 = un64*b + un1 - q1*v; // Multiply and subtract. + // Multiply and subtract + un21 = un64 * b + un1 - q1 * v; - q0 = un21/vn1; // Compute the second - rhat = un21 - q0*vn1; // quotient digit, q0. -again2: - if (q0 >= b || q0*vn0 > b*rhat + un0) { + // Compute the second quotient digit + q0 = un21 / vn1; + rhat = un21 - q0 * vn1; + + while (q0 >= b || q0 * vn0 > b * rhat + un0) { q0 = q0 - 1; rhat = rhat + vn1; - if (rhat < b) goto again2;} + if (rhat >= b) + break; + } - if (r != NULL) // If remainder is wanted, - *r = (un21*b + un0 - q0*v) >> s; // return it. - return q1*b + q0; + *r = (un21 * b + un0 - q0 * v) >> s; + return q1 * b + q0; +#endif } -#endif -#if LIBDIVIDE_ASSERTIONS_ON -#define LIBDIVIDE_ASSERT(x) do { if (! (x)) { fprintf(stderr, "Assertion failure on line %ld: %s\n", (long)__LINE__, #x); exit(-1); } } while (0) +// Bitshift a u128 in place, left (signed_shift > 0) or right (signed_shift < 0) +static inline void libdivide_u128_shift(uint64_t *u1, uint64_t *u0, int32_t signed_shift) { + if (signed_shift > 0) { + uint32_t shift = signed_shift; + *u1 <<= shift; + *u1 |= *u0 >> (64 - shift); + *u0 <<= shift; + } + else if (signed_shift < 0) { + uint32_t shift = -signed_shift; + *u0 >>= shift; + *u0 |= *u1 << (64 - shift); + *u1 >>= shift; + } +} + +// Computes a 128 / 128 -> 64 bit division, with a 128 bit remainder. +static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64_t v_hi, uint64_t v_lo, uint64_t *r_hi, uint64_t *r_lo) { +#if defined(HAS_INT128_T) && \ + defined(HAS_INT128_DIV) + __uint128_t ufull = u_hi; + __uint128_t vfull = v_hi; + ufull = (ufull << 64) | u_lo; + vfull = (vfull << 64) | v_lo; + uint64_t res = (uint64_t)(ufull / vfull); + __uint128_t remainder = ufull - (vfull * res); + *r_lo = (uint64_t)remainder; + *r_hi = (uint64_t)(remainder >> 64); + return res; #else -#define LIBDIVIDE_ASSERT(x) -#endif + // Adapted from "Unsigned Doubleword Division" in Hacker's Delight + // We want to compute u / v + typedef struct { uint64_t hi; uint64_t lo; } u128_t; + u128_t u = {u_hi, u_lo}; + u128_t v = {v_hi, v_lo}; -#ifndef LIBDIVIDE_HEADER_ONLY + if (v.hi == 0) { + // divisor v is a 64 bit value, so we just need one 128/64 division + // Note that we are simpler than Hacker's Delight here, because we know + // the quotient fits in 64 bits whereas Hacker's Delight demands a full + // 128 bit quotient + *r_hi = 0; + return libdivide_128_div_64_to_64(u.hi, u.lo, v.lo, r_lo); + } + // Here v >= 2**64 + // We know that v.hi != 0, so count leading zeros is OK + // We have 0 <= n <= 63 + uint32_t n = libdivide_count_leading_zeros64(v.hi); + + // Normalize the divisor so its MSB is 1 + u128_t v1t = v; + libdivide_u128_shift(&v1t.hi, &v1t.lo, n); + uint64_t v1 = v1t.hi; // i.e. v1 = v1t >> 64 + + // To ensure no overflow + u128_t u1 = u; + libdivide_u128_shift(&u1.hi, &u1.lo, -1); + + // Get quotient from divide unsigned insn. + uint64_t rem_ignored; + uint64_t q1 = libdivide_128_div_64_to_64(u1.hi, u1.lo, v1, &rem_ignored); + + // Undo normalization and division of u by 2. + u128_t q0 = {0, q1}; + libdivide_u128_shift(&q0.hi, &q0.lo, n); + libdivide_u128_shift(&q0.hi, &q0.lo, -63); + + // Make q0 correct or too small by 1 + // Equivalent to `if (q0 != 0) q0 = q0 - 1;` + if (q0.hi != 0 || q0.lo != 0) { + q0.hi -= (q0.lo == 0); // borrow + q0.lo -= 1; + } + + // Now q0 is correct. + // Compute q0 * v as q0v + // = (q0.hi << 64 + q0.lo) * (v.hi << 64 + v.lo) + // = (q0.hi * v.hi << 128) + (q0.hi * v.lo << 64) + + // (q0.lo * v.hi << 64) + q0.lo * v.lo) + // Each term is 128 bit + // High half of full product (upper 128 bits!) are dropped + u128_t q0v = {0, 0}; + q0v.hi = q0.hi*v.lo + q0.lo*v.hi + libdivide_mullhi_u64(q0.lo, v.lo); + q0v.lo = q0.lo*v.lo; + + // Compute u - q0v as u_q0v + // This is the remainder + u128_t u_q0v = u; + u_q0v.hi -= q0v.hi + (u.lo < q0v.lo); // second term is borrow + u_q0v.lo -= q0v.lo; + + // Check if u_q0v >= v + // This checks if our remainder is larger than the divisor + if ((u_q0v.hi > v.hi) || + (u_q0v.hi == v.hi && u_q0v.lo >= v.lo)) { + // Increment q0 + q0.lo += 1; + q0.hi += (q0.lo == 0); // carry + + // Subtract v from remainder + u_q0v.hi -= v.hi + (u_q0v.lo < v.lo); + u_q0v.lo -= v.lo; + } + + *r_hi = u_q0v.hi; + *r_lo = u_q0v.lo; + + LIBDIVIDE_ASSERT(q0.hi == 0); + return q0.lo; +#endif +} ////////// UINT32 -struct libdivide_u32_t libdivide_u32_gen(uint32_t d) { - struct libdivide_u32_t result; - if ((d & (d - 1)) == 0) { - result.magic = 0; - result.more = libdivide__count_trailing_zeros32(d) | LIBDIVIDE_U32_SHIFT_PATH; +static inline struct libdivide_u32_t libdivide_internal_u32_gen(uint32_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); } - else { - const uint32_t floor_log_2_d = 31 - libdivide__count_leading_zeros32(d); + struct libdivide_u32_t result; + uint32_t floor_log_2_d = 31 - libdivide_count_leading_zeros32(d); + + // Power of 2 + if ((d & (d - 1)) == 0) { + // We need to subtract 1 from the shift value in case of an unsigned + // branchfree divider because there is a hardcoded right shift by 1 + // in its division algorithm. Because of this we also need to add back + // 1 in its recovery algorithm. + result.magic = 0; + result.more = (uint8_t)(floor_log_2_d - (branchfree != 0)); + } else { uint8_t more; uint32_t rem, proposed_m; proposed_m = libdivide_64_div_32_to_32(1U << floor_log_2_d, 0, d, &rem); @@ -631,570 +569,1358 @@ struct libdivide_u32_t libdivide_u32_gen(uint32_t d) { LIBDIVIDE_ASSERT(rem > 0 && rem < d); const uint32_t e = d - rem; - /* This power works if e < 2**floor_log_2_d. */ - if (e < (1U << floor_log_2_d)) { - /* This power works */ + // This power works if e < 2**floor_log_2_d. + if (!branchfree && (e < (1U << floor_log_2_d))) { + // This power works more = floor_log_2_d; - } - else { - /* We have to use the general 33-bit algorithm. We need to compute (2**power) / d. However, we already have (2**(power-1))/d and its remainder. By doubling both, and then correcting the remainder, we can compute the larger division. */ - proposed_m += proposed_m; //don't care about overflow here - in fact, we expect it + } else { + // We have to use the general 33-bit algorithm. We need to compute + // (2**power) / d. However, we already have (2**(power-1))/d and + // its remainder. By doubling both, and then correcting the + // remainder, we can compute the larger division. + // don't care about overflow here - in fact, we expect it + proposed_m += proposed_m; const uint32_t twice_rem = rem + rem; if (twice_rem >= d || twice_rem < rem) proposed_m += 1; more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; } result.magic = 1 + proposed_m; result.more = more; - //result.more's shift should in general be ceil_log_2_d. But if we used the smaller power, we subtract one from the shift because we're using the smaller power. If we're using the larger power, we subtract one from the shift because it's taken care of by the add indicator. So floor_log_2_d happens to be correct in both cases. - + // result.more's shift should in general be ceil_log_2_d. But if we + // used the smaller power, we subtract one from the shift because we're + // using the smaller power. If we're using the larger power, we + // subtract one from the shift because it's taken care of by the add + // indicator. So floor_log_2_d happens to be correct in both cases. } return result; } +struct libdivide_u32_t libdivide_u32_gen(uint32_t d) { + return libdivide_internal_u32_gen(d, 0); +} + +struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } + struct libdivide_u32_t tmp = libdivide_internal_u32_gen(d, 1); + struct libdivide_u32_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_32_SHIFT_MASK)}; + return ret; +} + uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) { uint8_t more = denom->more; - if (more & LIBDIVIDE_U32_SHIFT_PATH) { - return numer >> (more & LIBDIVIDE_32_SHIFT_MASK); + if (!denom->magic) { + return numer >> more; } else { - uint32_t q = libdivide__mullhi_u32(denom->magic, numer); + uint32_t q = libdivide_mullhi_u32(denom->magic, numer); if (more & LIBDIVIDE_ADD_MARKER) { uint32_t t = ((numer - q) >> 1) + q; return t >> (more & LIBDIVIDE_32_SHIFT_MASK); } else { - return q >> more; //all upper bits are 0 - don't need to mask them off + // All upper bits are 0, + // don't need to mask them off. + return q >> more; } } } - -int libdivide_u32_get_algorithm(const struct libdivide_u32_t *denom) { - uint8_t more = denom->more; - if (more & LIBDIVIDE_U32_SHIFT_PATH) return 0; - else if (! (more & LIBDIVIDE_ADD_MARKER)) return 1; - else return 2; -} - -uint32_t libdivide_u32_do_alg0(uint32_t numer, const struct libdivide_u32_t *denom) { - return numer >> (denom->more & LIBDIVIDE_32_SHIFT_MASK); -} - -uint32_t libdivide_u32_do_alg1(uint32_t numer, const struct libdivide_u32_t *denom) { - uint32_t q = libdivide__mullhi_u32(denom->magic, numer); - return q >> denom->more; -} - -uint32_t libdivide_u32_do_alg2(uint32_t numer, const struct libdivide_u32_t *denom) { - // denom->add != 0 - uint32_t q = libdivide__mullhi_u32(denom->magic, numer); +uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom) { + uint32_t q = libdivide_mullhi_u32(denom->magic, numer); uint32_t t = ((numer - q) >> 1) + q; - return t >> (denom->more & LIBDIVIDE_32_SHIFT_MASK); + return t >> denom->more; } - - - -#if LIBDIVIDE_USE_SSE2 -__m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom) { +uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) { uint8_t more = denom->more; - if (more & LIBDIVIDE_U32_SHIFT_PATH) { - return _mm_srl_epi32(numers, libdivide_u32_to_m128i(more & LIBDIVIDE_32_SHIFT_MASK)); - } - else { - __m128i q = libdivide__mullhi_u32_flat_vector(numers, _mm_set1_epi32(denom->magic)); - if (more & LIBDIVIDE_ADD_MARKER) { - //uint32_t t = ((numer - q) >> 1) + q; - //return t >> denom->shift; - __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); - return _mm_srl_epi32(t, libdivide_u32_to_m128i(more & LIBDIVIDE_32_SHIFT_MASK)); + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; - } - else { - //q >> denom->shift - return _mm_srl_epi32(q, libdivide_u32_to_m128i(more)); - } + if (!denom->magic) { + return 1U << shift; + } else if (!(more & LIBDIVIDE_ADD_MARKER)) { + // We compute q = n/d = n*m / 2^(32 + shift) + // Therefore we have d = 2^(32 + shift) / m + // We need to ceil it. + // We know d is not a power of 2, so m is not a power of 2, + // so we can just add 1 to the floor + uint32_t hi_dividend = 1U << shift; + uint32_t rem_ignored; + return 1 + libdivide_64_div_32_to_32(hi_dividend, 0, denom->magic, &rem_ignored); + } else { + // Here we wish to compute d = 2^(32+shift+1)/(m+2^32). + // Notice (m + 2^32) is a 33 bit number. Use 64 bit division for now + // Also note that shift may be as high as 31, so shift + 1 will + // overflow. So we have to compute it as 2^(32+shift)/(m+2^32), and + // then double the quotient and remainder. + uint64_t half_n = 1ULL << (32 + shift); + uint64_t d = (1ULL << 32) | denom->magic; + // Note that the quotient is guaranteed <= 32 bits, but the remainder + // may need 33! + uint32_t half_q = (uint32_t)(half_n / d); + uint64_t rem = half_n % d; + // We computed 2^(32+shift)/(m+2^32) + // Need to double it, and then add 1 to the quotient if doubling th + // remainder would increase the quotient. + // Note that rem<<1 cannot overflow, since rem < d and d is 33 bits + uint32_t full_q = half_q + half_q + ((rem<<1) >= d); + + // We rounded down in gen (hence +1) + return full_q + 1; } } -__m128i libdivide_u32_do_vector_alg0(__m128i numers, const struct libdivide_u32_t *denom) { - return _mm_srl_epi32(numers, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_32_SHIFT_MASK)); -} +uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; -__m128i libdivide_u32_do_vector_alg1(__m128i numers, const struct libdivide_u32_t *denom) { - __m128i q = libdivide__mullhi_u32_flat_vector(numers, _mm_set1_epi32(denom->magic)); - return _mm_srl_epi32(q, libdivide_u32_to_m128i(denom->more)); -} + if (!denom->magic) { + return 1U << (shift + 1); + } else { + // Here we wish to compute d = 2^(32+shift+1)/(m+2^32). + // Notice (m + 2^32) is a 33 bit number. Use 64 bit division for now + // Also note that shift may be as high as 31, so shift + 1 will + // overflow. So we have to compute it as 2^(32+shift)/(m+2^32), and + // then double the quotient and remainder. + uint64_t half_n = 1ULL << (32 + shift); + uint64_t d = (1ULL << 32) | denom->magic; + // Note that the quotient is guaranteed <= 32 bits, but the remainder + // may need 33! + uint32_t half_q = (uint32_t)(half_n / d); + uint64_t rem = half_n % d; + // We computed 2^(32+shift)/(m+2^32) + // Need to double it, and then add 1 to the quotient if doubling th + // remainder would increase the quotient. + // Note that rem<<1 cannot overflow, since rem < d and d is 33 bits + uint32_t full_q = half_q + half_q + ((rem<<1) >= d); -__m128i libdivide_u32_do_vector_alg2(__m128i numers, const struct libdivide_u32_t *denom) { - __m128i q = libdivide__mullhi_u32_flat_vector(numers, _mm_set1_epi32(denom->magic)); - __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); - return _mm_srl_epi32(t, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_32_SHIFT_MASK)); + // We rounded down in gen (hence +1) + return full_q + 1; + } } -#endif - /////////// UINT64 -struct libdivide_u64_t libdivide_u64_gen(uint64_t d) { - struct libdivide_u64_t result; - if ((d & (d - 1)) == 0) { - result.more = libdivide__count_trailing_zeros64(d) | LIBDIVIDE_U64_SHIFT_PATH; - result.magic = 0; +static inline struct libdivide_u64_t libdivide_internal_u64_gen(uint64_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); } - else { - const uint32_t floor_log_2_d = 63 - libdivide__count_leading_zeros64(d); + struct libdivide_u64_t result; + uint32_t floor_log_2_d = 63 - libdivide_count_leading_zeros64(d); + + // Power of 2 + if ((d & (d - 1)) == 0) { + // We need to subtract 1 from the shift value in case of an unsigned + // branchfree divider because there is a hardcoded right shift by 1 + // in its division algorithm. Because of this we also need to add back + // 1 in its recovery algorithm. + result.magic = 0; + result.more = (uint8_t)(floor_log_2_d - (branchfree != 0)); + } else { uint64_t proposed_m, rem; uint8_t more; - proposed_m = libdivide_128_div_64_to_64(1ULL << floor_log_2_d, 0, d, &rem); //== (1 << (64 + floor_log_2_d)) / d + // (1 << (64 + floor_log_2_d)) / d + proposed_m = libdivide_128_div_64_to_64(1ULL << floor_log_2_d, 0, d, &rem); LIBDIVIDE_ASSERT(rem > 0 && rem < d); const uint64_t e = d - rem; - /* This power works if e < 2**floor_log_2_d. */ - if (e < (1ULL << floor_log_2_d)) { - /* This power works */ + // This power works if e < 2**floor_log_2_d. + if (!branchfree && e < (1ULL << floor_log_2_d)) { + // This power works more = floor_log_2_d; - } - else { - /* We have to use the general 65-bit algorithm. We need to compute (2**power) / d. However, we already have (2**(power-1))/d and its remainder. By doubling both, and then correcting the remainder, we can compute the larger division. */ - proposed_m += proposed_m; //don't care about overflow here - in fact, we expect it + } else { + // We have to use the general 65-bit algorithm. We need to compute + // (2**power) / d. However, we already have (2**(power-1))/d and + // its remainder. By doubling both, and then correcting the + // remainder, we can compute the larger division. + // don't care about overflow here - in fact, we expect it + proposed_m += proposed_m; const uint64_t twice_rem = rem + rem; if (twice_rem >= d || twice_rem < rem) proposed_m += 1; - more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; } result.magic = 1 + proposed_m; result.more = more; - //result.more's shift should in general be ceil_log_2_d. But if we used the smaller power, we subtract one from the shift because we're using the smaller power. If we're using the larger power, we subtract one from the shift because it's taken care of by the add indicator. So floor_log_2_d happens to be correct in both cases, which is why we do it outside of the if statement. + // result.more's shift should in general be ceil_log_2_d. But if we + // used the smaller power, we subtract one from the shift because we're + // using the smaller power. If we're using the larger power, we + // subtract one from the shift because it's taken care of by the add + // indicator. So floor_log_2_d happens to be correct in both cases, + // which is why we do it outside of the if statement. } return result; } +struct libdivide_u64_t libdivide_u64_gen(uint64_t d) { + return libdivide_internal_u64_gen(d, 0); +} + +struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } + struct libdivide_u64_t tmp = libdivide_internal_u64_gen(d, 1); + struct libdivide_u64_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_64_SHIFT_MASK)}; + return ret; +} + uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) { uint8_t more = denom->more; - if (more & LIBDIVIDE_U64_SHIFT_PATH) { - return numer >> (more & LIBDIVIDE_64_SHIFT_MASK); + if (!denom->magic) { + return numer >> more; } else { - uint64_t q = libdivide__mullhi_u64(denom->magic, numer); + uint64_t q = libdivide_mullhi_u64(denom->magic, numer); if (more & LIBDIVIDE_ADD_MARKER) { uint64_t t = ((numer - q) >> 1) + q; return t >> (more & LIBDIVIDE_64_SHIFT_MASK); } else { - return q >> more; //all upper bits are 0 - don't need to mask them off + // All upper bits are 0, + // don't need to mask them off. + return q >> more; } } } - -int libdivide_u64_get_algorithm(const struct libdivide_u64_t *denom) { - uint8_t more = denom->more; - if (more & LIBDIVIDE_U64_SHIFT_PATH) return 0; - else if (! (more & LIBDIVIDE_ADD_MARKER)) return 1; - else return 2; -} - -uint64_t libdivide_u64_do_alg0(uint64_t numer, const struct libdivide_u64_t *denom) { - return numer >> (denom->more & LIBDIVIDE_64_SHIFT_MASK); -} - -uint64_t libdivide_u64_do_alg1(uint64_t numer, const struct libdivide_u64_t *denom) { - uint64_t q = libdivide__mullhi_u64(denom->magic, numer); - return q >> denom->more; -} - -uint64_t libdivide_u64_do_alg2(uint64_t numer, const struct libdivide_u64_t *denom) { - uint64_t q = libdivide__mullhi_u64(denom->magic, numer); +uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom) { + uint64_t q = libdivide_mullhi_u64(denom->magic, numer); uint64_t t = ((numer - q) >> 1) + q; - return t >> (denom->more & LIBDIVIDE_64_SHIFT_MASK); + return t >> denom->more; } -#if LIBDIVIDE_USE_SSE2 -__m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t * denom) { +uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) { uint8_t more = denom->more; - if (more & LIBDIVIDE_U64_SHIFT_PATH) { - return _mm_srl_epi64(numers, libdivide_u32_to_m128i(more & LIBDIVIDE_64_SHIFT_MASK)); - } - else { - __m128i q = libdivide_mullhi_u64_flat_vector(numers, libdivide__u64_to_m128(denom->magic)); - if (more & LIBDIVIDE_ADD_MARKER) { - //uint32_t t = ((numer - q) >> 1) + q; - //return t >> denom->shift; - __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); - return _mm_srl_epi64(t, libdivide_u32_to_m128i(more & LIBDIVIDE_64_SHIFT_MASK)); - } - else { - //q >> denom->shift - return _mm_srl_epi64(q, libdivide_u32_to_m128i(more)); - } + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + + if (!denom->magic) { + return 1ULL << shift; + } else if (!(more & LIBDIVIDE_ADD_MARKER)) { + // We compute q = n/d = n*m / 2^(64 + shift) + // Therefore we have d = 2^(64 + shift) / m + // We need to ceil it. + // We know d is not a power of 2, so m is not a power of 2, + // so we can just add 1 to the floor + uint64_t hi_dividend = 1ULL << shift; + uint64_t rem_ignored; + return 1 + libdivide_128_div_64_to_64(hi_dividend, 0, denom->magic, &rem_ignored); + } else { + // Here we wish to compute d = 2^(64+shift+1)/(m+2^64). + // Notice (m + 2^64) is a 65 bit number. This gets hairy. See + // libdivide_u32_recover for more on what we do here. + // TODO: do something better than 128 bit math + + // Full n is a (potentially) 129 bit value + // half_n is a 128 bit value + // Compute the hi half of half_n. Low half is 0. + uint64_t half_n_hi = 1ULL << shift, half_n_lo = 0; + // d is a 65 bit value. The high bit is always set to 1. + const uint64_t d_hi = 1, d_lo = denom->magic; + // Note that the quotient is guaranteed <= 64 bits, + // but the remainder may need 65! + uint64_t r_hi, r_lo; + uint64_t half_q = libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); + // We computed 2^(64+shift)/(m+2^64) + // Double the remainder ('dr') and check if that is larger than d + // Note that d is a 65 bit value, so r1 is small and so r1 + r1 + // cannot overflow + uint64_t dr_lo = r_lo + r_lo; + uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry + int dr_exceeds_d = (dr_hi > d_hi) || (dr_hi == d_hi && dr_lo >= d_lo); + uint64_t full_q = half_q + half_q + (dr_exceeds_d ? 1 : 0); + return full_q + 1; } } -__m128i libdivide_u64_do_vector_alg0(__m128i numers, const struct libdivide_u64_t *denom) { - return _mm_srl_epi64(numers, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_64_SHIFT_MASK)); +uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + + if (!denom->magic) { + return 1ULL << (shift + 1); + } else { + // Here we wish to compute d = 2^(64+shift+1)/(m+2^64). + // Notice (m + 2^64) is a 65 bit number. This gets hairy. See + // libdivide_u32_recover for more on what we do here. + // TODO: do something better than 128 bit math + + // Full n is a (potentially) 129 bit value + // half_n is a 128 bit value + // Compute the hi half of half_n. Low half is 0. + uint64_t half_n_hi = 1ULL << shift, half_n_lo = 0; + // d is a 65 bit value. The high bit is always set to 1. + const uint64_t d_hi = 1, d_lo = denom->magic; + // Note that the quotient is guaranteed <= 64 bits, + // but the remainder may need 65! + uint64_t r_hi, r_lo; + uint64_t half_q = libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); + // We computed 2^(64+shift)/(m+2^64) + // Double the remainder ('dr') and check if that is larger than d + // Note that d is a 65 bit value, so r1 is small and so r1 + r1 + // cannot overflow + uint64_t dr_lo = r_lo + r_lo; + uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry + int dr_exceeds_d = (dr_hi > d_hi) || (dr_hi == d_hi && dr_lo >= d_lo); + uint64_t full_q = half_q + half_q + (dr_exceeds_d ? 1 : 0); + return full_q + 1; + } } -__m128i libdivide_u64_do_vector_alg1(__m128i numers, const struct libdivide_u64_t *denom) { - __m128i q = libdivide_mullhi_u64_flat_vector(numers, libdivide__u64_to_m128(denom->magic)); - return _mm_srl_epi64(q, libdivide_u32_to_m128i(denom->more)); -} - -__m128i libdivide_u64_do_vector_alg2(__m128i numers, const struct libdivide_u64_t *denom) { - __m128i q = libdivide_mullhi_u64_flat_vector(numers, libdivide__u64_to_m128(denom->magic)); - __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); - return _mm_srl_epi64(t, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_64_SHIFT_MASK)); -} - - -#endif - /////////// SINT32 +static inline struct libdivide_s32_t libdivide_internal_s32_gen(int32_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } -static inline int32_t libdivide__mullhi_s32(int32_t x, int32_t y) { - int64_t xl = x, yl = y; - int64_t rl = xl * yl; - return (int32_t)(rl >> 32); //needs to be arithmetic shift -} - -struct libdivide_s32_t libdivide_s32_gen(int32_t d) { struct libdivide_s32_t result; - /* If d is a power of 2, or negative a power of 2, we have to use a shift. This is especially important because the magic algorithm fails for -1. To check if d is a power of 2 or its inverse, it suffices to check whether its absolute value has exactly one bit set. This works even for INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set and is a power of 2. */ - uint32_t absD = (uint32_t)(d < 0 ? -d : d); //gcc optimizes this to the fast abs trick - if ((absD & (absD - 1)) == 0) { //check if exactly one bit is set, don't care if absD is 0 since that's divide by zero + // If d is a power of 2, or negative a power of 2, we have to use a shift. + // This is especially important because the magic algorithm fails for -1. + // To check if d is a power of 2 or its inverse, it suffices to check + // whether its absolute value has exactly one bit set. This works even for + // INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set + // and is a power of 2. + uint32_t ud = (uint32_t)d; + uint32_t absD = (d < 0) ? -ud : ud; + uint32_t floor_log_2_d = 31 - libdivide_count_leading_zeros32(absD); + // check if exactly one bit is set, + // don't care if absD is 0 since that's divide by zero + if ((absD & (absD - 1)) == 0) { + // Branchfree and normal paths are exactly the same result.magic = 0; - result.more = libdivide__count_trailing_zeros32(absD) | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0) | LIBDIVIDE_S32_SHIFT_PATH; - } - else { - const uint32_t floor_log_2_d = 31 - libdivide__count_leading_zeros32(absD); + result.more = floor_log_2_d | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); + } else { LIBDIVIDE_ASSERT(floor_log_2_d >= 1); uint8_t more; - //the dividend here is 2**(floor_log_2_d + 31), so the low 32 bit word is 0 and the high word is floor_log_2_d - 1 + // the dividend here is 2**(floor_log_2_d + 31), so the low 32 bit word + // is 0 and the high word is floor_log_2_d - 1 uint32_t rem, proposed_m; proposed_m = libdivide_64_div_32_to_32(1U << (floor_log_2_d - 1), 0, absD, &rem); const uint32_t e = absD - rem; - /* We are going to start with a power of floor_log_2_d - 1. This works if works if e < 2**floor_log_2_d. */ - if (e < (1U << floor_log_2_d)) { - /* This power works */ + // We are going to start with a power of floor_log_2_d - 1. + // This works if works if e < 2**floor_log_2_d. + if (!branchfree && e < (1U << floor_log_2_d)) { + // This power works more = floor_log_2_d - 1; - } - else { - /* We need to go one higher. This should not make proposed_m overflow, but it will make it negative when interpreted as an int32_t. */ + } else { + // We need to go one higher. This should not make proposed_m + // overflow, but it will make it negative when interpreted as an + // int32_t. proposed_m += proposed_m; const uint32_t twice_rem = rem + rem; if (twice_rem >= absD || twice_rem < rem) proposed_m += 1; - more = floor_log_2_d | LIBDIVIDE_ADD_MARKER | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); //use the general algorithm + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; } - proposed_m += 1; - result.magic = (d < 0 ? -(int32_t)proposed_m : (int32_t)proposed_m); - result.more = more; + proposed_m += 1; + int32_t magic = (int32_t)proposed_m; + + // Mark if we are negative. Note we only negate the magic number in the + // branchfull case. + if (d < 0) { + more |= LIBDIVIDE_NEGATIVE_DIVISOR; + if (!branchfree) { + magic = -magic; + } + } + + result.more = more; + result.magic = magic; } return result; } +struct libdivide_s32_t libdivide_s32_gen(int32_t d) { + return libdivide_internal_s32_gen(d, 0); +} + +struct libdivide_s32_branchfree_t libdivide_s32_branchfree_gen(int32_t d) { + struct libdivide_s32_t tmp = libdivide_internal_s32_gen(d, 1); + struct libdivide_s32_branchfree_t result = {tmp.magic, tmp.more}; + return result; +} + int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) { uint8_t more = denom->more; - if (more & LIBDIVIDE_S32_SHIFT_PATH) { - uint8_t shifter = more & LIBDIVIDE_32_SHIFT_MASK; - int32_t q = numer + ((numer >> 31) & ((1 << shifter) - 1)); - q = q >> shifter; - int32_t shiftMask = (int8_t)more >> 7; //must be arithmetic shift and then sign-extend - q = (q ^ shiftMask) - shiftMask; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + + if (!denom->magic) { + uint32_t sign = (int8_t)more >> 7; + uint32_t mask = (1U << shift) - 1; + uint32_t uq = numer + ((numer >> 31) & mask); + int32_t q = (int32_t)uq; + q >>= shift; + q = (q ^ sign) - sign; return q; - } - else { - int32_t q = libdivide__mullhi_s32(denom->magic, numer); + } else { + uint32_t uq = (uint32_t)libdivide_mullhi_s32(denom->magic, numer); if (more & LIBDIVIDE_ADD_MARKER) { - int32_t sign = (int8_t)more >> 7; //must be arithmetic shift and then sign extend - q += ((numer ^ sign) - sign); + // must be arithmetic shift and then sign extend + int32_t sign = (int8_t)more >> 7; + // q += (more < 0 ? -numer : numer) + // cast required to avoid UB + uq += ((uint32_t)numer ^ sign) - sign; } - q >>= more & LIBDIVIDE_32_SHIFT_MASK; + int32_t q = (int32_t)uq; + q >>= shift; q += (q < 0); return q; } } -int libdivide_s32_get_algorithm(const struct libdivide_s32_t *denom) { +int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom) { uint8_t more = denom->more; - int positiveDivisor = ! (more & LIBDIVIDE_NEGATIVE_DIVISOR); - if (more & LIBDIVIDE_S32_SHIFT_PATH) return (positiveDivisor ? 0 : 1); - else if (more & LIBDIVIDE_ADD_MARKER) return (positiveDivisor ? 2 : 3); - else return 4; -} - -int32_t libdivide_s32_do_alg0(int32_t numer, const struct libdivide_s32_t *denom) { - uint8_t shifter = denom->more & LIBDIVIDE_32_SHIFT_MASK; - int32_t q = numer + ((numer >> 31) & ((1 << shifter) - 1)); - return q >> shifter; -} - -int32_t libdivide_s32_do_alg1(int32_t numer, const struct libdivide_s32_t *denom) { - uint8_t shifter = denom->more & LIBDIVIDE_32_SHIFT_MASK; - int32_t q = numer + ((numer >> 31) & ((1 << shifter) - 1)); - return - (q >> shifter); -} - -int32_t libdivide_s32_do_alg2(int32_t numer, const struct libdivide_s32_t *denom) { - int32_t q = libdivide__mullhi_s32(denom->magic, numer); + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift and then sign extend + int32_t sign = (int8_t)more >> 7; + int32_t magic = denom->magic; + int32_t q = libdivide_mullhi_s32(magic, numer); q += numer; - q >>= denom->more & LIBDIVIDE_32_SHIFT_MASK; - q += (q < 0); + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is a power of + // 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + uint32_t q_sign = (uint32_t)(q >> 31); + q += q_sign & ((1U << shift) - is_power_of_2); + + // Now arithmetic right shift + q >>= shift; + // Negate if needed + q = (q ^ sign) - sign; + return q; } -int32_t libdivide_s32_do_alg3(int32_t numer, const struct libdivide_s32_t *denom) { - int32_t q = libdivide__mullhi_s32(denom->magic, numer); - q -= numer; - q >>= denom->more & LIBDIVIDE_32_SHIFT_MASK; - q += (q < 0); - return q; -} - -int32_t libdivide_s32_do_alg4(int32_t numer, const struct libdivide_s32_t *denom) { - int32_t q = libdivide__mullhi_s32(denom->magic, numer); - q >>= denom->more & LIBDIVIDE_32_SHIFT_MASK; - q += (q < 0); - return q; -} - -#if LIBDIVIDE_USE_SSE2 -__m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t * denom) { +int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) { uint8_t more = denom->more; - if (more & LIBDIVIDE_S32_SHIFT_PATH) { - uint32_t shifter = more & LIBDIVIDE_32_SHIFT_MASK; - __m128i roundToZeroTweak = _mm_set1_epi32((1 << shifter) - 1); //could use _mm_srli_epi32 with an all -1 register - __m128i q = _mm_add_epi32(numers, _mm_and_si128(_mm_srai_epi32(numers, 31), roundToZeroTweak)); //q = numer + ((numer >> 31) & roundToZeroTweak); - q = _mm_sra_epi32(q, libdivide_u32_to_m128i(shifter)); // q = q >> shifter - __m128i shiftMask = _mm_set1_epi32((int32_t)((int8_t)more >> 7)); //set all bits of shift mask = to the sign bit of more - q = _mm_sub_epi32(_mm_xor_si128(q, shiftMask), shiftMask); //q = (q ^ shiftMask) - shiftMask; - return q; - } - else { - __m128i q = libdivide_mullhi_s32_flat_vector(numers, _mm_set1_epi32(denom->magic)); - if (more & LIBDIVIDE_ADD_MARKER) { - __m128i sign = _mm_set1_epi32((int32_t)(int8_t)more >> 7); //must be arithmetic shift - q = _mm_add_epi32(q, _mm_sub_epi32(_mm_xor_si128(numers, sign), sign)); // q += ((numer ^ sign) - sign); + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + if (!denom->magic) { + uint32_t absD = 1U << shift; + if (more & LIBDIVIDE_NEGATIVE_DIVISOR) { + absD = -absD; } - q = _mm_sra_epi32(q, libdivide_u32_to_m128i(more & LIBDIVIDE_32_SHIFT_MASK)); //q >>= shift - q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); // q += (q < 0) - return q; + return (int32_t)absD; + } else { + // Unsigned math is much easier + // We negate the magic number only in the branchfull case, and we don't + // know which case we're in. However we have enough information to + // determine the correct sign of the magic number. The divisor was + // negative if LIBDIVIDE_NEGATIVE_DIVISOR is set. If ADD_MARKER is set, + // the magic number's sign is opposite that of the divisor. + // We want to compute the positive magic number. + int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) + ? denom->magic > 0 : denom->magic < 0; + + // Handle the power of 2 case (including branchfree) + if (denom->magic == 0) { + int32_t result = 1U << shift; + return negative_divisor ? -result : result; + } + + uint32_t d = (uint32_t)(magic_was_negated ? -denom->magic : denom->magic); + uint64_t n = 1ULL << (32 + shift); // this shift cannot exceed 30 + uint32_t q = (uint32_t)(n / d); + int32_t result = (int32_t)q; + result += 1; + return negative_divisor ? -result : result; } } -__m128i libdivide_s32_do_vector_alg0(__m128i numers, const struct libdivide_s32_t *denom) { - uint8_t shifter = denom->more & LIBDIVIDE_32_SHIFT_MASK; - __m128i roundToZeroTweak = _mm_set1_epi32((1 << shifter) - 1); - __m128i q = _mm_add_epi32(numers, _mm_and_si128(_mm_srai_epi32(numers, 31), roundToZeroTweak)); - return _mm_sra_epi32(q, libdivide_u32_to_m128i(shifter)); +int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom) { + return libdivide_s32_recover((const struct libdivide_s32_t *)denom); } -__m128i libdivide_s32_do_vector_alg1(__m128i numers, const struct libdivide_s32_t *denom) { - uint8_t shifter = denom->more & LIBDIVIDE_32_SHIFT_MASK; - __m128i roundToZeroTweak = _mm_set1_epi32((1 << shifter) - 1); - __m128i q = _mm_add_epi32(numers, _mm_and_si128(_mm_srai_epi32(numers, 31), roundToZeroTweak)); - return _mm_sub_epi32(_mm_setzero_si128(), _mm_sra_epi32(q, libdivide_u32_to_m128i(shifter))); -} - -__m128i libdivide_s32_do_vector_alg2(__m128i numers, const struct libdivide_s32_t *denom) { - __m128i q = libdivide_mullhi_s32_flat_vector(numers, _mm_set1_epi32(denom->magic)); - q = _mm_add_epi32(q, numers); - q = _mm_sra_epi32(q, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_32_SHIFT_MASK)); - q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); - return q; -} - -__m128i libdivide_s32_do_vector_alg3(__m128i numers, const struct libdivide_s32_t *denom) { - __m128i q = libdivide_mullhi_s32_flat_vector(numers, _mm_set1_epi32(denom->magic)); - q = _mm_sub_epi32(q, numers); - q = _mm_sra_epi32(q, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_32_SHIFT_MASK)); - q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); - return q; -} - -__m128i libdivide_s32_do_vector_alg4(__m128i numers, const struct libdivide_s32_t *denom) { - __m128i q = libdivide_mullhi_s32_flat_vector(numers, _mm_set1_epi32(denom->magic)); - q = _mm_sra_epi32(q, libdivide_u32_to_m128i(denom->more)); //q >>= shift - q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); // q += (q < 0) - return q; -} -#endif - ///////////// SINT64 +static inline struct libdivide_s64_t libdivide_internal_s64_gen(int64_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } -struct libdivide_s64_t libdivide_s64_gen(int64_t d) { struct libdivide_s64_t result; - /* If d is a power of 2, or negative a power of 2, we have to use a shift. This is especially important because the magic algorithm fails for -1. To check if d is a power of 2 or its inverse, it suffices to check whether its absolute value has exactly one bit set. This works even for INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set and is a power of 2. */ - const uint64_t absD = (uint64_t)(d < 0 ? -d : d); //gcc optimizes this to the fast abs trick - if ((absD & (absD - 1)) == 0) { //check if exactly one bit is set, don't care if absD is 0 since that's divide by zero - result.more = libdivide__count_trailing_zeros64(absD) | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); + // If d is a power of 2, or negative a power of 2, we have to use a shift. + // This is especially important because the magic algorithm fails for -1. + // To check if d is a power of 2 or its inverse, it suffices to check + // whether its absolute value has exactly one bit set. This works even for + // INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set + // and is a power of 2. + uint64_t ud = (uint64_t)d; + uint64_t absD = (d < 0) ? -ud : ud; + uint32_t floor_log_2_d = 63 - libdivide_count_leading_zeros64(absD); + // check if exactly one bit is set, + // don't care if absD is 0 since that's divide by zero + if ((absD & (absD - 1)) == 0) { + // Branchfree and non-branchfree cases are the same result.magic = 0; - } - else { - const uint32_t floor_log_2_d = 63 - libdivide__count_leading_zeros64(absD); - - //the dividend here is 2**(floor_log_2_d + 63), so the low 64 bit word is 0 and the high word is floor_log_2_d - 1 + result.more = floor_log_2_d | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); + } else { + // the dividend here is 2**(floor_log_2_d + 63), so the low 64 bit word + // is 0 and the high word is floor_log_2_d - 1 uint8_t more; uint64_t rem, proposed_m; proposed_m = libdivide_128_div_64_to_64(1ULL << (floor_log_2_d - 1), 0, absD, &rem); const uint64_t e = absD - rem; - /* We are going to start with a power of floor_log_2_d - 1. This works if works if e < 2**floor_log_2_d. */ - if (e < (1ULL << floor_log_2_d)) { - /* This power works */ + // We are going to start with a power of floor_log_2_d - 1. + // This works if works if e < 2**floor_log_2_d. + if (!branchfree && e < (1ULL << floor_log_2_d)) { + // This power works more = floor_log_2_d - 1; - } - else { - /* We need to go one higher. This should not make proposed_m overflow, but it will make it negative when interpreted as an int32_t. */ + } else { + // We need to go one higher. This should not make proposed_m + // overflow, but it will make it negative when interpreted as an + // int32_t. proposed_m += proposed_m; const uint64_t twice_rem = rem + rem; if (twice_rem >= absD || twice_rem < rem) proposed_m += 1; - more = floor_log_2_d | LIBDIVIDE_ADD_MARKER | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); + // note that we only set the LIBDIVIDE_NEGATIVE_DIVISOR bit if we + // also set ADD_MARKER this is an annoying optimization that + // enables algorithm #4 to avoid the mask. However we always set it + // in the branchfree case + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; } proposed_m += 1; + int64_t magic = (int64_t)proposed_m; + + // Mark if we are negative + if (d < 0) { + more |= LIBDIVIDE_NEGATIVE_DIVISOR; + if (!branchfree) { + magic = -magic; + } + } + result.more = more; - result.magic = (d < 0 ? -(int64_t)proposed_m : (int64_t)proposed_m); + result.magic = magic; } return result; } +struct libdivide_s64_t libdivide_s64_gen(int64_t d) { + return libdivide_internal_s64_gen(d, 0); +} + +struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d) { + struct libdivide_s64_t tmp = libdivide_internal_s64_gen(d, 1); + struct libdivide_s64_branchfree_t ret = {tmp.magic, tmp.more}; + return ret; +} + int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) { uint8_t more = denom->more; - int64_t magic = denom->magic; - if (magic == 0) { //shift path - uint32_t shifter = more & LIBDIVIDE_64_SHIFT_MASK; - int64_t q = numer + ((numer >> 63) & ((1LL << shifter) - 1)); - q = q >> shifter; - int64_t shiftMask = (int8_t)more >> 7; //must be arithmetic shift and then sign-extend - q = (q ^ shiftMask) - shiftMask; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + + if (!denom->magic) { // shift path + uint64_t mask = (1ULL << shift) - 1; + uint64_t uq = numer + ((numer >> 63) & mask); + int64_t q = (int64_t)uq; + q >>= shift; + // must be arithmetic shift and then sign-extend + int64_t sign = (int8_t)more >> 7; + q = (q ^ sign) - sign; return q; - } - else { - int64_t q = libdivide__mullhi_s64(magic, numer); + } else { + uint64_t uq = (uint64_t)libdivide_mullhi_s64(denom->magic, numer); if (more & LIBDIVIDE_ADD_MARKER) { - int64_t sign = (int8_t)more >> 7; //must be arithmetic shift and then sign extend - q += ((numer ^ sign) - sign); + // must be arithmetic shift and then sign extend + int64_t sign = (int8_t)more >> 7; + // q += (more < 0 ? -numer : numer) + // cast required to avoid UB + uq += ((uint64_t)numer ^ sign) - sign; } - q >>= more & LIBDIVIDE_64_SHIFT_MASK; + int64_t q = (int64_t)uq; + q >>= shift; q += (q < 0); return q; } } - -int libdivide_s64_get_algorithm(const struct libdivide_s64_t *denom) { - uint8_t more = denom->more; - int positiveDivisor = ! (more & LIBDIVIDE_NEGATIVE_DIVISOR); - if (denom->magic == 0) return (positiveDivisor ? 0 : 1); //shift path - else if (more & LIBDIVIDE_ADD_MARKER) return (positiveDivisor ? 2 : 3); - else return 4; -} - -int64_t libdivide_s64_do_alg0(int64_t numer, const struct libdivide_s64_t *denom) { - uint32_t shifter = denom->more & LIBDIVIDE_64_SHIFT_MASK; - int64_t q = numer + ((numer >> 63) & ((1LL << shifter) - 1)); - return q >> shifter; -} - -int64_t libdivide_s64_do_alg1(int64_t numer, const struct libdivide_s64_t *denom) { - //denom->shifter != -1 && demo->shiftMask != 0 - uint32_t shifter = denom->more & LIBDIVIDE_64_SHIFT_MASK; - int64_t q = numer + ((numer >> 63) & ((1LL << shifter) - 1)); - return - (q >> shifter); -} - -int64_t libdivide_s64_do_alg2(int64_t numer, const struct libdivide_s64_t *denom) { - int64_t q = libdivide__mullhi_s64(denom->magic, numer); - q += numer; - q >>= denom->more & LIBDIVIDE_64_SHIFT_MASK; - q += (q < 0); - return q; -} - -int64_t libdivide_s64_do_alg3(int64_t numer, const struct libdivide_s64_t *denom) { - int64_t q = libdivide__mullhi_s64(denom->magic, numer); - q -= numer; - q >>= denom->more & LIBDIVIDE_64_SHIFT_MASK; - q += (q < 0); - return q; -} - -int64_t libdivide_s64_do_alg4(int64_t numer, const struct libdivide_s64_t *denom) { - int64_t q = libdivide__mullhi_s64(denom->magic, numer); - q >>= denom->more; - q += (q < 0); - return q; -} - - -#if LIBDIVIDE_USE_SSE2 -__m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t * denom) { +int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom) { uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift and then sign extend + int64_t sign = (int8_t)more >> 7; int64_t magic = denom->magic; - if (magic == 0) { //shift path - uint32_t shifter = more & LIBDIVIDE_64_SHIFT_MASK; - __m128i roundToZeroTweak = libdivide__u64_to_m128((1LL << shifter) - 1); - __m128i q = _mm_add_epi64(numers, _mm_and_si128(libdivide_s64_signbits(numers), roundToZeroTweak)); //q = numer + ((numer >> 63) & roundToZeroTweak); - q = libdivide_s64_shift_right_vector(q, shifter); // q = q >> shifter - __m128i shiftMask = _mm_set1_epi32((int32_t)((int8_t)more >> 7)); - q = _mm_sub_epi64(_mm_xor_si128(q, shiftMask), shiftMask); //q = (q ^ shiftMask) - shiftMask; + int64_t q = libdivide_mullhi_s64(magic, numer); + q += numer; + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is a power of + // 2, or (2**shift) if it is not a power of 2. + uint64_t is_power_of_2 = (magic == 0); + uint64_t q_sign = (uint64_t)(q >> 63); + q += q_sign & ((1ULL << shift) - is_power_of_2); + + // Arithmetic right shift + q >>= shift; + // Negate if needed + q = (q ^ sign) - sign; + + return q; +} + +int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + if (denom->magic == 0) { // shift path + uint64_t absD = 1ULL << shift; + if (more & LIBDIVIDE_NEGATIVE_DIVISOR) { + absD = -absD; + } + return (int64_t)absD; + } else { + // Unsigned math is much easier + int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) + ? denom->magic > 0 : denom->magic < 0; + + uint64_t d = (uint64_t)(magic_was_negated ? -denom->magic : denom->magic); + uint64_t n_hi = 1ULL << shift, n_lo = 0; + uint64_t rem_ignored; + uint64_t q = libdivide_128_div_64_to_64(n_hi, n_lo, d, &rem_ignored); + int64_t result = (int64_t)(q + 1); + if (negative_divisor) { + result = -result; + } + return result; + } +} + +int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom) { + return libdivide_s64_recover((const struct libdivide_s64_t *)denom); +} + +#if defined(LIBDIVIDE_AVX512) + +static inline __m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom); +static inline __m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom); +static inline __m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom); +static inline __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom); + +static inline __m512i libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m512i libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m512i libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m512i libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom); + +//////// Internal Utility Functions + +static inline __m512i libdivide_s64_signbits(__m512i v) {; + return _mm512_srai_epi64(v, 63); +} + +static inline __m512i libdivide_s64_shift_right_vector(__m512i v, int amt) { + return _mm512_srai_epi64(v, amt); +} + +// Here, b is assumed to contain one 32-bit value repeated. +static inline __m512i libdivide_mullhi_u32_vector(__m512i a, __m512i b) { + __m512i hi_product_0Z2Z = _mm512_srli_epi64(_mm512_mul_epu32(a, b), 32); + __m512i a1X3X = _mm512_srli_epi64(a, 32); + __m512i mask = _mm512_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0); + __m512i hi_product_Z1Z3 = _mm512_and_si512(_mm512_mul_epu32(a1X3X, b), mask); + return _mm512_or_si512(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// b is one 32-bit value repeated. +static inline __m512i libdivide_mullhi_s32_vector(__m512i a, __m512i b) { + __m512i hi_product_0Z2Z = _mm512_srli_epi64(_mm512_mul_epi32(a, b), 32); + __m512i a1X3X = _mm512_srli_epi64(a, 32); + __m512i mask = _mm512_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0); + __m512i hi_product_Z1Z3 = _mm512_and_si512(_mm512_mul_epi32(a1X3X, b), mask); + return _mm512_or_si512(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// Here, y is assumed to contain one 64-bit value repeated. +// https://stackoverflow.com/a/28827013 +static inline __m512i libdivide_mullhi_u64_vector(__m512i x, __m512i y) { + __m512i lomask = _mm512_set1_epi64(0xffffffff); + __m512i xh = _mm512_shuffle_epi32(x, (_MM_PERM_ENUM) 0xB1); + __m512i yh = _mm512_shuffle_epi32(y, (_MM_PERM_ENUM) 0xB1); + __m512i w0 = _mm512_mul_epu32(x, y); + __m512i w1 = _mm512_mul_epu32(x, yh); + __m512i w2 = _mm512_mul_epu32(xh, y); + __m512i w3 = _mm512_mul_epu32(xh, yh); + __m512i w0h = _mm512_srli_epi64(w0, 32); + __m512i s1 = _mm512_add_epi64(w1, w0h); + __m512i s1l = _mm512_and_si512(s1, lomask); + __m512i s1h = _mm512_srli_epi64(s1, 32); + __m512i s2 = _mm512_add_epi64(w2, s1l); + __m512i s2h = _mm512_srli_epi64(s2, 32); + __m512i hi = _mm512_add_epi64(w3, s1h); + hi = _mm512_add_epi64(hi, s2h); + + return hi; +} + +// y is one 64-bit value repeated. +static inline __m512i libdivide_mullhi_s64_vector(__m512i x, __m512i y) { + __m512i p = libdivide_mullhi_u64_vector(x, y); + __m512i t1 = _mm512_and_si512(libdivide_s64_signbits(x), y); + __m512i t2 = _mm512_and_si512(libdivide_s64_signbits(y), x); + p = _mm512_sub_epi64(p, t1); + p = _mm512_sub_epi64(p, t2); + return p; +} + +////////// UINT32 + +__m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm512_srli_epi32(numers, more); + } + else { + __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + __m512i t = _mm512_add_epi32(_mm512_srli_epi32(_mm512_sub_epi32(numers, q), 1), q); + return _mm512_srli_epi32(t, shift); + } + else { + return _mm512_srli_epi32(q, more); + } + } +} + +__m512i libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom) { + __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic)); + __m512i t = _mm512_add_epi32(_mm512_srli_epi32(_mm512_sub_epi32(numers, q), 1), q); + return _mm512_srli_epi32(t, denom->more); +} + +////////// UINT64 + +__m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm512_srli_epi64(numers, more); + } + else { + __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + __m512i t = _mm512_add_epi64(_mm512_srli_epi64(_mm512_sub_epi64(numers, q), 1), q); + return _mm512_srli_epi64(t, shift); + } + else { + return _mm512_srli_epi64(q, more); + } + } +} + +__m512i libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom) { + __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic)); + __m512i t = _mm512_add_epi64(_mm512_srli_epi64(_mm512_sub_epi64(numers, q), 1), q); + return _mm512_srli_epi64(t, denom->more); +} + +////////// SINT32 + +__m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32_t mask = (1U << shift) - 1; + __m512i roundToZeroTweak = _mm512_set1_epi32(mask); + // q = numer + ((numer >> 31) & roundToZeroTweak); + __m512i q = _mm512_add_epi32(numers, _mm512_and_si512(_mm512_srai_epi32(numers, 31), roundToZeroTweak)); + q = _mm512_srai_epi32(q, shift); + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm512_sub_epi32(_mm512_xor_si512(q, sign), sign); return q; } else { - __m128i q = libdivide_mullhi_s64_flat_vector(numers, libdivide__u64_to_m128(magic)); + __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { - __m128i sign = _mm_set1_epi32((int32_t)((int8_t)more >> 7)); //must be arithmetic shift - q = _mm_add_epi64(q, _mm_sub_epi64(_mm_xor_si128(numers, sign), sign)); // q += ((numer ^ sign) - sign); + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm512_add_epi32(q, _mm512_sub_epi32(_mm512_xor_si512(numers, sign), sign)); } - q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); //q >>= denom->mult_path.shift + // q >>= shift + q = _mm512_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); + q = _mm512_add_epi32(q, _mm512_srli_epi32(q, 31)); // q += (q < 0) + return q; + } +} + +__m512i libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom) { + int32_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(magic)); + q = _mm512_add_epi32(q, numers); // q += numers + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + __m512i q_sign = _mm512_srai_epi32(q, 31); // q_sign = q >> 31 + __m512i mask = _mm512_set1_epi32((1U << shift) - is_power_of_2); + q = _mm512_add_epi32(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm512_srai_epi32(q, shift); // q >>= shift + q = _mm512_sub_epi32(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +////////// SINT64 + +__m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + int64_t magic = denom->magic; + if (magic == 0) { // shift path + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64_t mask = (1ULL << shift) - 1; + __m512i roundToZeroTweak = _mm512_set1_epi64(mask); + // q = numer + ((numer >> 63) & roundToZeroTweak); + __m512i q = _mm512_add_epi64(numers, _mm512_and_si512(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vector(q, shift); + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm512_sub_epi64(_mm512_xor_si512(q, sign), sign); + return q; + } + else { + __m512i q = libdivide_mullhi_s64_vector(numers, _mm512_set1_epi64(magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm512_add_epi64(q, _mm512_sub_epi64(_mm512_xor_si512(numers, sign), sign)); + } + // q >>= denom->mult_path.shift + q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm512_add_epi64(q, _mm512_srli_epi64(q, 63)); // q += (q < 0) + return q; + } +} + +__m512i libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom) { + int64_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + + // libdivide_mullhi_s64(numers, magic); + __m512i q = libdivide_mullhi_s64_vector(numers, _mm512_set1_epi64(magic)); + q = _mm512_add_epi64(q, numers); // q += numers + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2. + uint32_t is_power_of_2 = (magic == 0); + __m512i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m512i mask = _mm512_set1_epi64((1ULL << shift) - is_power_of_2); + q = _mm512_add_epi64(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift + q = _mm512_sub_epi64(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +#elif defined(LIBDIVIDE_AVX2) + +static inline __m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom); +static inline __m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom); +static inline __m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom); +static inline __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom); + +static inline __m256i libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m256i libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m256i libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m256i libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom); + +//////// Internal Utility Functions + +// Implementation of _mm256_srai_epi64(v, 63) (from AVX512). +static inline __m256i libdivide_s64_signbits(__m256i v) { + __m256i hiBitsDuped = _mm256_shuffle_epi32(v, _MM_SHUFFLE(3, 3, 1, 1)); + __m256i signBits = _mm256_srai_epi32(hiBitsDuped, 31); + return signBits; +} + +// Implementation of _mm256_srai_epi64 (from AVX512). +static inline __m256i libdivide_s64_shift_right_vector(__m256i v, int amt) { + const int b = 64 - amt; + __m256i m = _mm256_set1_epi64x(1ULL << (b - 1)); + __m256i x = _mm256_srli_epi64(v, amt); + __m256i result = _mm256_sub_epi64(_mm256_xor_si256(x, m), m); + return result; +} + +// Here, b is assumed to contain one 32-bit value repeated. +static inline __m256i libdivide_mullhi_u32_vector(__m256i a, __m256i b) { + __m256i hi_product_0Z2Z = _mm256_srli_epi64(_mm256_mul_epu32(a, b), 32); + __m256i a1X3X = _mm256_srli_epi64(a, 32); + __m256i mask = _mm256_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0); + __m256i hi_product_Z1Z3 = _mm256_and_si256(_mm256_mul_epu32(a1X3X, b), mask); + return _mm256_or_si256(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// b is one 32-bit value repeated. +static inline __m256i libdivide_mullhi_s32_vector(__m256i a, __m256i b) { + __m256i hi_product_0Z2Z = _mm256_srli_epi64(_mm256_mul_epi32(a, b), 32); + __m256i a1X3X = _mm256_srli_epi64(a, 32); + __m256i mask = _mm256_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0); + __m256i hi_product_Z1Z3 = _mm256_and_si256(_mm256_mul_epi32(a1X3X, b), mask); + return _mm256_or_si256(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// Here, y is assumed to contain one 64-bit value repeated. +// https://stackoverflow.com/a/28827013 +static inline __m256i libdivide_mullhi_u64_vector(__m256i x, __m256i y) { + __m256i lomask = _mm256_set1_epi64x(0xffffffff); + __m256i xh = _mm256_shuffle_epi32(x, 0xB1); // x0l, x0h, x1l, x1h + __m256i yh = _mm256_shuffle_epi32(y, 0xB1); // y0l, y0h, y1l, y1h + __m256i w0 = _mm256_mul_epu32(x, y); // x0l*y0l, x1l*y1l + __m256i w1 = _mm256_mul_epu32(x, yh); // x0l*y0h, x1l*y1h + __m256i w2 = _mm256_mul_epu32(xh, y); // x0h*y0l, x1h*y0l + __m256i w3 = _mm256_mul_epu32(xh, yh); // x0h*y0h, x1h*y1h + __m256i w0h = _mm256_srli_epi64(w0, 32); + __m256i s1 = _mm256_add_epi64(w1, w0h); + __m256i s1l = _mm256_and_si256(s1, lomask); + __m256i s1h = _mm256_srli_epi64(s1, 32); + __m256i s2 = _mm256_add_epi64(w2, s1l); + __m256i s2h = _mm256_srli_epi64(s2, 32); + __m256i hi = _mm256_add_epi64(w3, s1h); + hi = _mm256_add_epi64(hi, s2h); + + return hi; +} + +// y is one 64-bit value repeated. +static inline __m256i libdivide_mullhi_s64_vector(__m256i x, __m256i y) { + __m256i p = libdivide_mullhi_u64_vector(x, y); + __m256i t1 = _mm256_and_si256(libdivide_s64_signbits(x), y); + __m256i t2 = _mm256_and_si256(libdivide_s64_signbits(y), x); + p = _mm256_sub_epi64(p, t1); + p = _mm256_sub_epi64(p, t2); + return p; +} + +////////// UINT32 + +__m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm256_srli_epi32(numers, more); + } + else { + __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + __m256i t = _mm256_add_epi32(_mm256_srli_epi32(_mm256_sub_epi32(numers, q), 1), q); + return _mm256_srli_epi32(t, shift); + } + else { + return _mm256_srli_epi32(q, more); + } + } +} + +__m256i libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom) { + __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic)); + __m256i t = _mm256_add_epi32(_mm256_srli_epi32(_mm256_sub_epi32(numers, q), 1), q); + return _mm256_srli_epi32(t, denom->more); +} + +////////// UINT64 + +__m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm256_srli_epi64(numers, more); + } + else { + __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + __m256i t = _mm256_add_epi64(_mm256_srli_epi64(_mm256_sub_epi64(numers, q), 1), q); + return _mm256_srli_epi64(t, shift); + } + else { + return _mm256_srli_epi64(q, more); + } + } +} + +__m256i libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom) { + __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic)); + __m256i t = _mm256_add_epi64(_mm256_srli_epi64(_mm256_sub_epi64(numers, q), 1), q); + return _mm256_srli_epi64(t, denom->more); +} + +////////// SINT32 + +__m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32_t mask = (1U << shift) - 1; + __m256i roundToZeroTweak = _mm256_set1_epi32(mask); + // q = numer + ((numer >> 31) & roundToZeroTweak); + __m256i q = _mm256_add_epi32(numers, _mm256_and_si256(_mm256_srai_epi32(numers, 31), roundToZeroTweak)); + q = _mm256_srai_epi32(q, shift); + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm256_sub_epi32(_mm256_xor_si256(q, sign), sign); + return q; + } + else { + __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm256_add_epi32(q, _mm256_sub_epi32(_mm256_xor_si256(numers, sign), sign)); + } + // q >>= shift + q = _mm256_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); + q = _mm256_add_epi32(q, _mm256_srli_epi32(q, 31)); // q += (q < 0) + return q; + } +} + +__m256i libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom) { + int32_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(magic)); + q = _mm256_add_epi32(q, numers); // q += numers + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + __m256i q_sign = _mm256_srai_epi32(q, 31); // q_sign = q >> 31 + __m256i mask = _mm256_set1_epi32((1U << shift) - is_power_of_2); + q = _mm256_add_epi32(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm256_srai_epi32(q, shift); // q >>= shift + q = _mm256_sub_epi32(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +////////// SINT64 + +__m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + int64_t magic = denom->magic; + if (magic == 0) { // shift path + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64_t mask = (1ULL << shift) - 1; + __m256i roundToZeroTweak = _mm256_set1_epi64x(mask); + // q = numer + ((numer >> 63) & roundToZeroTweak); + __m256i q = _mm256_add_epi64(numers, _mm256_and_si256(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vector(q, shift); + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm256_sub_epi64(_mm256_xor_si256(q, sign), sign); + return q; + } + else { + __m256i q = libdivide_mullhi_s64_vector(numers, _mm256_set1_epi64x(magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm256_add_epi64(q, _mm256_sub_epi64(_mm256_xor_si256(numers, sign), sign)); + } + // q >>= denom->mult_path.shift + q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm256_add_epi64(q, _mm256_srli_epi64(q, 63)); // q += (q < 0) + return q; + } +} + +__m256i libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom) { + int64_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + + // libdivide_mullhi_s64(numers, magic); + __m256i q = libdivide_mullhi_s64_vector(numers, _mm256_set1_epi64x(magic)); + q = _mm256_add_epi64(q, numers); // q += numers + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2. + uint32_t is_power_of_2 = (magic == 0); + __m256i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m256i mask = _mm256_set1_epi64x((1ULL << shift) - is_power_of_2); + q = _mm256_add_epi64(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift + q = _mm256_sub_epi64(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +#elif defined(LIBDIVIDE_SSE2) + +static inline __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom); +static inline __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom); +static inline __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom); +static inline __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom); + +static inline __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom); + +//////// Internal Utility Functions + +// Implementation of _mm_srai_epi64(v, 63) (from AVX512). +static inline __m128i libdivide_s64_signbits(__m128i v) { + __m128i hiBitsDuped = _mm_shuffle_epi32(v, _MM_SHUFFLE(3, 3, 1, 1)); + __m128i signBits = _mm_srai_epi32(hiBitsDuped, 31); + return signBits; +} + +// Implementation of _mm_srai_epi64 (from AVX512). +static inline __m128i libdivide_s64_shift_right_vector(__m128i v, int amt) { + const int b = 64 - amt; + __m128i m = _mm_set1_epi64x(1ULL << (b - 1)); + __m128i x = _mm_srli_epi64(v, amt); + __m128i result = _mm_sub_epi64(_mm_xor_si128(x, m), m); + return result; +} + +// Here, b is assumed to contain one 32-bit value repeated. +static inline __m128i libdivide_mullhi_u32_vector(__m128i a, __m128i b) { + __m128i hi_product_0Z2Z = _mm_srli_epi64(_mm_mul_epu32(a, b), 32); + __m128i a1X3X = _mm_srli_epi64(a, 32); + __m128i mask = _mm_set_epi32(-1, 0, -1, 0); + __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epu32(a1X3X, b), mask); + return _mm_or_si128(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// SSE2 does not have a signed multiplication instruction, but we can convert +// unsigned to signed pretty efficiently. Again, b is just a 32 bit value +// repeated four times. +static inline __m128i libdivide_mullhi_s32_vector(__m128i a, __m128i b) { + __m128i p = libdivide_mullhi_u32_vector(a, b); + // t1 = (a >> 31) & y, arithmetic shift + __m128i t1 = _mm_and_si128(_mm_srai_epi32(a, 31), b); + __m128i t2 = _mm_and_si128(_mm_srai_epi32(b, 31), a); + p = _mm_sub_epi32(p, t1); + p = _mm_sub_epi32(p, t2); + return p; +} + +// Here, y is assumed to contain one 64-bit value repeated. +// https://stackoverflow.com/a/28827013 +static inline __m128i libdivide_mullhi_u64_vector(__m128i x, __m128i y) { + __m128i lomask = _mm_set1_epi64x(0xffffffff); + __m128i xh = _mm_shuffle_epi32(x, 0xB1); // x0l, x0h, x1l, x1h + __m128i yh = _mm_shuffle_epi32(y, 0xB1); // y0l, y0h, y1l, y1h + __m128i w0 = _mm_mul_epu32(x, y); // x0l*y0l, x1l*y1l + __m128i w1 = _mm_mul_epu32(x, yh); // x0l*y0h, x1l*y1h + __m128i w2 = _mm_mul_epu32(xh, y); // x0h*y0l, x1h*y0l + __m128i w3 = _mm_mul_epu32(xh, yh); // x0h*y0h, x1h*y1h + __m128i w0h = _mm_srli_epi64(w0, 32); + __m128i s1 = _mm_add_epi64(w1, w0h); + __m128i s1l = _mm_and_si128(s1, lomask); + __m128i s1h = _mm_srli_epi64(s1, 32); + __m128i s2 = _mm_add_epi64(w2, s1l); + __m128i s2h = _mm_srli_epi64(s2, 32); + __m128i hi = _mm_add_epi64(w3, s1h); + hi = _mm_add_epi64(hi, s2h); + + return hi; +} + +// y is one 64-bit value repeated. +static inline __m128i libdivide_mullhi_s64_vector(__m128i x, __m128i y) { + __m128i p = libdivide_mullhi_u64_vector(x, y); + __m128i t1 = _mm_and_si128(libdivide_s64_signbits(x), y); + __m128i t2 = _mm_and_si128(libdivide_s64_signbits(y), x); + p = _mm_sub_epi64(p, t1); + p = _mm_sub_epi64(p, t2); + return p; +} + +////////// UINT32 + +__m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm_srli_epi32(numers, more); + } + else { + __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); + return _mm_srli_epi32(t, shift); + } + else { + return _mm_srli_epi32(q, more); + } + } +} + +__m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom) { + __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic)); + __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); + return _mm_srli_epi32(t, denom->more); +} + +////////// UINT64 + +__m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm_srli_epi64(numers, more); + } + else { + __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); + return _mm_srli_epi64(t, shift); + } + else { + return _mm_srli_epi64(q, more); + } + } +} + +__m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom) { + __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic)); + __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); + return _mm_srli_epi64(t, denom->more); +} + +////////// SINT32 + +__m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32_t mask = (1U << shift) - 1; + __m128i roundToZeroTweak = _mm_set1_epi32(mask); + // q = numer + ((numer >> 31) & roundToZeroTweak); + __m128i q = _mm_add_epi32(numers, _mm_and_si128(_mm_srai_epi32(numers, 31), roundToZeroTweak)); + q = _mm_srai_epi32(q, shift); + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); + return q; + } + else { + __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm_add_epi32(q, _mm_sub_epi32(_mm_xor_si128(numers, sign), sign)); + } + // q >>= shift + q = _mm_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); + q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); // q += (q < 0) + return q; + } +} + +__m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom) { + int32_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(magic)); + q = _mm_add_epi32(q, numers); // q += numers + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + __m128i q_sign = _mm_srai_epi32(q, 31); // q_sign = q >> 31 + __m128i mask = _mm_set1_epi32((1U << shift) - is_power_of_2); + q = _mm_add_epi32(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm_srai_epi32(q, shift); // q >>= shift + q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +////////// SINT64 + +__m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + int64_t magic = denom->magic; + if (magic == 0) { // shift path + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64_t mask = (1ULL << shift) - 1; + __m128i roundToZeroTweak = _mm_set1_epi64x(mask); + // q = numer + ((numer >> 63) & roundToZeroTweak); + __m128i q = _mm_add_epi64(numers, _mm_and_si128(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vector(q, shift); + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); + return q; + } + else { + __m128i q = libdivide_mullhi_s64_vector(numers, _mm_set1_epi64x(magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm_add_epi64(q, _mm_sub_epi64(_mm_xor_si128(numers, sign), sign)); + } + // q >>= denom->mult_path.shift + q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); // q += (q < 0) return q; } } -__m128i libdivide_s64_do_vector_alg0(__m128i numers, const struct libdivide_s64_t *denom) { - uint32_t shifter = denom->more & LIBDIVIDE_64_SHIFT_MASK; - __m128i roundToZeroTweak = libdivide__u64_to_m128((1LL << shifter) - 1); - __m128i q = _mm_add_epi64(numers, _mm_and_si128(libdivide_s64_signbits(numers), roundToZeroTweak)); - q = libdivide_s64_shift_right_vector(q, shifter); - return q; -} +__m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom) { + int64_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); -__m128i libdivide_s64_do_vector_alg1(__m128i numers, const struct libdivide_s64_t *denom) { - uint32_t shifter = denom->more & LIBDIVIDE_64_SHIFT_MASK; - __m128i roundToZeroTweak = libdivide__u64_to_m128((1LL << shifter) - 1); - __m128i q = _mm_add_epi64(numers, _mm_and_si128(libdivide_s64_signbits(numers), roundToZeroTweak)); - q = libdivide_s64_shift_right_vector(q, shifter); - return _mm_sub_epi64(_mm_setzero_si128(), q); -} + // libdivide_mullhi_s64(numers, magic); + __m128i q = libdivide_mullhi_s64_vector(numers, _mm_set1_epi64x(magic)); + q = _mm_add_epi64(q, numers); // q += numers -__m128i libdivide_s64_do_vector_alg2(__m128i numers, const struct libdivide_s64_t *denom) { - __m128i q = libdivide_mullhi_s64_flat_vector(numers, libdivide__u64_to_m128(denom->magic)); - q = _mm_add_epi64(q, numers); - q = libdivide_s64_shift_right_vector(q, denom->more & LIBDIVIDE_64_SHIFT_MASK); - q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); // q += (q < 0) - return q; -} - -__m128i libdivide_s64_do_vector_alg3(__m128i numers, const struct libdivide_s64_t *denom) { - __m128i q = libdivide_mullhi_s64_flat_vector(numers, libdivide__u64_to_m128(denom->magic)); - q = _mm_sub_epi64(q, numers); - q = libdivide_s64_shift_right_vector(q, denom->more & LIBDIVIDE_64_SHIFT_MASK); - q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); // q += (q < 0) - return q; -} - -__m128i libdivide_s64_do_vector_alg4(__m128i numers, const struct libdivide_s64_t *denom) { - __m128i q = libdivide_mullhi_s64_flat_vector(numers, libdivide__u64_to_m128(denom->magic)); - q = libdivide_s64_shift_right_vector(q, denom->more); - q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2. + uint32_t is_power_of_2 = (magic == 0); + __m128i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m128i mask = _mm_set1_epi64x((1ULL << shift) - is_power_of_2); + q = _mm_add_epi64(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift + q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign return q; } @@ -1204,228 +1930,143 @@ __m128i libdivide_s64_do_vector_alg4(__m128i numers, const struct libdivide_s64_ #ifdef __cplusplus -/* The C++ template design here is a total mess. This needs to be fixed by someone better at templates than I. The current design is: - -- The base is a template divider_base that takes the integer type, the libdivide struct, a generating function, a get algorithm function, a do function, and either a do vector function or a dummy int. -- The base has storage for the libdivide struct. This is the only storage (so the C++ class should be no larger than the libdivide struct). - -- Above that, there's divider_mid. This is an empty struct by default, but it is specialized against our four int types. divider_mid contains a template struct algo, that contains a typedef for a specialization of divider_base. struct algo is specialized to take an "algorithm number," where -1 means to use the general algorithm. - -- Publicly we have class divider, which inherits from divider_mid::algo. This also take an algorithm number, which defaults to -1 (the general algorithm). -- divider has a operator / which allows you to use a divider as the divisor in a quotient expression. - -*/ - -namespace libdivide_internal { - -#if LIBDIVIDE_USE_SSE2 -#define MAYBE_VECTOR(x) x -#define MAYBE_VECTOR_PARAM __m128i vector_func(__m128i, const DenomType *) -#else -#define MAYBE_VECTOR(x) 0 -#define MAYBE_VECTOR_PARAM int vector_func -#endif - - /* Some bogus unswitch functions for unsigned types so the same (presumably templated) code can work for both signed and unsigned. */ - uint32_t crash_u32(uint32_t, const libdivide_u32_t *) { abort(); } - uint64_t crash_u64(uint64_t, const libdivide_u64_t *) { abort(); } -#ifdef __APPLE__ - UInt64 crash_u64(UInt64, const libdivide_u64_t *) { abort(); } -#endif -#if LIBDIVIDE_USE_SSE2 - __m128i crash_u32_vector(__m128i, const libdivide_u32_t *) { abort(); } - __m128i crash_u64_vector(__m128i, const libdivide_u64_t *) { abort(); } -#endif - - template - class divider_base { - public: - DenomType denom; - divider_base(IntType d) : denom(gen_func(d)) { } - divider_base(const DenomType & d) : denom(d) { } - - IntType perform_divide(IntType val) const { return do_func(val, &denom); } -#if LIBDIVIDE_USE_SSE2 - __m128i perform_divide_vector(__m128i val) const { return vector_func(val, &denom); } -#endif - - int get_algorithm() const { return get_algo(&denom); } - }; - - - template struct divider_mid { }; - - template<> struct divider_mid { - typedef uint32_t IntType; - typedef struct libdivide_u32_t DenomType; - template struct denom { - typedef divider_base divider; - }; - - template struct algo { }; - template struct algo<-1, J> { typedef denom::divider divider; }; - template struct algo<0, J> { typedef denom::divider divider; }; - template struct algo<1, J> { typedef denom::divider divider; }; - template struct algo<2, J> { typedef denom::divider divider; }; - - /* Define two more bogus ones so that the same (templated, presumably) code can handle both signed and unsigned */ - template struct algo<3, J> { typedef denom::divider divider; }; - template struct algo<4, J> { typedef denom::divider divider; }; - - }; - - template<> struct divider_mid { - typedef int32_t IntType; - typedef struct libdivide_s32_t DenomType; - template struct denom { - typedef divider_base divider; - }; - - - template struct algo { }; - template struct algo<-1, J> { typedef denom::divider divider; }; - template struct algo<0, J> { typedef denom::divider divider; }; - template struct algo<1, J> { typedef denom::divider divider; }; - template struct algo<2, J> { typedef denom::divider divider; }; - template struct algo<3, J> { typedef denom::divider divider; }; - template struct algo<4, J> { typedef denom::divider divider; }; - - }; - -#ifdef __APPLE__ - template<> struct divider_mid { - typedef Int64 IntType; - typedef struct libdivide_s64_t DenomType; - template struct denom { - typedef divider_base divider; - }; - - template struct algo { }; - template struct algo<-1, J> { typedef denom::divider divider; }; - template struct algo<0, J> { typedef denom::divider divider; }; - template struct algo<1, J> { typedef denom::divider divider; }; - template struct algo<2, J> { typedef denom::divider divider; }; - template struct algo<3, J> { typedef denom::divider divider; }; - template struct algo<4, J> { typedef denom::divider divider; }; - }; - - template<> struct divider_mid { - typedef UInt64 IntType; - typedef struct libdivide_u64_t DenomType; - template struct denom { - typedef divider_base divider; - }; - - template struct algo { }; - template struct algo<-1, J> { typedef denom::divider divider; }; - template struct algo<0, J> { typedef denom::divider divider; }; - template struct algo<1, J> { typedef denom::divider divider; }; - template struct algo<2, J> { typedef denom::divider divider; }; - - /* Define two more bogus ones so that the same (templated, presumably) code can handle both signed and unsigned */ - template struct algo<3, J> { typedef denom::divider divider; }; - template struct algo<4, J> { typedef denom::divider divider; }; - - - }; -#endif - - template<> struct divider_mid { - typedef uint64_t IntType; - typedef struct libdivide_u64_t DenomType; - template struct denom { - typedef divider_base divider; - }; - - template struct algo { }; - template struct algo<-1, J> { typedef denom::divider divider; }; - template struct algo<0, J> { typedef denom::divider divider; }; - template struct algo<1, J> { typedef denom::divider divider; }; - template struct algo<2, J> { typedef denom::divider divider; }; - - /* Define two more bogus ones so that the same (templated, presumably) code can handle both signed and unsigned */ - template struct algo<3, J> { typedef denom::divider divider; }; - template struct algo<4, J> { typedef denom::divider divider; }; - - - }; - - template<> struct divider_mid { - typedef int64_t IntType; - typedef struct libdivide_s64_t DenomType; - template struct denom { - typedef divider_base divider; - }; - - template struct algo { }; - template struct algo<-1, J> { typedef denom::divider divider; }; - template struct algo<0, J> { typedef denom::divider divider; }; - template struct algo<1, J> { typedef denom::divider divider; }; - template struct algo<2, J> { typedef denom::divider divider; }; - template struct algo<3, J> { typedef denom::divider divider; }; - template struct algo<4, J> { typedef denom::divider divider; }; - }; - -} - -template -class divider -{ - private: - typename libdivide_internal::divider_mid::template algo::divider sub; - template friend divider unswitch(const divider & d); - divider(const typename libdivide_internal::divider_mid::DenomType & denom) : sub(denom) { } - - public: - - /* Ordinary constructor, that takes the divisor as a parameter. */ - divider(T n) : sub(n) { } - - /* Default constructor, that divides by 1 */ - divider() : sub(1) { } - - /* Divides the parameter by the divisor, returning the quotient */ - T perform_divide(T val) const { return sub.perform_divide(val); } - -#if LIBDIVIDE_USE_SSE2 - /* Treats the vector as either two or four packed values (depending on the size), and divides each of them by the divisor, returning the packed quotients. */ - __m128i perform_divide_vector(__m128i val) const { return sub.perform_divide_vector(val); } -#endif - - /* Returns the index of algorithm, for use in the unswitch function */ - int get_algorithm() const { return sub.get_algorithm(); } // returns the algorithm for unswitching - - /* operator== */ - bool operator==(const divider & him) const { return sub.denom.magic == him.sub.denom.magic && sub.denom.more == him.sub.denom.more; } - - bool operator!=(const divider & him) const { return ! (*this == him); } +// The C++ divider class is templated on both an integer type +// (like uint64_t) and an algorithm type. +// * BRANCHFULL is the default algorithm type. +// * BRANCHFREE is the branchfree algorithm type. +enum { + BRANCHFULL, + BRANCHFREE }; -/* Returns a divider specialized for the given algorithm. */ -template -divider unswitch(const divider & d) { return divider(d.sub.denom); } - -/* Overload of the / operator for scalar division. */ -template -int_type operator/(int_type numer, const divider & denom) { - return denom.perform_divide(numer); -} - -#if LIBDIVIDE_USE_SSE2 -/* Overload of the / operator for vector division. */ -template -__m128i operator/(__m128i numer, const divider & denom) { - return denom.perform_divide_vector(numer); -} +#if defined(LIBDIVIDE_AVX512) + #define LIBDIVIDE_VECTOR_TYPE __m512i +#elif defined(LIBDIVIDE_AVX2) + #define LIBDIVIDE_VECTOR_TYPE __m256i +#elif defined(LIBDIVIDE_SSE2) + #define LIBDIVIDE_VECTOR_TYPE __m128i #endif - -#endif //__cplusplus - -#endif //LIBDIVIDE_HEADER_ONLY -#ifdef __cplusplus -} //close namespace libdivide -} //close anonymous namespace +#if !defined(LIBDIVIDE_VECTOR_TYPE) + #define LIBDIVIDE_DIVIDE_VECTOR(ALGO) +#else + #define LIBDIVIDE_DIVIDE_VECTOR(ALGO) \ + LIBDIVIDE_VECTOR_TYPE divide(LIBDIVIDE_VECTOR_TYPE n) const { \ + return libdivide_##ALGO##_do_vector(n, &denom); \ + } #endif -#pragma GCC diagnostic pop +// The DISPATCHER_GEN() macro generates C++ methods (for the given integer +// and algorithm types) that redirect to libdivide's C API. +#define DISPATCHER_GEN(T, ALGO) \ + libdivide_##ALGO##_t denom; \ + dispatcher() { } \ + dispatcher(T d) \ + : denom(libdivide_##ALGO##_gen(d)) \ + { } \ + T divide(T n) const { \ + return libdivide_##ALGO##_do(n, &denom); \ + } \ + LIBDIVIDE_DIVIDE_VECTOR(ALGO) \ + T recover() const { \ + return libdivide_##ALGO##_recover(&denom); \ + } + +// The dispatcher selects a specific division algorithm for a given +// type and ALGO using partial template specialization. +template struct dispatcher { }; + +template<> struct dispatcher { DISPATCHER_GEN(int32_t, s32) }; +template<> struct dispatcher { DISPATCHER_GEN(int32_t, s32_branchfree) }; +template<> struct dispatcher { DISPATCHER_GEN(uint32_t, u32) }; +template<> struct dispatcher { DISPATCHER_GEN(uint32_t, u32_branchfree) }; +template<> struct dispatcher { DISPATCHER_GEN(int64_t, s64) }; +template<> struct dispatcher { DISPATCHER_GEN(int64_t, s64_branchfree) }; +template<> struct dispatcher { DISPATCHER_GEN(uint64_t, u64) }; +template<> struct dispatcher { DISPATCHER_GEN(uint64_t, u64_branchfree) }; + +// This is the main divider class for use by the user (C++ API). +// The actual division algorithm is selected using the dispatcher struct +// based on the integer and algorithm template parameters. +template +class divider { +public: + // We leave the default constructor empty so that creating + // an array of dividers and then initializing them + // later doesn't slow us down. + divider() { } + + // Constructor that takes the divisor as a parameter + divider(T d) : div(d) { } + + // Divides n by the divisor + T divide(T n) const { + return div.divide(n); + } + + // Recovers the divisor, returns the value that was + // used to initialize this divider object. + T recover() const { + return div.recover(); + } + + bool operator==(const divider& other) const { + return div.denom.magic == other.denom.magic && + div.denom.more == other.denom.more; + } + + bool operator!=(const divider& other) const { + return !(*this == other); + } + +#if defined(LIBDIVIDE_VECTOR_TYPE) + // Treats the vector as packed integer values with the same type as + // the divider (e.g. s32, u32, s64, u64) and divides each of + // them by the divider, returning the packed quotients. + LIBDIVIDE_VECTOR_TYPE divide(LIBDIVIDE_VECTOR_TYPE n) const { + return div.divide(n); + } +#endif + +private: + // Storage for the actual divisor + dispatcher::value, + std::is_signed::value, sizeof(T), ALGO> div; +}; + +// Overload of operator / for scalar division +template +T operator/(T n, const divider& div) { + return div.divide(n); +} + +// Overload of operator /= for scalar division +template +T& operator/=(T& n, const divider& div) { + n = div.divide(n); + return n; +} + +#if defined(LIBDIVIDE_VECTOR_TYPE) + // Overload of operator / for vector division + template + LIBDIVIDE_VECTOR_TYPE operator/(LIBDIVIDE_VECTOR_TYPE n, const divider& div) { + return div.divide(n); + } + // Overload of operator /= for vector division + template + LIBDIVIDE_VECTOR_TYPE& operator/=(LIBDIVIDE_VECTOR_TYPE& n, const divider& div) { + n = div.divide(n); + return n; + } +#endif + +// libdivdie::branchfree_divider +template +using branchfree_divider = divider; + +} // namespace libdivide + +#endif // __cplusplus + +#endif // LIBDIVIDE_H diff --git a/contrib/libgsasl b/contrib/libgsasl index 42ef2068704..140fb582505 160000 --- a/contrib/libgsasl +++ b/contrib/libgsasl @@ -1 +1 @@ -Subproject commit 42ef20687042637252e64df1934b6d47771486d1 +Subproject commit 140fb58250588c8323285b75fcf127c4adc33dfa diff --git a/contrib/libpcg-random/README.md b/contrib/libpcg-random/README.md deleted file mode 100644 index c6e579cc2cc..00000000000 --- a/contrib/libpcg-random/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# PCG Random Number Generation, C++ Edition - -[PCG-Random website]: http://www.pcg-random.org - -This code provides an implementation of the PCG family of random number -generators, which are fast, statistically excellent, and offer a number of -useful features. - -Full details can be found at the [PCG-Random website]. This version -of the code provides many family members -- if you just want one -simple generator, you may prefer the minimal C version of the library. - -There are two kinds of generator, normal generators and extended generators. -Extended generators provide *k* dimensional equidistribution and can perform -party tricks, but generally speaking most people only need the normal -generators. - -There are two ways to access the generators, using a convenience typedef -or by using the underlying templates directly (similar to C++11's `std::mt19937` typedef vs its `std::mersenne_twister_engine` template). For most users, the convenience typedef is what you want, and probably you're fine with `pcg32` for 32-bit numbers. If you want 64-bit numbers, either use `pcg64` (or, if you're on a 32-bit system, making 64 bits from two calls to `pcg32_k2` may be faster). - -## Documentation and Examples - -Visit [PCG-Random website] for information on how to use this library, or look -at the sample code in the `sample` directory -- hopefully it should be fairly -self explanatory. - -## Building - -The code is written in C++11, as an include-only library (i.e., there is -nothing you need to build). There are some provided demo programs and tests -however. On a Unix-style system (e.g., Linux, Mac OS X) you should be able -to just type - - make - -To build the demo programs. - -## Testing - -Run - - make test - -## Directory Structure - -The directories are arranged as follows: - -* `include` -- contains `pcg_random.hpp` and supporting include files -* `test-high` -- test code for the high-level API where the functions have - shorter, less scary-looking names. -* `sample` -- sample code, some similar to the code in `test-high` but more - human readable, some other examples too diff --git a/contrib/llvm b/contrib/llvm index 5dab18f4861..3d6c7e91676 160000 --- a/contrib/llvm +++ b/contrib/llvm @@ -1 +1 @@ -Subproject commit 5dab18f4861677548b8f7f6815f49384480ecead +Subproject commit 3d6c7e916760b395908f28a1c885c8334d4fa98b diff --git a/contrib/msgpack-c b/contrib/msgpack-c new file mode 160000 index 00000000000..46684265d50 --- /dev/null +++ b/contrib/msgpack-c @@ -0,0 +1 @@ +Subproject commit 46684265d50b5d1b062d4c5c428ba08462844b1d diff --git a/contrib/pdqsort/pdqsort.h b/contrib/pdqsort/pdqsort.h index 31eb06fece4..01e82b710ee 100644 --- a/contrib/pdqsort/pdqsort.h +++ b/contrib/pdqsort/pdqsort.h @@ -124,11 +124,9 @@ namespace pdqsort_detail { inline bool partial_insertion_sort(Iter begin, Iter end, Compare comp) { typedef typename std::iterator_traits::value_type T; if (begin == end) return true; - - int limit = 0; - for (Iter cur = begin + 1; cur != end; ++cur) { - if (limit > partial_insertion_sort_limit) return false; + std::size_t limit = 0; + for (Iter cur = begin + 1; cur != end; ++cur) { Iter sift = cur; Iter sift_1 = cur - 1; @@ -142,6 +140,8 @@ namespace pdqsort_detail { *sift = PDQSORT_PREFER_MOVE(tmp); limit += cur - sift; } + + if (limit > partial_insertion_sort_limit) return false; } return true; @@ -232,7 +232,7 @@ namespace pdqsort_detail { unsigned char* offsets_r = align_cacheline(offsets_r_storage); int num_l, num_r, start_l, start_r; num_l = num_r = start_l = start_r = 0; - + while (last - first > 2 * block_size) { // Fill up offset blocks with elements that are on the wrong side. if (num_l == 0) { @@ -275,7 +275,7 @@ namespace pdqsort_detail { } int l_size = 0, r_size = 0; - int unknown_left = (last - first) - ((num_r || num_l) ? block_size : 0); + int unknown_left = (int)(last - first) - ((num_r || num_l) ? block_size : 0); if (num_r) { // Handle leftover block by assigning the unknown elements to the other block. l_size = unknown_left; @@ -311,7 +311,7 @@ namespace pdqsort_detail { start_l += num; start_r += num; if (num_l == 0) first += l_size; if (num_r == 0) last -= r_size; - + // We have now fully identified [first, last)'s proper position. Swap the last elements. if (num_l) { offsets_l += start_l; @@ -340,7 +340,7 @@ namespace pdqsort_detail { template inline std::pair partition_right(Iter begin, Iter end, Compare comp) { typedef typename std::iterator_traits::value_type T; - + // Move pivot into local for speed. T pivot(PDQSORT_PREFER_MOVE(*begin)); @@ -359,7 +359,7 @@ namespace pdqsort_detail { // If the first pair of elements that should be swapped to partition are the same element, // the passed in sequence already was correctly partitioned. bool already_partitioned = first >= last; - + // Keep swapping pairs of elements that are on the wrong side of the pivot. Previously // swapped pairs guard the searches, which is why the first iteration is special-cased // above. @@ -388,7 +388,7 @@ namespace pdqsort_detail { T pivot(PDQSORT_PREFER_MOVE(*begin)); Iter first = begin; Iter last = end; - + while (comp(pivot, *--last)); if (last + 1 == end) while (first < last && !comp(pivot, *++first)); @@ -475,11 +475,11 @@ namespace pdqsort_detail { std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); } } - + if (r_size >= insertion_sort_threshold) { std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); std::iter_swap(end - 1, end - r_size / 4); - + if (r_size > ninther_threshold) { std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); @@ -493,7 +493,7 @@ namespace pdqsort_detail { if (already_partitioned && partial_insertion_sort(begin, pivot_pos, comp) && partial_insertion_sort(pivot_pos + 1, end, comp)) return; } - + // Sort the left partition first using recursion and do tail recursion elimination for // the right-hand partition. pdqsort_loop(begin, pivot_pos, comp, bad_allowed, leftmost); diff --git a/contrib/poco b/contrib/poco index ddca76ba495..7d605a1ae5d 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit ddca76ba4956cb57150082394536cc43ff28f6fa +Subproject commit 7d605a1ae5d878294f91f68feb62ae51e9a04426 diff --git a/contrib/rapidjson b/contrib/rapidjson index 01950eb7ace..8f4c021fa2f 160000 --- a/contrib/rapidjson +++ b/contrib/rapidjson @@ -1 +1 @@ -Subproject commit 01950eb7acec78818d68b762efc869bba2420d82 +Subproject commit 8f4c021fa2f1e001d2376095928fc0532adf2ae6 diff --git a/debian/control b/debian/control index 3ce12b504c2..58efd711d27 100644 --- a/debian/control +++ b/debian/control @@ -28,7 +28,7 @@ Description: Client binary for ClickHouse Package: clickhouse-common-static Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, tzdata +Depends: ${shlibs:Depends}, ${misc:Depends} Suggests: clickhouse-common-static-dbg Replaces: clickhouse-common, clickhouse-server-base Provides: clickhouse-common, clickhouse-server-base diff --git a/docker/README.md b/docker/README.md index 083c6a1dc28..ec52ddd143e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,5 +1,5 @@ ## ClickHouse Dockerfiles -This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They updated each release. +This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They are updated in each release. Also there is bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`. diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index e76d9c290c0..f2dc484a438 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -1,5 +1,9 @@ FROM ubuntu:19.10 +RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list + RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get install --yes --no-install-recommends \ @@ -19,10 +23,10 @@ RUN apt-get update -y \ python-termcolor \ sudo \ tzdata \ - clang \ - clang-tidy \ - lld \ - lldb + clang-10 \ + clang-tidy-10 \ + lld-10 \ + lldb-10 COPY build.sh / diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 67c8c125c91..b696e0688ae 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -16,8 +16,7 @@ RUN apt-get update \ apt-get install --allow-unauthenticated --yes --no-install-recommends \ clickhouse-client=$version \ clickhouse-common-static=$version \ - locales \ - tzdata \ + locales && rm -rf /var/lib/apt/lists/* /var/cache/debconf \ && apt-get clean diff --git a/docker/images.json b/docker/images.json index c7dfc82d906..450f5ae3966 100644 --- a/docker/images.json +++ b/docker/images.json @@ -1,10 +1,10 @@ { "docker/packager/deb": "yandex/clickhouse-deb-builder", "docker/packager/binary": "yandex/clickhouse-binary-builder", + "docker/test/coverage": "yandex/clickhouse-coverage", "docker/test/compatibility/centos": "yandex/clickhouse-test-old-centos", "docker/test/compatibility/ubuntu": "yandex/clickhouse-test-old-ubuntu", - "docker/test/integration": "yandex/clickhouse-integration-test", - "docker/test/performance": "yandex/clickhouse-performance-test", + "docker/test/integration/base": "yandex/clickhouse-integration-test", "docker/test/performance-comparison": "yandex/clickhouse-performance-comparison", "docker/test/pvs": "yandex/clickhouse-pvs-test", "docker/test/stateful": "yandex/clickhouse-stateful-test", @@ -14,5 +14,6 @@ "docker/test/unit": "yandex/clickhouse-unit-test", "docker/test/stress": "yandex/clickhouse-stress-test", "docker/test/split_build_smoke_test": "yandex/clickhouse-split-build-smoke-test", - "tests/integration/image": "yandex/clickhouse-integration-tests-runner" + "docker/test/codebrowser": "yandex/clickhouse-codebrowser", + "docker/test/integration/runner": "yandex/clickhouse-integration-tests-runner" } diff --git a/docker/packager/README.md b/docker/packager/README.md index e02a45fdaea..5d9751a0fbd 100644 --- a/docker/packager/README.md +++ b/docker/packager/README.md @@ -3,10 +3,10 @@ compilers and build settings. Correctly configured Docker daemon is single depen Usage: -Build deb package with `gcc-8` in `debug` mode: +Build deb package with `gcc-9` in `debug` mode: ``` $ mkdir deb/test_output -$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=gcc-8 --build-type=debug +$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=gcc-9 --build-type=debug $ ls -l deb/test_output -rw-r--r-- 1 root root 3730 clickhouse-client_18.14.2+debug_all.deb -rw-r--r-- 1 root root 84221888 clickhouse-common-static_18.14.2+debug_amd64.deb @@ -18,11 +18,11 @@ $ ls -l deb/test_output ``` -Build ClickHouse binary with `clang-6.0` and `address` sanitizer in `relwithdebuginfo` +Build ClickHouse binary with `clang-9.0` and `address` sanitizer in `relwithdebuginfo` mode: ``` $ mkdir $HOME/some_clickhouse -$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-6.0 --sanitizer=address +$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-9.0 --sanitizer=address $ ls -l $HOME/some_clickhouse -rwxr-xr-x 1 root root 787061952 clickhouse lrwxrwxrwx 1 root root 10 clickhouse-benchmark -> clickhouse diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 54755d7c2f5..03c807cfaae 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -2,6 +2,10 @@ # docker build -t yandex/clickhouse-binary-builder . FROM ubuntu:19.10 +RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list + RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get --allow-unauthenticated install --yes --no-install-recommends \ @@ -23,6 +27,9 @@ RUN apt-get update -y \ curl \ gcc-9 \ g++-9 \ + clang-10 \ + lld-10 \ + clang-tidy-10 \ clang-9 \ lld-9 \ clang-tidy-9 \ @@ -43,10 +50,10 @@ RUN apt-get update -y \ build-essential # This symlink required by gcc to find lld compiler -RUN ln -s /usr/bin/lld-9 /usr/bin/ld.lld +RUN ln -s /usr/bin/lld-10 /usr/bin/ld.lld -ENV CC=clang-9 -ENV CXX=clang++-9 +ENV CC=clang-10 +ENV CXX=clang++-10 # libtapi is required to support .tbh format from recent MacOS SDKs RUN git clone https://github.com/tpoechtrager/apple-libtapi.git diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index bedde0a2013..b8e7b06bf6b 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,6 +1,10 @@ # docker build -t yandex/clickhouse-deb-builder . FROM ubuntu:19.10 +RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list + RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get --allow-unauthenticated install --yes --no-install-recommends \ @@ -19,6 +23,9 @@ RUN apt-get --allow-unauthenticated update -y \ apt-get --allow-unauthenticated install --yes --no-install-recommends \ gcc-9 \ g++-9 \ + clang-10 \ + lld-10 \ + clang-tidy-10 \ clang-9 \ lld-9 \ clang-tidy-9 \ @@ -48,6 +55,7 @@ RUN apt-get --allow-unauthenticated update -y \ libltdl-dev \ libre2-dev \ libjemalloc-dev \ + libmsgpack-dev \ unixodbc-dev \ odbcinst \ tzdata \ @@ -68,7 +76,7 @@ RUN chmod +x dpkg-deb RUN cp dpkg-deb /usr/bin # This symlink required by gcc to find lld compiler -RUN ln -s /usr/bin/lld-9 /usr/bin/ld.lld +RUN ln -s /usr/bin/lld-10 /usr/bin/ld.lld COPY build.sh / diff --git a/docker/packager/freebsd/Vagrantfile b/docker/packager/freebsd/Vagrantfile deleted file mode 100644 index 765f46d5604..00000000000 --- a/docker/packager/freebsd/Vagrantfile +++ /dev/null @@ -1,4 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = "robot-clickhouse/clickhouse-freebsd" - config.vm.synced_folder ".", "/vagrant", disabled: true -end diff --git a/docker/packager/packager b/docker/packager/packager index 360a358c6e5..71380b92fac 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -11,48 +11,8 @@ SCRIPT_PATH = os.path.realpath(__file__) IMAGE_MAP = { "deb": "yandex/clickhouse-deb-builder", "binary": "yandex/clickhouse-binary-builder", - "freebsd": os.path.join(os.path.dirname(SCRIPT_PATH), "freebsd"), } -class Vagrant(object): - def __init__(self, path_to_vagrant_file): - self.prefix = "VAGRANT_CWD=" + path_to_vagrant_file - - def __enter__(self): - subprocess.check_call("{} vagrant up".format(self.prefix), shell=True) - self.ssh_path = "/tmp/vagrant-ssh" - subprocess.check_call("{} vagrant ssh-config > {}".format(self.prefix, self.ssh_path), shell=True) - return self - - def copy_to_image(self, local_path, remote_path): - cmd = "scp -F {ssh} -r {lpath} default:{rpath}".format(ssh=self.ssh_path, lpath=local_path, rpath=remote_path) - logging.info("Copying to image %s", cmd) - subprocess.check_call( - cmd, - shell=True - ) - - def copy_from_image(self, remote_path, local_path): - cmd = "scp -F {ssh} -r default:{rpath} {lpath}".format(ssh=self.ssh_path, rpath=remote_path, lpath=local_path) - logging.info("Copying from image %s", cmd) - subprocess.check_call( - cmd, - shell=True - ) - - def execute_cmd(self, cmd): - cmd = '{} vagrant ssh -c "{}"'.format(self.prefix, cmd) - logging.info("Executin cmd %s", cmd) - subprocess.check_call( - cmd, - shell=True - ) - - def __exit__(self, exc_type, exc_val, exc_tb): - logging.info("Destroying image") - subprocess.check_call("{} vagrant destroy --force".format(self.prefix), shell=True) - - def check_image_exists_locally(image_name): try: output = subprocess.check_output("docker images -q {} 2> /dev/null".format(image_name), shell=True) @@ -94,15 +54,6 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache subprocess.check_call(cmd, shell=True) -def run_vagrant_box_with_env(image_path, output_dir, ch_root): - with Vagrant(image_path) as vagrant: - logging.info("Copying folder to vagrant machine") - vagrant.copy_to_image(ch_root, "~/ClickHouse") - logging.info("Running build") - vagrant.execute_cmd("cd ~/ClickHouse && cmake . && ninja") - logging.info("Copying binary back") - vagrant.copy_from_image("~/ClickHouse/programs/clickhouse", output_dir) - def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage): CLANG_PREFIX = "clang" DARWIN_SUFFIX = "-darwin" @@ -210,11 +161,11 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') parser = argparse.ArgumentParser(description="ClickHouse building script using prebuilt Docker image") # 'performance' creates a combined .tgz with server and configs to be used for performance test. - parser.add_argument("--package-type", choices=['deb', 'binary', 'performance', 'freebsd'], required=True) + parser.add_argument("--package-type", choices=['deb', 'binary', 'performance'], required=True) parser.add_argument("--clickhouse-repo-path", default="../../") parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") - parser.add_argument("--compiler", choices=("clang-8", "clang-8-darwin", "clang-8-aarch64", "gcc-8", "gcc-9", "clang-9"), default="gcc-8") + parser.add_argument("--compiler", choices=("clang-8", "clang-8-darwin", "clang-9-aarch64", "clang-9-freebsd", "gcc-8", "gcc-9", "clang-9"), default="gcc-8") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--split-binary", action="store_true") @@ -252,9 +203,5 @@ if __name__ == "__main__": args.build_type, args.compiler, args.sanitizer, args.package_type, image_type, args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy, args.version, args.author, args.official, args.alien_pkgs, args.with_coverage) - if image_type != "freebsd": - run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) - else: - logging.info("Running freebsd build, arguments will be ignored") - run_vagrant_box_with_env(image_name, args.output_dir, ch_root) + run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) logging.info("Output placed into {}".format(args.output_dir)) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index a8b382e3f08..be7ff06eba5 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -19,7 +19,6 @@ RUN apt-get update \ clickhouse-client=$version \ clickhouse-server=$version \ locales \ - tzdata \ wget \ && rm -rf \ /var/lib/apt/lists/* \ diff --git a/docker/test/integration/README.md b/docker/test/integration/README.md new file mode 100644 index 00000000000..4aa10d6db80 --- /dev/null +++ b/docker/test/integration/README.md @@ -0,0 +1,6 @@ +## Docker containers for integration tests +- `base` container with required packages +- `runner` container with that runs integration tests in docker +- `compose` contains docker_compose YaML files that are used in tests + +How to run integration tests is described in tests/integration/README.md \ No newline at end of file diff --git a/docker/test/integration/Dockerfile b/docker/test/integration/base/Dockerfile similarity index 100% rename from docker/test/integration/Dockerfile rename to docker/test/integration/base/Dockerfile diff --git a/tests/integration/helpers/docker_compose_hdfs.yml b/docker/test/integration/compose/docker_compose_hdfs.yml similarity index 100% rename from tests/integration/helpers/docker_compose_hdfs.yml rename to docker/test/integration/compose/docker_compose_hdfs.yml diff --git a/tests/integration/helpers/docker_compose_kafka.yml b/docker/test/integration/compose/docker_compose_kafka.yml similarity index 100% rename from tests/integration/helpers/docker_compose_kafka.yml rename to docker/test/integration/compose/docker_compose_kafka.yml diff --git a/tests/integration/helpers/docker_compose_minio.yml b/docker/test/integration/compose/docker_compose_minio.yml similarity index 100% rename from tests/integration/helpers/docker_compose_minio.yml rename to docker/test/integration/compose/docker_compose_minio.yml diff --git a/tests/integration/helpers/docker_compose_mongo.yml b/docker/test/integration/compose/docker_compose_mongo.yml similarity index 100% rename from tests/integration/helpers/docker_compose_mongo.yml rename to docker/test/integration/compose/docker_compose_mongo.yml diff --git a/tests/integration/helpers/docker_compose_mysql.yml b/docker/test/integration/compose/docker_compose_mysql.yml similarity index 100% rename from tests/integration/helpers/docker_compose_mysql.yml rename to docker/test/integration/compose/docker_compose_mysql.yml diff --git a/tests/integration/helpers/docker_compose_net.yml b/docker/test/integration/compose/docker_compose_net.yml similarity index 100% rename from tests/integration/helpers/docker_compose_net.yml rename to docker/test/integration/compose/docker_compose_net.yml diff --git a/tests/integration/helpers/docker_compose_postgres.yml b/docker/test/integration/compose/docker_compose_postgres.yml similarity index 100% rename from tests/integration/helpers/docker_compose_postgres.yml rename to docker/test/integration/compose/docker_compose_postgres.yml diff --git a/tests/integration/helpers/docker_compose_redis.yml b/docker/test/integration/compose/docker_compose_redis.yml similarity index 100% rename from tests/integration/helpers/docker_compose_redis.yml rename to docker/test/integration/compose/docker_compose_redis.yml diff --git a/tests/integration/helpers/docker_compose_zookeeper.yml b/docker/test/integration/compose/docker_compose_zookeeper.yml similarity index 100% rename from tests/integration/helpers/docker_compose_zookeeper.yml rename to docker/test/integration/compose/docker_compose_zookeeper.yml diff --git a/tests/integration/helpers/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile similarity index 100% rename from tests/integration/helpers/helper_container/Dockerfile rename to docker/test/integration/helper_container/Dockerfile diff --git a/tests/integration/image/Dockerfile b/docker/test/integration/runner/Dockerfile similarity index 70% rename from tests/integration/image/Dockerfile rename to docker/test/integration/runner/Dockerfile index 173551eb14c..648b98fa60d 100644 --- a/tests/integration/image/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -41,32 +41,32 @@ ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce RUN set -eux; \ - \ + \ # this "case" statement is generated via "update.sh" - \ - if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \ - exit 1; \ - fi; \ - \ - tar --extract \ - --file docker.tgz \ - --strip-components 1 \ - --directory /usr/local/bin/ \ - ; \ - rm docker.tgz; \ - \ - dockerd --version; \ - docker --version + \ + if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \ + echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \ + exit 1; \ + fi; \ + \ + tar --extract \ + --file docker.tgz \ + --strip-components 1 \ + --directory /usr/local/bin/ \ + ; \ + rm docker.tgz; \ + \ + dockerd --version; \ + docker --version COPY modprobe.sh /usr/local/bin/modprobe COPY dockerd-entrypoint.sh /usr/local/bin/ RUN set -x \ - && addgroup --system dockremap \ + && addgroup --system dockremap \ && adduser --system dockremap \ - && adduser dockremap dockremap \ - && echo 'dockremap:165536:65536' >> /etc/subuid \ + && adduser dockremap dockremap \ + && echo 'dockremap:165536:65536' >> /etc/subuid \ && echo 'dockremap:165536:65536' >> /etc/subgid VOLUME /var/lib/docker diff --git a/tests/integration/image/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh similarity index 100% rename from tests/integration/image/dockerd-entrypoint.sh rename to docker/test/integration/runner/dockerd-entrypoint.sh diff --git a/tests/integration/image/modprobe.sh b/docker/test/integration/runner/modprobe.sh similarity index 80% rename from tests/integration/image/modprobe.sh rename to docker/test/integration/runner/modprobe.sh index b357d893fda..141e45d0ec8 100755 --- a/tests/integration/image/modprobe.sh +++ b/docker/test/integration/runner/modprobe.sh @@ -9,10 +9,10 @@ set -eu # Docker often uses "modprobe -va foo bar baz" # so we ignore modules that start with "-" for module; do - if [ "${module#-}" = "$module" ]; then - ip link show "$module" || true - lsmod | grep "$module" || true - fi + if [ "${module#-}" = "$module" ]; then + ip link show "$module" || true + lsmod | grep "$module" || true + fi done # remove /usr/local/... from PATH so we can exec the real modprobe as a last resort diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index dd9edb0a95f..abdf8130c14 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update \ bash \ curl \ g++ \ + gdb \ git \ libc6-dbg \ moreutils \ diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index f89631522f4..fadd49759ed 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -2,7 +2,7 @@ set -ex set -o pipefail trap "exit" INT TERM -trap "kill $(jobs -pr) ||:" EXIT +trap 'kill $(jobs -pr) ||:' EXIT stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -14,26 +14,26 @@ function configure rm right/config/config.d/text_log.xml ||: cp -rv right/config left ||: - sed -i 's/9000/9001/g' left/config/config.xml - sed -i 's/9000/9002/g' right/config/config.xml + sed -i 's/900./9001/g' left/config/config.xml + sed -i 's/900./9002/g' right/config/config.xml # Start a temporary server to rename the tables - while killall clickhouse; do echo . ; sleep 1 ; done + while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed set -m # Spawn temporary in its own process groups - left/clickhouse server --config-file=left/config/config.xml -- --path db0 &> setup-server-log.log & + left/clickhouse-server --config-file=left/config/config.xml -- --path db0 &> setup-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid set +m - while ! left/clickhouse client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done + while ! clickhouse-client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done echo server for setup started - left/clickhouse client --port 9001 --query "create database test" ||: - left/clickhouse client --port 9001 --query "rename table datasets.hits_v1 to test.hits" ||: + clickhouse-client --port 9001 --query "create database test" ||: + clickhouse-client --port 9001 --query "rename table datasets.hits_v1 to test.hits" ||: - while killall clickhouse; do echo . ; sleep 1 ; done + while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed # Remove logs etc, because they will be updated, and sharing them between @@ -42,41 +42,50 @@ function configure rm db0/metadata/system/* -rf ||: # Make copies of the original db for both servers. Use hardlinks instead - # of copying. + # of copying. Be careful to remove preprocessed configs and system tables,or + # it can lead to weird effects. rm -r left/db ||: rm -r right/db ||: + rm -r db0/preprocessed_configs ||: + rm -r db/{data,metadata}/system ||: cp -al db0/ left/db/ cp -al db0/ right/db/ } function restart { - while killall clickhouse; do echo . ; sleep 1 ; done + while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed set -m # Spawn servers in their own process groups - left/clickhouse server --config-file=left/config/config.xml -- --path left/db &>> left-server-log.log & + left/clickhouse-server --config-file=left/config/config.xml -- --path left/db &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - right/clickhouse server --config-file=right/config/config.xml -- --path right/db &>> right-server-log.log & + right/clickhouse-server --config-file=right/config/config.xml -- --path right/db &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid set +m - while ! left/clickhouse client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done + while ! clickhouse-client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done echo left ok - while ! right/clickhouse client --port 9002 --query "select 1" ; do kill -0 $right_pid ; echo . ; sleep 1 ; done + while ! clickhouse-client --port 9002 --query "select 1" ; do kill -0 $right_pid ; echo . ; sleep 1 ; done echo right ok - left/clickhouse client --port 9001 --query "select * from system.tables where database != 'system'" - left/clickhouse client --port 9001 --query "select * from system.build_options" - right/clickhouse client --port 9002 --query "select * from system.tables where database != 'system'" - right/clickhouse client --port 9002 --query "select * from system.build_options" + clickhouse-client --port 9001 --query "select * from system.tables where database != 'system'" + clickhouse-client --port 9001 --query "select * from system.build_options" + clickhouse-client --port 9002 --query "select * from system.tables where database != 'system'" + clickhouse-client --port 9002 --query "select * from system.build_options" + + # Check again that both servers we started are running -- this is important + # for running locally, when there might be some other servers started and we + # will connect to them instead. + kill -0 $left_pid + kill -0 $right_pid } function run_tests @@ -91,7 +100,7 @@ function run_tests # changes. test_prefix=$([ "$PR_TO_TEST" == "0" ] && echo left || echo right)/performance - for x in {test-times,skipped-tests}.tsv + for x in {test-times,skipped-tests,wall-clock-times}.tsv do rm -v "$x" ||: touch "$x" @@ -127,7 +136,7 @@ function run_tests # FIXME remove some broken long tests for test_name in {IPv4,IPv6,modulo,parse_engine_file,number_formatting_formats,select_format,arithmetic,cryptographic_hashes,logical_functions_{medium,small}} do - printf "$test_name\tMarked as broken (see compare.sh)\n" >> skipped-tests.tsv + printf "%s\tMarked as broken (see compare.sh)\n" "$test_name">> skipped-tests.tsv rm "$test_prefix/$test_name.xml" ||: done test_files=$(ls "$test_prefix"/*.xml) @@ -138,9 +147,9 @@ function run_tests for test in $test_files do # Check that both servers are alive, to fail faster if they die. - left/clickhouse client --port 9001 --query "select 1 format Null" \ + clickhouse-client --port 9001 --query "select 1 format Null" \ || { echo $test_name >> left-server-died.log ; restart ; continue ; } - right/clickhouse client --port 9002 --query "select 1 format Null" \ + clickhouse-client --port 9002 --query "select 1 format Null" \ || { echo $test_name >> right-server-died.log ; restart ; continue ; } test_name=$(basename "$test" ".xml") @@ -158,7 +167,7 @@ function run_tests skipped=$(grep ^skipped "$test_name-raw.tsv" | cut -f2-) if [ "$skipped" != "" ] then - printf "$test_name""\t""$skipped""\n" >> skipped-tests.tsv + printf "%s\t%s\n" "$test_name" "$skipped">> skipped-tests.tsv fi done @@ -167,27 +176,49 @@ function run_tests wait } +function get_profiles_watchdog +{ + sleep 3000 + + echo "The trace collection did not finish in time." >> profile-errors.log + + for pid in $(pgrep -f clickhouse) + do + gdb -p $pid --batch --ex "info proc all" --ex "thread apply all bt" --ex quit &> "$pid.gdb.log" & + done + wait + + for i in {1..10} + do + if ! pkill -f clickhouse + then + break + fi + sleep 1 + done +} + function get_profiles { # Collect the profiles - left/clickhouse client --port 9001 --query "set query_profiler_cpu_time_period_ns = 0" - left/clickhouse client --port 9001 --query "set query_profiler_real_time_period_ns = 0" - right/clickhouse client --port 9001 --query "set query_profiler_cpu_time_period_ns = 0" - right/clickhouse client --port 9001 --query "set query_profiler_real_time_period_ns = 0" - left/clickhouse client --port 9001 --query "system flush logs" - right/clickhouse client --port 9002 --query "system flush logs" + clickhouse-client --port 9001 --query "set query_profiler_cpu_time_period_ns = 0" + clickhouse-client --port 9001 --query "set query_profiler_real_time_period_ns = 0" + clickhouse-client --port 9001 --query "set query_profiler_cpu_time_period_ns = 0" + clickhouse-client --port 9001 --query "set query_profiler_real_time_period_ns = 0" + clickhouse-client --port 9001 --query "system flush logs" + clickhouse-client --port 9002 --query "system flush logs" - left/clickhouse client --port 9001 --query "select * from system.query_log where type = 2 format TSVWithNamesAndTypes" > left-query-log.tsv ||: & - left/clickhouse client --port 9001 --query "select * from system.query_thread_log format TSVWithNamesAndTypes" > left-query-thread-log.tsv ||: & - left/clickhouse client --port 9001 --query "select * from system.trace_log format TSVWithNamesAndTypes" > left-trace-log.tsv ||: & - left/clickhouse client --port 9001 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > left-addresses.tsv ||: & - left/clickhouse client --port 9001 --query "select * from system.metric_log format TSVWithNamesAndTypes" > left-metric-log.tsv ||: & + clickhouse-client --port 9001 --query "select * from system.query_log where type = 2 format TSVWithNamesAndTypes" > left-query-log.tsv ||: & + clickhouse-client --port 9001 --query "select * from system.query_thread_log format TSVWithNamesAndTypes" > left-query-thread-log.tsv ||: & + clickhouse-client --port 9001 --query "select * from system.trace_log format TSVWithNamesAndTypes" > left-trace-log.tsv ||: & + clickhouse-client --port 9001 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > left-addresses.tsv ||: & + clickhouse-client --port 9001 --query "select * from system.metric_log format TSVWithNamesAndTypes" > left-metric-log.tsv ||: & - right/clickhouse client --port 9002 --query "select * from system.query_log where type = 2 format TSVWithNamesAndTypes" > right-query-log.tsv ||: & - right/clickhouse client --port 9002 --query "select * from system.query_thread_log format TSVWithNamesAndTypes" > right-query-thread-log.tsv ||: & - right/clickhouse client --port 9002 --query "select * from system.trace_log format TSVWithNamesAndTypes" > right-trace-log.tsv ||: & - right/clickhouse client --port 9002 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > right-addresses.tsv ||: & - right/clickhouse client --port 9002 --query "select * from system.metric_log format TSVWithNamesAndTypes" > right-metric-log.tsv ||: & + clickhouse-client --port 9002 --query "select * from system.query_log where type = 2 format TSVWithNamesAndTypes" > right-query-log.tsv ||: & + clickhouse-client --port 9002 --query "select * from system.query_thread_log format TSVWithNamesAndTypes" > right-query-thread-log.tsv ||: & + clickhouse-client --port 9002 --query "select * from system.trace_log format TSVWithNamesAndTypes" > right-trace-log.tsv ||: & + clickhouse-client --port 9002 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > right-addresses.tsv ||: & + clickhouse-client --port 9002 --query "select * from system.metric_log format TSVWithNamesAndTypes" > right-metric-log.tsv ||: & wait } @@ -195,9 +226,9 @@ function get_profiles # Build and analyze randomization distribution for all queries. function analyze_queries { - find . -maxdepth 1 -name "*-queries.tsv" -print | \ - xargs -n1 -I% basename % -queries.tsv | \ - parallel --verbose right/clickhouse local --file "{}-queries.tsv" \ + find . -maxdepth 1 -name "*-queries.tsv" -print0 | \ + xargs -0 -n1 -I% basename % -queries.tsv | \ + parallel --verbose clickhouse-local --file "{}-queries.tsv" \ --structure "\"query text, run int, version UInt32, time float\"" \ --query "\"$(cat "$script_dir/eqmed.sql")\"" \ ">" {}-report.tsv @@ -219,33 +250,39 @@ done rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||: -right/clickhouse local --query " +cat profile-errors.log >> report-errors.rep + +clickhouse-local --query " create table queries engine File(TSVWithNamesAndTypes, 'queries.rep') as select -- FIXME Comparison mode doesn't make sense for queries that complete -- immediately, so for now we pretend they don't exist. We don't want to -- remove them altogether because we want to be able to detect regressions, -- but the right way to do this is not yet clear. - left + right < 0.05 as short, + (left + right) / 2 < 0.02 as short, - not short and abs(diff) < 0.10 and rd[3] > 0.10 as unstable, - - -- Do not consider changed the queries with 5% RD below 5% -- e.g., we're - -- likely to observe a difference > 5% in less than 5% cases. - -- Not sure it is correct, but empirically it filters out a lot of noise. - not short and abs(diff) > 0.15 and abs(diff) > rd[3] and rd[1] > 0.05 as changed, + -- Difference > 5% and > rd(99%) -- changed. We can't filter out flaky + -- queries by rd(5%), because it can be zero when the difference is smaller + -- than a typical distribution width. The difference is still real though. + not short and abs(diff) > 0.05 and abs(diff) > rd[4] as changed, + + -- Not changed but rd(99%) > 5% -- unstable. + not short and not changed and rd[4] > 0.05 as unstable, + left, right, diff, rd, replaceAll(_file, '-report.tsv', '') test, - query + + -- Truncate long queries. + if(length(query) < 300, query, substr(query, 1, 298) || '...') query from file('*-report.tsv', TSV, 'left float, right float, diff float, rd Array(float), query text'); create table changed_perf_tsv engine File(TSV, 'changed-perf.tsv') as select left, right, diff, rd, test, query from queries where changed - order by rd[3] desc; + order by abs(diff) desc; create table unstable_queries_tsv engine File(TSV, 'unstable-queries.tsv') as select left, right, diff, rd, test, query from queries where unstable - order by rd[3] desc; + order by rd[4] desc; create table unstable_tests_tsv engine File(TSV, 'bad-tests.tsv') as select test, sum(unstable) u, sum(changed) c, u + c s from queries @@ -291,10 +328,10 @@ create table all_tests_tsv engine File(TSV, 'all-queries.tsv') as for version in {right,left} do -right/clickhouse local --query " +clickhouse-local --query " create view queries as select * from file('queries.rep', TSVWithNamesAndTypes, - 'short int, unstable int, changed int, left float, right float, + 'short int, changed int, unstable int, left float, right float, diff float, rd Array(float), test text, query text'); create view query_log as select * @@ -359,8 +396,8 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes, create table metric_devation engine File(TSVWithNamesAndTypes, 'metric-deviation.$version.rep') as - select floor((q[3] - q[1])/q[2], 3) d, - quantilesExact(0, 0.5, 1)(value) q, metric, query + select query, floor((q[3] - q[1])/q[2], 3) d, + quantilesExact(0, 0.5, 1)(value) q, metric from (select * from unstable_run_metrics union all select * from unstable_run_traces union all select * from unstable_run_metrics_2) mm @@ -394,11 +431,17 @@ do for query in $(cut -d' ' -f1 "stacks.$version.rep" | sort | uniq) do query_file=$(echo "$query" | cut -c-120 | sed 's/[/]/_/g') + + # Build separate .svg flamegraph for each query. grep -F "$query " "stacks.$version.rep" \ | cut -d' ' -f 2- \ | sed 's/\t/ /g' \ | tee "$query_file.stacks.$version.rep" \ | ~/fg/flamegraph.pl > "$query_file.$version.svg" & + + # Copy metric stats into separate files as well. + grep -F "$query " "metric-deviation.$version.rep" \ + | cut -f2- > "$query_file.$version.metrics.rep" & done done wait @@ -406,9 +449,13 @@ unset IFS # Remember that grep sets error code when nothing is found, hence the bayan # operator. -grep -H -m2 -i '\(Exception\|Error\):[^:]' ./*-err.log | sed 's/:/\t/' > run-errors.tsv ||: +grep -H -m2 -i '\(Exception\|Error\):[^:]' ./*-err.log | sed 's/:/\t/' >> run-errors.tsv ||: } +# Check that local and client are in PATH +clickhouse-local --version > /dev/null +clickhouse-client --version > /dev/null + case "$stage" in "") ;& @@ -423,10 +470,28 @@ case "$stage" in time run_tests ||: ;& "get_profiles") + # Getting profiles inexplicably hangs sometimes, so try to save some logs if + # this happens again. Give the servers 5 minutes to collect all info, then + # trace and kill. Start in a subshell, so that both function don't interfere + # with each other's jobs through `wait`. Also make the subshell have its own + # process group, so that we can then kill it with all its child processes. + # Somehow it doesn't kill the children by itself when dying. + set -m + ( get_profiles_watchdog ) & + watchdog_pid=$! + set +m + # Check that the watchdog started OK. + kill -0 $watchdog_pid + # If the tests fail with OOM or something, still try to restart the servers # to collect the logs. Prefer not to restart, because addresses might change - # and we won't be able to process trace_log data. - time get_profiles || restart || get_profiles ||: + # and we won't be able to process trace_log data. Start in a subshell, so that + # it doesn't interfere with the watchdog through `wait`. + ( time get_profiles || restart || get_profiles ||: ) + + # Kill the whole process group, because somehow when the subshell is killed, + # the sleep inside remains alive and orphaned. + while env kill -- -$watchdog_pid ; do sleep 1; done # Stop the servers to free memory for the subsequent query analysis. while killall clickhouse; do echo . ; sleep 1 ; done @@ -442,3 +507,7 @@ case "$stage" in time "$script_dir/report.py" > report.html ;& esac + +# Print some final debug info to help debug Weirdness, of which there is plenty. +jobs +pstree -apgT diff --git a/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml b/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml index 863a40718d9..e41ab8eb75d 100644 --- a/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml +++ b/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml @@ -1,4 +1,9 @@ - + + + + + :: + true diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index fc4622fdf39..bf066997f7f 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -2,9 +2,11 @@ set -ex set -o pipefail trap "exit" INT TERM -trap "kill $(jobs -pr) ||:" EXIT +trap 'kill $(jobs -pr) ||:' EXIT mkdir db0 ||: +mkdir left ||: +mkdir right ||: left_pr=$1 left_sha=$2 @@ -22,11 +24,6 @@ dataset_paths["values"]="https://clickhouse-datasets.s3.yandex.net/values_with_e function download { - rm -r left ||: - mkdir left ||: - rm -r right ||: - mkdir right ||: - # might have the same version on left and right if ! [ "$left_sha" = "$right_sha" ] then @@ -39,7 +36,11 @@ function download for dataset_name in $datasets do dataset_path="${dataset_paths[$dataset_name]}" - [ "$dataset_path" != "" ] + if [ "$dataset_path" = "" ] + then + >&2 echo "Unknown dataset '$dataset_name'" + exit 1 + fi cd db0 && wget -nv -nd -c "$dataset_path" -O- | tar -xv & done diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index dc0480715d0..f316a659f3c 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -87,9 +87,6 @@ git -C ch diff --name-only "$SHA_TO_TEST" "$(git -C ch merge-base "$SHA_TO_TEST" # Set python output encoding so that we can print queries with Russian letters. export PYTHONIOENCODING=utf-8 -# Use a default number of runs if not told otherwise -export CHPC_RUNS=${CHPC_RUNS:-7} - # By default, use the main comparison script from the tested package, so that we # can change it in PRs. script_path="right/scripts" @@ -101,14 +98,15 @@ fi # Even if we have some errors, try our best to save the logs. set +e -# Older version use 'kill 0', so put the script into a separate process group -# FIXME remove set +m in April 2020 -set +m +# Use clickhouse-client and clickhouse-local from the right server. +PATH="$(readlink -f right/)":"$PATH" +export PATH + +# Start the main comparison script. { \ time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \ time stage=configure "$script_path"/compare.sh ; \ } 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee compare.log -set -m # Stop the servers to free memory. Normally they are restarted before getting # the profile info, so they shouldn't use much, but if the comparison script diff --git a/docker/test/performance-comparison/manual-run.sh b/docker/test/performance-comparison/manual-run.sh new file mode 100755 index 00000000000..84a96223b0e --- /dev/null +++ b/docker/test/performance-comparison/manual-run.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -ex +set -o pipefail +trap "exit" INT TERM +trap 'kill $(jobs -pr) ||:' EXIT + +stage=${stage:-} +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +repo_dir=${repo_dir:-$(readlink -f "$script_dir/../../..")} + + +function download +{ + rm -r left right db0 ||: + mkdir left right db0 ||: + + "$script_dir/download.sh" ||: & + cp -vP "$repo_dir"/../build-gcc9-rel/programs/clickhouse* right & + cp -vP "$repo_dir"/../build-clang10-rel/programs/clickhouse* left & + wait +} + +function configure +{ + # Test files + cp -av "$repo_dir/tests/performance" right + cp -av "$repo_dir/tests/performance" left + + # Configs + cp -av "$script_dir/config" right + cp -av "$script_dir/config" left + cp -av "$repo_dir"/programs/server/config* right/config + cp -av "$repo_dir"/programs/server/user* right/config + cp -av "$repo_dir"/programs/server/config* left/config + cp -av "$repo_dir"/programs/server/user* left/config + + tree left +} + +function run +{ + left/clickhouse-local --query "select * from system.build_options format PrettySpace" | sed 's/ *$//' | fold -w 80 -s > left-commit.txt + right/clickhouse-local --query "select * from system.build_options format PrettySpace" | sed 's/ *$//' | fold -w 80 -s > right-commit.txt + + PATH=right:"$PATH" stage=configure "$script_dir/compare.sh" &> >(tee compare.log) +} + +download +configure +run + +rm output.7z +7z a output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs} + diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 55d93f89c6e..06f8ca8d784 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -25,7 +25,7 @@ parser = argparse.ArgumentParser(description='Run performance test.') parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') parser.add_argument('--host', nargs='*', default=['localhost'], help="Server hostname(s). Corresponds to '--port' options.") parser.add_argument('--port', nargs='*', default=[9000], help="Server port(s). Corresponds to '--host' options.") -parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.') +parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 13)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.') parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') args = parser.parse_args() @@ -140,9 +140,16 @@ report_stage_end('substitute2') for q in test_queries: # Prewarm: run once on both servers. Helps to bring the data into memory, # precompile the queries, etc. - for conn_index, c in enumerate(connections): - res = c.execute(q, query_id = 'prewarm {} {}'.format(0, q)) - print('prewarm\t' + tsv_escape(q) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) + try: + for conn_index, c in enumerate(connections): + res = c.execute(q, query_id = 'prewarm {} {}'.format(0, q)) + print('prewarm\t' + tsv_escape(q) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) + except: + # If prewarm fails for some query -- skip it, and try to test the others. + # This might happen if the new test introduces some function that the + # old server doesn't support. Still, report it as an error. + print(traceback.format_exc(), file=sys.stderr) + continue # Now, perform measured runs. # Track the time spent by the client to process this query, so that we can notice diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 84b0239ccda..43cdfc91b63 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -169,12 +169,14 @@ if args.report == 'main': attrs = ['' for c in columns] for row in rows: - if float(row[2]) < 0.: - faster_queries += 1 - attrs[2] = 'style="background: #adbdff"' - else: - slower_queries += 1 - attrs[2] = 'style="background: #ffb0a0"' + attrs[2] = '' + if abs(float(row[2])) > 0.10: + if float(row[2]) < 0.: + faster_queries += 1 + attrs[2] = 'style="background: #adbdff"' + else: + slower_queries += 1 + attrs[2] = 'style="background: #ffb0a0"' print(tableRow(row, attrs)) @@ -256,17 +258,18 @@ if args.report == 'main': print(tableStart('Test times')) print(tableHeader(columns)) - + + runs = 13 # FIXME pass this as an argument attrs = ['' for c in columns] for r in rows: - if float(r[6]) > 22: + if float(r[6]) > 3 * runs: # FIXME should be 15s max -- investigate parallel_insert slow_average_tests += 1 attrs[6] = 'style="background: #ffb0a0"' else: attrs[6] = '' - if float(r[5]) > 30: + if float(r[5]) > 4 * runs: slow_average_tests += 1 attrs[5] = 'style="background: #ffb0a0"' else: @@ -366,10 +369,20 @@ elif args.report == 'all-queries': attrs = ['' for c in columns] for r in rows: - if float(r[2]) > 0.05: - attrs[3] = 'style="background: #ffb0a0"' - elif float(r[2]) < -0.05: - attrs[3] = 'style="background: #adbdff"' + rd = ast.literal_eval(r[4]) + # Note the zero-based array index, this is rd[4] in SQL. + threshold = rd[3] + if threshold > 0.2: + attrs[4] = 'style="background: #ffb0a0"' + else: + attrs[4] = '' + + diff = float(r[2]) + if abs(diff) > threshold and threshold >= 0.05: + if diff > 0.: + attrs[3] = 'style="background: #ffb0a0"' + else: + attrs[3] = 'style="background: #adbdff"' else: attrs[3] = '' diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile index 78fb6990773..a3639fb1551 100644 --- a/docker/test/pvs/Dockerfile +++ b/docker/test/pvs/Dockerfile @@ -20,9 +20,9 @@ RUN apt-get --allow-unauthenticated update -y \ # apt-get --allow-unauthenticated install --yes --no-install-recommends \ # pvs-studio -ENV PKG_VERSION="pvs-studio-7.04.34029.84-amd64.deb" +ENV PKG_VERSION="pvs-studio-7.07.37949.43-amd64.deb" -RUN wget -q http://files.viva64.com/beta/$PKG_VERSION +RUN wget -q http://files.viva64.com/$PKG_VERSION RUN sudo dpkg -i $PKG_VERSION CMD cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 7fade12466d..a4c5ab5a554 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -70,6 +70,7 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/; \ diff --git a/docker/test/stateless_with_coverage/run.sh b/docker/test/stateless_with_coverage/run.sh index 0a914030c4a..238c7567694 100755 --- a/docker/test/stateless_with_coverage/run.sh +++ b/docker/test/stateless_with_coverage/run.sh @@ -14,6 +14,11 @@ kill_clickhouse () { sleep 10 fi done + + echo "Will try to send second kill signal for sure" + kill `pgrep -u clickhouse` 2>/dev/null + sleep 5 + echo "clickhouse pids" `ps aux | grep clickhouse` | ts '%Y-%m-%d %H:%M:%S' } start_clickhouse () { @@ -50,6 +55,7 @@ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/con ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/; \ diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 4d983ec4695..f022411a031 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -36,6 +36,7 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ ln -s /usr/lib/llvm-9/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; \ echo "TSAN_OPTIONS='halt_on_error=1 history_size=7 ignore_noninstrumented_modules=1 verbosity=1'" >> /etc/environment; \ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ + echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment; \ service clickhouse-server start && sleep 5 \ && /s3downloader --dataset-names $DATASETS \ && chmod 777 -R /var/lib/clickhouse \ diff --git a/docs/README.md b/docs/README.md index f075716a0b0..a7473094ad7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -135,16 +135,13 @@ When adding a new file: $ ln -sr en/new/file.md lang/new/file.md ``` -- Reference the file from `toc_{en,ru,zh,ja,fa}.yaml` files with the pages index. - - ### Adding a New Language 1. Create a new docs subfolder named using the [ISO-639-1 language code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes). 2. Add Markdown files with the translation, mirroring the folder structure of other languages. -3. Commit and open a pull request with the new content. +3. Commit and open a pull-request with the new content. When everything is ready, we will add the new language to the website. @@ -206,4 +203,4 @@ Templates: ## How to Build Documentation -You can build your documentation manually by following the instructions in [docs/tools/README.md](docs/tools/README.md). Also, our CI runs the documentation build after the `documentation` label is added to PR. You can see the results of a build in the GitHub interface. If you have no permissions to add labels, a reviewer of your PR will add it. +You can build your documentation manually by following the instructions in [docs/tools/README.md](../docs/tools/README.md). Also, our CI runs the documentation build after the `documentation` label is added to PR. You can see the results of a build in the GitHub interface. If you have no permissions to add labels, a reviewer of your PR will add it. diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md index 3f7ead86219..5255d254100 100644 --- a/docs/en/commercial/cloud.md +++ b/docs/en/commercial/cloud.md @@ -1,3 +1,8 @@ +--- +toc_title: Cloud +toc_priority: 1 +--- + # ClickHouse Cloud Service Providers {#clickhouse-cloud-service-providers} !!! info "Info" diff --git a/docs/en/commercial/support.md b/docs/en/commercial/support.md new file mode 100644 index 00000000000..e7b4503a534 --- /dev/null +++ b/docs/en/commercial/support.md @@ -0,0 +1,21 @@ +--- +toc_title: Support +toc_priority: 3 +--- + +# ClickHouse Commercial Support Service Providers {#clickhouse-commercial-support-service-providers} + +!!! info "Info" + If you have launched a ClickHouse commercial support service, feel free to [open a pull-request](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/support.md) adding it to the following list. + +## Altinity {#altinity} + +[Service description](https://www.altinity.com/24x7-support) + +## Mafiree {#mafiree} + +[Service description](http://mafiree.com/clickhouse-analytics-services.php) + +## MinervaDB {#minervadb} + +[Service description](https://minervadb.com/index.php/clickhouse-consulting-and-support-by-minervadb/) diff --git a/docs/en/development/browse_code.md b/docs/en/development/browse_code.md index 69b15df3629..3e7b259bb39 100644 --- a/docs/en/development/browse_code.md +++ b/docs/en/development/browse_code.md @@ -1,11 +1,11 @@ --- toc_priority: 63 -toc_title: Browse ClickHouse Source Code +toc_title: Browse Source Code --- # Browse ClickHouse Source Code {#browse-clickhouse-source-code} -You can use **Woboq** online code browser available [here](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/src/index.html). It provides code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. +You can use **Woboq** online code browser available [here](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). It provides code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 02620b92367..45adb221b5b 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -15,7 +15,7 @@ Tests are located in `queries` directory. There are two subdirectories: `statele Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery --testmode`. `.sh` test is a script that is run by itself. -To run all tests, use `testskhouse-test` tool. Look `--help` for the list of possible options. You can simply run all tests or run subset of tests filtered by substring in test name: `./clickhouse-test substring`. +To run all tests, use `clickhouse-test` tool. Look `--help` for the list of possible options. You can simply run all tests or run subset of tests filtered by substring in test name: `./clickhouse-test substring`. The most simple way to invoke functional tests is to copy `clickhouse-client` to `/usr/bin/`, run `clickhouse-server` and then run `./clickhouse-test` from its own directory. @@ -34,13 +34,13 @@ disable these groups of tests using `--no-zookeeper`, `--no-shard` and ## Known Bugs {#known-bugs} -If we know some bugs that can be easily reproduced by functional tests, we place prepared functional tests in `queries/bugs` directory. These tests will be moved to `teststests_stateless` when bugs are fixed. +If we know some bugs that can be easily reproduced by functional tests, we place prepared functional tests in `tests/queries/bugs` directory. These tests will be moved to `tests/queries/0_stateless` when bugs are fixed. ## Integration Tests {#integration-tests} Integration tests allow to test ClickHouse in clustered configuration and ClickHouse interaction with other servers like MySQL, Postgres, MongoDB. They are useful to emulate network splits, packet drops, etc. These tests are run under Docker and create multiple containers with various software. -See `testsgration/README.md` on how to run these tests. +See `tests/integration/README.md` on how to run these tests. Note that integration of ClickHouse with third-party drivers is not tested. Also we currently don’t have integration tests with our JDBC and ODBC drivers. @@ -54,7 +54,7 @@ It’s not necessarily to have unit tests if the code is already covered by func Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `clickhouse performance-test` tool (that is embedded in `clickhouse` binary). See `--help` for invocation. -Each test run one or miltiple queries (possibly with combinations of parameters) in a loop with some conditions for stop (like “maximum execution speed is not changing in three seconds”) and measure some metrics about query performance (like “maximum execution speed”). Some tests can contain preconditions on preloaded test dataset. +Each test run one or multiple queries (possibly with combinations of parameters) in a loop with some conditions for stop (like “maximum execution speed is not changing in three seconds”) and measure some metrics about query performance (like “maximum execution speed”). Some tests can contain preconditions on preloaded test dataset. If you want to improve performance of ClickHouse in some scenario, and if improvements can be observed on simple queries, it is highly recommended to write a performance test. It always makes sense to use `perf top` or other perf tools during your tests. @@ -64,13 +64,13 @@ Some programs in `tests` directory are not prepared tests, but are test tools. F You can also place pair of files `.sh` and `.reference` along with the tool to run it on some predefined input - then script result can be compared to `.reference` file. These kind of tests are not automated. -## Miscellanous Tests {#miscellanous-tests} +## Miscellaneous Tests {#miscellaneous-tests} There are tests for external dictionaries located at `tests/external_dictionaries` and for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests. There is separate test for quorum inserts. This test run ClickHouse cluster on separate servers and emulate various failure cases: network split, packet drop (between ClickHouse nodes, between ClickHouse and ZooKeeper, between ClickHouse server and client, etc.), `kill -9`, `kill -STOP` and `kill -CONT` , like [Jepsen](https://aphyr.com/tags/Jepsen). Then the test checks that all acknowledged inserts was written and all rejected inserts was not. -Quorum test was written by separate team before ClickHouse was open-sourced. This team no longer work with ClickHouse. Test was accidentially written in Java. For these reasons, quorum test must be rewritten and moved to integration tests. +Quorum test was written by separate team before ClickHouse was open-sourced. This team no longer work with ClickHouse. Test was accidentally written in Java. For these reasons, quorum test must be rewritten and moved to integration tests. ## Manual Testing {#manual-testing} diff --git a/docs/en/engines/database_engines/mysql.md b/docs/en/engines/database_engines/mysql.md index 678c174e1fb..467a3aa032d 100644 --- a/docs/en/engines/database_engines/mysql.md +++ b/docs/en/engines/database_engines/mysql.md @@ -3,7 +3,7 @@ toc_priority: 30 toc_title: MySQL --- -# Mysql {#mysql} +# MySQL {#mysql} Allows to connect to databases on a remote MySQL server and perform `INSERT` and `SELECT` queries to exchange data between ClickHouse and MySQL. @@ -19,7 +19,7 @@ You cannot perform the following queries: ``` sql CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', 'database', 'user', 'password') +ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') ``` **Engine Parameters** diff --git a/docs/en/engines/table_engines/mergetree_family/aggregatingmergetree.md b/docs/en/engines/table_engines/mergetree_family/aggregatingmergetree.md index 2103efe98dc..9e310d313b9 100644 --- a/docs/en/engines/table_engines/mergetree_family/aggregatingmergetree.md +++ b/docs/en/engines/table_engines/mergetree_family/aggregatingmergetree.md @@ -9,7 +9,10 @@ The engine inherits from [MergeTree](mergetree.md#table_engines-mergetree), alte You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views. -The engine processes all columns with [AggregateFunction](../../../sql_reference/data_types/aggregatefunction.md) type. +The engine processes all columns with the following types: + +- [AggregateFunction](../../../sql_reference/data_types/aggregatefunction.md) +- [SimpleAggregateFunction](../../../sql_reference/data_types/simpleaggregatefunction.md) It is appropriate to use `AggregatingMergeTree` if it reduces the number of rows by orders. diff --git a/docs/en/engines/table_engines/mergetree_family/mergetree.md b/docs/en/engines/table_engines/mergetree_family/mergetree.md index fc0b6e63158..c44a993a55c 100644 --- a/docs/en/engines/table_engines/mergetree_family/mergetree.md +++ b/docs/en/engines/table_engines/mergetree_family/mergetree.md @@ -505,14 +505,14 @@ Configuration structure: - /mnt/fast_ssd/clickhouse + /mnt/fast_ssd/clickhouse/ - /mnt/hdd1/clickhouse + /mnt/hdd1/clickhouse/ 10485760 - /mnt/hdd2/clickhouse + /mnt/hdd2/clickhouse/ 10485760 diff --git a/docs/en/getting_started/install.md b/docs/en/getting_started/install.md index 316cc5a47ef..3a9cac802fc 100644 --- a/docs/en/getting_started/install.md +++ b/docs/en/getting_started/install.md @@ -38,7 +38,7 @@ sudo apt-get update sudo apt-get install clickhouse-client clickhouse-server ``` -You can also download and install packages manually from here: https://repo.yandex.ru/clickhouse/deb/stable/main/. +You can also download and install packages manually from [here](https://repo.yandex.ru/clickhouse/deb/stable/main/). #### Packages {#packages} @@ -67,7 +67,7 @@ Then run these commands to install packages: sudo yum install clickhouse-server clickhouse-client ``` -You can also download and install packages manually from here: https://repo.clickhouse.tech/rpm/stable/x86\_64. +You can also download and install packages manually from [here](https://repo.clickhouse.tech/rpm/stable/x86_64). ### From Tgz Archives {#from-tgz-archives} diff --git a/docs/en/getting_started/tutorial.md b/docs/en/getting_started/tutorial.md index 08cca45d21d..9763f814d59 100644 --- a/docs/en/getting_started/tutorial.md +++ b/docs/en/getting_started/tutorial.md @@ -108,7 +108,7 @@ Syntax for creating tables is way more complicated compared to databases (see [r 1. Name of table to create. 2. Table schema, i.e. list of columns and their [data types](../sql_reference/data_types/index.md). -3. [Table engine](../engines/table_engines/index.md) and it’s settings, which determines all the details on how queries to this table will be physically executed. +3. [Table engine](../engines/table_engines/index.md) and its settings, which determines all the details on how queries to this table will be physically executed. Yandex.Metrica is a web analytics service, and sample dataset doesn’t cover its full functionality, so there are only two tables to create: diff --git a/docs/en/index.md b/docs/en/index.md index 5d40ae9177c..67c4ca4d53b 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -1,5 +1,5 @@ --- -toc_priority: 3 +toc_priority: 0 toc_title: Overview --- @@ -78,48 +78,6 @@ See the difference? For example, the query “count the number of records for each advertising platform” requires reading one “advertising platform ID” column, which takes up 1 byte uncompressed. If most of the traffic was not from advertising platforms, you can expect at least 10-fold compression of this column. When using a quick compression algorithm, data decompression is possible at a speed of at least several gigabytes of uncompressed data per second. In other words, this query can be processed at a speed of approximately several billion rows per second on a single server. This speed is actually achieved in practice. -
- -Example - -``` bash -$ clickhouse-client -ClickHouse client version 0.0.52053. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.52053. -``` - -``` sql -SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 -``` - -``` text -┌─CounterID─┬──count()─┐ -│ 114208 │ 56057344 │ -│ 115080 │ 51619590 │ -│ 3228 │ 44658301 │ -│ 38230 │ 42045932 │ -│ 145263 │ 42042158 │ -│ 91244 │ 38297270 │ -│ 154139 │ 26647572 │ -│ 150748 │ 24112755 │ -│ 242232 │ 21302571 │ -│ 338158 │ 13507087 │ -│ 62180 │ 12229491 │ -│ 82264 │ 12187441 │ -│ 232261 │ 12148031 │ -│ 146272 │ 11438516 │ -│ 168777 │ 11403636 │ -│ 4120072 │ 11227824 │ -│ 10938808 │ 10519739 │ -│ 74088 │ 9047015 │ -│ 115079 │ 8837972 │ -│ 337234 │ 8205961 │ -└───────────┴──────────┘ -``` - -
- ### CPU {#cpu} Since executing a query requires processing a large number of rows, it helps to dispatch all operations for entire vectors instead of for separate rows, or to implement the query engine so that there is almost no dispatching cost. If you don’t do this, with any half-decent disk subsystem, the query interpreter inevitably stalls the CPU. It makes sense to both store data in columns and process it, when possible, by columns. diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index ff0b7fe2e19..9f4b0f2fa65 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -24,7 +24,10 @@ toc_title: Integrations - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) - Message queues - [Kafka](https://kafka.apache.org) - - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/)) +- Stream processing + - [Flink](https://flink.apache.org) + - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) - Object storages - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup) @@ -72,6 +75,9 @@ toc_title: Integrations - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (uses [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [pandas](https://pandas.pydata.org) - [pandahouse](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - [RClickHouse](https://github.com/IMSMWU/RClickHouse) (uses [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index f7006ae15c8..d2fcb112e67 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -28,7 +28,7 @@ toc_title: Adopters | [ContentSquare](https://contentsquare.com) | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | [Cloudflare](https://cloudflare.com) | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | | [Corunet](https://coru.net/) | Analytics | Main product | — | — | [Slides in English, April 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [CraiditX 氪信](https://creditx.com) | Finance AI | Analysis | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| [CraiditX 氪信](https://www.creditx.com) | Finance AI | Analysis | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | | [Criteo/Storetail](https://www.criteo.com/) | Retail | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | | [Deutsche Bank](https://db.com) | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | | [Diva-e](https://www.diva-e.com) | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | @@ -49,7 +49,7 @@ toc_title: Adopters | [Pragma Innovation](http://www.pragma-innovation.fr/) | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | | [QINGCLOUD](https://www.qingcloud.com/) | Cloud services | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | | [Qrator](https://qrator.net) | DDoS protection | Main product | — | — | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [Beijing PERCENT Information Technology Co., Ltd.](https://www.percent.cn/) | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| [Percent 百分点](https://www.percent.cn/) | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | | [Rambler](https://rambler.ru) | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | | [Tencent](https://www.tencent.com) | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | | [Traffic Stars](https://trafficstars.com/) | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | @@ -67,7 +67,7 @@ toc_title: Adopters | [Uber](https://www.uber.com) | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | | [VKontakte](https://vk.com) | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | | [Wisebits](https://wisebits.com/) | IT Solutions | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Xiaoxin Tech.](https://www.xiaoheiban.cn/) | Education | Common purpose | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| [Xiaoxin Tech](http://www.xiaoxintech.cn/) | Education | Common purpose | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | | [Ximalaya](https://www.ximalaya.com/) | Audio sharing | OLAP | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | | [Yandex Cloud](https://cloud.yandex.ru/services/managed-clickhouse) | Public Cloud | Main product | — | — | [Talk in Russian, December 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | | [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Business Intelligence | Main product | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | @@ -75,6 +75,11 @@ toc_title: Adopters | [Yandex Metrica](https://metrica.yandex.com) | Web analytics | Main product | 360 servers in one cluster, 1862 servers in one department | 66.41 PiB / 5.68 PiB | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | | [ЦВТ](https://htc-cs.ru/) | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | | [МКБ](https://mkb.ru/) | Bank | Web-system monitoring | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | -| [金数据](https://jinshuju.net) | BI Analytics | Main product | — | — | [Slides in Chinese, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| [Jinshuju 金数据](https://jinshuju.net) | BI Analytics | Main product | — | — | [Slides in Chinese, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| [Instana](https://www.instana.com) | APM Platform | Main product | — | — | [Twitter post](https://twitter.com/mieldonkers/status/1248884119158882304) | +| [Wargaming](https://wargaming.com/en/) | Games | | — | — | [Interview](https://habr.com/en/post/496954/) | +| [Crazypanda](https://crazypanda.ru/en/) | Games | | — | — | Live session on ClickHouse meetup | +| [FunCorp](https://fun.co/rp) | Games | | — | — | [Article](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) | + [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/en/introduction/distinctive_features.md b/docs/en/introduction/distinctive_features.md index 31770b0bf79..1244e80a253 100644 --- a/docs/en/introduction/distinctive_features.md +++ b/docs/en/introduction/distinctive_features.md @@ -66,4 +66,10 @@ ClickHouse uses asynchronous multi-master replication. After being written to an For more information, see the section [Data replication](../engines/table_engines/mergetree_family/replication.md). +## Features that Can Be Considered Disadvantages {#clickhouse-features-that-can-be-considered-disadvantages} + +1. No full-fledged transactions. +2. Lack of ability to modify or delete already inserted data with high rate and low latency. There are batch deletes and updates available to clean up or modify data, for example to comply with [GDPR](https://gdpr-info.eu). +3. The sparse index makes ClickHouse not so suitable for point queries retrieving single rows by their keys. + [Original article](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/en/introduction/features_considered_disadvantages.md b/docs/en/introduction/features_considered_disadvantages.md deleted file mode 100644 index e295b5570ab..00000000000 --- a/docs/en/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -toc_priority: 5 -toc_title: ClickHouse Features that Can Be Considered Disadvantages ---- - -# ClickHouse Features that Can Be Considered Disadvantages {#clickhouse-features-that-can-be-considered-disadvantages} - -1. No full-fledged transactions. -2. Lack of ability to modify or delete already inserted data with high rate and low latency. There are batch deletes and updates available to clean up or modify data, for example to comply with [GDPR](https://gdpr-info.eu). -3. The sparse index makes ClickHouse not so suitable for point queries retrieving single rows by their keys. - -[Original article](https://clickhouse.tech/docs/en/introduction/features_considered_disadvantages/) diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md index 065121f10ec..73aacc293b1 100644 --- a/docs/en/introduction/performance.md +++ b/docs/en/introduction/performance.md @@ -5,7 +5,7 @@ toc_title: Performance # Performance {#performance} -According to internal testing results at Yandex, ClickHouse shows the best performance (both the highest throughput for long queries and the lowest latency on short queries) for comparable operating scenarios among systems of its class that were available for testing. You can view the test results on a [separate page](https://clickhouse.tech/benchmark.html). +According to internal testing results at Yandex, ClickHouse shows the best performance (both the highest throughput for long queries and the lowest latency on short queries) for comparable operating scenarios among systems of its class that were available for testing. You can view the test results on a [separate page](https://clickhouse.tech/benchmark/dbms/). Numerous independent benchmarks came to similar conclusions. They are not difficult to find using an internet search, or you can see [our small collection of related links](https://clickhouse.tech/#independent-benchmarks). diff --git a/docs/en/operations/monitoring.md b/docs/en/operations/monitoring.md index 363e9cc4bff..dee1255569b 100644 --- a/docs/en/operations/monitoring.md +++ b/docs/en/operations/monitoring.md @@ -37,6 +37,8 @@ You can find metrics in the [system.metrics](../operations/system_tables.md#syst You can configure ClickHouse to export metrics to [Graphite](https://github.com/graphite-project). See the [Graphite section](server_configuration_parameters/settings.md#server_configuration_parameters-graphite) in the ClickHouse server configuration file. Before configuring export of metrics, you should set up Graphite by following their official [guide](https://graphite.readthedocs.io/en/latest/install.html). +You can configure ClickHouse to export metrics to [Prometheus](https://prometheus.io). See the [Prometheus section](server_configuration_parameters/settings.md#server_configuration_parameters-prometheus) in the ClickHouse server configuration file. Before configuring export of metrics, you should set up Prometheus by following their official [guide](https://prometheus.io/docs/prometheus/latest/installation/). + Additionally, you can monitor server availability through the HTTP API. Send the `HTTP GET` request to `/ping`. If the server is available, it responds with `200 OK`. To monitor servers in a cluster configuration, you should set the [max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) parameter and use the HTTP resource `/replicas_status`. A request to `/replicas_status` returns `200 OK` if the replica is available and is not delayed behind the other replicas. If a replica is delayed, it returns `503 HTTP_SERVICE_UNAVAILABLE` with information about the gap. diff --git a/docs/en/operations/performance_test.md b/docs/en/operations/performance_test.md index d955b50fa02..9114cbde71a 100644 --- a/docs/en/operations/performance_test.md +++ b/docs/en/operations/performance_test.md @@ -24,7 +24,7 @@ With this instruction you can run basic ClickHouse performance test on any serve # Then do: chmod a+x clickhouse -1. Download configs: +5. Download configs: @@ -34,7 +34,7 @@ With this instruction you can run basic ClickHouse performance test on any serve wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml -1. Download benchmark files: +6. Download benchmark files: @@ -42,7 +42,7 @@ With this instruction you can run basic ClickHouse performance test on any serve chmod a+x benchmark-new.sh wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql -1. Download test data according to the [Yandex.Metrica dataset](../getting_started/example_datasets/metrica.md) instruction (“hits” table containing 100 million rows). +7. Download test data according to the [Yandex.Metrica dataset](../getting_started/example_datasets/metrica.md) instruction (“hits” table containing 100 million rows). @@ -50,31 +50,31 @@ With this instruction you can run basic ClickHouse performance test on any serve tar xvf hits_100m_obfuscated_v1.tar.xz -C . mv hits_100m_obfuscated_v1/* . -1. Run the server: +8. Run the server: ./clickhouse server -1. Check the data: ssh to the server in another terminal +9. Check the data: ssh to the server in another terminal ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" 100000000 -1. Edit the benchmark-new.sh, change “clickhouse-client” to “./clickhouse client” and add “–max\_memory\_usage 100000000000” parameter. +10. Edit the benchmark-new.sh, change `clickhouse-client` to `./clickhouse client` and add `–-max_memory_usage 100000000000` parameter. mcedit benchmark-new.sh -1. Run the benchmark: +11. Run the benchmark: ./benchmark-new.sh hits_100m_obfuscated -1. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com +12. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com -All the results are published here: https://clickhouse.tech/benchmark\_hardware.html +All the results are published here: https://clickhouse.tech/benchmark/hardware/ diff --git a/docs/en/operations/server_configuration_parameters/settings.md b/docs/en/operations/server_configuration_parameters/settings.md index 85744a039f4..02c00fababf 100644 --- a/docs/en/operations/server_configuration_parameters/settings.md +++ b/docs/en/operations/server_configuration_parameters/settings.md @@ -536,6 +536,30 @@ The path to the directory containing data. /var/lib/clickhouse/ ``` +## prometheus {#server_configuration_parameters-prometheus} + +Exposing metrics data for scraping from [Prometheus](https://prometheus.io). + +Settings: + +- `endpoint` – HTTP endpoint for scraping metrics by prometheus server. Start from '/'. +- `port` – Port for `endpoint`. +- `metrics` – Flag that sets to expose metrics from the [system.metrics](../system_tables.md#system_tables-metrics) table. +- `events` – Flag that sets to expose metrics from the [system.events](../system_tables.md#system_tables-events) table. +- `asynchronous_metrics` – Flag that sets to expose current metrics values from the [system.asynchronous\_metrics](../system_tables.md#system_tables-asynchronous_metrics) table. + +**Example** + +``` xml + + /metrics + 8001 + true + true + true + +``` + ## query\_log {#server_configuration_parameters-query-log} Setting for logging queries received with the [log\_queries=1](../settings/settings.md) setting. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 69c444ebaef..46cd00cb661 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -220,21 +220,28 @@ Ok. ## input\_format\_values\_deduce\_templates\_of\_expressions {#settings-input_format_values_deduce_templates_of_expressions} -Enables or disables template deduction for an SQL expressions in [Values](../../interfaces/formats.md#data-format-values) format. It allows to parse and interpret expressions in `Values` much faster if expressions in consecutive rows have the same structure. ClickHouse will try to deduce template of an expression, parse the following rows using this template and evaluate the expression on a batch of successfully parsed rows. For the following query: +Enables or disables template deduction for SQL expressions in [Values](../../interfaces/formats.md#data-format-values) format. It allows parsing and interpreting expressions in `Values` much faster if expressions in consecutive rows have the same structure. ClickHouse tries to deduce template of an expression, parse the following rows using this template and evaluate the expression on a batch of successfully parsed rows. + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 1. + +For the following query: ``` sql INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), (upper('Values')), ... ``` -- if `input_format_values_interpret_expressions=1` and `format_values_deduce_templates_of_expressions=0` expressions will be interpreted separately for each row (this is very slow for large number of rows) -- if `input_format_values_interpret_expressions=0` and `format_values_deduce_templates_of_expressions=1` expressions in the first, second and third rows will be parsed using template `lower(String)` and interpreted together, expression is the forth row will be parsed with another template (`upper(String)`) -- if `input_format_values_interpret_expressions=1` and `format_values_deduce_templates_of_expressions=1` - the same as in previous case, but also allows fallback to interpreting expressions separately if it’s not possible to deduce template. - -Enabled by default. +- If `input_format_values_interpret_expressions=1` and `format_values_deduce_templates_of_expressions=0`, expressions are interpreted separately for each row (this is very slow for large number of rows). +- If `input_format_values_interpret_expressions=0` and `format_values_deduce_templates_of_expressions=1`, expressions in the first, second and third rows are parsed using template `lower(String)` and interpreted together, expression in the forth row is parsed with another template (`upper(String)`). +- If `input_format_values_interpret_expressions=1` and `format_values_deduce_templates_of_expressions=1`, the same as in previous case, but also allows fallback to interpreting expressions separately if it’s not possible to deduce template. ## input\_format\_values\_accurate\_types\_of\_literals {#settings-input-format-values-accurate-types-of-literals} -This setting is used only when `input_format_values_deduce_templates_of_expressions = 1`. It can happen, that expressions for some column have the same structure, but contain numeric literals of different types, e.g +This setting is used only when `input_format_values_deduce_templates_of_expressions = 1`. It can happen, that expressions for some column have the same structure, but contain numeric literals of different types, e.g. ``` sql (..., abs(0), ...), -- UInt64 literal @@ -242,9 +249,17 @@ This setting is used only when `input_format_values_deduce_templates_of_expressi (..., abs(-1), ...), -- Int64 literal ``` -When this setting is enabled, ClickHouse will check the actual type of literal and will use an expression template of the corresponding type. In some cases, it may significantly slow down expression evaluation in `Values`. -When disabled, ClickHouse may use more general type for some literals (e.g. `Float64` or `Int64` instead of `UInt64` for `42`), but it may cause overflow and precision issues. -Enabled by default. +Possible values: + +- 0 — Disabled. + + In this case, ClickHouse may use a more general type for some literals (e.g., `Float64` or `Int64` instead of `UInt64` for `42`), but it may cause overflow and precision issues. + +- 1 — Enabled. + + In this case, ClickHouse checks the actual type of literal and uses an expression template of the corresponding type. In some cases, it may significantly slow down expression evaluation in `Values`. + +Default value: 1. ## input\_format\_defaults\_for\_omitted\_fields {#session_settings-input_format_defaults_for_omitted_fields} @@ -507,6 +522,24 @@ Example: log_queries=1 ``` +## log\_queries\_min\_type {#settings-log-queries-min-type} + +`query_log` minimal type to log. + +Possible values: +- `QUERY_START` (`=1`) +- `QUERY_FINISH` (`=2`) +- `EXCEPTION_BEFORE_START` (`=3`) +- `EXCEPTION_WHILE_PROCESSING` (`=4`) + +Default value: `QUERY_START`. + +Can be used to limit which entiries will goes to `query_log`, say you are interesting only in errors, then you can use `EXCEPTION_WHILE_PROCESSING`: + +``` text +log_queries_min_type='EXCEPTION_WHILE_PROCESSING' +``` + ## log\_query\_threads {#settings-log-query-threads} Setting up query threads logging. @@ -531,6 +564,28 @@ Default value: 1,048,576. The default is slightly more than `max_block_size`. The reason for this is because certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion and a large enough block size allow sorting more data in RAM. +## min_insert_block_size_rows {#min-insert-block-size-rows} + +Sets minimum number of rows in block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. + +Possible values: + +- Positive integer. +- 0 — Squashing disabled. + +Default value: 1048576. + +## min_insert_block_size_bytes {#min-insert-block-size-bytes} + +Sets minimum number of bytes in block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. + +Possible values: + +- Positive integer. +- 0 — Squashing disabled. + +Default value: 268435456. + ## max\_replica\_delay\_for\_distributed\_queries {#settings-max_replica_delay_for_distributed_queries} Disables lagging replicas for distributed queries. See [Replication](../../engines/table_engines/mergetree_family/replication.md). diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 1c7d9546ff8..22e688c59cd 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -147,27 +147,68 @@ This system table is used for implementing the `SHOW DATABASES` query. Contains information about detached parts of [MergeTree](../engines/table_engines/mergetree_family/mergetree.md) tables. The `reason` column specifies why the part was detached. For user-detached parts, the reason is empty. Such parts can be attached with [ALTER TABLE ATTACH PARTITION\|PART](../sql_reference/statements/alter.md#alter_attach-partition) command. For the description of other columns, see [system.parts](#system_tables-parts). If part name is invalid, values of some columns may be `NULL`. Such parts can be deleted with [ALTER TABLE DROP DETACHED PART](../sql_reference/statements/alter.md#alter_drop-detached). -## system.dictionaries {#system-dictionaries} +## system.dictionaries {#system_tables-dictionaries} -Contains information about external dictionaries. +Contains information about [external dictionaries](../sql_reference/dictionaries/external_dictionaries/external_dicts.md). Columns: -- `name` (String) — Dictionary name. -- `type` (String) — Dictionary type: Flat, Hashed, Cache. -- `origin` (String) — Path to the configuration file that describes the dictionary. -- `attribute.names` (Array(String)) — Array of attribute names provided by the dictionary. -- `attribute.types` (Array(String)) — Corresponding array of attribute types that are provided by the dictionary. -- `has_hierarchy` (UInt8) — Whether the dictionary is hierarchical. -- `bytes_allocated` (UInt64) — The amount of RAM the dictionary uses. -- `hit_rate` (Float64) — For cache dictionaries, the percentage of uses for which the value was in the cache. -- `element_count` (UInt64) — The number of items stored in the dictionary. -- `load_factor` (Float64) — The percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). -- `creation_time` (DateTime) — The time when the dictionary was created or last successfully reloaded. -- `last_exception` (String) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn’t be created. -- `source` (String) — Text describing the data source for the dictionary. +- `database` ([String](../sql_reference/data_types/string.md)) — Name of the database containing the dictionary created by DDL query. Empty string for other dictionaries. +- `name` ([String](../sql_reference/data_types/string.md)) — [Dictionary name](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md). +- `status` ([Enum8](../sql_reference/data_types/enum.md)) — Dictionary status. Possible values: + - `NOT_LOADED` — Dictionary was not loaded because it was not used. + - `LOADED` — Dictionary loaded successfully. + - `FAILED` — Unable to load the dictionary as a result of an error. + - `LOADING` — Dictionary is loading now. + - `LOADED_AND_RELOADING` — Dictionary is loaded successfully, and is being reloaded right now (frequent reasons: [SYSTEM RELOAD DICTIONARY](../sql_reference/statements/system.md#query_language-system-reload-dictionary) query, timeout, dictionary config has changed). + - `FAILED_AND_RELOADING` — Could not load the dictionary as a result of an error and is loading now. +- `origin` ([String](../sql_reference/data_types/string.md)) — Path to the configuration file that describes the dictionary. +- `type` ([String](../sql_reference/data_types/string.md)) — Type of a dictionary allocation. [Storing Dictionaries in Memory](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md). +- `key` — [Key type](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-key): Numeric Key ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) or Сomposite key ([String](../sql_reference/data_types/string.md)) — form "(type 1, type 2, ..., type n)". +- `attribute.names` ([Array](../sql_reference/data_types/array.md)([String](../sql_reference/data_types/string.md))) — Array of [attribute names](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes) provided by the dictionary. +- `attribute.types` ([Array](../sql_reference/data_types/array.md)([String](../sql_reference/data_types/string.md))) — Corresponding array of [attribute types](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes) that are provided by the dictionary. +- `bytes_allocated` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary. +- `query_count` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot. +- `hit_rate` ([Float64](../sql_reference/data_types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache. +- `element_count` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Number of items stored in the dictionary. +- `load_factor` ([Float64](../sql_reference/data_types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). +- `source` ([String](../sql_reference/data_types/string.md)) — Text describing the [data source](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md) for the dictionary. +- `lifetime_min` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Minimum [lifetime](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md) of the dictionary in memory, after which ClickHouse tries to reload the dictionary (if `invalidate_query` is set, then only if it has changed). Set in seconds. +- `lifetime_max` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Maximum [lifetime](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md) of the dictionary in memory, after which ClickHouse tries to reload the dictionary (if `invalidate_query` is set, then only if it has changed). Set in seconds. +- `loading_start_time` ([DateTime](../sql_reference/data_types/datetime.md)) — Start time for loading the dictionary. +- `last_successful_update_time` ([DateTime](../sql_reference/data_types/datetime.md)) — End time for loading or updating the dictionary. Helps to monitor some troubles with external sources and investigate causes. +- `loading_duration` ([Float32](../sql_reference/data_types/float.md)) — Duration of a dictionary loading. +- `last_exception` ([String](../sql_reference/data_types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn't be created. -Note that the amount of memory used by the dictionary is not proportional to the number of items stored in it. So for flat and cached dictionaries, all the memory cells are pre-assigned, regardless of how full the dictionary actually is. + +**Example** + +Configure the dictionary. + +```sql +CREATE DICTIONARY dictdb.dict +( + `key` Int64 DEFAULT -1, + `value_default` String DEFAULT 'world', + `value_expression` String DEFAULT 'xxx' EXPRESSION 'toString(127 * 172)' +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dicttbl' DB 'dictdb')) +LIFETIME(MIN 0 MAX 1) +LAYOUT(FLAT()) +``` + +Make sure that the dictionary is loaded. + +```sql +SELECT * FROM system.dictionaries +``` + +```text +┌─database─┬─name─┬─status─┬─origin──────┬─type─┬─key────┬─attribute.names──────────────────────┬─attribute.types─────┬─bytes_allocated─┬─query_count─┬─hit_rate─┬─element_count─┬───────────load_factor─┬─source─────────────────────┬─lifetime_min─┬─lifetime_max─┬──loading_start_time─┌──last_successful_update_time─┬──────loading_duration─┬─last_exception─┐ +│ dictdb │ dict │ LOADED │ dictdb.dict │ Flat │ UInt64 │ ['value_default','value_expression'] │ ['String','String'] │ 74032 │ 0 │ 1 │ 1 │ 0.0004887585532746823 │ ClickHouse: dictdb.dicttbl │ 0 │ 1 │ 2020-03-04 04:17:34 │ 2020-03-04 04:30:34 │ 0.002 │ │ +└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ +``` ## system.events {#system_tables-events} @@ -668,7 +709,7 @@ When the table is deleted manually, it will be automatically created on the fly. You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log) server setting (see the `partition_by` parameter). -## system.trace\_log {#system_tables-trace_log} +## system.trace_log {#system_tables-trace_log} Contains stack traces collected by the sampling query profiler. @@ -678,24 +719,26 @@ To analyze logs, use the `addressToLine`, `addressToSymbol` and `demangle` intro Columns: -- `event_date`([Date](../sql_reference/data_types/date.md)) — Date of sampling moment. +- `event_date` ([Date](../sql_reference/data_types/date.md)) — Date of sampling moment. -- `event_time`([DateTime](../sql_reference/data_types/datetime.md)) — Timestamp of sampling moment. +- `event_time` ([DateTime](../sql_reference/data_types/datetime.md)) — Timestamp of the sampling moment. -- `revision`([UInt32](../sql_reference/data_types/int_uint.md)) — ClickHouse server build revision. +- `timestamp_ns` ([UInt64](../sql_reference/data_types/int_uint.md)) — Timestamp of the sampling moment in nanoseconds. + +- `revision` ([UInt32](../sql_reference/data_types/int_uint.md)) — ClickHouse server build revision. When connecting to server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server. -- `timer_type`([Enum8](../sql_reference/data_types/enum.md)) — Timer type: +- `timer_type` ([Enum8](../sql_reference/data_types/enum.md)) — Timer type: - `Real` represents wall-clock time. - `CPU` represents CPU time. -- `thread_number`([UInt32](../sql_reference/data_types/int_uint.md)) — Thread identifier. +- `thread_number` ([UInt32](../sql_reference/data_types/int_uint.md)) — Thread identifier. -- `query_id`([String](../sql_reference/data_types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) system table. +- `query_id` ([String](../sql_reference/data_types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) system table. -- `trace`([Array(UInt64)](../sql_reference/data_types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. +- `trace` ([Array(UInt64)](../sql_reference/data_types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. **Example** diff --git a/docs/en/sql_reference/data_types/simpleaggregatefunction.md b/docs/en/sql_reference/data_types/simpleaggregatefunction.md new file mode 100644 index 00000000000..81036aec4c8 --- /dev/null +++ b/docs/en/sql_reference/data_types/simpleaggregatefunction.md @@ -0,0 +1,34 @@ +# SimpleAggregateFunction(name, types\_of\_arguments…) {#data-type-simpleaggregatefunction} + +`SimpleAggregateFunction` data type stores current value of the aggregate function, and does not store its full state as [`AggregateFunction`](aggregatefunction.md) does. This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. This property guarantees that partial aggregation results are enough to compute the combined one, so we don't have to store and process any extra data. + +The following aggregate functions are supported: + +- [`any`](../../sql_reference/aggregate_functions/reference.md#agg_function-any) +- [`anyLast`](../../sql_reference/aggregate_functions/reference.md#anylastx) +- [`min`](../../sql_reference/aggregate_functions/reference.md#agg_function-min) +- [`max`](../../sql_reference/aggregate_functions/reference.md#agg_function-max) +- [`sum`](../../sql_reference/aggregate_functions/reference.md#agg_function-sum) +- [`groupBitAnd`](../../sql_reference/aggregate_functions/reference.md#groupbitand) +- [`groupBitOr`](../../sql_reference/aggregate_functions/reference.md#groupbitor) +- [`groupBitXor`](../../sql_reference/aggregate_functions/reference.md#groupbitxor) + + +Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way as `Type`, so you do not need to apply functions with `-Merge`/`-State` suffixes. `SimpleAggregateFunction` has better performance than `AggregateFunction` with same aggregation function. + +**Parameters** + +- Name of the aggregate function. +- Types of the aggregate function arguments. + +**Example** + +``` sql +CREATE TABLE t +( + column1 SimpleAggregateFunction(sum, UInt64), + column2 SimpleAggregateFunction(any, String) +) ENGINE = ... +``` + +[Original article](https://clickhouse.tech/docs/en/data_types/simpleaggregatefunction/) diff --git a/docs/en/sql_reference/data_types/special_data_types/set.md b/docs/en/sql_reference/data_types/special_data_types/set.md index 61f3c302ce4..05ca97910b6 100644 --- a/docs/en/sql_reference/data_types/special_data_types/set.md +++ b/docs/en/sql_reference/data_types/special_data_types/set.md @@ -5,6 +5,6 @@ toc_title: Set # Set {#set} -Used for the right half of an [IN](../../../sql_reference/statements/select.md#select-in-operators) expression. +Used for the right half of an [IN](../../statements/select.md#select-in-operators) expression. [Original article](https://clickhouse.tech/docs/en/data_types/special_data_types/set/) diff --git a/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md b/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md index 645c6347f66..97d5b6e4474 100644 --- a/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md +++ b/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md @@ -49,6 +49,11 @@ or LIFETIME(MIN 300 MAX 360) ``` +If `0` and `0`, ClickHouse does not reload the dictionary by timeout. +In this case, ClickHouse can reload the dictionary earlier if the dictionary configuration file was changed or the `SYSTEM RELOAD DICTIONARY` command was executed. + +When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [source](external_dicts_dict_sources.md): + When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [source](external_dicts_dict_sources.md): - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated. diff --git a/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md b/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md index 1d3b78635d0..ca8aef24ea1 100644 --- a/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md +++ b/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md @@ -35,6 +35,25 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration The source is configured in the `source` section. +For source types +[Local file](#dicts-external_dicts_dict_sources-local_file), +[Executable file](#dicts-external_dicts_dict_sources-executable), +[HTTP(s)](#dicts-external_dicts_dict_sources-http), +[ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) +optional settings are available: + +``` xml + + + /opt/dictionaries/os.tsv + TabSeparated + + + 0 + + +``` + Types of sources (`source_type`): - [Local file](#dicts-external_dicts_dict_sources-local_file) diff --git a/docs/en/sql_reference/functions/array_functions.md b/docs/en/sql_reference/functions/array_functions.md index 0b6b41d04ac..77a955fa18b 100644 --- a/docs/en/sql_reference/functions/array_functions.md +++ b/docs/en/sql_reference/functions/array_functions.md @@ -989,7 +989,7 @@ Result: ## arrayZip {#arrayzip} -Combine multiple Array type columns into one Array\[Tuple(…)\] column +Combines multiple arrays into a single array. The resulting array contains the corresponding elements of the source arrays grouped into tuples in the listed order of arguments. **Syntax** @@ -999,28 +999,33 @@ arrayZip(arr1, arr2, ..., arrN) **Parameters** -`arr` — Any number of [array](../../sql_reference/data_types/array.md) type columns to combine. +- `arrN` — [Array](../data_types/array.md). + +The function can take any number of arrays of different types. All the input arrays must be of equal size. **Returned value** -The result of Array\[Tuple(…)\] type after the combination of these arrays +- Array with elements from the source arrays grouped into [tuples](../data_types/tuple.md). Data types in the tuple are the same as types of the input arrays and in the same order as arrays are passed. + +Type: [Array](../data_types/array.md). **Example** Query: ``` sql -SELECT arrayZip(['a', 'b', 'c'], ['d', 'e', 'f']); +SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]) ``` Result: ``` text -┌─arrayZip(['a', 'b', 'c'], ['d', 'e', 'f'])─┐ -│ [('a','d'),('b','e'),('c','f')] │ -└────────────────────────────────────────────┘ +┌─arrayZip(['a', 'b', 'c'], [5, 2, 1])─┐ +│ [('a',5),('b',2),('c',1)] │ +└──────────────────────────────────────┘ ``` + ## arrayAUC {#arrayauc} Calculate AUC (Area Under the Curve, which is a concept in machine learning, see more details: https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve). diff --git a/docs/en/sql_reference/functions/hash_functions.md b/docs/en/sql_reference/functions/hash_functions.md index d6f4bacd9d6..772c3c92d27 100644 --- a/docs/en/sql_reference/functions/hash_functions.md +++ b/docs/en/sql_reference/functions/hash_functions.md @@ -345,6 +345,44 @@ SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23: └──────────────────────┴────────┘ ``` +## gccMurmurHash {#gccmurmurhash} + +Calculates a 64-bit [MurmurHash2](https://github.com/aappleby/smhasher) hash value using the same hash seed as [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191). It is portable between CLang and GCC builds. + +**Syntax** + +```sql +gccMurmurHash(par1, ...); +``` + +**Parameters** + +- `par1, ...` — A variable number of parameters that can be any of the [supported data types](../../sql_reference/data_types/index.md#data_types). + +**Returned value** + +- Calculated hash value. + +Type: [UInt64](../../sql_reference/data_types/int_uint.md). + +**Example** + +Query: + +```sql +SELECT + gccMurmurHash(1, 2, 3) AS res1, + gccMurmurHash(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))) AS res2 +``` + +Result: + +``` text +┌─────────────────res1─┬────────────────res2─┐ +│ 12384823029245979431 │ 1188926775431157506 │ +└──────────────────────┴─────────────────────┘ +``` + ## murmurHash3\_32, murmurHash3\_64 {#murmurhash3-32-murmurhash3-64} Produces a [MurmurHash3](https://github.com/aappleby/smhasher) hash value. @@ -441,4 +479,5 @@ Result: - [xxHash](http://cyan4973.github.io/xxHash/). + [Original article](https://clickhouse.tech/docs/en/query_language/functions/hash_functions/) diff --git a/docs/en/sql_reference/functions/json_functions.md b/docs/en/sql_reference/functions/json_functions.md index f1a6b6cc44a..ca1690130e6 100644 --- a/docs/en/sql_reference/functions/json_functions.md +++ b/docs/en/sql_reference/functions/json_functions.md @@ -194,17 +194,17 @@ SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \ ## JSONExtractKeysAndValues(json\[, indices\_or\_keys…\], value\_type) {#jsonextractkeysandvaluesjson-indices-or-keys-value-type} -Parse key-value pairs from a JSON where the values are of the given ClickHouse data type. +Parses key-value pairs from a JSON where the values are of the given ClickHouse data type. Example: ``` sql -SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)]; +SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)] ``` ## JSONExtractRaw(json\[, indices\_or\_keys\]…) {#jsonextractrawjson-indices-or-keys} -Returns a part of JSON. +Returns a part of JSON as unparsed string. If the part does not exist or has a wrong type, an empty string will be returned. @@ -214,7 +214,7 @@ Example: SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' ``` -## JSONExtractArrayRaw(json\[, indices\_or\_keys\]…) {#jsonextractarrayrawjson-indices-or-keys} +## JSONExtractArrayRaw(json\[, indices\_or\_keys…\]) {#jsonextractarrayrawjson-indices-or-keys} Returns an array with elements of JSON array, each represented as unparsed string. @@ -227,3 +227,13 @@ SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ``` [Original article](https://clickhouse.tech/docs/en/query_language/functions/json_functions/) + +## JSONExtractKeysAndValuesRaw(json\[, indices\_or\_keys…\]) {#jsonextractkeysandvaluesrawjson-indices-or-keys} + +Parses key-value pairs from a JSON and returns an array of such pairs, each value represented as unparsed string. + +Example: + +``` sql +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}') = [('a','"hello"'),('b','[-100,200,300]')] +``` diff --git a/docs/en/sql_reference/operators.md b/docs/en/sql_reference/operators.md index 418a9e32771..8ae9e460d87 100644 --- a/docs/en/sql_reference/operators.md +++ b/docs/en/sql_reference/operators.md @@ -5,8 +5,7 @@ toc_title: Operators # Operators {#operators} -All operators are transformed to their corresponding functions at the query parsing stage in accordance with their precedence and associativity. -Groups of operators are listed in order of priority (the higher it is in the list, the earlier the operator is connected to its arguments). +ClickHouse transforms operators to their corresponding functions at the query parsing stage according to their priority, precedence, and associativity. ## Access Operators {#access-operators} @@ -78,7 +77,7 @@ Groups of operators are listed in order of priority (the higher it is in the lis EXTRACT(part FROM date); ``` -Extracts a part from a given date. For example, you can retrieve a month from a given date, or a second from a time. +Extract parts from a given date. For example, you can retrieve a month from a given date, or a second from a time. The `part` parameter specifies which part of the date to retrieve. The following values are available: @@ -151,7 +150,7 @@ Types of intervals: - `YEAR` !!! warning "Warning" - Intervals with different types can’t be combined. You can’t use expressions like `INTERVAL 4 DAY 1 HOUR`. Express intervals in units that are smaller or equal the the smallest unit of the interval, for example `INTERVAL 25 HOUR`. You can use consequtive operations like in the example below. + Intervals with different types can’t be combined. You can’t use expressions like `INTERVAL 4 DAY 1 HOUR`. Specify intervals in units that are smaller or equal to the smallest unit of the interval, for example, `INTERVAL 25 HOUR`. You can use consecutive operations, like in the example below. Example: @@ -214,7 +213,7 @@ The `transform` function does not work with `NULL`. `x -> expr` – The `lambda(x, expr) function.` -The following operators do not have a priority, since they are brackets: +The following operators do not have a priority since they are brackets: ## Array Creation Operator {#array-creation-operator} @@ -229,7 +228,7 @@ The following operators do not have a priority, since they are brackets: All binary operators have left associativity. For example, `1 + 2 + 3` is transformed to `plus(plus(1, 2), 3)`. Sometimes this doesn’t work the way you expect. For example, `SELECT 4 > 2 > 3` will result in 0. -For efficiency, the `and` and `or` functions accept any number of arguments. The corresponding chains of `AND` and `OR` operators are transformed to a single call of these functions. +For efficiency, the `and` and `or` functions accept any number of arguments. The corresponding chains of `AND` and `OR` operators are transformed into a single call of these functions. ## Checking for `NULL` {#checking-for-null} diff --git a/docs/en/sql_reference/statements/alter.md b/docs/en/sql_reference/statements/alter.md index 885dc138a51..aa04ae65514 100644 --- a/docs/en/sql_reference/statements/alter.md +++ b/docs/en/sql_reference/statements/alter.md @@ -204,7 +204,6 @@ The following operations with [partitions](../../engines/table_engines/mergetree - [DETACH PARTITION](#alter_detach-partition) – Moves a partition to the `detached` directory and forget it. - [DROP PARTITION](#alter_drop-partition) – Deletes a partition. - [ATTACH PART\|PARTITION](#alter_attach-partition) – Adds a part or partition from the `detached` directory to the table. -- [REPLACE PARTITION](#alter_replace-partition) - Copies the data partition from one table to another. - [ATTACH PARTITION FROM](#alter_attach-partition-from) – Copies the data partition from one table to another and adds. - [REPLACE PARTITION](#alter_replace-partition) - Copies the data partition from one table to another and replaces. - [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) (\#alter\_move\_to\_table-partition) - Move the data partition from one table to another. diff --git a/docs/en/sql_reference/statements/create.md b/docs/en/sql_reference/statements/create.md index fbfe3eb2cfb..430bcacbc34 100644 --- a/docs/en/sql_reference/statements/create.md +++ b/docs/en/sql_reference/statements/create.md @@ -15,24 +15,18 @@ CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(.. ### Clauses {#clauses} -- `IF NOT EXISTS` +- `IF NOT EXISTS` + If the `db_name` database already exists, then ClickHouse doesn't create a new database and: + - Doesn't throw an exception if clause is specified. + - Throws an exception if clause isn't specified. - If the `db_name` database already exists, then ClickHouse doesn't create a new database and: - - - Doesn't throw an exception if clause is specified. - - Throws an exception if clause isn't specified. - -- `ON CLUSTER` - - ClickHouse creates the `db_name` database on all the servers of a specified cluster. +- `ON CLUSTER` + ClickHouse creates the `db_name` database on all the servers of a specified cluster. - `ENGINE` - - - [MySQL](../engines/database_engines/mysql.md) - - Allows you to retrieve data from the remote MySQL server. - - By default, ClickHouse uses its own [database engine](../engines/database_engines/index.md). + - [MySQL](../../engines/database_engines/mysql.md) + Allows you to retrieve data from the remote MySQL server. + By default, ClickHouse uses its own [database engine](../../engines/database_engines/index.md). ## CREATE TABLE {#create-table-query} @@ -185,7 +179,7 @@ CREATE TABLE codec_example ENGINE = MergeTree() ``` -#### Common Purpose Codecs {#create-query-common-purpose-codecs} +#### General Purpose Codecs {#create-query-general-purpose-codecs} Codecs: diff --git a/docs/en/sql_reference/statements/index.md b/docs/en/sql_reference/statements/index.md index 1adf93a153e..507d858c14a 100644 --- a/docs/en/sql_reference/statements/index.md +++ b/docs/en/sql_reference/statements/index.md @@ -2,4 +2,3 @@ toc_folder_title: Statements toc_priority: 31 --- - diff --git a/docs/en/sql_reference/statements/select.md b/docs/en/sql_reference/statements/select.md index 83ead5b26e1..9232011a8e2 100644 --- a/docs/en/sql_reference/statements/select.md +++ b/docs/en/sql_reference/statements/select.md @@ -248,7 +248,7 @@ Here, a sample of 10% is taken from the second half of the data. ### ARRAY JOIN Clause {#select-array-join-clause} -Allows executing `JOIN` with an array or nested data structure. The intent is similar to the [arrayJoin](../../sql_reference/functions/array_join.md#functions_arrayjoin) function, but its functionality is broader. +Allows executing `JOIN` with an array or nested data structure. The intent is similar to the [arrayJoin](../functions/array_join.md#functions_arrayjoin) function, but its functionality is broader. ``` sql SELECT @@ -602,7 +602,777 @@ USING (equi_column1, ... equi_columnN, asof_column) For example, consider the following tables: -\`\`\` text -table\_1 table\_2 + table_1 table_2 + event | ev_time | user_id event | ev_time | user_id + ----------|---------|---------- ----------|---------|---------- + ... ... + event_1_1 | 12:00 | 42 event_2_1 | 11:59 | 42 + ... event_2_2 | 12:30 | 42 + event_1_2 | 13:00 | 42 event_2_3 | 13:00 | 42 + ... ... -event \| ev\_time \| user\_id event \| ev\_time \| user\_id + +`ASOF JOIN` can take the timestamp of a user event from `table_1` and find an event in `table_2` where the timestamp is closest to the timestamp of the event from `table_1` corresponding to the closest match condition. Equal timestamp values are the closest if available. Here, the `user_id` column can be used for joining on equality and the `ev_time` column can be used for joining on the closest match. In our example, `event_1_1` can be joined with `event_2_1` and `event_1_2` can be joined with `event_2_3`, but `event_2_2` can’t be joined. + +!!! note "Note" + `ASOF` join is **not** supported in the [Join](../../engines/table_engines/special/join.md) table engine. + +To set the default strictness value, use the session configuration parameter [join\_default\_strictness](../../operations/settings/settings.md#settings-join_default_strictness). + +#### GLOBAL JOIN {#global-join} + +When using a normal `JOIN`, the query is sent to remote servers. Subqueries are run on each of them in order to make the right table, and the join is performed with this table. In other words, the right table is formed on each server separately. + +When using `GLOBAL ... JOIN`, first the requestor server runs a subquery to calculate the right table. This temporary table is passed to each remote server, and queries are run on them using the temporary data that was transmitted. + +Be careful when using `GLOBAL`. For more information, see the section [Distributed subqueries](#select-distributed-subqueries). + +#### Usage Recommendations {#usage-recommendations} + +When running a `JOIN`, there is no optimization of the order of execution in relation to other stages of the query. The join (a search in the right table) is run before filtering in `WHERE` and before aggregation. In order to explicitly set the processing order, we recommend running a `JOIN` subquery with a subquery. + +Example: + +``` sql +SELECT + CounterID, + hits, + visits +FROM +( + SELECT + CounterID, + count() AS hits + FROM test.hits + GROUP BY CounterID +) ANY LEFT JOIN +( + SELECT + CounterID, + sum(Sign) AS visits + FROM test.visits + GROUP BY CounterID +) USING CounterID +ORDER BY hits DESC +LIMIT 10 +``` + +``` text +┌─CounterID─┬───hits─┬─visits─┐ +│ 1143050 │ 523264 │ 13665 │ +│ 731962 │ 475698 │ 102716 │ +│ 722545 │ 337212 │ 108187 │ +│ 722889 │ 252197 │ 10547 │ +│ 2237260 │ 196036 │ 9522 │ +│ 23057320 │ 147211 │ 7689 │ +│ 722818 │ 90109 │ 17847 │ +│ 48221 │ 85379 │ 4652 │ +│ 19762435 │ 77807 │ 7026 │ +│ 722884 │ 77492 │ 11056 │ +└───────────┴────────┴────────┘ +``` + +Subqueries don’t allow you to set names or use them for referencing a column from a specific subquery. +The columns specified in `USING` must have the same names in both subqueries, and the other columns must be named differently. You can use aliases to change the names of columns in subqueries (the example uses the aliases `hits` and `visits`). + +The `USING` clause specifies one or more columns to join, which establishes the equality of these columns. The list of columns is set without brackets. More complex join conditions are not supported. + +The right table (the subquery result) resides in RAM. If there isn’t enough memory, you can’t run a `JOIN`. + +Each time a query is run with the same `JOIN`, the subquery is run again because the result is not cached. To avoid this, use the special [Join](../../engines/table_engines/special/join.md) table engine, which is a prepared array for joining that is always in RAM. + +In some cases, it is more efficient to use `IN` instead of `JOIN`. +Among the various types of `JOIN`, the most efficient is `ANY LEFT JOIN`, then `ANY INNER JOIN`. The least efficient are `ALL LEFT JOIN` and `ALL INNER JOIN`. + +If you need a `JOIN` for joining with dimension tables (these are relatively small tables that contain dimension properties, such as names for advertising campaigns), a `JOIN` might not be very convenient due to the fact that the right table is re-accessed for every query. For such cases, there is an “external dictionaries” feature that you should use instead of `JOIN`. For more information, see the section [External dictionaries](../dictionaries/external_dictionaries/external_dicts.md). + +**Memory Limitations** + +ClickHouse uses the [hash join](https://en.wikipedia.org/wiki/Hash_join) algorithm. ClickHouse takes the `` and creates a hash table for it in RAM. If you need to restrict join operation memory consumption use the following settings: + +- [max\_rows\_in\_join](../../operations/settings/query_complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table. +- [max\_bytes\_in\_join](../../operations/settings/query_complexity.md#settings-max_bytes_in_join) — Limits size of the hash table. + +When any of these limits is reached, ClickHouse acts as the [join\_overflow\_mode](../../operations/settings/query_complexity.md#settings-join_overflow_mode) setting instructs. + +#### Processing of Empty or NULL Cells {#processing-of-empty-or-null-cells} + +While joining tables, the empty cells may appear. The setting [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls) define how ClickHouse fills these cells. + +If the `JOIN` keys are [Nullable](../data_types/nullable.md) fields, the rows where at least one of the keys has the value [NULL](../syntax.md#null-literal) are not joined. + +#### Syntax Limitations {#syntax-limitations} + +For multiple `JOIN` clauses in a single `SELECT` query: + +- Taking all the columns via `*` is available only if tables are joined, not subqueries. +- The `PREWHERE` clause is not available. + +For `ON`, `WHERE`, and `GROUP BY` clauses: + +- Arbitrary expressions cannot be used in `ON`, `WHERE`, and `GROUP BY` clauses, but you can define an expression in a `SELECT` clause and then use it in these clauses via an alias. + +### WHERE Clause {#select-where} + +If there is a WHERE clause, it must contain an expression with the UInt8 type. This is usually an expression with comparison and logical operators. +This expression will be used for filtering data before all other transformations. + +If indexes are supported by the database table engine, the expression is evaluated on the ability to use indexes. + +### PREWHERE Clause {#prewhere-clause} + +This clause has the same meaning as the WHERE clause. The difference is in which data is read from the table. +When using PREWHERE, first only the columns necessary for executing PREWHERE are read. Then the other columns are read that are needed for running the query, but only those blocks where the PREWHERE expression is true. + +It makes sense to use PREWHERE if there are filtration conditions that are used by a minority of the columns in the query, but that provide strong data filtration. This reduces the volume of data to read. + +For example, it is useful to write PREWHERE for queries that extract a large number of columns, but that only have filtration for a few columns. + +PREWHERE is only supported by tables from the `*MergeTree` family. + +A query may simultaneously specify PREWHERE and WHERE. In this case, PREWHERE precedes WHERE. + +If the ‘optimize\_move\_to\_prewhere’ setting is set to 1 and PREWHERE is omitted, the system uses heuristics to automatically move parts of expressions from WHERE to PREWHERE. + +### GROUP BY Clause {#select-group-by-clause} + +This is one of the most important parts of a column-oriented DBMS. + +If there is a GROUP BY clause, it must contain a list of expressions. Each expression will be referred to here as a “key”. +All the expressions in the SELECT, HAVING, and ORDER BY clauses must be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. + +If a query contains only table columns inside aggregate functions, the GROUP BY clause can be omitted, and aggregation by an empty set of keys is assumed. + +Example: + +``` sql +SELECT + count(), + median(FetchTiming > 60 ? 60 : FetchTiming), + count() - sum(Refresh) +FROM hits +``` + +However, in contrast to standard SQL, if the table doesn’t have any rows (either there aren’t any at all, or there aren’t any after using WHERE to filter), an empty result is returned, and not the result from one of the rows containing the initial values of aggregate functions. + +As opposed to MySQL (and conforming to standard SQL), you can’t get some value of some column that is not in a key or aggregate function (except constant expressions). To work around this, you can use the ‘any’ aggregate function (get the first encountered value) or ‘min/max’. + +Example: + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + count(), + any(Title) AS title -- getting the first occurred page header for each domain. +FROM hits +GROUP BY domain +``` + +For every different key value encountered, GROUP BY calculates a set of aggregate function values. + +GROUP BY is not supported for array columns. + +A constant can’t be specified as arguments for aggregate functions. Example: sum(1). Instead of this, you can get rid of the constant. Example: `count()`. + +#### NULL processing {#null-processing} + +For grouping, ClickHouse interprets [NULL](../syntax.md) as a value, and `NULL=NULL`. + +Here’s an example to show what this means. + +Assume you have this table: + +``` text +┌─x─┬────y─┐ +│ 1 │ 2 │ +│ 2 │ ᴺᵁᴸᴸ │ +│ 3 │ 2 │ +│ 3 │ 3 │ +│ 3 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +The query `SELECT sum(x), y FROM t_null_big GROUP BY y` results in: + +``` text +┌─sum(x)─┬────y─┐ +│ 4 │ 2 │ +│ 3 │ 3 │ +│ 5 │ ᴺᵁᴸᴸ │ +└────────┴──────┘ +``` + +You can see that `GROUP BY` for `y = NULL` summed up `x`, as if `NULL` is this value. + +If you pass several keys to `GROUP BY`, the result will give you all the combinations of the selection, as if `NULL` were a specific value. + +#### WITH TOTALS Modifier {#with-totals-modifier} + +If the WITH TOTALS modifier is specified, another row will be calculated. This row will have key columns containing default values (zeros or empty lines), and columns of aggregate functions with the values calculated across all the rows (the “total” values). + +This extra row is output in JSON\*, TabSeparated\*, and Pretty\* formats, separately from the other rows. In the other formats, this row is not output. + +In JSON\* formats, this row is output as a separate ‘totals’ field. In TabSeparated\* formats, the row comes after the main result, preceded by an empty row (after the other data). In Pretty\* formats, the row is output as a separate table after the main result. + +`WITH TOTALS` can be run in different ways when HAVING is present. The behavior depends on the ‘totals\_mode’ setting. +By default, `totals_mode = 'before_having'`. In this case, ‘totals’ is calculated across all rows, including the ones that don’t pass through HAVING and ‘max\_rows\_to\_group\_by’. + +The other alternatives include only the rows that pass through HAVING in ‘totals’, and behave differently with the setting `max_rows_to_group_by` and `group_by_overflow_mode = 'any'`. + +`after_having_exclusive` – Don’t include rows that didn’t pass through `max_rows_to_group_by`. In other words, ‘totals’ will have less than or the same number of rows as it would if `max_rows_to_group_by` were omitted. + +`after_having_inclusive` – Include all the rows that didn’t pass through ‘max\_rows\_to\_group\_by’ in ‘totals’. In other words, ‘totals’ will have more than or the same number of rows as it would if `max_rows_to_group_by` were omitted. + +`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn’t pass through ‘max\_rows\_to\_group\_by’ in ‘totals’. Otherwise, do not include them. + +`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. + +If `max_rows_to_group_by` and `group_by_overflow_mode = 'any'` are not used, all variations of `after_having` are the same, and you can use any of them (for example, `after_having_auto`). + +You can use WITH TOTALS in subqueries, including subqueries in the JOIN clause (in this case, the respective total values are combined). + +#### GROUP BY in External Memory {#select-group-by-in-external-memory} + +You can enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`. +The [max\_bytes\_before\_external\_group\_by](../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled. + +When using `max_bytes_before_external_group_by`, we recommend that you set `max_memory_usage` about twice as high. This is necessary because there are two stages to aggregation: reading the date and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn’t dumped, then stage 2 might require up to the same amount of memory as in stage 1. + +For example, if [max\_memory\_usage](../../operations/settings/settings.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and max\_memory\_usage to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`. + +With distributed query processing, external aggregation is performed on remote servers. In order for the requester server to use only a small amount of RAM, set `distributed_aggregation_memory_efficient` to 1. + +When merging data flushed to the disk, as well as when merging results from remote servers when the `distributed_aggregation_memory_efficient` setting is enabled, consumes up to `1/256 * the_number_of_threads` from the total amount of RAM. + +When external aggregation is enabled, if there was less than `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). + +If you have an `ORDER BY` with a `LIMIT` after `GROUP BY`, then the amount of used RAM depends on the amount of data in `LIMIT`, not in the whole table. But if the `ORDER BY` doesn’t have `LIMIT`, don’t forget to enable external sorting (`max_bytes_before_external_sort`). + +### LIMIT BY Clause {#limit-by-clause} + +A query with the `LIMIT n BY expressions` clause selects the first `n` rows for each distinct value of `expressions`. The key for `LIMIT BY` can contain any number of [expressions](../syntax.md#syntax-expressions). + +ClickHouse supports the following syntax: + +- `LIMIT [offset_value, ]n BY expressions` +- `LIMIT n OFFSET offset_value BY expressions` + +During query processing, ClickHouse selects data ordered by sorting key. The sorting key is set explicitly using an [ORDER BY](#select-order-by) clause or implicitly as a property of the table engine. Then ClickHouse applies `LIMIT n BY expressions` and returns the first `n` rows for each distinct combination of `expressions`. If `OFFSET` is specified, then for each data block that belongs to a distinct combination of `expressions`, ClickHouse skips `offset_value` number of rows from the beginning of the block and returns a maximum of `n` rows as a result. If `offset_value` is bigger than the number of rows in the data block, ClickHouse returns zero rows from the block. + +`LIMIT BY` is not related to `LIMIT`. They can both be used in the same query. + +**Examples** + +Sample table: + +``` sql +CREATE TABLE limit_by(id Int, val Int) ENGINE = Memory; +INSERT INTO limit_by values(1, 10), (1, 11), (1, 12), (2, 20), (2, 21); +``` + +Queries: + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 10 │ +│ 1 │ 11 │ +│ 2 │ 20 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +``` sql +SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id +``` + +``` text +┌─id─┬─val─┐ +│ 1 │ 11 │ +│ 1 │ 12 │ +│ 2 │ 21 │ +└────┴─────┘ +``` + +The `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` query returns the same result. + +The following query returns the top 5 referrers for each `domain, device_type` pair with a maximum of 100 rows in total (`LIMIT n BY + LIMIT`). + +``` sql +SELECT + domainWithoutWWW(URL) AS domain, + domainWithoutWWW(REFERRER_URL) AS referrer, + device_type, + count() cnt +FROM hits +GROUP BY domain, referrer, device_type +ORDER BY cnt DESC +LIMIT 5 BY domain, device_type +LIMIT 100 +``` + +### HAVING Clause {#having-clause} + +Allows filtering the result received after GROUP BY, similar to the WHERE clause. +WHERE and HAVING differ in that WHERE is performed before aggregation (GROUP BY), while HAVING is performed after it. +If aggregation is not performed, HAVING can’t be used. + +### ORDER BY Clause {#select-order-by} + +The ORDER BY clause contains a list of expressions, which can each be assigned DESC or ASC (the sorting direction). If the direction is not specified, ASC is assumed. ASC is sorted in ascending order, and DESC in descending order. The sorting direction applies to a single expression, not to the entire list. Example: `ORDER BY Visits DESC, SearchPhrase` + +For sorting by String values, you can specify collation (comparison). Example: `ORDER BY SearchPhrase COLLATE 'tr'` - for sorting by keyword in ascending order, using the Turkish alphabet, case insensitive, assuming that strings are UTF-8 encoded. COLLATE can be specified or not for each expression in ORDER BY independently. If ASC or DESC is specified, COLLATE is specified after it. When using COLLATE, sorting is always case-insensitive. + +We only recommend using COLLATE for final sorting of a small number of rows, since sorting with COLLATE is less efficient than normal sorting by bytes. + +Rows that have identical values for the list of sorting expressions are output in an arbitrary order, which can also be nondeterministic (different each time). +If the ORDER BY clause is omitted, the order of the rows is also undefined, and may be nondeterministic as well. + +`NaN` and `NULL` sorting order: + +- With the modifier `NULLS FIRST` — First `NULL`, then `NaN`, then other values. +- With the modifier `NULLS LAST` — First the values, then `NaN`, then `NULL`. +- Default — The same as with the `NULLS LAST` modifier. + +Example: + +For the table + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 2 │ +│ 1 │ nan │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ nan │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +Run the query `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` to get: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 7 │ ᴺᵁᴸᴸ │ +│ 1 │ nan │ +│ 6 │ nan │ +│ 2 │ 2 │ +│ 2 │ 2 │ +│ 3 │ 4 │ +│ 5 │ 6 │ +│ 6 │ 7 │ +│ 8 │ 9 │ +└───┴──────┘ +``` + +When floating point numbers are sorted, NaNs are separate from the other values. Regardless of the sorting order, NaNs come at the end. In other words, for ascending sorting they are placed as if they are larger than all the other numbers, while for descending sorting they are placed as if they are smaller than the rest. + +Less RAM is used if a small enough LIMIT is specified in addition to ORDER BY. Otherwise, the amount of memory spent is proportional to the volume of data for sorting. For distributed query processing, if GROUP BY is omitted, sorting is partially done on remote servers, and the results are merged on the requestor server. This means that for distributed sorting, the volume of data to sort can be greater than the amount of memory on a single server. + +If there is not enough RAM, it is possible to perform sorting in external memory (creating temporary files on a disk). Use the setting `max_bytes_before_external_sort` for this purpose. If it is set to 0 (the default), external sorting is disabled. If it is enabled, when the volume of data to sort reaches the specified number of bytes, the collected data is sorted and dumped into a temporary file. After all data is read, all the sorted files are merged and the results are output. Files are written to the /var/lib/clickhouse/tmp/ directory in the config (by default, but you can use the ‘tmp\_path’ parameter to change this setting). + +Running a query may use more memory than ‘max\_bytes\_before\_external\_sort’. For this reason, this setting must have a value significantly smaller than ‘max\_memory\_usage’. As an example, if your server has 128 GB of RAM and you need to run a single query, set ‘max\_memory\_usage’ to 100 GB, and ‘max\_bytes\_before\_external\_sort’ to 80 GB. + +External sorting works much less effectively than sorting in RAM. + +### SELECT Clause {#select-select} + +[Expressions](../syntax.md#syntax-expressions) specified in the `SELECT` clause are calculated after all the operations in the clauses described above are finished. These expressions work as if they apply to separate rows in the result. If expressions in the `SELECT` clause contain aggregate functions, then ClickHouse processes aggregate functions and expressions used as their arguments during the [GROUP BY](#select-group-by-clause) aggregation. + +If you want to include all columns in the result, use the asterisk (`*`) symbol. For example, `SELECT * FROM ...`. + +To match some columns in the result with a [re2](https://en.wikipedia.org/wiki/RE2_(software)) regular expression, you can use the `COLUMNS` expression. + +``` sql +COLUMNS('regexp') +``` + +For example, consider the table: + +``` sql +CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog +``` + +The following query selects data from all the columns containing the `a` symbol in their name. + +``` sql +SELECT COLUMNS('a') FROM col_names +``` + +``` text +┌─aa─┬─ab─┐ +│ 1 │ 1 │ +└────┴────┘ +``` + +The selected columns are returned not in the alphabetical order. + +You can use multiple `COLUMNS` expressions in a query and apply functions to them. + +For example: + +``` sql +SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names +``` + +``` text +┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ +│ 1 │ 1 │ 1 │ Int8 │ +└────┴────┴────┴────────────────┘ +``` + +Each column returned by the `COLUMNS` expression is passed to the function as a separate argument. Also you can pass other arguments to the function if it supports them. Be careful when using functions. If a function doesn’t support the number of arguments you have passed to it, ClickHouse throws an exception. + +For example: + +``` sql +SELECT COLUMNS('a') + COLUMNS('c') FROM col_names +``` + +``` text +Received exception from server (version 19.14.1): +Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. +``` + +In this example, `COLUMNS('a')` returns two columns: `aa` and `ab`. `COLUMNS('c')` returns the `bc` column. The `+` operator can’t apply to 3 arguments, so ClickHouse throws an exception with the relevant message. + +Columns that matched the `COLUMNS` expression can have different data types. If `COLUMNS` doesn’t match any columns and is the only expression in `SELECT`, ClickHouse throws an exception. + +### DISTINCT Clause {#select-distinct} + +If DISTINCT is specified, only a single row will remain out of all the sets of fully matching rows in the result. +The result will be the same as if GROUP BY were specified across all the fields specified in SELECT without aggregate functions. But there are several differences from GROUP BY: + +- DISTINCT can be applied together with GROUP BY. +- When ORDER BY is omitted and LIMIT is defined, the query stops running immediately after the required number of different rows has been read. +- Data blocks are output as they are processed, without waiting for the entire query to finish running. + +DISTINCT is not supported if SELECT has at least one array column. + +`DISTINCT` works with [NULL](../syntax.md) as if `NULL` were a specific value, and `NULL=NULL`. In other words, in the `DISTINCT` results, different combinations with `NULL` only occur once. + +ClickHouse supports using the `DISTINCT` and `ORDER BY` clauses for different columns in one query. The `DISTINCT` clause is executed before the `ORDER BY` clause. + +Example table: + +``` text +┌─a─┬─b─┐ +│ 2 │ 1 │ +│ 1 │ 2 │ +│ 3 │ 3 │ +│ 2 │ 4 │ +└───┴───┘ +``` + +When selecting data with the `SELECT DISTINCT a FROM t1 ORDER BY b ASC` query, we get the following result: + +``` text +┌─a─┐ +│ 2 │ +│ 1 │ +│ 3 │ +└───┘ +``` + +If we change the sorting direction `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, we get the following result: + +``` text +┌─a─┐ +│ 3 │ +│ 1 │ +│ 2 │ +└───┘ +``` + +Row `2, 4` was cut before sorting. + +Take this implementation specificity into account when programming queries. + +### LIMIT Clause {#limit-clause} + +`LIMIT m` allows you to select the first `m` rows from the result. + +`LIMIT n, m` allows you to select the first `m` rows from the result after skipping the first `n` rows. The `LIMIT m OFFSET n` syntax is also supported. + +`n` and `m` must be non-negative integers. + +If there isn’t an `ORDER BY` clause that explicitly sorts results, the result may be arbitrary and nondeterministic. + +### UNION ALL Clause {#union-all-clause} + +You can use UNION ALL to combine any number of queries. Example: + +``` sql +SELECT CounterID, 1 AS table, toInt64(count()) AS c + FROM test.hits + GROUP BY CounterID + +UNION ALL + +SELECT CounterID, 2 AS table, sum(Sign) AS c + FROM test.visits + GROUP BY CounterID + HAVING c > 0 +``` + +Only UNION ALL is supported. The regular UNION (UNION DISTINCT) is not supported. If you need UNION DISTINCT, you can write SELECT DISTINCT from a subquery containing UNION ALL. + +Queries that are parts of UNION ALL can be run simultaneously, and their results can be mixed together. + +The structure of results (the number and type of columns) must match for the queries. But the column names can differ. In this case, the column names for the final result will be taken from the first query. Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION ALL` has a `Nullable` type field. + +Queries that are parts of UNION ALL can’t be enclosed in brackets. ORDER BY and LIMIT are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with UNION ALL in a subquery in the FROM clause. + +### INTO OUTFILE Clause {#into-outfile-clause} + +Add the `INTO OUTFILE filename` clause (where filename is a string literal) to redirect query output to the specified file. +In contrast to MySQL, the file is created on the client side. The query will fail if a file with the same filename already exists. +This functionality is available in the command-line client and clickhouse-local (a query sent via HTTP interface will fail). + +The default output format is TabSeparated (the same as in the command-line client batch mode). + +### FORMAT Clause {#format-clause} + +Specify ‘FORMAT format’ to get data in any specified format. +You can use this for convenience, or for creating dumps. +For more information, see the section “Formats”. +If the FORMAT clause is omitted, the default format is used, which depends on both the settings and the interface used for accessing the DB. For the HTTP interface and the command-line client in batch mode, the default format is TabSeparated. For the command-line client in interactive mode, the default format is PrettyCompact (it has attractive and compact tables). + +When using the command-line client, data is passed to the client in an internal efficient format. The client independently interprets the FORMAT clause of the query and formats the data itself (thus relieving the network and the server from the load). + +### IN Operators {#select-in-operators} + +The `IN`, `NOT IN`, `GLOBAL IN`, and `GLOBAL NOT IN` operators are covered separately, since their functionality is quite rich. + +The left side of the operator is either a single column or a tuple. + +Examples: + +``` sql +SELECT UserID IN (123, 456) FROM ... +SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... +``` + +If the left side is a single column that is in the index, and the right side is a set of constants, the system uses the index for processing the query. + +Don’t list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section “External data for query processing”), then use a subquery. + +The right side of the operator can be a set of constant expressions, a set of tuples with constant expressions (shown in the examples above), or the name of a database table or SELECT subquery in brackets. + +If the right side of the operator is the name of a table (for example, `UserID IN users`), this is equivalent to the subquery `UserID IN (SELECT * FROM users)`. Use this when working with external data that is sent along with the query. For example, the query can be sent together with a set of user IDs loaded to the ‘users’ temporary table, which should be filtered. + +If the right side of the operator is a table name that has the Set engine (a prepared data set that is always in RAM), the data set will not be created over again for each query. + +The subquery may specify more than one column for filtering tuples. +Example: + +``` sql +SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... +``` + +The columns to the left and right of the IN operator should have the same type. + +The IN operator and subquery may occur in any part of the query, including in aggregate functions and lambda functions. +Example: + +``` sql +SELECT + EventDate, + avg(UserID IN + ( + SELECT UserID + FROM test.hits + WHERE EventDate = toDate('2014-03-17') + )) AS ratio +FROM test.hits +GROUP BY EventDate +ORDER BY EventDate ASC +``` + +``` text +┌──EventDate─┬────ratio─┐ +│ 2014-03-17 │ 1 │ +│ 2014-03-18 │ 0.807696 │ +│ 2014-03-19 │ 0.755406 │ +│ 2014-03-20 │ 0.723218 │ +│ 2014-03-21 │ 0.697021 │ +│ 2014-03-22 │ 0.647851 │ +│ 2014-03-23 │ 0.648416 │ +└────────────┴──────────┘ +``` + +For each day after March 17th, count the percentage of pageviews made by users who visited the site on March 17th. +A subquery in the IN clause is always run just one time on a single server. There are no dependent subqueries. + +#### NULL processing {#null-processing-1} + +During request processing, the IN operator assumes that the result of an operation with [NULL](../syntax.md) is always equal to `0`, regardless of whether `NULL` is on the right or left side of the operator. `NULL` values are not included in any dataset, do not correspond to each other and cannot be compared. + +Here is an example with the `t_null` table: + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Running the query `SELECT x FROM t_null WHERE y IN (NULL,3)` gives you the following result: + +``` text +┌─x─┐ +│ 2 │ +└───┘ +``` + +You can see that the row in which `y = NULL` is thrown out of the query results. This is because ClickHouse can’t decide whether `NULL` is included in the `(NULL,3)` set, returns `0` as the result of the operation, and `SELECT` excludes this row from the final output. + +``` sql +SELECT y IN (NULL, 3) +FROM t_null +``` + +``` text +┌─in(y, tuple(NULL, 3))─┐ +│ 0 │ +│ 1 │ +└───────────────────────┘ +``` + +#### Distributed Subqueries {#select-distributed-subqueries} + +There are two options for IN-s with subqueries (similar to JOINs): normal `IN` / `JOIN` and `GLOBAL IN` / `GLOBAL JOIN`. They differ in how they are run for distributed query processing. + +!!! attention "Attention" + Remember that the algorithms described below may work differently depending on the [settings](../../operations/settings/settings.md) `distributed_product_mode` setting. + +When using the regular IN, the query is sent to remote servers, and each of them runs the subqueries in the `IN` or `JOIN` clause. + +When using `GLOBAL IN` / `GLOBAL JOINs`, first all the subqueries are run for `GLOBAL IN` / `GLOBAL JOINs`, and the results are collected in temporary tables. Then the temporary tables are sent to each remote server, where the queries are run using this temporary data. + +For a non-distributed query, use the regular `IN` / `JOIN`. + +Be careful when using subqueries in the `IN` / `JOIN` clauses for distributed query processing. + +Let’s look at some examples. Assume that each server in the cluster has a normal **local\_table**. Each server also has a **distributed\_table** table with the **Distributed** type, which looks at all the servers in the cluster. + +For a query to the **distributed\_table**, the query will be sent to all the remote servers and run on them using the **local\_table**. + +For example, the query + +``` sql +SELECT uniq(UserID) FROM distributed_table +``` + +will be sent to all remote servers as + +``` sql +SELECT uniq(UserID) FROM local_table +``` + +and run on each of them in parallel, until it reaches the stage where intermediate results can be combined. Then the intermediate results will be returned to the requestor server and merged on it, and the final result will be sent to the client. + +Now let’s examine a query with IN: + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +- Calculation of the intersection of audiences of two sites. + +This query will be sent to all remote servers as + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +In other words, the data set in the IN clause will be collected on each server independently, only across the data that is stored locally on each of the servers. + +This will work correctly and optimally if you are prepared for this case and have spread data across the cluster servers such that the data for a single UserID resides entirely on a single server. In this case, all the necessary data will be available locally on each server. Otherwise, the result will be inaccurate. We refer to this variation of the query as “local IN”. + +To correct how the query works when data is spread randomly across the cluster servers, you could specify **distributed\_table** inside a subquery. The query would look like this: + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +This query will be sent to all remote servers as + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +The subquery will begin running on each remote server. Since the subquery uses a distributed table, the subquery that is on each remote server will be resent to every remote server as + +``` sql +SELECT UserID FROM local_table WHERE CounterID = 34 +``` + +For example, if you have a cluster of 100 servers, executing the entire query will require 10,000 elementary requests, which is generally considered unacceptable. + +In such cases, you should always use GLOBAL IN instead of IN. Let’s look at how it works for the query + +``` sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +The requestor server will run the subquery + +``` sql +SELECT UserID FROM distributed_table WHERE CounterID = 34 +``` + +and the result will be put in a temporary table in RAM. Then the request will be sent to each remote server as + +``` sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 +``` + +and the temporary table `_data1` will be sent to every remote server with the query (the name of the temporary table is implementation-defined). + +This is more optimal than using the normal IN. However, keep the following points in mind: + +1. When creating a temporary table, data is not made unique. To reduce the volume of data transmitted over the network, specify DISTINCT in the subquery. (You don’t need to do this for a normal IN.) +2. The temporary table will be sent to all the remote servers. Transmission does not account for network topology. For example, if 10 remote servers reside in a datacenter that is very remote in relation to the requestor server, the data will be sent 10 times over the channel to the remote datacenter. Try to avoid large data sets when using GLOBAL IN. +3. When transmitting data to remote servers, restrictions on network bandwidth are not configurable. You might overload the network. +4. Try to distribute data across servers so that you don’t need to use GLOBAL IN on a regular basis. +5. If you need to use GLOBAL IN often, plan the location of the ClickHouse cluster so that a single group of replicas resides in no more than one data center with a fast network between them, so that a query can be processed entirely within a single data center. + +It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers. + +### Extreme Values {#extreme-values} + +In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. + +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../interfaces/formats.md), separate from the other rows. They are not output for other formats. + +In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. + +Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`. + +### Notes {#notes} + +The `GROUP BY` and `ORDER BY` clauses do not support positional arguments. This contradicts MySQL, but conforms to standard SQL. +For example, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). + +You can use synonyms (`AS` aliases) in any part of a query. + +You can put an asterisk in any part of a query instead of an expression. When the query is analyzed, the asterisk is expanded to a list of all table columns (excluding the `MATERIALIZED` and `ALIAS` columns). There are only a few cases when using an asterisk is justified: + +- When creating a table dump. +- For tables containing just a few columns, such as system tables. +- For getting information about what columns are in a table. In this case, set `LIMIT 1`. But it is better to use the `DESC TABLE` query. +- When there is strong filtration on a small number of columns using `PREWHERE`. +- In subqueries (since columns that aren’t needed for the external query are excluded from subqueries). + +In all other cases, we don’t recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. In other words using the asterisk is not recommended. + +[Original article](https://clickhouse.tech/docs/en/query_language/select/) diff --git a/docs/en/sql_reference/syntax.md b/docs/en/sql_reference/syntax.md index 12a4b9df7ef..10e8c421f6d 100644 --- a/docs/en/sql_reference/syntax.md +++ b/docs/en/sql_reference/syntax.md @@ -16,11 +16,11 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') The `INSERT INTO t VALUES` fragment is parsed by the full parser, and the data `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` is parsed by the fast stream parser. You can also turn on the full parser for the data by using the [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) setting. When `input_format_values_interpret_expressions = 1`, ClickHouse first tries to parse values with the fast stream parser. If it fails, ClickHouse tries to use the full parser for the data, treating it like an SQL [expression](#syntax-expressions). Data can have any format. When a query is received, the server calculates no more than [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) bytes of the request in RAM (by default, 1 MB), and the rest is stream parsed. -This means the system doesn’t have problems with large `INSERT` queries, like MySQL does. +It allows for avoiding issues with large `INSERT` queries. When using the `Values` format in an `INSERT` query, it may seem that data is parsed the same as expressions in a `SELECT` query, but this is not true. The `Values` format is much more limited. -Next we will cover the full parser. For more information about format parsers, see the [Formats](../interfaces/formats.md) section. +The rest of this article covers the full parser. For more information about format parsers, see the [Formats](../interfaces/formats.md) section. ## Spaces {#spaces} @@ -28,33 +28,33 @@ There may be any number of space symbols between syntactical constructions (incl ## Comments {#comments} -SQL-style and C-style comments are supported. -SQL-style comments: from `--` to the end of the line. The space after `--` can be omitted. -Comments in C-style: from `/*` to `*/`. These comments can be multiline. Spaces are not required here, either. +ClickHouse supports either SQL-style and C-style comments. +SQL-style comments start with `--` and continue to the end of the line, a space after `--` can be omitted. +C-style are from `/*` to `*/`and can be multiline, spaces are not required either. ## Keywords {#syntax-keywords} Keywords are case-insensitive when they correspond to: - SQL standard. For example, `SELECT`, `select` and `SeLeCt` are all valid. -- Implementation in some popular DBMS (MySQL or Postgres). For example, `DateTime` is same as `datetime`. +- Implementation in some popular DBMS (MySQL or Postgres). For example, `DateTime` is the same as `datetime`. Whether data type name is case-sensitive can be checked in the `system.data_type_families` table. -In contrast to standard SQL all other keywords (including functions names) are **case-sensitive**. +In contrast to standard SQL, all other keywords (including functions names) are **case-sensitive**. -Keywords are not reserved (they are just parsed as keywords in the corresponding context). If you use [identifiers](#syntax-identifiers) the same as the keywords, enclose them into quotes. For example, the query `SELECT "FROM" FROM table_name` is valid if the table `table_name` has column with the name `"FROM"`. +Keywords are not reserved; they are treated as such only in the corresponding context. If you use [identifiers](#syntax-identifiers) with the same name as the keywords, enclose them into double-quotes or backticks. For example, the query `SELECT "FROM" FROM table_name` is valid if the table `table_name` has column with the name `"FROM"`. ## Identifiers {#syntax-identifiers} Identifiers are: -- Cluster, database, table, partition and column names. +- Cluster, database, table, partition, and column names. - Functions. - Data types. - [Expression aliases](#syntax-expression_aliases). -Identifiers can be quoted or non-quoted. It is recommended to use non-quoted identifiers. +Identifiers can be quoted or non-quoted. The latter is preferred. Non-quoted identifiers must match the regex `^[a-zA-Z_][0-9a-zA-Z_]*$` and can not be equal to [keywords](#syntax-keywords). Examples: `x, _1, X_y__Z123_.` @@ -62,34 +62,34 @@ If you want to use identifiers the same as keywords or you want to use other sym ## Literals {#literals} -There are: numeric, string, compound and `NULL` literals. +There are numeric, string, compound, and `NULL` literals. ### Numeric {#numeric} -A numeric literal tries to be parsed: +Numeric literal tries to be parsed: -- First as a 64-bit signed number, using the [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) function. +- First, as a 64-bit signed number, using the [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) function. - If unsuccessful, as a 64-bit unsigned number, using the [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) function. - If unsuccessful, as a floating-point number using the [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) function. -- Otherwise, an error is returned. +- Otherwise, it returns an error. -The corresponding value will have the smallest type that the value fits in. +Literal value has the smallest type that the value fits in. For example, 1 is parsed as `UInt8`, but 256 is parsed as `UInt16`. For more information, see [Data types](../sql_reference/data_types/index.md). Examples: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. ### String {#syntax-string-literal} -Only string literals in single quotes are supported. The enclosed characters can be backslash-escaped. The following escape sequences have a corresponding special value: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. In all other cases, escape sequences in the format `\c`, where `c` is any character, are converted to `c`. This means that you can use the sequences `\'`and`\\`. The value will have the [String](../sql_reference/data_types/string.md) type. +Only string literals in single quotes are supported. The enclosed characters can be backslash-escaped. The following escape sequences have a corresponding special value: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. In all other cases, escape sequences in the format `\c`, where `c` is any character, are converted to `c`. It means that you can use the sequences `\'`and`\\`. The value will have the [String](../sql_reference/data_types/string.md) type. -The minimum set of characters that you need to escape in string literals: `'` and `\`. Single quote can be escaped with the single quote, literals `'It\'s'` and `'It''s'` are equal. +In string literals, you need to escape at least `'` and `\`. Single quotes can be escaped with the single quote, literals `'It\'s'` and `'It''s'` are equal. ### Compound {#compound} -Constructions are supported for arrays: `[1, 2, 3]` and tuples: `(1, 'Hello, world!', 2)`.. -Actually, these are not literals, but expressions with the array creation operator and the tuple creation operator, respectively. +Arrays are constructed with square brackets `[1, 2, 3]`. Nuples are constructed with round brackets `(1, 'Hello, world!', 2)`. +Technically these are not literals, but expressions with the array creation operator and the tuple creation operator, respectively. An array must consist of at least one item, and a tuple must have at least two items. -Tuples have a special purpose for use in the `IN` clause of a `SELECT` query. Tuples can be obtained as the result of a query, but they can’t be saved to a database (with the exception of [Memory](../engines/table_engines/special/memory.md) tables). +There's a separate case when tuples appear in the `IN` clause of a `SELECT` query. Query results can include tuples, but tuples can’t be saved to a database (except of tables with [Memory](../engines/table_engines/special/memory.md) engine). ### NULL {#null-literal} @@ -99,13 +99,13 @@ In order to store `NULL` in a table field, it must be of the [Nullable](../sql_r Depending on the data format (input or output), `NULL` may have a different representation. For more information, see the documentation for [data formats](../interfaces/formats.md#formats). -There are many nuances to processing `NULL`. For example, if at least one of the arguments of a comparison operation is `NULL`, the result of this operation will also be `NULL`. The same is true for multiplication, addition, and other operations. For more information, read the documentation for each operation. +There are many nuances to processing `NULL`. For example, if at least one of the arguments of a comparison operation is `NULL`, the result of this operation is also `NULL`. The same is true for multiplication, addition, and other operations. For more information, read the documentation for each operation. In queries, you can check `NULL` using the [IS NULL](operators.md#operator-is-null) and [IS NOT NULL](operators.md) operators and the related functions `isNull` and `isNotNull`. ## Functions {#functions} -Functions are written like an identifier with a list of arguments (possibly empty) in brackets. In contrast to standard SQL, the brackets are required, even for an empty arguments list. Example: `now()`. +Function calls are written like an identifier with a list of arguments (possibly empty) in round brackets. In contrast to standard SQL, the brackets are required, even for an empty argument list. Example: `now()`. There are regular and aggregate functions (see the section “Aggregate functions”). Some aggregate functions can contain two lists of arguments in brackets. Example: `quantile (0.9) (x)`. These aggregate functions are called “parametric” functions, and the arguments in the first list are called “parameters”. The syntax of aggregate functions without parameters is the same as for regular functions. ## Operators {#operators} @@ -115,11 +115,11 @@ For example, the expression `1 + 2 * 3 + 4` is transformed to `plus(plus(1, mult ## Data Types and Database Table Engines {#data_types-and-database-table-engines} -Data types and table engines in the `CREATE` query are written the same way as identifiers or functions. In other words, they may or may not contain an arguments list in brackets. For more information, see the sections “Data types,” “Table engines,” and “CREATE”. +Data types and table engines in the `CREATE` query are written the same way as identifiers or functions. In other words, they may or may not contain an argument list in brackets. For more information, see the sections “Data types,” “Table engines,” and “CREATE”. ## Expression Aliases {#syntax-expression_aliases} -An alias is a user-defined name for an expression in a query. +An alias is a user-defined name for expression in a query. ``` sql expr AS alias @@ -141,7 +141,7 @@ expr AS alias ### Notes on Usage {#notes-on-usage} -Aliases are global for a query or subquery and you can define an alias in any part of a query for any expression. For example, `SELECT (1 AS n) + 2, n`. +Aliases are global for a query or subquery, and you can define an alias in any part of a query for any expression. For example, `SELECT (1 AS n) + 2, n`. Aliases are not visible in subqueries and between subqueries. For example, while executing the query `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ClickHouse generates the exception `Unknown identifier: num`. @@ -182,4 +182,4 @@ An expression is a function, identifier, literal, application of an operator, ex A list of expressions is one or more expressions separated by commas. Functions and operators, in turn, can have expressions as arguments. -[Original article](https://clickhouse.tech/docs/en/query_language/syntax/) +[Original article](https://clickhouse.tech/docs/en/sql_reference/syntax/) diff --git a/docs/en/whats_new/changelog/2018.md b/docs/en/whats_new/changelog/2018.md index 108332ce56b..346c8d6b639 100644 --- a/docs/en/whats_new/changelog/2018.md +++ b/docs/en/whats_new/changelog/2018.md @@ -1058,4 +1058,4 @@ This release contains bug fixes for the previous release 1.1.54337: - When doing a rolling update on a cluster, at the point when some of the replicas are running the old version of ClickHouse and some are running the new version, replication is temporarily stopped and the message `unknown parameter 'shard'` appears in the log. Replication will continue after all replicas of the cluster are updated. - If different versions of ClickHouse are running on the cluster servers, it is possible that distributed queries using the following functions will have incorrect results: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. You should update all cluster nodes. -## [Changelog for 2017](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2017.md) +## [Changelog for 2017](./2017.md) diff --git a/docs/en/whats_new/changelog/2019.md b/docs/en/whats_new/changelog/2019.md index 5b0405f0b0a..b510c68ba87 100644 --- a/docs/en/whats_new/changelog/2019.md +++ b/docs/en/whats_new/changelog/2019.md @@ -2069,4 +2069,4 @@ This release contains exactly the same set of patches as 19.3.6. - Fixed misspells in comments and string literals under `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([maiha](https://github.com/maiha)) - Fixed typos in comments. [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty)) -## [Changelog for 2018](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2018.md) +## [Changelog for 2018](./2018.md) diff --git a/docs/en/whats_new/changelog/index.md b/docs/en/whats_new/changelog/index.md deleted file mode 100644 index bcfe62cbd0b..00000000000 --- a/docs/en/whats_new/changelog/index.md +++ /dev/null @@ -1,650 +0,0 @@ ---- -toc_folder_title: Changelog -toc_priority: 74 -toc_title: '2020' ---- - -## ClickHouse release v20.3 - -### ClickHouse release v20.3.4.10, 2020-03-20 - -#### Bug Fix -* This release also contains all bug fixes from 20.1.8.41 -* Fix missing `rows_before_limit_at_least` for queries over http (with processors pipeline). This fixes [#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - - -### ClickHouse release v20.3.3.6, 2020-03-17 - -#### Bug Fix -* This release also contains all bug fixes from 20.1.7.38 -* Fix bug in a replication that doesn't allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. -* Add setting `use_compact_format_in_distributed_parts_names` which allows to write files for `INSERT` queries into `Distributed` table with more compact format. This fixes [#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. - -### ClickHouse release v20.3.2.1, 2020-03-12 - -#### Backward Incompatible Change - -* Fixed the issue `file name too long` when sending data for `Distributed` tables for a large number of replicas. Fixed the issue that replica credentials were exposed in the server log. The format of directory name on disk was changed to `[shard{shard_index}[_replica{replica_index}]]`. [#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([Mikhail Korotov](https://github.com/millb)) After you upgrade to the new version, you will not be able to downgrade without manual intervention, because old server version does not recognize the new directory format. If you want to downgrade, you have to manually rename the corresponding directories to the old format. This change is relevant only if you have used asynchronous `INSERT`s to `Distributed` tables. In the version 20.3.3 we will introduce a setting that will allow you to enable the new format gradually. -* Changed the format of replication log entries for mutation commands. You have to wait for old mutations to process before installing the new version. -* Implement simple memory profiler that dumps stacktraces to `system.trace_log` every N bytes over soft allocation limit [#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([Ivan](https://github.com/abyss7)) [#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([alexey-milovidov](https://github.com/alexey-milovidov)) The column of `system.trace_log` was renamed from `timer_type` to `trace_type`. This will require changes in third-party performance analysis and flamegraph processing tools. -* Use OS thread id everywhere instead of internal thread number. This fixes [#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) Old `clickhouse-client` cannot receive logs that are send from the server when the setting `send_logs_level` is enabled, because the names and types of the structured log messages were changed. On the other hand, different server versions can send logs with different types to each other. When you don't use the `send_logs_level` setting, you should not care. [#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Remove `indexHint` function [#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Remove `findClusterIndex`, `findClusterValue` functions. This fixes [#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). If you were using these functions, send an email to `clickhouse-feedback@yandex-team.com` [#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Now it's not allowed to create columns or add columns with `SELECT` subquery as default expression. [#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([alesapin](https://github.com/alesapin)) -* Require aliases for subqueries in JOIN. [#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) -* Improved `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn't change type of column and `MODIFY` type doesn't loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alesapin](https://github.com/alesapin)) -* Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* The setting `experimental_use_processors` is enabled by default. This setting enables usage of the new query pipeline. This is internal refactoring and we expect no visible changes. If you will see any issues, set it to back zero. [#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### New Feature -* Add `Avro` and `AvroConfluent` input/output formats [#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([Andrew Onyshchuk](https://github.com/oandrew)) [#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([Andrew Onyshchuk](https://github.com/oandrew)) [#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Multi-threaded and non-blocking updates of expired keys in `cache` dictionaries (with optional permission to read old ones). [#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Add query `ALTER ... MATERIALIZE TTL`. It runs mutation that forces to remove expired data by TTL and recalculates meta-information about TTL in all parts. [#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([Anton Popov](https://github.com/CurtizJ)) -* Switch from HashJoin to MergeJoin (on disk) if needed [#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([Artem Zuikov](https://github.com/4ertus2)) -* Added `MOVE PARTITION` command for `ALTER TABLE` [#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Reloading storage configuration from configuration file on the fly. [#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Allowed to change `storage_policy` to not less rich one. [#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Added support for globs/wildcards for S3 storage and table function. [#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Implement `bitAnd`, `bitOr`, `bitXor`, `bitNot` for `FixedString(N)` datatype. [#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Added function `bitCount`. This fixes [#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([alexey-milovidov](https://github.com/alexey-milovidov)) [#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([ikopylov](https://github.com/ikopylov)) -* Add `generateRandom` table function to generate random rows with given schema. Allows to populate arbitrary test table with data. [#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([Ilya Yatsishin](https://github.com/qoega)) -* `JSONEachRowFormat`: support special case when objects enclosed in top-level array. [#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([Kruglov Pavel](https://github.com/Avogar)) -* Now it's possible to create a column with `DEFAULT` expression which depends on a column with default `ALIAS` expression. [#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([alesapin](https://github.com/alesapin)) -* Allow to specify `--limit` more than the source data size in `clickhouse-obfuscator`. The data will repeat itself with different random seed. [#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added `groupArraySample` function (similar to `groupArray`) with reservior sampling algorithm. [#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([Amos Bird](https://github.com/amosbird)) -* Now you can monitor the size of update queue in `cache`/`complex_key_cache` dictionaries via system metrics. [#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Allow to use CRLF as a line separator in CSV output format with setting `output_format_csv_crlf_end_of_line` is set to 1 [#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([Mikhail Korotov](https://github.com/millb)) -* Implement more functions of the [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` and `stringToH3` [#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([Nico Mandery](https://github.com/nmandery)) -* New setting introduced: `max_parser_depth` to control maximum stack size and allow large complex queries. This fixes [#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) and [#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([Maxim Smirnov](https://github.com/qMBQx8GH)) -* Add a setting `force_optimize_skip_unused_shards` setting to throw if skipping of unused shards is not possible [#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([Azat Khuzhin](https://github.com/azat)) -* Allow to configure multiple disks/volumes for storing data for send in `Distributed` engine [#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([Azat Khuzhin](https://github.com/azat)) -* Support storage policy (``) for storing temporary data. [#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([Azat Khuzhin](https://github.com/azat)) -* Added `X-ClickHouse-Exception-Code` HTTP header that is set if exception was thrown before sending data. This implements [#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([Mikhail Korotov](https://github.com/millb)) -* Added function `ifNotFinite`. It is just a syntactic sugar: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added `last_successful_update_time` column in `system.dictionaries` table [#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Add `blockSerializedSize` function (size on disk without compression) [#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([Azat Khuzhin](https://github.com/azat)) -* Add function `moduloOrZero` [#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([hcz](https://github.com/hczhcz)) -* Added system tables `system.zeros` and `system.zeros_mt` as well as tale functions `zeros()` and `zeros_mt()`. Tables (and table functions) contain single column with name `zero` and type `UInt8`. This column contains zeros. It is needed for test purposes as the fastest method to generate many rows. This fixes [#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Experimental Feature -* Add new compact format of parts in `MergeTree`-family tables in which all columns are stored in one file. It helps to increase performance of small and frequent inserts. The old format (one file per column) is now called wide. Data storing format is controlled by settings `min_bytes_for_wide_part` and `min_rows_for_wide_part`. [#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([Anton Popov](https://github.com/CurtizJ)) -* Support for S3 storage for `Log`, `TinyLog` and `StripeLog` tables. [#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([Pavel Kovalenko](https://github.com/Jokser)) - -#### Bug Fix -* Fixed inconsistent whitespaces in log messages. [#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix bug in which arrays of unnamed tuples were flattened as Nested structures on table creation. [#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([achulkov2](https://github.com/achulkov2)) -* Fixed the issue when "Too many open files" error may happen if there are too many files matching glob pattern in `File` table or `file` table function. Now files are opened lazily. This fixes [#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* DROP TEMPORARY TABLE now drops only temporary table. [#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([Vitaly Baranov](https://github.com/vitlibar)) -* Remove outdated partition when we shutdown the server or DETACH/ATTACH a table. [#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([Guillaume Tassery](https://github.com/YiuRULE)) -* For how the default disk calculates the free space from `data` subdirectory. Fixed the issue when the amount of free space is not calculated correctly if the `data` directory is mounted to a separate device (rare case). This fixes [#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([Mikhail Korotov](https://github.com/millb)) -* Allow comma (cross) join with IN () inside. [#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([Artem Zuikov](https://github.com/4ertus2)) -* Allow to rewrite CROSS to INNER JOIN if there's [NOT] LIKE operator in WHERE section. [#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix possible incorrect result after `GROUP BY` with enabled setting `distributed_aggregation_memory_efficient`. Fixes [#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Found keys were counted as missed in metrics of cache dictionaries. [#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Fix replication protocol incompatibility introduced in [#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) -* Fixed race condition on `queue_task_handle` at the startup of `ReplicatedMergeTree` tables. [#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* The token `NOT` didn't work in `SHOW TABLES NOT LIKE` query [#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added range check to function `h3EdgeLengthM`. Without this check, buffer overflow is possible. [#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed up a bug in batched calculations of ternary logical OPs on multiple arguments (more than 10). [#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) -* Fix error of PREWHERE optimization, which could lead to segfaults or `Inconsistent number of columns got from MergeTreeRangeReader` exception. [#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([Anton Popov](https://github.com/CurtizJ)) -* Fix unexpected `Timeout exceeded while reading from socket` exception, which randomly happens on secure connection before timeout actually exceeded and when query profiler is enabled. Also add `connect_timeout_with_failover_secure_ms` settings (default 100ms), which is similar to `connect_timeout_with_failover_ms`, but is used for secure connections (because SSL handshake is slower, than ordinary TCP connection) [#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([tavplubix](https://github.com/tavplubix)) -* Fix bug with mutations finalization, when mutation may hang in state with `parts_to_do=0` and `is_done=0`. [#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([alesapin](https://github.com/alesapin)) -* Use new ANY JOIN logic with `partial_merge_join` setting. It's possible to make `ANY|ALL|SEMI LEFT` and `ALL INNER` joins with `partial_merge_join=1` now. [#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([Artem Zuikov](https://github.com/4ertus2)) -* Shard now clamps the settings got from the initiator to the shard's constaints instead of throwing an exception. This fix allows to send queries to a shard with another constraints. [#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([Vitaly Baranov](https://github.com/vitlibar)) -* Fixed memory management problem in `MergeTreeReadPool`. [#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix `toDecimal*OrNull()` functions family when called with string `e`. Fixes [#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([Artem Zuikov](https://github.com/4ertus2)) -* Make sure that `FORMAT Null` sends no data to the client. [#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Fix bug that timestamp in `LiveViewBlockInputStream` will not updated. `LIVE VIEW` is an experimental feature. [#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([vxider](https://github.com/Vxider)) [#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([vxider](https://github.com/Vxider)) -* Fixed `ALTER MODIFY TTL` wrong behavior which did not allow to delete old TTL expressions. [#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fixed UBSan report in MergeTreeIndexSet. This fixes [#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed the behaviour of `match` and `extract` functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. This fixes [#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Avoid throwing from destructor in Apache Avro 3rd-party library. [#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([Andrew Onyshchuk](https://github.com/oandrew)) -* Don't commit a batch polled from `Kafka` partially as it can lead to holes in data. [#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([filimonov](https://github.com/filimonov)) -* Fix `joinGet` with nullable return types. https://github.com/ClickHouse/ClickHouse/issues/8919 [#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([Amos Bird](https://github.com/amosbird)) -* Fix data incompatibility when compressed with `T64` codec. [#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([Artem Zuikov](https://github.com/4ertus2)) Fix data type ids in `T64` compression codec that leads to wrong (de)compression in affected versions. [#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([Artem Zuikov](https://github.com/4ertus2)) -* Add setting `enable_early_constant_folding` and disable it in some cases that leads to errors. [#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix pushdown predicate optimizer with VIEW and enable the test [#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([Winter Zhang](https://github.com/zhang2014)) -* Fix segfault in `Merge` tables, that can happen when reading from `File` storages [#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([tavplubix](https://github.com/tavplubix)) -* Added a check for storage policy in `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. Otherwise it could make data of part inaccessible after restart and prevent ClickHouse to start. [#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix alters if there is TTL set for table. [#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([Anton Popov](https://github.com/CurtizJ)) -* Fix race condition that can happen when `SYSTEM RELOAD ALL DICTIONARIES` is executed while some dictionary is being modified/added/removed. [#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([Vitaly Baranov](https://github.com/vitlibar)) -* In previous versions `Memory` database engine use empty data path, so tables are created in `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([tavplubix](https://github.com/tavplubix)) -* Fixed wrong log messages about missing default disk or policy. [#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix not(has()) for the bloom_filter index of array types. [#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -* Allow first column(s) in a table with `Log` engine be an alias [#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([Ivan](https://github.com/abyss7)) -* Fix order of ranges while reading from `MergeTree` table in one thread. It could lead to exceptions from `MergeTreeRangeReader` or wrong query results. [#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([Anton Popov](https://github.com/CurtizJ)) -* Make `reinterpretAsFixedString` to return `FixedString` instead of `String`. [#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([Andrew Onyshchuk](https://github.com/oandrew)) -* Avoid extremely rare cases when the user can get wrong error message (`Success` instead of detailed error description). [#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Do not crash when using `Template` format with empty row template. [#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Metadata files for system tables could be created in wrong place [#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([tavplubix](https://github.com/tavplubix)) Fixes [#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). -* Fix data race on exception_ptr in cache dictionary [#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Do not throw an exception for query `ATTACH TABLE IF NOT EXISTS`. Previously it was thrown if table already exists, despite the `IF NOT EXISTS` clause. [#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([Anton Popov](https://github.com/CurtizJ)) -* Fixed missing closing paren in exception message. [#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Avoid message `Possible deadlock avoided` at the startup of clickhouse-client in interactive mode. [#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed the issue when padding at the end of base64 encoded value can be malformed. Update base64 library. This fixes [#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), closes [#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Prevent losing data in `Kafka` in rare cases when exception happens after reading suffix but before commit. Fixes [#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([filimonov](https://github.com/filimonov)) -* Fixed exception in `DROP TABLE IF EXISTS` [#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([Nikita Vasilev](https://github.com/nikvas0)) -* Fix crash when a user tries to `ALTER MODIFY SETTING` for old-formated `MergeTree` table engines family. [#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) -* Support for UInt64 numbers that don't fit in Int64 in JSON-related functions. Update SIMDJSON to master. This fixes [#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed execution of inversed predicates when non-strictly monotinic functional index is used. [#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([Alexander Kazakov](https://github.com/Akazz)) -* Don't try to fold `IN` constant in `GROUP BY` [#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([Amos Bird](https://github.com/amosbird)) -* Fix bug in `ALTER DELETE` mutations which leads to index corruption. This fixes [#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) and [#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). Additionally fix extremely rare race conditions in `ReplicatedMergeTree` `ALTER` queries. [#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([alesapin](https://github.com/alesapin)) -* When the setting `compile_expressions` is enabled, you can get `unexpected column` in `LLVMExecutableFunction` when we use `Nullable` type [#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Multiple fixes for `Kafka` engine: 1) fix duplicates that were appearing during consumer group rebalance. 2) Fix rare 'holes' appeared when data were polled from several partitions with one poll and committed partially (now we always process / commit the whole polled block of messages). 3) Fix flushes by block size (before that only flushing by timeout was working properly). 4) better subscription procedure (with assignment feedback). 5) Make tests work faster (with default intervals and timeouts). Due to the fact that data was not flushed by block size before (as it should according to documentation), that PR may lead to some performance degradation with default settings (due to more often & tinier flushes which are less optimal). If you encounter the performance issue after that change - please increase `kafka_max_block_size` in the table to the bigger value ( for example `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). Fixes [#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([filimonov](https://github.com/filimonov)) -* Fix `Parameter out of bound` exception in some queries after PREWHERE optimizations. [#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) -* Fixed the case of mixed-constness of arguments of function `arrayZip`. [#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* When executing `CREATE` query, fold constant expressions in storage engine arguments. Replace empty database name with current database. Fixes [#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([tavplubix](https://github.com/tavplubix)) -* Now it's not possible to create or add columns with simple cyclic aliases like `a DEFAULT b, b DEFAULT a`. [#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) -* Fixed a bug with double move which may corrupt original part. This is relevant if you use `ALTER TABLE MOVE` [#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Allow `interval` identifier to correctly parse without backticks. Fixed issue when a query cannot be executed even if the `interval` identifier is enclosed in backticks or double quotes. This fixes [#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed fuzz test and incorrect behaviour of `bitTestAll`/`bitTestAny` functions. [#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix possible crash/wrong number of rows in `LIMIT n WITH TIES` when there are a lot of rows equal to n'th row. [#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -* Fix mutations with parts written with enabled `insert_quorum`. [#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([alesapin](https://github.com/alesapin)) -* Fix data race at destruction of `Poco::HTTPServer`. It could happen when server is started and immediately shut down. [#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) -* Fix bug in which a misleading error message was shown when running `SHOW CREATE TABLE a_table_that_does_not_exist`. [#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2](https://github.com/achulkov2)) -* Fixed `Parameters are out of bound` exception in some rare cases when we have a constant in the `SELECT` clause when we have an `ORDER BY` and a `LIMIT` clause. [#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Fix mutations finalization, when already done mutation can have status `is_done=0`. [#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([alesapin](https://github.com/alesapin)) -* Prevent from executing `ALTER ADD INDEX` for MergeTree tables with old syntax, because it doesn't work. [#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) -* During server startup do not access table, which `LIVE VIEW` depends on, so server will be able to start. Also remove `LIVE VIEW` dependencies when detaching `LIVE VIEW`. `LIVE VIEW` is an experimental feature. [#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) -* Fix possible segfault in `MergeTreeRangeReader`, while executing `PREWHERE`. [#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([Anton Popov](https://github.com/CurtizJ)) -* Fix possible mismatched checksums with column TTLs. [#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) -* Fixed a bug when parts were not being moved in background by TTL rules in case when there is only one volume. [#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fixed the issue `Method createColumn() is not implemented for data type Set`. This fixes [#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Now we will try finalize mutations more frequently. [#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) -* Fix `intDiv` by minus one constant [#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([hcz](https://github.com/hczhcz)) -* Fix possible race condition in `BlockIO`. [#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix bug leading to server termination when trying to use / drop `Kafka` table created with wrong parameters. [#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([filimonov](https://github.com/filimonov)) -* Added workaround if OS returns wrong result for `timer_create` function. [#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed error in usage of `min_marks_for_seek` parameter. Fixed the error message when there is no sharding key in Distributed table and we try to skip unused shards. [#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([Azat Khuzhin](https://github.com/azat)) - -#### Improvement -* Implement `ALTER MODIFY/DROP` queries on top of mutations for `ReplicatedMergeTree*` engines family. Now `ALTERS` blocks only at the metadata update stage, and don't block after that. [#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([alesapin](https://github.com/alesapin)) -* Add ability to rewrite CROSS to INNER JOINs with `WHERE` section containing unqialified names. [#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([Artem Zuikov](https://github.com/4ertus2)) -* Make `SHOW TABLES` and `SHOW DATABASES` queries support the `WHERE` expressions and `FROM`/`IN` [#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([sundyli](https://github.com/sundy-li)) -* Added a setting `deduplicate_blocks_in_dependent_materialized_views`. [#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([urykhy](https://github.com/urykhy)) -* After recent changes MySQL client started to print binary strings in hex thereby making them not readable ([#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). The workaround in ClickHouse is to mark string columns as UTF-8, which is not always, but usually the case. [#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([Yuriy Baranov](https://github.com/yurriy)) -* Add support of String and FixedString keys for `sumMap` [#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) -* Support string keys in SummingMergeTree maps [#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) -* Signal termination of thread to the thread pool even if the thread has thrown exception [#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -* Allow to set `query_id` in `clickhouse-benchmark` [#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([Anton Popov](https://github.com/CurtizJ)) -* Don't allow strange expressions in `ALTER TABLE ... PARTITION partition` query. This addresses [#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* The table `system.table_engines` now provides information about feature support (like `supports_ttl` or `supports_sort_order`). [#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([Max Akhmedov](https://github.com/zlobober)) -* Enable `system.metric_log` by default. It will contain rows with values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval (one second by default). The table is very small (usually in order of megabytes) and collecting this data by default is reasonable. [#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([Ivan](https://github.com/abyss7)) -* Now temporary `LIVE VIEW` is created by `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` instead of `CREATE TEMPORARY LIVE VIEW ...`, because the previous syntax was not consistent with `CREATE TEMPORARY TABLE ...` [#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([tavplubix](https://github.com/tavplubix)) -* Add text_log.level configuration parameter to limit entries that goes to `system.text_log` table [#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([Azat Khuzhin](https://github.com/azat)) -* Allow to put downloaded part to a disks/volumes according to TTL rules [#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([Vladimir Chebotarev](https://github.com/excitoon)) -* For external MySQL dictionaries, allow to mutualize MySQL connection pool to "share" them among dictionaries. This option significantly reduces the number of connections to MySQL servers. [#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) -* Show nearest query execution time for quantiles in `clickhouse-benchmark` output instead of interpolated values. It's better to show values that correspond to the execution time of some queries. [#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Possibility to add key & timestamp for the message when inserting data to Kafka. Fixes [#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([filimonov](https://github.com/filimonov)) -* If server is run from terminal, highlight thread number, query id and log priority by colors. This is for improved readability of correlated log messages for developers. [#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Better exception message while loading tables for `Ordinary` database. [#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Implement `arraySlice` for arrays with aggregate function states. This fixes [#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Allow constant functions and constant arrays to be used on the right side of IN operator. [#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([Anton Popov](https://github.com/CurtizJ)) -* If zookeeper exception has happened while fetching data for system.replicas, display it in a separate column. This implements [#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Atomically remove MergeTree data parts on destroy. [#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Support row-level security for Distributed tables. [#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([Ivan](https://github.com/abyss7)) -* Now we recognize suffix (like KB, KiB...) in settings values. [#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([Mikhail Korotov](https://github.com/millb)) -* Prevent out of memory while constructing result of a large JOIN. [#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([Artem Zuikov](https://github.com/4ertus2)) -* Added names of clusters to suggestions in interactive mode in `clickhouse-client`. [#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([Ivan](https://github.com/abyss7)) -* Added column `exception_code` in `system.query_log` table. [#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([Mikhail Korotov](https://github.com/millb)) -* Enabled MySQL compatibility server on port `9004` in the default server configuration file. Fixed password generation command in the example in configuration. [#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([Yuriy Baranov](https://github.com/yurriy)) -* Prevent abort on shutdown if the filesystem is readonly. This fixes [#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Better exception message when length is required in HTTP POST query. [#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add `_path` and `_file` virtual columns to `HDFS` and `File` engines and `hdfs` and `file` table functions [#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([Olga Khvostikova](https://github.com/stavrolia)) -* Fix error `Cannot find column` while inserting into `MATERIALIZED VIEW` in case if new column was added to view's internal table. [#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix progress over native client-server protocol, by send progress after final update (like logs). This may be relevant only to some third-party tools that are using native protocol. [#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([Azat Khuzhin](https://github.com/azat)) -* Add a system metric tracking the number of client connections using MySQL protocol ([#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([Eugene Klimov](https://github.com/Slach)) -* From now on, HTTP responses will have `X-ClickHouse-Timezone` header set to the same timezone value that `SELECT timezone()` would report. [#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([Denis Glazachev](https://github.com/traceon)) - -#### Performance Improvement -* Improve performance of analysing index with IN [#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([Anton Popov](https://github.com/CurtizJ)) -* Simpler and more efficient code in Logical Functions + code cleanups. A followup to [#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([Alexander Kazakov](https://github.com/Akazz)) -* Overall performance improvement (in range of 5%..200% for affected queries) by ensuring even more strict aliasing with C++20 features. [#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([Amos Bird](https://github.com/amosbird)) -* More strict aliasing for inner loops of comparison functions. [#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* More strict aliasing for inner loops of arithmetic functions. [#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* A ~3 times faster implementation for ColumnVector::replicate(), via which ColumnConst::convertToFullColumn() is implemented. Also will be useful in tests when materializing constants. [#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([Alexander Kazakov](https://github.com/Akazz)) -* Another minor performance improvement to `ColumnVector::replicate()` (this speeds up the `materialize` function and higher order functions) an even further improvement to [#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([Alexander Kazakov](https://github.com/Akazz)) -* Improved performance of `stochasticLinearRegression` aggregate function. This patch is contributed by Intel. [#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Improve performance of `reinterpretAsFixedString` function. [#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Do not send blocks to client for `Null` format in processors pipeline. [#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### Build/Testing/Packaging Improvement -* Exception handling now works correctly on Windows Subsystem for Linux. See https://github.com/ClickHouse-Extras/libunwind/pull/3 This fixes [#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) -* Replace `readline` with `replxx` for interactive line editing in `clickhouse-client` [#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([Ivan](https://github.com/abyss7)) -* Better build time and less template instantiations in FunctionsComparison. [#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added integration with `clang-tidy` in CI. See also [#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Now we link ClickHouse in CI using `lld` even for `gcc`. [#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([alesapin](https://github.com/alesapin)) -* Allow to randomize thread scheduling and insert glitches when `THREAD_FUZZER_*` environment variables are set. This helps testing. [#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Enable secure sockets in stateless tests [#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([tavplubix](https://github.com/tavplubix)) -* Make SPLIT_SHARED_LIBRARIES=OFF more robust [#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([Azat Khuzhin](https://github.com/azat)) -* Make "performance_introspection_and_logging" test reliable to random server stuck. This may happen in CI environment. See also [#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Validate XML in style check. [#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed race condition in test `00738_lock_for_inner_table`. This test relied on sleep. [#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Remove performance tests of type `once`. This is needed to run all performance tests in statistical comparison mode (more reliable). [#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added performance test for arithmetic functions. [#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added performance test for `sumMap` and `sumMapWithOverflow` aggregate functions. Follow-up for [#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Ensure style of ErrorCodes by style check. [#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add script for tests history. [#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([alesapin](https://github.com/alesapin)) -* Add GCC warning `-Wsuggest-override` to locate and fix all places where `override` keyword must be used. [#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -* Ignore weak symbol under Mac OS X because it must be defined [#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([Deleted user](https://github.com/ghost)) -* Normalize running time of some queries in performance tests. This is done in preparation to run all the performance tests in comparison mode. [#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix some tests to support pytest with query tests [#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([Ivan](https://github.com/abyss7)) -* Enable SSL in build with MSan, so server will not fail at startup when running stateless tests [#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([tavplubix](https://github.com/tavplubix)) -* Fix database substitution in test results [#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([Ilya Yatsishin](https://github.com/qoega)) -* Build fixes for miscellaneous platforms [#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([proller](https://github.com/proller)) [#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([proller](https://github.com/proller)) [#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([proller](https://github.com/proller)) -* Added disks section to stateless-with-coverage test docker image [#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([Pavel Kovalenko](https://github.com/Jokser)) -* Get rid of in-source-tree files when building with GRPC [#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([Amos Bird](https://github.com/amosbird)) -* Slightly faster build time by removing SessionCleaner from Context. Make the code of SessionCleaner more simple. [#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Updated checking for hung queries in clickhouse-test script [#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) -* Removed some useless files from repository. [#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Changed type of math perftests from `once` to `loop`. [#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html) -* Suppress some test failures under MSan. [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Speedup "exception while insert" test. This test often time out in debug-with-coverage build. [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Updated `libcxx` and `libcxxabi` to master. In preparation to [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix flacky test `00910_zookeeper_test_alter_compression_codecs`. [#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Clean up duplicated linker flags. Make sure the linker won't look up an unexpected symbol. [#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([Amos Bird](https://github.com/amosbird)) -* Add `clickhouse-odbc` driver into test images. This allows to test interaction of ClickHouse with ClickHouse via its own ODBC driver. [#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov)) -* Fix several bugs in unit tests. [#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([alesapin](https://github.com/alesapin)) -* Enable `-Wmissing-include-dirs` GCC warning to eliminate all non-existing includes - mostly as a result of CMake scripting errors [#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -* Describe reasons if query profiler cannot work. This is intended for [#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Update OpenSSL to upstream master. Fixed the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` and `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. The issue was present in version 20.1. [#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Update Dockerfile for server [#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([Ilya Mazaev](https://github.com/ne-ray)) -* Minor fixes in build-gcc-from-sources script [#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([Michael Nacharov](https://github.com/mnach)) -* Replace `numbers` to `zeros` in perftests where `number` column is not used. This will lead to more clean test results. [#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix stack overflow issue when using initializer_list in Column constructors. [#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([Deleted user](https://github.com/ghost)) -* Upgrade librdkafka to v1.3.0. Enable bundled `rdkafka` and `gsasl` libraries on Mac OS X. [#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([Andrew Onyshchuk](https://github.com/oandrew)) -* build fix on GCC 9.2.0 [#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([vxider](https://github.com/Vxider)) - - -## ClickHouse release v20.1 - -### ClickHouse release v20.1.8.41, 2020-03-20 - -#### Bug Fix -* Fix possible permanent `Cannot schedule a task` error (due to unhandled exception in `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). This fixes [#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) -* Fix excessive memory consumption in `ALTER` queries (mutations). This fixes [#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) and [#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alesapin](https://github.com/alesapin)) -* Fix bug in backquoting in external dictionaries DDL. This fixes [#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) - -### ClickHouse release v20.1.7.38, 2020-03-18 - -#### Bug Fix -* Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. I lead to exception while using this functions in remote queries. [#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). This issue was in all ClickHouse releases. -* Allow `ALTER ON CLUSTER` of `Distributed` tables with internal replication. This fixes [#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)). This issue was in all ClickHouse releases. -* Fix possible exceptions `Size of filter doesn't match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. Fixes [#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) -* Fixed the issue: timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)). This issue was in all ClickHouse releases. -* Now it's not possible to create or add columns with simple cyclic aliases like `a DEFAULT b, b DEFAULT a`. [#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) -* Fixed the issue when padding at the end of base64 encoded value can be malformed. Update base64 library. This fixes [#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), closes [#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix data race at destruction of `Poco::HTTPServer`. It could happen when server is started and immediately shut down. [#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) -* Fix possible crash/wrong number of rows in `LIMIT n WITH TIES` when there are a lot of rows equal to n'th row. [#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) -* Fix possible mismatched checksums with column TTLs. [#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) -* Fix crash when a user tries to `ALTER MODIFY SETTING` for old-formated `MergeTree` table engines family. [#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) -* Now we will try finalize mutations more frequently. [#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) -* Fix replication protocol incompatibility introduced in [#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) -* Fix not(has()) for the bloom_filter index of array types. [#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) -* Fixed the behaviour of `match` and `extract` functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. This fixes [#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Build/Testing/Packaging Improvement - -* Exception handling now works correctly on Windows Subsystem for Linux. See https://github.com/ClickHouse-Extras/libunwind/pull/3 This fixes [#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) - - -### ClickHouse release v20.1.6.30, 2020-03-05 - -#### Bug Fix - -* Fix data incompatibility when compressed with `T64` codec. -[#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [(abyss7)](https://github.com/abyss7) -* Fix order of ranges while reading from MergeTree table in one thread. Fixes [#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). -[#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(CurtizJ)](https://github.com/CurtizJ) -* Fix possible segfault in `MergeTreeRangeReader`, while executing `PREWHERE`. Fixes [#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). -[#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(CurtizJ)](https://github.com/CurtizJ) -* Fix `reinterpretAsFixedString` to return `FixedString` instead of `String`. -[#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [(oandrew)](https://github.com/oandrew) -* Fix `joinGet` with nullable return types. Fixes [#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) -[#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [(amosbird)](https://github.com/amosbird) -* Fix fuzz test and incorrect behaviour of bitTestAll/bitTestAny functions. -[#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [(alexey-milovidov)](https://github.com/alexey-milovidov) -* Fix the behaviour of match and extract functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. Fixes [#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) -[#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [(alexey-milovidov)](https://github.com/alexey-milovidov) -* Fixed execution of inversed predicates when non-strictly monotinic functional index is used. Fixes [#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) -[#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [(Akazz)](https://github.com/Akazz) -* Allow to rewrite `CROSS` to `INNER JOIN` if there's `[NOT] LIKE` operator in `WHERE` section. Fixes [#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) -[#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4ertus2)](https://github.com/4ertus2) -* Allow first column(s) in a table with Log engine be an alias. -[#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(abyss7)](https://github.com/abyss7) -* Allow comma join with `IN()` inside. Fixes [#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). -[#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4ertus2)](https://github.com/4ertus2) -* Improve `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn't change type of column and `MODIFY` type doesn't loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). -[#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) -* Fix mutations finalization, when already done mutation can have status is_done=0. -[#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) -* Support "Processors" pipeline for system.numbers and system.numbers_mt. This also fixes the bug when `max_execution_time` is not respected. -[#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [(KochetovNicolai)](https://github.com/KochetovNicolai) -* Fix wrong counting of `DictCacheKeysRequestedFound` metric. -[#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [(nikitamikhaylov)](https://github.com/nikitamikhaylov) -* Added a check for storage policy in `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` which otherwise could make data of part inaccessible after restart and prevent ClickHouse to start. -[#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [(excitoon)](https://github.com/excitoon) -* Fixed UBSan report in `MergeTreeIndexSet`. This fixes [#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) -[#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [(alexey-milovidov)](https://github.com/alexey-milovidov) -* Fix possible datarace in BlockIO. -[#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [(KochetovNicolai)](https://github.com/KochetovNicolai) -* Support for `UInt64` numbers that don't fit in Int64 in JSON-related functions. Update `SIMDJSON` to master. This fixes [#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) -[#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [(alexey-milovidov)](https://github.com/alexey-milovidov) -* Fix the issue when the amount of free space is not calculated correctly if the data directory is mounted to a separate device. For default disk calculate the free space from data subdirectory. This fixes [#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) -[#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [(millb)](https://github.com/millb) -* Fix the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` Update OpenSSL to upstream master. -[#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [(alexey-milovidov)](https://github.com/alexey-milovidov) -* When executing `CREATE` query, fold constant expressions in storage engine arguments. Replace empty database name with current database. Fixes [#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Also fix check for local address in ClickHouseDictionarySource. -[#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) -* Fix segfault in `StorageMerge`, which can happen when reading from StorageFile. -[#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [(tabplubix)](https://github.com/tavplubix) -* Prevent losing data in `Kafka` in rare cases when exception happens after reading suffix but before commit. Fixes [#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). Related: [#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) -[#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) -* Fix bug leading to server termination when trying to use / drop `Kafka` table created with wrong parameters. Fixes [#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Incorporates [#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). -[#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) - -#### New Feature -* Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity. -[#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) - -### ClickHouse release v20.1.2.4, 2020-01-22 - -#### Backward Incompatible Change -* Make the setting `merge_tree_uniform_read_distribution` obsolete. The server still recognizes this setting but it has no effect. [#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Changed return type of the function `greatCircleDistance` to `Float32` because now the result of calculation is `Float32`. [#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Now it's expected that query parameters are represented in "escaped" format. For example, to pass string `ab` you have to write `a\tb` or `a\b` and respectively, `a%5Ctb` or `a%5C%09b` in URL. This is needed to add the possibility to pass NULL as `\N`. This fixes [#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Enable `use_minimalistic_part_header_in_zookeeper` setting for `ReplicatedMergeTree` by default. This will significantly reduce amount of data stored in ZooKeeper. This setting is supported since version 19.1 and we already use it in production in multiple services without any issues for more than half a year. Disable this setting if you have a chance to downgrade to versions older than 19.1. [#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Data skipping indices are production ready and enabled by default. The settings `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` and `allow_experimental_multiple_joins_emulation` are now obsolete and do nothing. [#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add new `ANY JOIN` logic for `StorageJoin` consistent with `JOIN` operation. To upgrade without changes in behaviour you need add `SETTINGS any_join_distinct_right_table_keys = 1` to Engine Join tables metadata or recreate these tables after upgrade. [#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) -* Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -#### New Feature -* Added information about part paths to `system.merges`. [#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Add ability to execute `SYSTEM RELOAD DICTIONARY` query in `ON CLUSTER` mode. [#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Add ability to execute `CREATE DICTIONARY` queries in `ON CLUSTER` mode. [#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) -* Now user's profile in `users.xml` can inherit multiple profiles. [#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -* Added `system.stack_trace` table that allows to look at stack traces of all server threads. This is useful for developers to introspect server state. This fixes [#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add `DateTime64` datatype with configurable sub-second precision. [#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) -* Add table function `clusterAllReplicas` which allows to query all the nodes in the cluster. [#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) -* Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature. [#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) -* Speed up parsing of data files in `CSV`, `TSV` and `JSONEachRow` format by doing it in parallel. [#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Add function `bankerRound` which performs banker's rounding. [#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) -* Support more languages in embedded dictionary for region names: 'ru', 'en', 'ua', 'uk', 'by', 'kz', 'tr', 'de', 'uz', 'lv', 'lt', 'et', 'pt', 'he', 'vi'. [#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Improvements in consistency of `ANY JOIN` logic. Now `t1 ANY LEFT JOIN t2` equals `t2 ANY RIGHT JOIN t1`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -* Add setting `any_join_distinct_right_table_keys` which enables old behaviour for `ANY INNER JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -* Add new `SEMI` and `ANTI JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -* Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by asynchronous inserts into `Distributed` table. [#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Add optional reset column argument for `runningAccumulate` which allows to reset aggregation results for each new key value. [#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) -* Add ability to use ClickHouse as Prometheus endpoint. [#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) -* Add section `` in `config.xml` which restricts allowed hosts for remote table engines and table functions `URL`, `S3`, `HDFS`. [#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) -* Added function `greatCircleAngle` which calculates the distance on a sphere in degrees. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Changed Earth radius to be consistent with H3 library. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added `JSONCompactEachRow` and `JSONCompactEachRowWithNamesAndTypes` formats for input and output. [#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) -* Added feature for file-related table engines and table functions (`File`, `S3`, `URL`, `HDFS`) which allows to read and write `gzip` files based on additional engine parameter or file extension. [#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) -* Added the `randomASCII(length)` function, generating a string with a random set of [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) printable characters. [#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([BayoNet](https://github.com/BayoNet)) -* Added function `JSONExtractArrayRaw` which returns an array on unparsed json array elements from `JSON` string. [#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) -* Add `arrayZip` function which allows to combine multiple arrays of equal lengths into one array of tuples. [#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([Winter Zhang](https://github.com/zhang2014)) -* Add ability to move data between disks according to configured `TTL`-expressions for `*MergeTree` table engines family. [#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Added new aggregate function `avgWeighted` which allows to calculate weighted average. [#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) -* Now parallel parsing is enabled by default for `TSV`, `TSKV`, `CSV` and `JSONEachRow` formats. [#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Add several geo functions from `H3` library: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` and `h3kRing`. [#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) -* Added support for brotli (`br`) compression in file-related storages and table functions. This fixes [#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add `groupBit*` functions for the `SimpleAggregationFunction` type. [#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) - -#### Bug Fix -* Fix rename of tables with `Distributed` engine. Fixes issue [#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) -* Now dictionaries support `EXPRESSION` for attributes in arbitrary string in non-ClickHouse SQL dialect. [#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) -* Fix broken `INSERT SELECT FROM mysql(...)` query. This fixes [#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) and [#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -* Fix error "Mismatch column sizes" when inserting default `Tuple` from `JSONEachRow`. This fixes [#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) -* Now an exception will be thrown in case of using `WITH TIES` alongside `LIMIT BY`. Also add ability to use `TOP` with `LIMIT BY`. This fixes [#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Fix unintendent dependency from fresh glibc version in `clickhouse-odbc-bridge` binary. [#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Bird](https://github.com/amosbird)) -* Fix bug in check function of `*MergeTree` engines family. Now it doesn't fail in case when we have equal amount of rows in last granule and last mark (non-final). [#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) -* Fix insert into `Enum*` columns after `ALTER` query, when underlying numeric type is equal to table specified type. This fixes [#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) -* Allowed non-constant negative "size" argument for function `substring`. It was not allowed by mistake. This fixes [#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix parsing bug when wrong number of arguments passed to `(O|J)DBC` table engine. [#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) -* Using command name of the running clickhouse process when sending logs to syslog. In previous versions, empty string was used instead of command name. [#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) -* Fix check of allowed hosts for `localhost`. This PR fixes the solution provided in [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) -* Fix rare crash in `argMin` and `argMax` functions for long string arguments, when result is used in `runningAccumulate` function. This fixes [#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) -* Fix memory overcommit for tables with `Buffer` engine. [#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) -* Fixed potential bug in functions that can take `NULL` as one of the arguments and return non-NULL. [#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Better metrics calculations in thread pool for background processes for `MergeTree` table engines. [#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix function `IN` inside `WHERE` statement when row-level table filter is present. Fixes [#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) -* Now an exception is thrown if the integral value is not parsed completely for settings values. [#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) -* Fix exception when aggregate function is used in query to distributed table with more than two local shards. [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -* Now bloom filter can handle zero length arrays and doesn't perform redundant calculations. [#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) -* Fixed checking if a client host is allowed by matching the client host to `host_regexp` specified in `users.xml`. [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) -* Relax ambiguous column check that leads to false positives in multiple `JOIN ON` section. [#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) -* Fixed possible server crash (`std::terminate`) when the server cannot send or write data in `JSON` or `XML` format with values of `String` data type (that require `UTF-8` validation) or when compressing result data with Brotli algorithm or in some other rare cases. This fixes [#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix race condition in `StorageDistributedDirectoryMonitor` found by CI. This fixes [#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now background merges in `*MergeTree` table engines family preserve storage policy volume order more accurately. [#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Now table engine `Kafka` works properly with `Native` format. This fixes [#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -* Fixed formats with headers (like `CSVWithNames`) which were throwing exception about EOF for table engine `Kafka`. [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -* Fixed a bug with making set from subquery in right part of `IN` section. This fixes [#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) and [#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Fix possible crash while reading from storage `File`. [#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fixed reading of the files in `Parquet` format containing columns of type `list`. [#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) -* Fix error `Not found column` for distributed queries with `PREWHERE` condition dependent on sampling key if `max_parallel_replicas > 1`. [#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix error `Not found column` if query used `PREWHERE` dependent on table's alias and the result set was empty because of primary key condition. [#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fixed return type for functions `rand` and `randConstant` in case of `Nullable` argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Disabled predicate push-down for `WITH FILL` expression. This fixes [#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Winter Zhang](https://github.com/zhang2014)) -* Fixed incorrect `count()` result for `SummingMergeTree` when `FINAL` section is used. [#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Fix possible incorrect result for constant functions from remote servers. It happened for queries with functions like `version()`, `uptime()`, etc. which returns different constant values for different servers. This fixes [#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix complicated bug in push-down predicate optimization which leads to wrong results. This fixes a lot of issues on push-down predicate optimization. [#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Winter Zhang](https://github.com/zhang2014)) -* Fix crash in `CREATE TABLE .. AS dictionary` query. [#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) -* Several improvements ClickHouse grammar in `.g4` file. [#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) -* Fix bug that leads to crashes in `JOIN`s with tables with engine `Join`. This fixes [#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix redundant dictionaries reload on `CREATE DATABASE`. [#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) -* Limit maximum number of streams for read from `StorageFile` and `StorageHDFS`. Fixes https://github.com/ClickHouse/ClickHouse/issues/7650. [#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) -* Fix bug in `ALTER ... MODIFY ... CODEC` query, when user specify both default expression and codec. Fixes [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) -* Fix error in background merge of columns with `SimpleAggregateFunction(LowCardinality)` type. [#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fixed type check in function `toDateTime64`. [#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) -* Now server do not crash on `LEFT` or `FULL JOIN` with and Join engine and unsupported `join_use_nulls` settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) -* Now `DROP DICTIONARY IF EXISTS db.dict` query doesn't throw exception if `db` doesn't exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) -* Fix possible crashes in table functions (`file`, `mysql`, `remote`) caused by usage of reference to removed `IStorage` object. Fix incorrect parsing of columns specified at insertion into table function. [#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) -* Ensure network be up before starting `clickhouse-server`. This fixes [#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) -* Fix timeouts handling for secure connections, so queries doesn't hang indefenitely. This fixes [#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix `clickhouse-copier`'s redundant contention between concurrent workers. [#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -* Now mutations doesn't skip attached parts, even if their mutation version were larger than current mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) -* Ignore redundant copies of `*MergeTree` data parts after move to another disk and server restart. [#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix crash in `FULL JOIN` with `LowCardinality` in `JOIN` key. [#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) -* Forbidden to use column name more than once in insert query like `INSERT INTO tbl (x, y, x)`. This fixes [#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) -* Added fallback for detection the number of physical CPU cores for unknown CPUs (using the number of logical CPU cores). This fixes [#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix `There's no column` error for materialized and alias columns. [#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) -* Fixed sever crash when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier. Just like `EXISTS t`. This fixes [#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). This bug was introduced in version 19.17. [#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix rare bug with error `"Sizes of columns doesn't match"` that might appear when using `SimpleAggregateFunction` column. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -* Fix bug where user with empty `allow_databases` got access to all databases (and same for `allow_dictionaries`). [#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) -* Fix client crash when server already disconnected from client. [#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -* Fix `ORDER BY` behaviour in case of sorting by primary key prefix and non primary key suffix. [#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) -* Check if qualified column present in the table. This fixes [#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) -* Fixed behavior with `ALTER MOVE` ran immediately after merge finish moves superpart of specified. Fixes [#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix possible server crash while using `UNION` with different number of columns. Fixes [#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix size of result substring for function `substr` with negative size. [#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now server does not execute part mutation in `MergeTree` if there are not enough free threads in background pool. [#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) -* Fix a minor typo on formatting `UNION ALL` AST. [#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) -* Fixed incorrect bloom filter results for negative numbers. This fixes [#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([Winter Zhang](https://github.com/zhang2014)) -* Fixed potential buffer overflow in decompress. Malicious user can pass fabricated compressed data that will cause read after buffer. This issue was found by Eldar Zaitov from Yandex information security team. [#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix incorrect result because of integers overflow in `arrayIntersect`. [#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now `OPTIMIZE TABLE` query will not wait for offline replicas to perform the operation. [#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) -* Fixed `ALTER TTL` parser for `Replicated*MergeTree` tables. [#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix communication between server and client, so server read temporary tables info after query failure. [#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) -* Fix `bitmapAnd` function error when intersecting an aggregated bitmap and a scalar bitmap. [#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) -* Refine the definition of `ZXid` according to the ZooKeeper Programmer's Guide which fixes bug in `clickhouse-cluster-copier`. [#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -* `odbc` table function now respects `external_table_functions_use_nulls` setting. [#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) -* Fixed bug that lead to a rare data race. [#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) -* Now `SYSTEM RELOAD DICTIONARY` reloads a dictionary completely, ignoring `update_field`. This fixes [#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) -* Add ability to check if dictionary exists in create query. [#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alesapin](https://github.com/alesapin)) -* Fix `Float*` parsing in `Values` format. This fixes [#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -* Fix crash when we cannot reserve space in some background operations of `*MergeTree` table engines family. [#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix crash of merge operation when table contains `SimpleAggregateFunction(LowCardinality)` column. This fixes [#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) -* Restore support of all ICU locales and add the ability to apply collations for constant expressions. Also add language name to `system.collations` table. [#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) -* Fix bug when external dictionaries with zero minimal lifetime (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) don't update in background. [#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alesapin](https://github.com/alesapin)) -* Fix crash when external dictionary with ClickHouse source has subquery in query. [#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix incorrect parsing of file extension in table with engine `URL`. This fixes [#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) -* Fix `CHECK TABLE` query for `*MergeTree` tables without key. Fixes [#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) -* Fixed conversion of `Float64` to MySQL type. [#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) -* Now if table was not completely dropped because of server crash, server will try to restore and load it. [#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -* Fixed crash in table function `file` while inserting into file that doesn't exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -* Fix rare deadlock which can happen when `trace_log` is in enabled. [#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) -* Add ability to work with different types besides `Date` in `RangeHashed` external dictionary created from DDL query. Fixes [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) -* Fixes crash when `now64()` is called with result of another function. [#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) -* Fixed bug with detecting client IP for connections through mysql wire protocol. [#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) -* Fix empty array handling in `arraySplit` function. This fixes [#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) -* Fixed the issue when `pid-file` of another running `clickhouse-server` may be deleted. [#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) -* Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) -* Fixed error in function `arrayReduce` that may lead to "double free" and error in aggregate function combinator `Resample` that may lead to memory leak. Added aggregate function `aggThrow`. This function can be used for testing purposes. [#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Improvement -* Improved logging when working with `S3` table engine. [#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -* Printed help message when no arguments are passed when calling `clickhouse-local`. This fixes [#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) -* Add setting `mutations_sync` which allows to wait `ALTER UPDATE/DELETE` queries synchronously. [#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) -* Allow to set up relative `user_files_path` in `config.xml` (in the way similar to `format_schema_path`). [#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) -* Add exception for illegal types for conversion functions with `-OrZero` postfix. [#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) -* Simplify format of the header of data sending to a shard in a distributed query. [#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) -* `Live View` table engine refactoring. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) -* Add additional checks for external dictionaries created from DDL-queries. [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) -* Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now table the first argument of `joinGet` function can be table indentifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) -* Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) -* Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) -* `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) -* Stack traces now display physical addresses (offsets in object file) instead of virtual memory addresses (where the object file was loaded). That allows the use of `addr2line` when binary is position independent and ASLR is active. This fixes [#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Support new syntax for row-level security filters: `…
`. Fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) -* Now `cityHash` function can work with `Decimal` and `UUID` types. Fixes [#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) -* Removed fixed index granularity (it was 1024) from system logs because it's obsolete after implementation of adaptive granularity. [#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Enabled MySQL compatibility server when ClickHouse is compiled without SSL. [#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([Yuriy Baranov](https://github.com/yurriy)) -* Now server checksums distributed batches, which gives more verbose errors in case of corrupted data in batch. [#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) -* Support `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` and `ATTACH TABLE` for `MySQL` database engine. [#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([Winter Zhang](https://github.com/zhang2014)) -* Add authentication in S3 table function and table engine. [#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Added check for extra parts of `MergeTree` at different disks, in order to not allow to miss data parts at undefined disks. [#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Enable SSL support for Mac client and server. [#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([Ivan](https://github.com/abyss7)) -* Now ClickHouse can work as MySQL federated server (see https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) -* `clickhouse-client` now only enable `bracketed-paste` when multiquery is on and multiline is off. This fixes (#7757)[https://github.com/ClickHouse/ClickHouse/issues/7757]. [#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Bird](https://github.com/amosbird)) -* Support `Array(Decimal)` in `if` function. [#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) -* Support Decimals in `arrayDifference`, `arrayCumSum` and `arrayCumSumNegative` functions. [#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) -* Added `lifetime` column to `system.dictionaries` table. [#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) -* Improved check for existing parts on different disks for `*MergeTree` table engines. Addresses [#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Integration with `AWS SDK` for `S3` interactions which allows to use all S3 features out of the box. [#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) -* Added support for subqueries in `Live View` tables. [#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) -* Check for using `Date` or `DateTime` column from `TTL` expressions was removed. [#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Information about disk was added to `system.detached_parts` table. [#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Now settings `max_(table|partition)_size_to_drop` can be changed without a restart. [#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) -* Slightly better usability of error messages. Ask user not to remove the lines below `Stack trace:`. [#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Better reading messages from `Kafka` engine in various formats after [#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([Ivan](https://github.com/abyss7)) -* Better compatibility with MySQL clients which don't support `sha2_password` auth plugin. [#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) -* Support more column types in MySQL compatibility server. [#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) -* Implement `ORDER BY` optimization for `Merge`, `Buffer` and `Materilized View` storages with underlying `MergeTree` tables. [#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) -* Now we always use POSIX implementation of `getrandom` to have better compatibility with old kernels (< 3.17). [#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Bird](https://github.com/amosbird)) -* Better check for valid destination in a move TTL rule. [#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Better checks for broken insert batches for `Distributed` table engine. [#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) -* Add column with array of parts name which mutations must process in future to `system.mutations` table. [#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) -* Parallel merge sort optimization for processors. [#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* The settings `mark_cache_min_lifetime` is now obsolete and does nothing. In previous versions, mark cache can grow in memory larger than `mark_cache_size` to accomodate data within `mark_cache_min_lifetime` seconds. That was leading to confusion and higher memory usage than expected, that is especially bad on memory constrained systems. If you will see performance degradation after installing this release, you should increase the `mark_cache_size`. [#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Preparation to use `tid` everywhere. This is needed for [#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Performance Improvement -* Performance optimizations in processors pipeline. [#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Non-blocking updates of expired keys in cache dictionaries (with permission to read old ones). [#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Compile ClickHouse without `-fno-omit-frame-pointer` globally to spare one more register. [#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Bird](https://github.com/amosbird)) -* Speedup `greatCircleDistance` function and add performance tests for it. [#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) -* Improved performance of function `roundDown`. [#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Improved performance of `max`, `min`, `argMin`, `argMax` for `DateTime64` data type. [#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) -* Improved performance of sorting without a limit or with big limit and external sorting. [#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Improved performance of formatting floating point numbers up to 6 times. [#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Improved performance of `modulo` function. [#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Bird](https://github.com/amosbird)) -* Optimized `ORDER BY` and merging with single column key. [#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Better implementation for `arrayReduce`, `-Array` and `-State` combinators. [#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Bird](https://github.com/amosbird)) -* Now `PREWHERE` should be optimized to be at least as efficient as `WHERE`. [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) -* Improve the way `round` and `roundBankers` handling negative numbers. [#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) -* Improved decoding performance of `DoubleDelta` and `Gorilla` codecs by roughly 30-40%. This fixes [#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) -* Improved performance of `base64` related functions. [#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Added a function `geoDistance`. It is similar to `greatCircleDistance` but uses approximation to WGS-84 ellipsoid model. The performance of both functions are near the same. [#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Faster `min` and `max` aggregation functions for `Decimal` data type. [#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) -* Vectorize processing `arrayReduce`. [#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) -* `if` chains are now optimized as `multiIf`. [#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) -* Fix performance regression of `Kafka` table engine introduced in 19.15. This fixes [#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) -* Removed "pie" code generation that `gcc` from Debian packages occasionally brings by default. [#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Parallel parsing data formats [#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Enable optimized parser of `Values` with expressions by default (`input_format_values_deduce_templates_of_expressions=1`). [#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) - -#### Build/Testing/Packaging Improvement -* Build fixes for `ARM` and in minimal mode. [#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) -* Add coverage file flush for `clickhouse-server` when std::atexit is not called. Also slightly improved logging in stateless tests with coverage. [#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) -* Update LLVM library in contrib. Avoid using LLVM from OS packages. [#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Make bundled `curl` build fully quiet. [#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) -* Fix some `MemorySanitizer` warnings. [#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Use `add_warning` and `no_warning` macros in `CMakeLists.txt`. [#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([Ivan](https://github.com/abyss7)) -* Add support of Minio S3 Compatible object (https://min.io/) for better integration tests. [#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) -* Imported `libc` headers to contrib. It allows to make builds more consistent across various systems (only for `x86_64-linux-gnu`). [#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Remove `-fPIC` from some libraries. [#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Clean `CMakeLists.txt` for curl. See https://github.com/ClickHouse/ClickHouse/pull/8011#issuecomment-569478910 [#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Silent warnings in `CapNProto` library. [#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add performance tests for short string optimized hash tables. [#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Bird](https://github.com/amosbird)) -* Now ClickHouse will build on `AArch64` even if `MADV_FREE` is not available. This fixes [#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Bird](https://github.com/amosbird)) -* Update `zlib-ng` to fix memory sanitizer problems. [#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually doesn't work at all. This fixes [#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed build on some systems after enabling `libc++`. This supersedes [#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Make `Field` methods more type-safe to find more errors. [#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Added missing files to the `libc-headers` submodule. [#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix wrong `JSON` quoting in performance test output. [#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now stack trace is displayed for `std::exception` and `Poco::Exception`. In previous versions it was available only for `DB::Exception`. This improves diagnostics. [#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Porting `clock_gettime` and `clock_nanosleep` for fresh glibc versions. [#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Bird](https://github.com/amosbird)) -* Enable `part_log` in example config for developers. [#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix async nature of reload in `01036_no_superfluous_dict_reload_on_create_database*`. [#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) -* Fixed codec performance tests. [#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) -* Add install scripts for `.tgz` build and documentation for them. [#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) -* Removed old `ZSTD` test (it was created in year 2016 to reproduce the bug that pre 1.0 version of ZSTD has had). This fixes [#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed build on Mac OS Catalina. [#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) -* Increased number of rows in codec performance tests to make results noticeable. [#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) -* In debug builds, treat `LOGICAL_ERROR` exceptions as assertion failures, so that they are easier to notice. [#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Make formats-related performance test more deterministic. [#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Update `lz4` to fix a MemorySanitizer failure. [#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Suppress a known MemorySanitizer false positive in exception handling. [#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Update `gcc` and `g++` to version 9 in `build/docker/build.sh` [#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) -* Add performance test case to test that `PREWHERE` is worse than `WHERE`. [#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Bird](https://github.com/amosbird)) -* Progress towards fixing one flacky test. [#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Avoid MemorySanitizer report for data from `libunwind`. [#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Updated `libc++` to the latest version. [#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Build ICU library from sources. This fixes [#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Switched from `libressl` to `openssl`. ClickHouse should support TLS 1.3 and SNI after this change. This fixes [#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed UBSan report when using `chacha20_poly1305` from SSL (happens on connect to https://yandex.ru/). [#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix mode of default password file for `.deb` linux distros. [#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) -* Improved expression for getting `clickhouse-server` PID in `clickhouse-test`. [#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) -* Updated contrib/googletest to v1.10.0. [#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) -* Fixed ThreadSaninitizer report in `base64` library. Also updated this library to the latest version, but it doesn't matter. This fixes [#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix `00600_replace_running_query` for processors. [#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Remove support for `tcmalloc` to make `CMakeLists.txt` simpler. [#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Release gcc builds now use `libc++` instead of `libstdc++`. Recently `libc++` was used only with clang. This will improve consistency of build configurations and portability. [#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Enable ICU library for build with MemorySanitizer. [#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Suppress warnings from `CapNProto` library. [#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Removed special cases of code for `tcmalloc`, because it's no longer supported. [#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* In CI coverage task, kill the server gracefully to allow it to save the coverage report. This fixes incomplete coverage reports we've been seeing lately. [#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alesapin](https://github.com/alesapin)) -* Performance tests for all codecs against `Float64` and `UInt64` values. [#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) -* `termcap` is very much deprecated and lead to various problems (f.g. missing "up" cap and echoing `^J` instead of multi line) . Favor `terminfo` or bundled `ncurses`. [#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Bird](https://github.com/amosbird)) -* Fix `test_storage_s3` integration test. [#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Support `StorageFile(, null) ` to insert block into given format file without actually write to disk. This is required for performance tests. [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) -* Added argument `--print-time` to functional tests which prints execution time per test. [#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Added asserts to `KeyCondition` while evaluating RPN. This will fix warning from gcc-9. [#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Dump cmake options in CI builds. [#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Don't generate debug info for some fat libraries. [#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Make `log_to_console.xml` always log to stderr, regardless of is it interactive or not. [#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Removed some unused features from `clickhouse-performance-test` tool. [#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Now we will also search for `lld-X` with corresponding `clang-X` version. [#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alesapin](https://github.com/alesapin)) -* Parquet build improvement. [#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) -* More GCC warnings [#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -* Package for Arch Linux now allows to run ClickHouse server, and not only client. [#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix test with processors. Tiny performance fixes. [#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Update contrib/protobuf. [#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) -* In preparation of switching to c++20 as a new year celebration. "May the C++ force be with ClickHouse." [#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Bird](https://github.com/amosbird)) - -#### Experimental Feature -* Added experimental setting `min_bytes_to_use_mmap_io`. It allows to read big files without copying data from kernel to userspace. The setting is disabled by default. Recommended threshold is about 64 MB, because mmap/munmap is slow. [#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Reworked quotas as a part of access control system. Added new table `system.quotas`, new functions `currentQuota`, `currentQuotaKey`, new SQL syntax `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) -* Allow skipping unknown settings with warnings instead of throwing exceptions. [#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) -* Reworked row policies as a part of access control system. Added new table `system.row_policies`, new function `currentRowPolicies()`, new SQL syntax `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) - -#### Security Fix -* Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) diff --git a/docs/en/whats_new/changelog/index.md b/docs/en/whats_new/changelog/index.md new file mode 120000 index 00000000000..2bf6a6cbd6d --- /dev/null +++ b/docs/en/whats_new/changelog/index.md @@ -0,0 +1 @@ +../../../../CHANGELOG.md \ No newline at end of file diff --git a/docs/es/development/browse_code.md b/docs/es/development/browse_code.md index 6bbc9d57729..393577e8dca 100644 --- a/docs/es/development/browse_code.md +++ b/docs/es/development/browse_code.md @@ -7,7 +7,7 @@ toc_title: "Examinar el c\xF3digo fuente de ClickHouse" # Examinar el código fuente de ClickHouse {#browse-clickhouse-source-code} -Usted puede utilizar **Woboq** navegador de código en línea disponible [aqui](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/src/index.html). Proporciona navegación de código y resaltado semántico, búsqueda e indexación. La instantánea de código se actualiza diariamente. +Usted puede utilizar **Woboq** navegador de código en línea disponible [aqui](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). Proporciona navegación de código y resaltado semántico, búsqueda e indexación. La instantánea de código se actualiza diariamente. Además, puede navegar por las fuentes en [GitHub](https://github.com/ClickHouse/ClickHouse) como de costumbre. diff --git a/docs/es/development/tests.md b/docs/es/development/tests.md index 388931e9436..12afbb68f2e 100644 --- a/docs/es/development/tests.md +++ b/docs/es/development/tests.md @@ -17,7 +17,7 @@ Las pruebas se encuentran en `queries` directorio. Hay dos subdirectorios: `stat Cada prueba puede ser de dos tipos: `.sql` y `.sh`. `.sql` test es el script SQL simple que se canaliza a `clickhouse-client --multiquery --testmode`. `.sh` test es un script que se ejecuta por sí mismo. -Para ejecutar todas las pruebas, use `testskhouse-test` herramienta. Mira `--help` para la lista de posibles opciones. Simplemente puede ejecutar todas las pruebas o ejecutar un subconjunto de pruebas filtradas por subcadena en el nombre de la prueba: `./clickhouse-test substring`. +Para ejecutar todas las pruebas, use `clickhouse-test` herramienta. Mira `--help` para la lista de posibles opciones. Simplemente puede ejecutar todas las pruebas o ejecutar un subconjunto de pruebas filtradas por subcadena en el nombre de la prueba: `./clickhouse-test substring`. La forma más sencilla de invocar pruebas funcionales es copiar `clickhouse-client` a `/usr/bin/`, ejecutar `clickhouse-server` y luego ejecutar `./clickhouse-test` de su propio directorio. @@ -36,13 +36,13 @@ deshabilitar estos grupos de pruebas utilizando `--no-zookeeper`, `--no-shard` y ## Bugs conocidos {#known-bugs} -Si conocemos algunos errores que se pueden reproducir fácilmente mediante pruebas funcionales, colocamos pruebas funcionales preparadas en `queries/bugs` directorio. Estas pruebas se moverán a `teststests_stateless` cuando se corrigen errores. +Si conocemos algunos errores que se pueden reproducir fácilmente mediante pruebas funcionales, colocamos pruebas funcionales preparadas en `tests/queries/bugs` directorio. Estas pruebas se moverán a `tests/queries/0_stateless` cuando se corrigen errores. ## Pruebas de integración {#integration-tests} Las pruebas de integración permiten probar ClickHouse en la configuración agrupada y la interacción de ClickHouse con otros servidores como MySQL, Postgres, MongoDB. Son útiles para emular divisiones de red, caídas de paquetes, etc. Estas pruebas se ejecutan bajo Docker y crean múltiples contenedores con varios software. -Ver `testsgration/README.md` sobre cómo ejecutar estas pruebas. +Ver `tests/integration/README.md` sobre cómo ejecutar estas pruebas. Tenga en cuenta que la integración de ClickHouse con controladores de terceros no se ha probado. Además, actualmente no tenemos pruebas de integración con nuestros controladores JDBC y ODBC. diff --git a/docs/es/engines/table_engines/mergetree_family/mergetree.md b/docs/es/engines/table_engines/mergetree_family/mergetree.md index a043effa333..2d5ac62f704 100644 --- a/docs/es/engines/table_engines/mergetree_family/mergetree.md +++ b/docs/es/engines/table_engines/mergetree_family/mergetree.md @@ -507,14 +507,14 @@ Configuración de la estructura: - /mnt/fast_ssd/clickhouse + /mnt/fast_ssd/clickhouse/ - /mnt/hdd1/clickhouse + /mnt/hdd1/clickhouse/ 10485760 - /mnt/hdd2/clickhouse + /mnt/hdd2/clickhouse/ 10485760 diff --git a/docs/es/index.md b/docs/es/index.md index 2cbe375e8ee..1348ef7013d 100644 --- a/docs/es/index.md +++ b/docs/es/index.md @@ -80,48 +80,6 @@ Ver la diferencia? Por ejemplo, la consulta “count the number of records for each advertising platform” requiere leer uno “advertising platform ID” columna, que ocupa 1 byte sin comprimir. Si la mayor parte del tráfico no proviene de plataformas publicitarias, puede esperar al menos una compresión de 10 veces de esta columna. Cuando se utiliza un algoritmo de compresión rápida, la descompresión de datos es posible a una velocidad de al menos varios gigabytes de datos sin comprimir por segundo. En otras palabras, esta consulta se puede procesar a una velocidad de aproximadamente varios miles de millones de filas por segundo en un único servidor. Esta velocidad se logra realmente en la práctica. -
- -Ejemplo - -``` bash -$ clickhouse-client -ClickHouse client version 0.0.52053. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.52053. -``` - -``` sql -SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 -``` - -``` text -┌─CounterID─┬──count()─┐ -│ 114208 │ 56057344 │ -│ 115080 │ 51619590 │ -│ 3228 │ 44658301 │ -│ 38230 │ 42045932 │ -│ 145263 │ 42042158 │ -│ 91244 │ 38297270 │ -│ 154139 │ 26647572 │ -│ 150748 │ 24112755 │ -│ 242232 │ 21302571 │ -│ 338158 │ 13507087 │ -│ 62180 │ 12229491 │ -│ 82264 │ 12187441 │ -│ 232261 │ 12148031 │ -│ 146272 │ 11438516 │ -│ 168777 │ 11403636 │ -│ 4120072 │ 11227824 │ -│ 10938808 │ 10519739 │ -│ 74088 │ 9047015 │ -│ 115079 │ 8837972 │ -│ 337234 │ 8205961 │ -└───────────┴──────────┘ -``` - -
- ### CPU {#cpu} Dado que la ejecución de una consulta requiere procesar un gran número de filas, ayuda enviar todas las operaciones para vectores completos en lugar de para filas separadas, o implementar el motor de consultas para que casi no haya costo de envío. Si no hace esto, con cualquier subsistema de disco medio decente, el intérprete de consultas inevitablemente detiene la CPU. Tiene sentido almacenar datos en columnas y procesarlos, cuando sea posible, por columnas. diff --git a/docs/es/interfaces/third-party/integrations.md b/docs/es/interfaces/third-party/integrations.md index 00be86b187d..d706278a280 100644 --- a/docs/es/interfaces/third-party/integrations.md +++ b/docs/es/interfaces/third-party/integrations.md @@ -26,7 +26,10 @@ toc_title: "Integraci\xF3n" - [Método de codificación de datos:](https://github.com/zlzforever/ClickHouseMigrator) - Colas de mensajes - [Kafka](https://kafka.apache.org) - - [Método de codificación de datos:](https://github.com/housepower/clickhouse_sinker) (utilizar [Ir cliente](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (usos [Go client](https://github.com/ClickHouse/clickhouse-go/)) +- Procesamiento de flujo + - [Flink](https://flink.apache.org) + - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) - Almacenamiento de objetos - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - [Haga clic en el botón de copia de seguridad](https://github.com/AlexAkulov/clickhouse-backup) @@ -74,6 +77,9 @@ toc_title: "Integraci\xF3n" - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (utilizar [InformaciónSistema abierto.](https://github.com/Infinidat/infi.clickhouse_orm)) - [pandas](https://pandas.pydata.org) - [Pandahouse](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [Dplyr](https://db.rstudio.com/dplyr/) - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/IMSMWU/RClickhouse) (utilizar [Bienvenidos](https://github.com/artpaul/clickhouse-cpp)) diff --git a/docs/es/introduction/distinctive_features.md b/docs/es/introduction/distinctive_features.md index 5117fcf9324..82b757341be 100644 --- a/docs/es/introduction/distinctive_features.md +++ b/docs/es/introduction/distinctive_features.md @@ -68,4 +68,10 @@ ClickHouse utiliza la replicación multi-maestro asincrónica. Después de escri Para obtener más información, consulte la sección [Replicación de datos](../engines/table_engines/mergetree_family/replication.md). +## Características que pueden considerarse desventajas {#clickhouse-features-that-can-be-considered-disadvantages} + +1. No hay transacciones completas. +2. Falta de capacidad para modificar o eliminar datos ya insertados con alta tasa y baja latencia. Hay eliminaciones y actualizaciones por lotes disponibles para limpiar o modificar datos, por ejemplo, para cumplir con [GDPR](https://gdpr-info.eu). +3. El índice disperso hace que ClickHouse no sea tan adecuado para consultas de puntos que recuperan filas individuales por sus claves. + [Artículo Original](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/es/introduction/features_considered_disadvantages.md b/docs/es/introduction/features_considered_disadvantages.md deleted file mode 100644 index 60eabad3102..00000000000 --- a/docs/es/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: 3e185d24c9fe772c7cf03d5475247fb829a21dfa -toc_priority: 5 -toc_title: "Caracter\xEDsticas de ClickHouse que pueden considerarse desventajas" ---- - -# Características de ClickHouse que pueden considerarse desventajas {#clickhouse-features-that-can-be-considered-disadvantages} - -1. No hay transacciones completas. -2. Falta de capacidad para modificar o eliminar datos ya insertados con alta tasa y baja latencia. Hay eliminaciones y actualizaciones por lotes disponibles para limpiar o modificar datos, por ejemplo, para cumplir con [GDPR](https://gdpr-info.eu). -3. El índice disperso hace que ClickHouse no sea tan adecuado para consultas de puntos que recuperan filas individuales por sus claves. - -[Artículo Original](https://clickhouse.tech/docs/en/introduction/features_considered_disadvantages/) diff --git a/docs/es/introduction/performance.md b/docs/es/introduction/performance.md index b0b6cf0afa1..ad992dd13d6 100644 --- a/docs/es/introduction/performance.md +++ b/docs/es/introduction/performance.md @@ -7,7 +7,7 @@ toc_title: Rendimiento # Rendimiento {#performance} -De acuerdo con los resultados de las pruebas internas en Yandex, ClickHouse muestra el mejor rendimiento (tanto el mayor rendimiento para consultas largas como la menor latencia en consultas cortas) para escenarios operativos comparables entre los sistemas de su clase que estaban disponibles para pruebas. Puede ver los resultados de la prueba en un [página separada](https://clickhouse.tech/benchmark.html). +De acuerdo con los resultados de las pruebas internas en Yandex, ClickHouse muestra el mejor rendimiento (tanto el mayor rendimiento para consultas largas como la menor latencia en consultas cortas) para escenarios operativos comparables entre los sistemas de su clase que estaban disponibles para pruebas. Puede ver los resultados de la prueba en un [página separada](https://clickhouse.tech/benchmark/dbms/). Numerosos puntos de referencia independientes llegaron a conclusiones similares. No son difíciles de encontrar mediante una búsqueda en Internet, o se puede ver [nuestra pequeña colección de enlaces relacionados](https://clickhouse.tech/#independent-benchmarks). diff --git a/docs/es/whats_new/changelog/index.md b/docs/es/whats_new/changelog/index.md index 03c8ea00cbe..053f924099a 100644 --- a/docs/es/whats_new/changelog/index.md +++ b/docs/es/whats_new/changelog/index.md @@ -249,7 +249,7 @@ toc_title: '2020' - Comprobación actualizada de consultas colgadas en el script de prueba de clickhouse [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alejandro Kazakov](https://github.com/Akazz)) - Se eliminaron algunos archivos inútiles del repositorio. [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Tipo cambiado de perftests matemáticos de `once` a `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Agregue una imagen acoplable que permite construir un informe HTML del navegador de código interactivo para nuestra base de código. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) Ver [Navegador de código Woboq](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html) +- Agregue una imagen acoplable que permite construir un informe HTML del navegador de código interactivo para nuestra base de código. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) Ver [Navegador de código Woboq](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) - Suprima algunas fallas de prueba bajo MSan. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alejandro Kuzmenkov](https://github.com/akuzm)) - Aceleración “exception while insert” prueba. Esta prueba a menudo se agota en la compilación de depuración con cobertura. [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Actualizar `libcxx` y `libcxxabi` dominar. En preparación para [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) diff --git a/docs/fa/development/browse_code.md b/docs/fa/development/browse_code.md index 1609e4b1d77..0338511c44c 100644 --- a/docs/fa/development/browse_code.md +++ b/docs/fa/development/browse_code.md @@ -8,7 +8,7 @@ toc_title: "\u0645\u0631\u0648\u0631 \u06A9\u062F \u0645\u0646\u0628\u0639 \u06A # فهرست clickhouse کد منبع {#browse-clickhouse-source-code} -شما می توانید استفاده کنید **ووبوک** آنلاین کد مرورگر موجود [اینجا](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html). این فراهم می کند ناوبری کد و برجسته معنایی, جستجو و نمایه سازی. عکس فوری کد روزانه به روز می شود. +شما می توانید استفاده کنید **ووبوک** آنلاین کد مرورگر موجود [اینجا](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html). این فراهم می کند ناوبری کد و برجسته معنایی, جستجو و نمایه سازی. عکس فوری کد روزانه به روز می شود. همچنین شما می توانید فهرست منابع در [گیتهاب](https://github.com/ClickHouse/ClickHouse) به عنوان معمول است. diff --git a/docs/fa/development/tests.md b/docs/fa/development/tests.md index 874ac3063b9..922bc43fd46 100644 --- a/docs/fa/development/tests.md +++ b/docs/fa/development/tests.md @@ -18,7 +18,7 @@ toc_title: "\u0646\u062D\u0648\u0647 \u0627\u062C\u0631\u0627\u06CC \u062A\u0633 هر تست می تواند یکی از دو نوع باشد: `.sql` و `.sh`. `.sql` تست اسکریپت ساده مربع است که به لوله کشی است `clickhouse-client --multiquery --testmode`. `.sh` تست یک اسکریپت است که به خودی خود اجرا است. -برای اجرای تمام تست ها استفاده کنید `testskhouse-test` ابزار. نگاه کن `--help` برای لیستی از گزینه های ممکن. شما به سادگی می توانید تمام تست ها را اجرا کنید یا زیر مجموعه ای از تست های فیلتر شده توسط زیر رشته را در نام تست اجرا کنید: `./clickhouse-test substring`. +برای اجرای تمام تست ها استفاده کنید `clickhouse-test` ابزار. نگاه کن `--help` برای لیستی از گزینه های ممکن. شما به سادگی می توانید تمام تست ها را اجرا کنید یا زیر مجموعه ای از تست های فیلتر شده توسط زیر رشته را در نام تست اجرا کنید: `./clickhouse-test substring`. ساده ترین راه برای فراخوانی تست های کاربردی کپی است `clickhouse-client` به `/usr/bin/` فرار کن `clickhouse-server` و سپس اجرا کنید `./clickhouse-test` از دایرکتوری خود را. @@ -37,13 +37,13 @@ toc_title: "\u0646\u062D\u0648\u0647 \u0627\u062C\u0631\u0627\u06CC \u062A\u0633 ## اشکالات شناخته شده {#known-bugs} -اگر ما می دانیم برخی از اشکالات است که می تواند به راحتی توسط تست های کاربردی تکثیر, ما تست های عملکردی تهیه شده در `queries/bugs` فهرست راهنما. این تست خواهد شد به نقل مکان کرد `teststests_stateless` هنگامی که اشکالات ثابت هستند. +اگر ما می دانیم برخی از اشکالات است که می تواند به راحتی توسط تست های کاربردی تکثیر, ما تست های عملکردی تهیه شده در `tests/queries/bugs` فهرست راهنما. این تست خواهد شد به نقل مکان کرد `tests/queries/0_stateless` هنگامی که اشکالات ثابت هستند. ## تست های ادغام {#integration-tests} ادغام آزمون اجازه می دهد برای تست clickhouse در خوشه پیکربندی و clickhouse تعامل با سرور های دیگر مانند mysql, postgres, mongodb. مفید برای تقلید انشعابات شبکه قطره بسته و غیره هستند. این تست ها تحت کارگر بارانداز اجرا و ایجاد ظروف متعدد با نرم افزار های مختلف. -ببینید `testsgration/README.md` در مورد چگونگی اجرای این تست. +ببینید `tests/integration/README.md` در مورد چگونگی اجرای این تست. توجه داشته باشید که ادغام کلیک با رانندگان شخص ثالث تست نشده است. همچنین ما در حال حاضر تست های یکپارچه سازی با رانندگان جی بی سی و بی سی ما ندارد. diff --git a/docs/fa/engines/table_engines/mergetree_family/mergetree.md b/docs/fa/engines/table_engines/mergetree_family/mergetree.md index b6f65611596..10fe5d8ba5e 100644 --- a/docs/fa/engines/table_engines/mergetree_family/mergetree.md +++ b/docs/fa/engines/table_engines/mergetree_family/mergetree.md @@ -507,14 +507,14 @@ ALTER TABLE example_table - /mnt/fast_ssd/clickhouse + /mnt/fast_ssd/clickhouse/ - /mnt/hdd1/clickhouse + /mnt/hdd1/clickhouse/ 10485760 - /mnt/hdd2/clickhouse + /mnt/hdd2/clickhouse/ 10485760 diff --git a/docs/fa/index.md b/docs/fa/index.md index a8f1c201e37..f2b8f433edb 100644 --- a/docs/fa/index.md +++ b/docs/fa/index.md @@ -79,54 +79,6 @@ ClickHouse یک مدیریت دیتابیس (DBMS) ستون گرا برای پر برای مثال، query «تعداد رکوردها به ازای هر بستر نیازمندی» نیازمند خواندن ستون «آیدی بستر آگهی»، که 1 بایت بدون فشرده طول می کشد، خواهد بود. اگر بیشتر ترافیک مربوط به بستر های نیازمندی نبود، شما می توانید انتظار حداقل 10 برابر فشرده سازی این ستون را داشته باشید. زمانی که از الگوریتم فشرده سازی quick استفاده می کنید، عملیات decompression داده ها با سرعت حداقل چندین گیگابایت در ثانیه انجام می شود. به عبارت دیگر، این query توانایی پردازش تقریبا چندین میلیارد رکورد در ثانیه به ازای یک سرور را دارد. این سرعت در عمل واقعی و دست یافتنی است. -
- -مثال - - $ clickhouse-client - ClickHouse client version 0.0.52053. - Connecting to localhost:9000. - Connected to ClickHouse server version 0.0.52053. - - :) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 - - SELECT - CounterID, - count() - FROM hits - GROUP BY CounterID - ORDER BY count() DESC - LIMIT 20 - - ┌─CounterID─┬──count()─┐ - │ 114208 │ 56057344 │ - │ 115080 │ 51619590 │ - │ 3228 │ 44658301 │ - │ 38230 │ 42045932 │ - │ 145263 │ 42042158 │ - │ 91244 │ 38297270 │ - │ 154139 │ 26647572 │ - │ 150748 │ 24112755 │ - │ 242232 │ 21302571 │ - │ 338158 │ 13507087 │ - │ 62180 │ 12229491 │ - │ 82264 │ 12187441 │ - │ 232261 │ 12148031 │ - │ 146272 │ 11438516 │ - │ 168777 │ 11403636 │ - │ 4120072 │ 11227824 │ - │ 10938808 │ 10519739 │ - │ 74088 │ 9047015 │ - │ 115079 │ 8837972 │ - │ 337234 │ 8205961 │ - └───────────┴──────────┘ - - 20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) - - :) - -
- ### CPU {#cpu} از آنجایی که اجرای یک query نیازمند پردازش تعداد زیادی سطر می باشد، این کمک می کند تا تمام عملیات ها به جای ارسال به سطرهای جداگانه، برای کل بردار ارسال شود، یا برای ترکیب query engine به طوری که هیچ هزینه ی ارسالی وجود ندارد. اگر این کار رو نکنید، با هر half-decent disk subsystem، تفسیرگر query ناگزیر است که CPU را متوقف کند. این منطقی است که که در صورت امکان هر دو کار ذخیره سازی داده در ستون ها و پردازش ستون ها با هم انجام شود. diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md index ddbdffad169..0ad13d1fd9e 100644 --- a/docs/fa/interfaces/third-party/integrations.md +++ b/docs/fa/interfaces/third-party/integrations.md @@ -74,6 +74,9 @@ toc_title: "\u06CC\u06A9\u067E\u0627\u0631\u0686\u06AF\u06CC" - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (استفاده [اطالعات.کلیک \_شورم](https://github.com/Infinidat/infi.clickhouse_orm)) - [پانداها](https://pandas.pydata.org) - [پانداهاوس](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [هواپیمای دوباله](https://db.rstudio.com/dplyr/) - [خانه روستایی](https://github.com/IMSMWU/RClickhouse) (استفاده [صفحه اصلی](https://github.com/artpaul/clickhouse-cpp)) diff --git a/docs/fa/introduction/distinctive_features.md b/docs/fa/introduction/distinctive_features.md index a4313168796..71a8f3eb543 100644 --- a/docs/fa/introduction/distinctive_features.md +++ b/docs/fa/introduction/distinctive_features.md @@ -62,6 +62,12 @@ ClickHouse از روش asynchronous multimaster replication استفاده می برای اطلاعات بیشتر، به بخش [replication داده ها](../engines/table_engines/mergetree_family/replication.md) مراجعه کنید. +## ویژگی های از ClickHouse که می تواند معایبی باشد. {#wyjgy-hy-z-clickhouse-khh-my-twnd-m-yby-bshd} + +1. بدون پشتیبانی کامل از تراکنش +2. عدم توانایی برای تغییر و یا حذف داده های در حال حاضر وارد شده با سرعت بالا و تاخیر کم. برای پاک کردن و یا اصلاح داده ها، به عنوان مثال برای پیروی از [GDPR](https://gdpr-info.eu)، دسته ای پاک و به روزرسانی وجود دارد.حال توسعه می باشد. +3. Sparse index باعث می شود ClickHouse چندان مناسب اجرای پرسمان های point query برای دریافت یک ردیف از داده ها با استفاده از کلید آنها نباشد. + [مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/distinctive_features/) diff --git a/docs/fa/introduction/features_considered_disadvantages.md b/docs/fa/introduction/features_considered_disadvantages.md deleted file mode 100644 index 5a8ea156da5..00000000000 --- a/docs/fa/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1,11 +0,0 @@ -
- -# ویژگی های از ClickHouse که می تواند معایبی باشد. {#wyjgy-hy-z-clickhouse-khh-my-twnd-m-yby-bshd} - -1. بدون پشتیبانی کامل از تراکنش -2. عدم توانایی برای تغییر و یا حذف داده های در حال حاضر وارد شده با سرعت بالا و تاخیر کم. برای پاک کردن و یا اصلاح داده ها، به عنوان مثال برای پیروی از [GDPR](https://gdpr-info.eu)، دسته ای پاک و به روزرسانی وجود دارد.حال توسعه می باشد. -3. Sparse index باعث می شود ClickHouse چندان مناسب اجرای پرسمان های point query برای دریافت یک ردیف از داده ها با استفاده از کلید آنها نباشد. - -
- -[مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/features_considered_disadvantages/) diff --git a/docs/fa/introduction/performance.md b/docs/fa/introduction/performance.md index 5eb876f5ee1..8fa65ad1bb5 100644 --- a/docs/fa/introduction/performance.md +++ b/docs/fa/introduction/performance.md @@ -2,7 +2,7 @@ # Performance {#performance} -با توجه به نتایج تست های Yandex، ClickHouse بهترین عملکرد را برای سناریوهای عملیاتی قابل مقایسه با دیگر سیستم های در کلاس خود را از خود نشان داد. این تست ها شامل بالاترین توان عملیاتی برای query های طولانی، و کمترین latency برای query های کوتاه بود. نتایج این تست های در [صفحه ی جدا](https://clickhouse.tech/benchmark.html) موجود است. +با توجه به نتایج تست های Yandex، ClickHouse بهترین عملکرد را برای سناریوهای عملیاتی قابل مقایسه با دیگر سیستم های در کلاس خود را از خود نشان داد. این تست ها شامل بالاترین توان عملیاتی برای query های طولانی، و کمترین latency برای query های کوتاه بود. نتایج این تست های در [صفحه ی جدا](https://clickhouse.tech/benchmark/dbms/) موجود است. benchmark های زیادی وجود دارند که این نتایج را تایید می کنند. میتوانید این نتایج را جستجو کنید و یا [این لینک های benchmark](https://clickhouse.tech/#independent-benchmarks). مستقل را ببینید. diff --git a/docs/fa/operations/performance/sampling_query_profiler_example_result.txt b/docs/fa/operations/performance/sampling_query_profiler_example_result.txt index a5f6d71ca95..56c2fdf9c65 100644 --- a/docs/fa/operations/performance/sampling_query_profiler_example_result.txt +++ b/docs/fa/operations/performance/sampling_query_profiler_example_result.txt @@ -1,7 +1,3 @@ ---- -en_copy: true ---- - Row 1: ────── count(): 6344 diff --git a/docs/fa/whats_new/changelog/index.md b/docs/fa/whats_new/changelog/index.md index c7eb257e5a9..8dcc20e9409 100644 --- a/docs/fa/whats_new/changelog/index.md +++ b/docs/fa/whats_new/changelog/index.md @@ -249,7 +249,7 @@ toc_title: '2020' - به روز شده در چک کردن برای نمایش داده شد را قطع کرد در اسکریپت کلیک تست [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([الکساندر کازاکوف](https://github.com/Akazz)) - حذف برخی از فایل های بی فایده از مخزن. [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - نوع تغییر کامل ریاضی از `once` به `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([نیکولای کوچتو](https://github.com/KochetovNicolai)) -- اضافه کردن تصویر کارگر بارانداز که اجازه می دهد تا برای ساخت کد تعاملی مرورگر گزارش متنی برای کدهای ما. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([الساپین](https://github.com/alesapin)) ببینید [مرورگر کد ووبوک](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html) +- اضافه کردن تصویر کارگر بارانداز که اجازه می دهد تا برای ساخت کد تعاملی مرورگر گزارش متنی برای کدهای ما. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([الساپین](https://github.com/alesapin)) ببینید [مرورگر کد ووبوک](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) - سرکوب برخی از شکست تست تحت مسان. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([الکساندر کوزمنکوف](https://github.com/akuzm)) - افزایش سرعت “exception while insert” امتحان این تست اغلب زمان در اشکال زدایی با پوشش ساخت. [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([الکسی میلویدو](https://github.com/alexey-milovidov)) - به روز شده `libcxx` و `libcxxabi` به سلامتی استاد در تهیه به [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([الکسی میلویدو](https://github.com/alexey-milovidov)) diff --git a/docs/fr/development/browse_code.md b/docs/fr/development/browse_code.md index 62caa530b5d..0d708da5beb 100644 --- a/docs/fr/development/browse_code.md +++ b/docs/fr/development/browse_code.md @@ -7,7 +7,7 @@ toc_title: Parcourir Le Code Source De ClickHouse # Parcourir Le Code Source De ClickHouse {#browse-clickhouse-source-code} -Vous pouvez utiliser **Woboq** navigateur de code en ligne disponible [ici](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html). Il fournit la navigation de code et la mise en évidence sémantique, la recherche et l'indexation. L'instantané de code est mis à jour quotidiennement. +Vous pouvez utiliser **Woboq** navigateur de code en ligne disponible [ici](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html). Il fournit la navigation de code et la mise en évidence sémantique, la recherche et l'indexation. L'instantané de code est mis à jour quotidiennement. Aussi, vous pouvez parcourir les sources sur [GitHub](https://github.com/ClickHouse/ClickHouse) comme à l'habitude. diff --git a/docs/fr/development/tests.md b/docs/fr/development/tests.md index e5c8a50fa31..6637e9546fe 100644 --- a/docs/fr/development/tests.md +++ b/docs/fr/development/tests.md @@ -17,7 +17,7 @@ Les Tests sont situés dans `queries` répertoire. Il y a deux sous-répertoires Chaque test peut être de deux types: `.sql` et `.sh`. `.sql` test est le script SQL simple qui est canalisé vers `clickhouse-client --multiquery --testmode`. `.sh` test est un script qui est exécuté par lui-même. -Pour exécuter tous les tests, utilisez `testskhouse-test` outil. Regarder `--help` pour la liste des options possibles. Vous pouvez simplement exécuter tous les tests ou exécuter un sous ensemble de tests filtrés par sous chaîne dans le nom du test: `./clickhouse-test substring`. +Pour exécuter tous les tests, utilisez `clickhouse-test` outil. Regarder `--help` pour la liste des options possibles. Vous pouvez simplement exécuter tous les tests ou exécuter un sous ensemble de tests filtrés par sous chaîne dans le nom du test: `./clickhouse-test substring`. Le moyen le plus simple d'invoquer des tests fonctionnels est de copier `clickhouse-client` de `/usr/bin/`, exécuter `clickhouse-server` et puis exécutez `./clickhouse-test` à partir de son propre répertoire. @@ -36,13 +36,13 @@ désactivez ces groupes de tests en utilisant `--no-zookeeper`, `--no-shard` et ## Bugs connus {#known-bugs} -Si nous connaissons des bugs qui peuvent être facilement reproduits par des tests fonctionnels, nous plaçons des tests fonctionnels préparés dans `queries/bugs` répertoire. Ces tests seront déplacés à `teststests_stateless` quand les bugs sont corrigés. +Si nous connaissons des bugs qui peuvent être facilement reproduits par des tests fonctionnels, nous plaçons des tests fonctionnels préparés dans `tests/queries/bugs` répertoire. Ces tests seront déplacés à `tests/queries/0_stateless` quand les bugs sont corrigés. ## Les Tests D'Intégration {#integration-tests} Les tests d'intégration permettent de tester ClickHouse en configuration cluster et clickhouse interaction avec D'autres serveurs comme MySQL, Postgres, MongoDB. Ils sont utiles pour émuler les splits réseau, les chutes de paquets, etc. Ces tests sont exécutés sous Docker et créent plusieurs conteneurs avec divers logiciels. -Voir `testsgration/README.md` sur la façon d'exécuter ces tests. +Voir `tests/integration/README.md` sur la façon d'exécuter ces tests. Notez que l'intégration de ClickHouse avec des pilotes tiers n'est pas testée. De plus, nous n'avons actuellement pas de tests d'intégration avec nos pilotes JDBC et ODBC. diff --git a/docs/fr/engines/table_engines/mergetree_family/mergetree.md b/docs/fr/engines/table_engines/mergetree_family/mergetree.md index ada3437d714..1195751ca9c 100644 --- a/docs/fr/engines/table_engines/mergetree_family/mergetree.md +++ b/docs/fr/engines/table_engines/mergetree_family/mergetree.md @@ -507,14 +507,14 @@ Structure de Configuration: - /mnt/fast_ssd/clickhouse + /mnt/fast_ssd/clickhouse/ - /mnt/hdd1/clickhouse + /mnt/hdd1/clickhouse/ 10485760 - /mnt/hdd2/clickhouse + /mnt/hdd2/clickhouse/ 10485760 diff --git a/docs/fr/index.md b/docs/fr/index.md index 0e53167306e..bcdeda0a693 100644 --- a/docs/fr/index.md +++ b/docs/fr/index.md @@ -80,48 +80,6 @@ Vous voyez la différence? Par exemple, la requête “count the number of records for each advertising platform” nécessite la lecture d'un “advertising platform ID” colonne, qui prend 1 octet non compressé. Si la majeure partie du trafic ne provenait pas de plates-formes publicitaires, vous pouvez vous attendre à une compression d'au moins 10 fois de cette colonne. Lors de l'utilisation d'un algorithme de compression rapide, la décompression des données est possible à une vitesse d'au moins plusieurs gigaoctets de données non compressées par seconde. En d'autres termes, cette requête ne peut être traitée qu'à une vitesse d'environ plusieurs milliards de lignes par seconde sur un seul serveur. Cette vitesse est effectivement atteinte dans la pratique. -
- -Exemple - -``` bash -$ clickhouse-client -ClickHouse client version 0.0.52053. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.52053. -``` - -``` sql -SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 -``` - -``` text -┌─CounterID─┬──count()─┐ -│ 114208 │ 56057344 │ -│ 115080 │ 51619590 │ -│ 3228 │ 44658301 │ -│ 38230 │ 42045932 │ -│ 145263 │ 42042158 │ -│ 91244 │ 38297270 │ -│ 154139 │ 26647572 │ -│ 150748 │ 24112755 │ -│ 242232 │ 21302571 │ -│ 338158 │ 13507087 │ -│ 62180 │ 12229491 │ -│ 82264 │ 12187441 │ -│ 232261 │ 12148031 │ -│ 146272 │ 11438516 │ -│ 168777 │ 11403636 │ -│ 4120072 │ 11227824 │ -│ 10938808 │ 10519739 │ -│ 74088 │ 9047015 │ -│ 115079 │ 8837972 │ -│ 337234 │ 8205961 │ -└───────────┴──────────┘ -``` - -
- ### CPU {#cpu} Étant donné que l'exécution d'une requête nécessite le traitement d'un grand nombre de lignes, il est utile de répartir toutes les opérations pour des vecteurs entiers au lieu de lignes séparées, ou d'implémenter le moteur de requête de sorte qu'il n'y ait presque aucun coût d'expédition. Si vous ne le faites pas, avec un sous-système de disque à moitié décent, l'interpréteur de requête bloque inévitablement le processeur. Il est logique de stocker des données dans des colonnes et de les traiter, si possible, par des colonnes. diff --git a/docs/fr/interfaces/third-party/integrations.md b/docs/fr/interfaces/third-party/integrations.md index 76648d8613d..565be2b9604 100644 --- a/docs/fr/interfaces/third-party/integrations.md +++ b/docs/fr/interfaces/third-party/integrations.md @@ -74,6 +74,9 @@ toc_title: "Int\xE9gration" - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (utiliser [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [Panda](https://pandas.pydata.org) - [pandahouse](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) (utiliser [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) diff --git a/docs/fr/introduction/distinctive_features.md b/docs/fr/introduction/distinctive_features.md index dcea4046fcd..2c825cac85a 100644 --- a/docs/fr/introduction/distinctive_features.md +++ b/docs/fr/introduction/distinctive_features.md @@ -68,4 +68,10 @@ ClickHouse utilise la réplication multi-maître asynchrone. Après avoir été Pour plus d'informations, consultez la section [Réplication des données](../engines/table_engines/mergetree_family/replication.md). +## Caractéristiques de ClickHouse qui peuvent être considérées comme des inconvénients {#clickhouse-features-that-can-be-considered-disadvantages} + +1. Pas de transactions à part entière. +2. Manque de capacité à modifier ou supprimer des données déjà insérées avec un taux élevé et une faible latence. Des suppressions et des mises à jour par lots sont disponibles pour nettoyer ou modifier les données, par exemple pour [GDPR](https://gdpr-info.eu). +3. L'index clairsemé rend ClickHouse pas si approprié pour les requêtes ponctuelles récupérant des lignes simples par leurs clés. + [Article Original](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/fr/introduction/features_considered_disadvantages.md b/docs/fr/introduction/features_considered_disadvantages.md deleted file mode 100644 index dc9fe708fef..00000000000 --- a/docs/fr/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -machine_translated: true -machine_translated_rev: f865c9653f9df092694258e0ccdd733c339112f5 -toc_priority: 5 -toc_title: "Caract\xE9ristiques de ClickHouse qui peuvent \xEAtre consid\xE9r\xE9\ - es comme des inconv\xE9nients" ---- - -# Caractéristiques de ClickHouse qui peuvent être considérées comme des inconvénients {#clickhouse-features-that-can-be-considered-disadvantages} - -1. Pas de transactions à part entière. -2. Manque de capacité à modifier ou supprimer des données déjà insérées avec un taux élevé et une faible latence. Des suppressions et des mises à jour par lots sont disponibles pour nettoyer ou modifier les données, par exemple pour [GDPR](https://gdpr-info.eu). -3. L'index clairsemé rend ClickHouse pas si approprié pour les requêtes ponctuelles récupérant des lignes simples par leurs clés. - -[Article Original](https://clickhouse.tech/docs/en/introduction/features_considered_disadvantages/) diff --git a/docs/fr/introduction/performance.md b/docs/fr/introduction/performance.md index dcda1878d61..3e94830d7c0 100644 --- a/docs/fr/introduction/performance.md +++ b/docs/fr/introduction/performance.md @@ -7,7 +7,7 @@ toc_title: Performance # Performance {#performance} -Selon les résultats des tests internes chez Yandex, ClickHouse affiche les meilleures performances (à la fois le débit le plus élevé pour les requêtes longues et la latence la plus faible pour les requêtes courtes) pour des scénarios d'exploitation comparables parmi les systèmes de sa classe disponibles pour les tests. Vous pouvez afficher les résultats du test sur un [page séparée](https://clickhouse.tech/benchmark.html). +Selon les résultats des tests internes chez Yandex, ClickHouse affiche les meilleures performances (à la fois le débit le plus élevé pour les requêtes longues et la latence la plus faible pour les requêtes courtes) pour des scénarios d'exploitation comparables parmi les systèmes de sa classe disponibles pour les tests. Vous pouvez afficher les résultats du test sur un [page séparée](https://clickhouse.tech/benchmark/dbms/). De nombreux points de repère indépendants sont arrivés à des conclusions similaires. Ils ne sont pas difficiles à trouver en utilisant une recherche sur internet, ou vous pouvez voir [notre petite collection de liens](https://clickhouse.tech/#independent-benchmarks). diff --git a/docs/fr/whats_new/changelog/index.md b/docs/fr/whats_new/changelog/index.md index d45e36b1d8f..fe49ce8244c 100644 --- a/docs/fr/whats_new/changelog/index.md +++ b/docs/fr/whats_new/changelog/index.md @@ -249,7 +249,7 @@ toc_title: '2020' - Mise à jour de la vérification des requêtes suspendues dans le script clickhouse-test [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) - Suppression de certains fichiers inutiles du référentiel. [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Changement de type de math perftests de `once` de `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Ajouter une image docker qui permet de créer un rapport HTML interactif du navigateur de code pour notre base de code. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alésapine](https://github.com/alesapin)) Voir [Navigateur De Code Woboq](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/src/src/index.html) +- Ajouter une image docker qui permet de créer un rapport HTML interactif du navigateur de code pour notre base de code. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alésapine](https://github.com/alesapin)) Voir [Navigateur De Code Woboq](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/src/index.html) - Supprimer certains échecs de test sous MSan. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) - SpeedUp “exception while insert” test. Ce test expire souvent dans la construction debug-with-coverage. [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexeï-milovidov](https://github.com/alexey-milovidov)) - Mettre `libcxx` et `libcxxabi` maîtriser. En préparation à [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexeï-milovidov](https://github.com/alexey-milovidov)) diff --git a/docs/ja/development/browse_code.md b/docs/ja/development/browse_code.md index f8357fcca27..d66b14e400f 100644 --- a/docs/ja/development/browse_code.md +++ b/docs/ja/development/browse_code.md @@ -7,7 +7,7 @@ toc_title: "ClickHouse\u306E\u30BD\u30FC\u30B9\u30B3\u30FC\u30C9\u3092\u53C2\u71 # ClickHouseのソースコードを参照 {#browse-clickhouse-source-code} -を使用することができ **Woboq** オンラインのコードブラウザをご利用 [ここに](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html). このコードナビゲーションや意味のハイライト表示、検索インデックス. コードのスナップショットは随時更新中です。 +を使用することができ **Woboq** オンラインのコードブラウザをご利用 [ここに](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html). このコードナビゲーションや意味のハイライト表示、検索インデックス. コードのスナップショットは随時更新中です。 また、ソースを閲覧することもできます [GitHub](https://github.com/ClickHouse/ClickHouse) いつものように diff --git a/docs/ja/development/tests.md b/docs/ja/development/tests.md index 27b8870461e..b6c5abea621 100644 --- a/docs/ja/development/tests.md +++ b/docs/ja/development/tests.md @@ -17,7 +17,7 @@ toc_title: "ClickHouse\u30C6\u30B9\u30C8\u3092\u5B9F\u884C\u3059\u308B\u65B9\u6C それぞれの試験できるの種類: `.sql` と `.sh`. `.sql` testは、パイプ処理される単純なSQLスクリプトです `clickhouse-client --multiquery --testmode`. `.sh` テストは、単独で実行されるスクリプトです。 -すべてのテストを実行するには、 `testskhouse-test` ツール。 見て！ `--help` 可能なオプションのリストについて。 できるだけ実行すべての試験または実行のサブセットの試験フィルター部分文字列の試験名: `./clickhouse-test substring`. +すべてのテストを実行するには、 `clickhouse-test` ツール。 見て！ `--help` 可能なオプションのリストについて。 できるだけ実行すべての試験または実行のサブセットの試験フィルター部分文字列の試験名: `./clickhouse-test substring`. 機能テストを呼び出す最も簡単な方法は、コピーすることです `clickhouse-client` に `/usr/bin/`、実行 `clickhouse-server` そして、実行 `./clickhouse-test` 独自のディレクトリから。 @@ -36,13 +36,13 @@ toc_title: "ClickHouse\u30C6\u30B9\u30C8\u3092\u5B9F\u884C\u3059\u308B\u65B9\u6C ## 既知のバグ {#known-bugs} -機能テストで簡単に再現できるいくつかのバグを知っていれば、準備された機能テストを `queries/bugs` ディレクトリ。 これらのテストはに移動されます `teststests_stateless` バグが修正されたとき。 +機能テストで簡単に再現できるいくつかのバグを知っていれば、準備された機能テストを `tests/queries/bugs` ディレクトリ。 これらのテストはに移動されます `tests/queries/0_stateless` バグが修正されたとき。 ## 統合テスト {#integration-tests} 統合テストでは、クラスター化された設定でclickhouseをテストし、mysql、postgres、mongodbのような他のサーバーとのclickhouseの相互作用を可能にします。 それらはネットワークの割れ目、包みの低下、等を競争して有用である。 これらの試験する方向に作用しdockerを複数の容器を様々なソフトウェアです。 -見る `testsgration/README.md` これらのテストを実行する方法について。 +見る `tests/integration/README.md` これらのテストを実行する方法について。 ClickHouseとサードパーティドライバの統合はテストされていません。 また、現在、JDBCおよびODBCドライバとの統合テストはありません。 diff --git a/docs/ja/engines/table_engines/mergetree_family/mergetree.md b/docs/ja/engines/table_engines/mergetree_family/mergetree.md index 19ad4537492..4a1ff40ad0c 100644 --- a/docs/ja/engines/table_engines/mergetree_family/mergetree.md +++ b/docs/ja/engines/table_engines/mergetree_family/mergetree.md @@ -507,14 +507,14 @@ ALTER TABLE example_table - /mnt/fast_ssd/clickhouse + /mnt/fast_ssd/clickhouse/ - /mnt/hdd1/clickhouse + /mnt/hdd1/clickhouse/ 10485760 - /mnt/hdd2/clickhouse + /mnt/hdd2/clickhouse/ 10485760 diff --git a/docs/ja/index.md b/docs/ja/index.md index 07c2724eb0e..4dc166d7f5c 100644 --- a/docs/ja/index.md +++ b/docs/ja/index.md @@ -82,48 +82,6 @@ OLAPシナリオは、他の一般的なシナリオ（OLTPやKey-Valueアクセ たとえば、「各広告プラットフォームのレコード数をカウントする」クエリでは、1つの「広告プラットフォームID」列を読み取る必要がありますが、これは非圧縮では1バイトの領域を要します。トラフィックのほとんどが広告プラットフォームからのものではない場合、この列は少なくとも10倍の圧縮が期待できます。高速な圧縮アルゴリズムを使用すれば、1秒あたり少なくとも非圧縮データに換算して数ギガバイトの速度でデータを展開できます。つまり、このクエリは、単一のサーバーで1秒あたり約数十億行の速度で処理できます。この速度はまさに実際に達成されます。 -
- -Example - -``` bash -$ clickhouse-client -ClickHouse client version 0.0.52053. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.52053. -``` - -``` sql -SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 -``` - -``` text -┌─CounterID─┬──count()─┐ -│ 114208 │ 56057344 │ -│ 115080 │ 51619590 │ -│ 3228 │ 44658301 │ -│ 38230 │ 42045932 │ -│ 145263 │ 42042158 │ -│ 91244 │ 38297270 │ -│ 154139 │ 26647572 │ -│ 150748 │ 24112755 │ -│ 242232 │ 21302571 │ -│ 338158 │ 13507087 │ -│ 62180 │ 12229491 │ -│ 82264 │ 12187441 │ -│ 232261 │ 12148031 │ -│ 146272 │ 11438516 │ -│ 168777 │ 11403636 │ -│ 4120072 │ 11227824 │ -│ 10938808 │ 10519739 │ -│ 74088 │ 9047015 │ -│ 115079 │ 8837972 │ -│ 337234 │ 8205961 │ -└───────────┴──────────┘ -``` - -
- ### CPU {#cpu} クエリを実行するには大量の行を処理する必要があるため、個別の行ではなくベクター全体のすべての操作をディスパッチするか、ディスパッチコストがほとんどないようにクエリエンジンを実装すると効率的です。 適切なディスクサブシステムでこれを行わないと、クエリインタープリターが必然的にCPUを失速させます。 diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md index 7f5bfb93325..74d0a51e7de 100644 --- a/docs/ja/interfaces/third-party/integrations.md +++ b/docs/ja/interfaces/third-party/integrations.md @@ -74,6 +74,9 @@ toc_title: "\u7D71\u5408" - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (用途 [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [パンダ](https://pandas.pydata.org) - [パンダハウス](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - [Rクリックハウス](https://github.com/IMSMWU/RClickhouse) (用途 [クリックハウス-cpp](https://github.com/artpaul/clickhouse-cpp)) diff --git a/docs/ja/introduction/distinctive_features.md b/docs/ja/introduction/distinctive_features.md index 6cd0834708c..5c4b91759dc 100644 --- a/docs/ja/introduction/distinctive_features.md +++ b/docs/ja/introduction/distinctive_features.md @@ -63,4 +63,10 @@ ClickHouseには、精度を犠牲にしてパフォーマンスを得るため 詳細については、[データ複製](../engines/table_engines/mergetree_family/replication.md) セクションを参照してください。 +## 欠点と考えられるClickHouseの機能 {#qian-dian-tokao-erareruclickhousenoji-neng} + +1. 本格的なトランザクションはありません。 +2. 既に挿入されたデータの変更または削除を、高頻度かつ低遅延に行う機能はありません。 [GDPR](https://gdpr-info.eu)に準拠するなど、データをクリーンアップまたは変更するために、バッチ削除およびバッチ更新が利用可能です。 +3. インデックスが疎であるため、ClickHouseは、キーで単一行を取得するようなクエリにはあまり適していません。 + [Original article](https://clickhouse.yandex/docs/en/introduction/distinctive_features/) diff --git a/docs/ja/introduction/features_considered_disadvantages.md b/docs/ja/introduction/features_considered_disadvantages.md deleted file mode 100644 index 8c766e06fe8..00000000000 --- a/docs/ja/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -toc_priority: 5 -toc_title: 欠点と見なすことができるClickHouseの機能 ---- - -# 欠点と考えられるClickHouseの機能 {#qian-dian-tokao-erareruclickhousenoji-neng} - -1. 本格的なトランザクションはありません。 -2. 既に挿入されたデータの変更または削除を、高頻度かつ低遅延に行う機能はありません。 [GDPR](https://gdpr-info.eu)に準拠するなど、データをクリーンアップまたは変更するために、バッチ削除およびバッチ更新が利用可能です。 -3. インデックスが疎であるため、ClickHouseは、キーで単一行を取得するようなクエリにはあまり適していません。 - -[Original article](https://clickhouse.yandex/docs/en/introduction/features_considered_disadvantages/) diff --git a/docs/ja/introduction/performance.md b/docs/ja/introduction/performance.md index 8a99d15c01e..66ff8b2f4c0 100644 --- a/docs/ja/introduction/performance.md +++ b/docs/ja/introduction/performance.md @@ -5,7 +5,7 @@ toc_title: パフォーマンス # パフォーマンス {#pahuomansu} -Yandexの内部テスト結果によると、ClickHouseは、テスト可能なクラスのシステム間で同等の動作シナリオで最高のパフォーマンス(長時間のクエリで最も高いスループットと、短時間のクエリで最小のレイテンシの両方)を示します。 [別のページで](https://clickhouse.yandex/benchmark.html)テスト結果を表示できます 。 +Yandexの内部テスト結果によると、ClickHouseは、テスト可能なクラスのシステム間で同等の動作シナリオで最高のパフォーマンス(長時間のクエリで最も高いスループットと、短時間のクエリで最小のレイテンシの両方)を示します。 [別のページで](https://clickhouse.yandex/benchmark/dbms/)テスト結果を表示できます 。 これは、多数の独立したベンチマークでも確認されています。インターネット検索で見つけることは難しくありませんし、 [私達がまとめた関連リンク集](https://clickhouse.yandex/#independent-benchmarks) から見つけることもできます。 diff --git a/docs/ja/whats_new/changelog/index.md b/docs/ja/whats_new/changelog/index.md index dd382ebf5ce..0d90862ee11 100644 --- a/docs/ja/whats_new/changelog/index.md +++ b/docs/ja/whats_new/changelog/index.md @@ -249,7 +249,7 @@ toc_title: '2020' - Clickhouseテストスクリプトでハングクエリのチェックを更新 [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) - リポジトリか [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) - から数学perftestsの変更タイプ `once` に `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- 追加docker画像を構築ィコードのブラウザのhtmlレポート当社のコードベース. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin))見る [Woboqコードブラウザ](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html) +- 追加docker画像を構築ィコードのブラウザのhtmlレポート当社のコードベース. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin))見る [Woboqコードブラウザ](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) - MSanの下でいくつかのテストの失敗を抑制. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) - スピードアップ “exception while insert” テスト。 このテス [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) - 更新 `libcxx` と `libcxxabi` マスターに。 準備のために [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) diff --git a/docs/redirects.txt b/docs/redirects.txt index c0f3c81813f..1a9455355db 100644 --- a/docs/redirects.txt +++ b/docs/redirects.txt @@ -43,6 +43,7 @@ dicts/external_dicts_dict_sources.md query_language/dicts/external_dicts_dict_so dicts/external_dicts_dict_structure.md query_language/dicts/external_dicts_dict_structure.md dicts/index.md query_language/dicts/index.md dicts/internal_dicts.md query_language/dicts/internal_dicts.md +extended_roadmap.md whats_new/extended_roadmap.md formats.md interfaces/formats.md formats/capnproto.md interfaces/formats.md formats/csv.md interfaces/formats.md @@ -98,6 +99,7 @@ functions/ym_dict_functions.md query_language/functions/ym_dict_functions.md interfaces/http_interface.md interfaces/http.md interfaces/third-party_client_libraries.md interfaces/third-party/client_libraries.md interfaces/third-party_gui.md interfaces/third-party/gui.md +introduction/features_considered_disadvantages.md introduction/distinctive_features.md introduction/possible_silly_questions.md faq/general.md introduction/ya_metrika_task.md introduction/history.md operations/performance/sampling_query_profiler.md operations/optimizing_performance/sampling_query_profiler.md diff --git a/docs/ru/changelog/2017.md b/docs/ru/changelog/2017.md deleted file mode 100644 index 95156754100..00000000000 --- a/docs/ru/changelog/2017.md +++ /dev/null @@ -1,265 +0,0 @@ ---- -en_copy: true ---- - -### ClickHouse release 1.1.54327, 2017-12-21 {#clickhouse-release-1-1-54327-2017-12-21} - -This release contains bug fixes for the previous release 1.1.54318: - -- Fixed bug with possible race condition in replication that could lead to data loss. This issue affects versions 1.1.54310 and 1.1.54318. If you use one of these versions with Replicated tables, the update is strongly recommended. This issue shows in logs in Warning messages like `Part ... from own log doesn't exist.` The issue is relevant even if you don’t see these messages in logs. - -### ClickHouse release 1.1.54318, 2017-11-30 {#clickhouse-release-1-1-54318-2017-11-30} - -This release contains bug fixes for the previous release 1.1.54310: - -- Fixed incorrect row deletions during merges in the SummingMergeTree engine -- Fixed a memory leak in unreplicated MergeTree engines -- Fixed performance degradation with frequent inserts in MergeTree engines -- Fixed an issue that was causing the replication queue to stop running -- Fixed rotation and archiving of server logs - -### ClickHouse release 1.1.54310, 2017-11-01 {#clickhouse-release-1-1-54310-2017-11-01} - -#### New features: {#new-features} - -- Custom partitioning key for the MergeTree family of table engines. -- [Kafka](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) table engine. -- Added support for loading [CatBoost](https://catboost.yandex/) models and applying them to data stored in ClickHouse. -- Added support for time zones with non-integer offsets from UTC. -- Added support for arithmetic operations with time intervals. -- The range of values for the Date and DateTime types is extended to the year 2105. -- Added the `CREATE MATERIALIZED VIEW x TO y` query (specifies an existing table for storing the data of a materialized view). -- Added the `ATTACH TABLE` query without arguments. -- The processing logic for Nested columns with names ending in -Map in a SummingMergeTree table was extracted to the sumMap aggregate function. You can now specify such columns explicitly. -- Max size of the IP trie dictionary is increased to 128M entries. -- Added the getSizeOfEnumType function. -- Added the sumWithOverflow aggregate function. -- Added support for the Cap’n Proto input format. -- You can now customize compression level when using the zstd algorithm. - -#### Backward incompatible changes: {#backward-incompatible-changes} - -- Creation of temporary tables with an engine other than Memory is not allowed. -- Explicit creation of tables with the View or MaterializedView engine is not allowed. -- During table creation, a new check verifies that the sampling key expression is included in the primary key. - -#### Bug fixes: {#bug-fixes} - -- Fixed hangups when synchronously inserting into a Distributed table. -- Fixed nonatomic adding and removing of parts in Replicated tables. -- Data inserted into a materialized view is not subjected to unnecessary deduplication. -- Executing a query to a Distributed table for which the local replica is lagging and remote replicas are unavailable does not result in an error anymore. -- Users don’t need access permissions to the `default` database to create temporary tables anymore. -- Fixed crashing when specifying the Array type without arguments. -- Fixed hangups when the disk volume containing server logs is full. -- Fixed an overflow in the toRelativeWeekNum function for the first week of the Unix epoch. - -#### Build improvements: {#build-improvements} - -- Several third-party libraries (notably Poco) were updated and converted to git submodules. - -### ClickHouse release 1.1.54304, 2017-10-19 {#clickhouse-release-1-1-54304-2017-10-19} - -#### New features: {#new-features-1} - -- TLS support in the native protocol (to enable, set `tcp_ssl_port` in `config.xml` ). - -#### Bug fixes: {#bug-fixes-1} - -- `ALTER` for replicated tables now tries to start running as soon as possible. -- Fixed crashing when reading data with the setting `preferred_block_size_bytes=0.` -- Fixed crashes of `clickhouse-client` when pressing `Page Down` -- Correct interpretation of certain complex queries with `GLOBAL IN` and `UNION ALL` -- `FREEZE PARTITION` always works atomically now. -- Empty POST requests now return a response with code 411. -- Fixed interpretation errors for expressions like `CAST(1 AS Nullable(UInt8)).` -- Fixed an error when reading `Array(Nullable(String))` columns from `MergeTree` tables. -- Fixed crashing when parsing queries like `SELECT dummy AS dummy, dummy AS b` -- Users are updated correctly with invalid `users.xml` -- Correct handling when an executable dictionary returns a non-zero response code. - -### ClickHouse release 1.1.54292, 2017-09-20 {#clickhouse-release-1-1-54292-2017-09-20} - -#### New features: {#new-features-2} - -- Added the `pointInPolygon` function for working with coordinates on a coordinate plane. -- Added the `sumMap` aggregate function for calculating the sum of arrays, similar to `SummingMergeTree`. -- Added the `trunc` function. Improved performance of the rounding functions (`round`, `floor`, `ceil`, `roundToExp2`) and corrected the logic of how they work. Changed the logic of the `roundToExp2` function for fractions and negative numbers. -- The ClickHouse executable file is now less dependent on the libc version. The same ClickHouse executable file can run on a wide variety of Linux systems. There is still a dependency when using compiled queries (with the setting `compile = 1` , which is not used by default). -- Reduced the time needed for dynamic compilation of queries. - -#### Bug fixes: {#bug-fixes-2} - -- Fixed an error that sometimes produced `part ... intersects previous part` messages and weakened replica consistency. -- Fixed an error that caused the server to lock up if ZooKeeper was unavailable during shutdown. -- Removed excessive logging when restoring replicas. -- Fixed an error in the UNION ALL implementation. -- Fixed an error in the concat function that occurred if the first column in a block has the Array type. -- Progress is now displayed correctly in the system.merges table. - -### ClickHouse release 1.1.54289, 2017-09-13 {#clickhouse-release-1-1-54289-2017-09-13} - -#### New features: {#new-features-3} - -- `SYSTEM` queries for server administration: `SYSTEM RELOAD DICTIONARY`, `SYSTEM RELOAD DICTIONARIES`, `SYSTEM DROP DNS CACHE`, `SYSTEM SHUTDOWN`, `SYSTEM KILL`. -- Added functions for working with arrays: `concat`, `arraySlice`, `arrayPushBack`, `arrayPushFront`, `arrayPopBack`, `arrayPopFront`. -- Added `root` and `identity` parameters for the ZooKeeper configuration. This allows you to isolate individual users on the same ZooKeeper cluster. -- Added aggregate functions `groupBitAnd`, `groupBitOr`, and `groupBitXor` (for compatibility, they are also available under the names `BIT_AND`, `BIT_OR`, and `BIT_XOR`). -- External dictionaries can be loaded from MySQL by specifying a socket in the filesystem. -- External dictionaries can be loaded from MySQL over SSL (`ssl_cert`, `ssl_key`, `ssl_ca` parameters). -- Added the `max_network_bandwidth_for_user` setting to restrict the overall bandwidth use for queries per user. -- Support for `DROP TABLE` for temporary tables. -- Support for reading `DateTime` values in Unix timestamp format from the `CSV` and `JSONEachRow` formats. -- Lagging replicas in distributed queries are now excluded by default (the default threshold is 5 minutes). -- FIFO locking is used during ALTER: an ALTER query isn’t blocked indefinitely for continuously running queries. -- Option to set `umask` in the config file. -- Improved performance for queries with `DISTINCT` . - -#### Bug fixes: {#bug-fixes-3} - -- Improved the process for deleting old nodes in ZooKeeper. Previously, old nodes sometimes didn’t get deleted if there were very frequent inserts, which caused the server to be slow to shut down, among other things. -- Fixed randomization when choosing hosts for the connection to ZooKeeper. -- Fixed the exclusion of lagging replicas in distributed queries if the replica is localhost. -- Fixed an error where a data part in a `ReplicatedMergeTree` table could be broken after running `ALTER MODIFY` on an element in a `Nested` structure. -- Fixed an error that could cause SELECT queries to “hang”. -- Improvements to distributed DDL queries. -- Fixed the query `CREATE TABLE ... AS `. -- Resolved the deadlock in the `ALTER ... CLEAR COLUMN IN PARTITION` query for `Buffer` tables. -- Fixed the invalid default value for `Enum` s (0 instead of the minimum) when using the `JSONEachRow` and `TSKV` formats. -- Resolved the appearance of zombie processes when using a dictionary with an `executable` source. -- Fixed segfault for the HEAD query. - -#### Improved workflow for developing and assembling ClickHouse: {#improved-workflow-for-developing-and-assembling-clickhouse} - -- You can use `pbuilder` to build ClickHouse. -- You can use `libc++` instead of `libstdc++` for builds on Linux. -- Added instructions for using static code analysis tools: `Coverage`, `clang-tidy`, `cppcheck`. - -#### Please note when upgrading: {#please-note-when-upgrading} - -- There is now a higher default value for the MergeTree setting `max_bytes_to_merge_at_max_space_in_pool` (the maximum total size of data parts to merge, in bytes): it has increased from 100 GiB to 150 GiB. This might result in large merges running after the server upgrade, which could cause an increased load on the disk subsystem. If the free space available on the server is less than twice the total amount of the merges that are running, this will cause all other merges to stop running, including merges of small data parts. As a result, INSERT queries will fail with the message “Merges are processing significantly slower than inserts.” Use the `SELECT * FROM system.merges` query to monitor the situation. You can also check the `DiskSpaceReservedForMerge` metric in the `system.metrics` table, or in Graphite. You don’t need to do anything to fix this, since the issue will resolve itself once the large merges finish. If you find this unacceptable, you can restore the previous value for the `max_bytes_to_merge_at_max_space_in_pool` setting. To do this, go to the section in config.xml, set ``` ``107374182400 ``` and restart the server. - -### ClickHouse release 1.1.54284, 2017-08-29 {#clickhouse-release-1-1-54284-2017-08-29} - -- This is a bugfix release for the previous 1.1.54282 release. It fixes leaks in the parts directory in ZooKeeper. - -### ClickHouse release 1.1.54282, 2017-08-23 {#clickhouse-release-1-1-54282-2017-08-23} - -This release contains bug fixes for the previous release 1.1.54276: - -- Fixed `DB::Exception: Assertion violation: !_path.empty()` when inserting into a Distributed table. -- Fixed parsing when inserting in RowBinary format if input data starts with’;’. -- Errors during runtime compilation of certain aggregate functions (e.g. `groupArray()`). - -### Clickhouse Release 1.1.54276, 2017-08-16 {#clickhouse-release-1-1-54276-2017-08-16} - -#### New features: {#new-features-4} - -- Added an optional WITH section for a SELECT query. Example query: `WITH 1+1 AS a SELECT a, a*a` -- INSERT can be performed synchronously in a Distributed table: OK is returned only after all the data is saved on all the shards. This is activated by the setting insert\_distributed\_sync=1. -- Added the UUID data type for working with 16-byte identifiers. -- Added aliases of CHAR, FLOAT and other types for compatibility with the Tableau. -- Added the functions toYYYYMM, toYYYYMMDD, and toYYYYMMDDhhmmss for converting time into numbers. -- You can use IP addresses (together with the hostname) to identify servers for clustered DDL queries. -- Added support for non-constant arguments and negative offsets in the function `substring(str, pos, len).` -- Added the max\_size parameter for the `groupArray(max_size)(column)` aggregate function, and optimized its performance. - -#### Main changes: {#main-changes} - -- Security improvements: all server files are created with 0640 permissions (can be changed via config parameter). -- Improved error messages for queries with invalid syntax. -- Significantly reduced memory consumption and improved performance when merging large sections of MergeTree data. -- Significantly increased the performance of data merges for the ReplacingMergeTree engine. -- Improved performance for asynchronous inserts from a Distributed table by combining multiple source inserts. To enable this functionality, use the setting distributed\_directory\_monitor\_batch\_inserts=1. - -#### Backward incompatible changes: {#backward-incompatible-changes-1} - -- Changed the binary format of aggregate states of `groupArray(array_column)` functions for arrays. - -#### Complete list of changes: {#complete-list-of-changes} - -- Added the `output_format_json_quote_denormals` setting, which enables outputting nan and inf values in JSON format. -- Optimized stream allocation when reading from a Distributed table. -- Settings can be configured in readonly mode if the value doesn’t change. -- Added the ability to retrieve non-integer granules of the MergeTree engine in order to meet restrictions on the block size specified in the preferred\_block\_size\_bytes setting. The purpose is to reduce the consumption of RAM and increase cache locality when processing queries from tables with large columns. -- Efficient use of indexes that contain expressions like `toStartOfHour(x)` for conditions like `toStartOfHour(x) op сonstexpr.` -- Added new settings for MergeTree engines (the merge\_tree section in config.xml): - - replicated\_deduplication\_window\_seconds sets the number of seconds allowed for deduplicating inserts in Replicated tables. - - cleanup\_delay\_period sets how often to start cleanup to remove outdated data. - - replicated\_can\_become\_leader can prevent a replica from becoming the leader (and assigning merges). -- Accelerated cleanup to remove outdated data from ZooKeeper. -- Multiple improvements and fixes for clustered DDL queries. Of particular interest is the new setting distributed\_ddl\_task\_timeout, which limits the time to wait for a response from the servers in the cluster. If a ddl request has not been performed on all hosts, a response will contain a timeout error and a request will be executed in an async mode. -- Improved display of stack traces in the server logs. -- Added the “none” value for the compression method. -- You can use multiple dictionaries\_config sections in config.xml. -- It is possible to connect to MySQL through a socket in the file system. -- The system.parts table has a new column with information about the size of marks, in bytes. - -#### Bug fixes: {#bug-fixes-4} - -- Distributed tables using a Merge table now work correctly for a SELECT query with a condition on the `_table` field. -- Fixed a rare race condition in ReplicatedMergeTree when checking data parts. -- Fixed possible freezing on “leader election” when starting a server. -- The max\_replica\_delay\_for\_distributed\_queries setting was ignored when using a local replica of the data source. This has been fixed. -- Fixed incorrect behavior of `ALTER TABLE CLEAR COLUMN IN PARTITION` when attempting to clean a non-existing column. -- Fixed an exception in the multiIf function when using empty arrays or strings. -- Fixed excessive memory allocations when deserializing Native format. -- Fixed incorrect auto-update of Trie dictionaries. -- Fixed an exception when running queries with a GROUP BY clause from a Merge table when using SAMPLE. -- Fixed a crash of GROUP BY when using distributed\_aggregation\_memory\_efficient=1. -- Now you can specify the database.table in the right side of IN and JOIN. -- Too many threads were used for parallel aggregation. This has been fixed. -- Fixed how the “if” function works with FixedString arguments. -- SELECT worked incorrectly from a Distributed table for shards with a weight of 0. This has been fixed. -- Running `CREATE VIEW IF EXISTS no longer causes crashes.` -- Fixed incorrect behavior when input\_format\_skip\_unknown\_fields=1 is set and there are negative numbers. -- Fixed an infinite loop in the `dictGetHierarchy()` function if there is some invalid data in the dictionary. -- Fixed `Syntax error: unexpected (...)` errors when running distributed queries with subqueries in an IN or JOIN clause and Merge tables. -- Fixed an incorrect interpretation of a SELECT query from Dictionary tables. -- Fixed the “Cannot mremap” error when using arrays in IN and JOIN clauses with more than 2 billion elements. -- Fixed the failover for dictionaries with MySQL as the source. - -#### Improved workflow for developing and assembling ClickHouse: {#improved-workflow-for-developing-and-assembling-clickhouse-1} - -- Builds can be assembled in Arcadia. -- You can use gcc 7 to compile ClickHouse. -- Parallel builds using ccache+distcc are faster now. - -### ClickHouse release 1.1.54245, 2017-07-04 {#clickhouse-release-1-1-54245-2017-07-04} - -#### New features: {#new-features-5} - -- Distributed DDL (for example, `CREATE TABLE ON CLUSTER`) -- The replicated query `ALTER TABLE CLEAR COLUMN IN PARTITION.` -- The engine for Dictionary tables (access to dictionary data in the form of a table). -- Dictionary database engine (this type of database automatically has Dictionary tables available for all the connected external dictionaries). -- You can check for updates to the dictionary by sending a request to the source. -- Qualified column names -- Quoting identifiers using double quotation marks. -- Sessions in the HTTP interface. -- The OPTIMIZE query for a Replicated table can can run not only on the leader. - -#### Backward incompatible changes: {#backward-incompatible-changes-2} - -- Removed SET GLOBAL. - -#### Minor changes: {#minor-changes} - -- Now after an alert is triggered, the log prints the full stack trace. -- Relaxed the verification of the number of damaged/extra data parts at startup (there were too many false positives). - -#### Bug fixes: {#bug-fixes-5} - -- Fixed a bad connection “sticking” when inserting into a Distributed table. -- GLOBAL IN now works for a query from a Merge table that looks at a Distributed table. -- The incorrect number of cores was detected on a Google Compute Engine virtual machine. This has been fixed. -- Changes in how an executable source of cached external dictionaries works. -- Fixed the comparison of strings containing null characters. -- Fixed the comparison of Float32 primary key fields with constants. -- Previously, an incorrect estimate of the size of a field could lead to overly large allocations. -- Fixed a crash when querying a Nullable column added to a table using ALTER. -- Fixed a crash when sorting by a Nullable column, if the number of rows is less than LIMIT. -- Fixed an ORDER BY subquery consisting of only constant values. -- Previously, a Replicated table could remain in the invalid state after a failed DROP TABLE. -- Aliases for scalar subqueries with empty results are no longer lost. -- Now a query that used compilation does not fail with an error if the .so file gets damaged. diff --git a/docs/ru/changelog/2018.md b/docs/ru/changelog/2018.md deleted file mode 100644 index 49bef18cbf3..00000000000 --- a/docs/ru/changelog/2018.md +++ /dev/null @@ -1,1060 +0,0 @@ ---- -en_copy: true ---- - -## ClickHouse release 18.16 {#clickhouse-release-18-16} - -### ClickHouse release 18.16.1, 2018-12-21 {#clickhouse-release-18-16-1-2018-12-21} - -#### Bug fixes: {#bug-fixes} - -- Fixed an error that led to problems with updating dictionaries with the ODBC source. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) -- JIT compilation of aggregate functions now works with LowCardinality columns. [\#3838](https://github.com/ClickHouse/ClickHouse/issues/3838) - -#### Improvements: {#improvements} - -- Added the `low_cardinality_allow_in_native_format` setting (enabled by default). When disabled, LowCardinality columns will be converted to ordinary columns for SELECT queries and ordinary columns will be expected for INSERT queries. [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) - -#### Build improvements: {#build-improvements} - -- Fixes for builds on macOS and ARM. - -### ClickHouse release 18.16.0, 2018-12-14 {#clickhouse-release-18-16-0-2018-12-14} - -#### New features: {#new-features} - -- `DEFAULT` expressions are evaluated for missing fields when loading data in semi-structured input formats (`JSONEachRow`, `TSKV`). The feature is enabled with the `insert_sample_with_metadata` setting. [\#3555](https://github.com/ClickHouse/ClickHouse/pull/3555) -- The `ALTER TABLE` query now has the `MODIFY ORDER BY` action for changing the sorting key when adding or removing a table column. This is useful for tables in the `MergeTree` family that perform additional tasks when merging based on this sorting key, such as `SummingMergeTree`, `AggregatingMergeTree`, and so on. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) [\#3755](https://github.com/ClickHouse/ClickHouse/pull/3755) -- For tables in the `MergeTree` family, now you can specify a different sorting key (`ORDER BY`) and index (`PRIMARY KEY`). The sorting key can be longer than the index. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) -- Added the `hdfs` table function and the `HDFS` table engine for importing and exporting data to HDFS. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/3617) -- Added functions for working with base64: `base64Encode`, `base64Decode`, `tryBase64Decode`. [Alexander Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3350) -- Now you can use a parameter to configure the precision of the `uniqCombined` aggregate function (select the number of HyperLogLog cells). [\#3406](https://github.com/ClickHouse/ClickHouse/pull/3406) -- Added the `system.contributors` table that contains the names of everyone who made commits in ClickHouse. [\#3452](https://github.com/ClickHouse/ClickHouse/pull/3452) -- Added the ability to omit the partition for the `ALTER TABLE ... FREEZE` query in order to back up all partitions at once. [\#3514](https://github.com/ClickHouse/ClickHouse/pull/3514) -- Added `dictGet` and `dictGetOrDefault` functions that don’t require specifying the type of return value. The type is determined automatically from the dictionary description. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3564) -- Now you can specify comments for a column in the table description and change it using `ALTER`. [\#3377](https://github.com/ClickHouse/ClickHouse/pull/3377) -- Reading is supported for `Join` type tables with simple keys. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3728) -- Now you can specify the options `join_use_nulls`, `max_rows_in_join`, `max_bytes_in_join`, and `join_overflow_mode` when creating a `Join` type table. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3728) -- Added the `joinGet` function that allows you to use a `Join` type table like a dictionary. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3728) -- Added the `partition_key`, `sorting_key`, `primary_key`, and `sampling_key` columns to the `system.tables` table in order to provide information about table keys. [\#3609](https://github.com/ClickHouse/ClickHouse/pull/3609) -- Added the `is_in_partition_key`, `is_in_sorting_key`, `is_in_primary_key`, and `is_in_sampling_key` columns to the `system.columns` table. [\#3609](https://github.com/ClickHouse/ClickHouse/pull/3609) -- Added the `min_time` and `max_time` columns to the `system.parts` table. These columns are populated when the partitioning key is an expression consisting of `DateTime` columns. [Emmanuel Donin de Rosière](https://github.com/ClickHouse/ClickHouse/pull/3800) - -#### Bug fixes: {#bug-fixes-1} - -- Fixes and performance improvements for the `LowCardinality` data type. `GROUP BY` using `LowCardinality(Nullable(...))`. Getting the values of `extremes`. Processing high-order functions. `LEFT ARRAY JOIN`. Distributed `GROUP BY`. Functions that return `Array`. Execution of `ORDER BY`. Writing to `Distributed` tables (nicelulu). Backward compatibility for `INSERT` queries from old clients that implement the `Native` protocol. Support for `LowCardinality` for `JOIN`. Improved performance when working in a single stream. [\#3823](https://github.com/ClickHouse/ClickHouse/pull/3823) [\#3803](https://github.com/ClickHouse/ClickHouse/pull/3803) [\#3799](https://github.com/ClickHouse/ClickHouse/pull/3799) [\#3769](https://github.com/ClickHouse/ClickHouse/pull/3769) [\#3744](https://github.com/ClickHouse/ClickHouse/pull/3744) [\#3681](https://github.com/ClickHouse/ClickHouse/pull/3681) [\#3651](https://github.com/ClickHouse/ClickHouse/pull/3651) [\#3649](https://github.com/ClickHouse/ClickHouse/pull/3649) [\#3641](https://github.com/ClickHouse/ClickHouse/pull/3641) [\#3632](https://github.com/ClickHouse/ClickHouse/pull/3632) [\#3568](https://github.com/ClickHouse/ClickHouse/pull/3568) [\#3523](https://github.com/ClickHouse/ClickHouse/pull/3523) [\#3518](https://github.com/ClickHouse/ClickHouse/pull/3518) -- Fixed how the `select_sequential_consistency` option works. Previously, when this setting was enabled, an incomplete result was sometimes returned after beginning to write to a new partition. [\#2863](https://github.com/ClickHouse/ClickHouse/pull/2863) -- Databases are correctly specified when executing DDL `ON CLUSTER` queries and `ALTER UPDATE/DELETE`. [\#3772](https://github.com/ClickHouse/ClickHouse/pull/3772) [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) -- Databases are correctly specified for subqueries inside a VIEW. [\#3521](https://github.com/ClickHouse/ClickHouse/pull/3521) -- Fixed a bug in `PREWHERE` with `FINAL` for `VersionedCollapsingMergeTree`. [7167bfd7](https://github.com/ClickHouse/ClickHouse/commit/7167bfd7b365538f7a91c4307ad77e552ab4e8c1) -- Now you can use `KILL QUERY` to cancel queries that have not started yet because they are waiting for the table to be locked. [\#3517](https://github.com/ClickHouse/ClickHouse/pull/3517) -- Corrected date and time calculations if the clocks were moved back at midnight (this happens in Iran, and happened in Moscow from 1981 to 1983). Previously, this led to the time being reset a day earlier than necessary, and also caused incorrect formatting of the date and time in text format. [\#3819](https://github.com/ClickHouse/ClickHouse/pull/3819) -- Fixed bugs in some cases of `VIEW` and subqueries that omit the database. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3521) -- Fixed a race condition when simultaneously reading from a `MATERIALIZED VIEW` and deleting a `MATERIALIZED VIEW` due to not locking the internal `MATERIALIZED VIEW`. [\#3404](https://github.com/ClickHouse/ClickHouse/pull/3404) [\#3694](https://github.com/ClickHouse/ClickHouse/pull/3694) -- Fixed the error `Lock handler cannot be nullptr.` [\#3689](https://github.com/ClickHouse/ClickHouse/pull/3689) -- Fixed query processing when the `compile_expressions` option is enabled (it’s enabled by default). Nondeterministic constant expressions like the `now` function are no longer unfolded. [\#3457](https://github.com/ClickHouse/ClickHouse/pull/3457) -- Fixed a crash when specifying a non-constant scale argument in `toDecimal32/64/128` functions. -- Fixed an error when trying to insert an array with `NULL` elements in the `Values` format into a column of type `Array` without `Nullable` (if `input_format_values_interpret_expressions` = 1). [\#3487](https://github.com/ClickHouse/ClickHouse/pull/3487) [\#3503](https://github.com/ClickHouse/ClickHouse/pull/3503) -- Fixed continuous error logging in `DDLWorker` if ZooKeeper is not available. [8f50c620](https://github.com/ClickHouse/ClickHouse/commit/8f50c620334988b28018213ec0092fe6423847e2) -- Fixed the return type for `quantile*` functions from `Date` and `DateTime` types of arguments. [\#3580](https://github.com/ClickHouse/ClickHouse/pull/3580) -- Fixed the `WITH` clause if it specifies a simple alias without expressions. [\#3570](https://github.com/ClickHouse/ClickHouse/pull/3570) -- Fixed processing of queries with named sub-queries and qualified column names when `enable_optimize_predicate_expression` is enabled. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3588) -- Fixed the error `Attempt to attach to nullptr thread group` when working with materialized views. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3623) -- Fixed a crash when passing certain incorrect arguments to the `arrayReverse` function. [73e3a7b6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) -- Fixed the buffer overflow in the `extractURLParameter` function. Improved performance. Added correct processing of strings containing zero bytes. [141e9799](https://github.com/ClickHouse/ClickHouse/commit/141e9799e49201d84ea8e951d1bed4fb6d3dacb5) -- Fixed buffer overflow in the `lowerUTF8` and `upperUTF8` functions. Removed the ability to execute these functions over `FixedString` type arguments. [\#3662](https://github.com/ClickHouse/ClickHouse/pull/3662) -- Fixed a rare race condition when deleting `MergeTree` tables. [\#3680](https://github.com/ClickHouse/ClickHouse/pull/3680) -- Fixed a race condition when reading from `Buffer` tables and simultaneously performing `ALTER` or `DROP` on the target tables. [\#3719](https://github.com/ClickHouse/ClickHouse/pull/3719) -- Fixed a segfault if the `max_temporary_non_const_columns` limit was exceeded. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) - -#### Improvements: {#improvements-1} - -- The server does not write the processed configuration files to the `/etc/clickhouse-server/` directory. Instead, it saves them in the `preprocessed_configs` directory inside `path`. This means that the `/etc/clickhouse-server/` directory doesn’t have write access for the `clickhouse` user, which improves security. [\#2443](https://github.com/ClickHouse/ClickHouse/pull/2443) -- The `min_merge_bytes_to_use_direct_io` option is set to 10 GiB by default. A merge that forms large parts of tables from the MergeTree family will be performed in `O_DIRECT` mode, which prevents excessive page cache eviction. [\#3504](https://github.com/ClickHouse/ClickHouse/pull/3504) -- Accelerated server start when there is a very large number of tables. [\#3398](https://github.com/ClickHouse/ClickHouse/pull/3398) -- Added a connection pool and HTTP `Keep-Alive` for connections between replicas. [\#3594](https://github.com/ClickHouse/ClickHouse/pull/3594) -- If the query syntax is invalid, the `400 Bad Request` code is returned in the `HTTP` interface (500 was returned previously). [31bc680a](https://github.com/ClickHouse/ClickHouse/commit/31bc680ac5f4bb1d0360a8ba4696fa84bb47d6ab) -- The `join_default_strictness` option is set to `ALL` by default for compatibility. [120e2cbe](https://github.com/ClickHouse/ClickHouse/commit/120e2cbe2ff4fbad626c28042d9b28781c805afe) -- Removed logging to `stderr` from the `re2` library for invalid or complex regular expressions. [\#3723](https://github.com/ClickHouse/ClickHouse/pull/3723) -- Added for the `Kafka` table engine: checks for subscriptions before beginning to read from Kafka; the kafka\_max\_block\_size setting for the table. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3396) -- The `cityHash64`, `farmHash64`, `metroHash64`, `sipHash64`, `halfMD5`, `murmurHash2_32`, `murmurHash2_64`, `murmurHash3_32`, and `murmurHash3_64` functions now work for any number of arguments and for arguments in the form of tuples. [\#3451](https://github.com/ClickHouse/ClickHouse/pull/3451) [\#3519](https://github.com/ClickHouse/ClickHouse/pull/3519) -- The `arrayReverse` function now works with any types of arrays. [73e3a7b6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) -- Added an optional parameter: the slot size for the `timeSlots` function. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3724) -- For `FULL` and `RIGHT JOIN`, the `max_block_size` setting is used for a stream of non-joined data from the right table. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3699) -- Added the `--secure` command line parameter in `clickhouse-benchmark` and `clickhouse-performance-test` to enable TLS. [\#3688](https://github.com/ClickHouse/ClickHouse/pull/3688) [\#3690](https://github.com/ClickHouse/ClickHouse/pull/3690) -- Type conversion when the structure of a `Buffer` type table does not match the structure of the destination table. [Vitaly Baranov](https://github.com/ClickHouse/ClickHouse/pull/3603) -- Added the `tcp_keep_alive_timeout` option to enable keep-alive packets after inactivity for the specified time interval. [\#3441](https://github.com/ClickHouse/ClickHouse/pull/3441) -- Removed unnecessary quoting of values for the partition key in the `system.parts` table if it consists of a single column. [\#3652](https://github.com/ClickHouse/ClickHouse/pull/3652) -- The modulo function works for `Date` and `DateTime` data types. [\#3385](https://github.com/ClickHouse/ClickHouse/pull/3385) -- Added synonyms for the `POWER`, `LN`, `LCASE`, `UCASE`, `REPLACE`, `LOCATE`, `SUBSTR`, and `MID` functions. [\#3774](https://github.com/ClickHouse/ClickHouse/pull/3774) [\#3763](https://github.com/ClickHouse/ClickHouse/pull/3763) Some function names are case-insensitive for compatibility with the SQL standard. Added syntactic sugar `SUBSTRING(expr FROM start FOR length)` for compatibility with SQL. [\#3804](https://github.com/ClickHouse/ClickHouse/pull/3804) -- Added the ability to `mlock` memory pages corresponding to `clickhouse-server` executable code to prevent it from being forced out of memory. This feature is disabled by default. [\#3553](https://github.com/ClickHouse/ClickHouse/pull/3553) -- Improved performance when reading from `O_DIRECT` (with the `min_bytes_to_use_direct_io` option enabled). [\#3405](https://github.com/ClickHouse/ClickHouse/pull/3405) -- Improved performance of the `dictGet...OrDefault` function for a constant key argument and a non-constant default argument. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3563) -- The `firstSignificantSubdomain` function now processes the domains `gov`, `mil`, and `edu`. [Igor Hatarist](https://github.com/ClickHouse/ClickHouse/pull/3601) Improved performance. [\#3628](https://github.com/ClickHouse/ClickHouse/pull/3628) -- Ability to specify custom environment variables for starting `clickhouse-server` using the `SYS-V init.d` script by defining `CLICKHOUSE_PROGRAM_ENV` in `/etc/default/clickhouse`. - [Pavlo Bashynskyi](https://github.com/ClickHouse/ClickHouse/pull/3612) -- Correct return code for the clickhouse-server init script. [\#3516](https://github.com/ClickHouse/ClickHouse/pull/3516) -- The `system.metrics` table now has the `VersionInteger` metric, and `system.build_options` has the added line `VERSION_INTEGER`, which contains the numeric form of the ClickHouse version, such as `18016000`. [\#3644](https://github.com/ClickHouse/ClickHouse/pull/3644) -- Removed the ability to compare the `Date` type with a number to avoid potential errors like `date = 2018-12-17`, where quotes around the date are omitted by mistake. [\#3687](https://github.com/ClickHouse/ClickHouse/pull/3687) -- Fixed the behavior of stateful functions like `rowNumberInAllBlocks`. They previously output a result that was one number larger due to starting during query analysis. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3729) -- If the `force_restore_data` file can’t be deleted, an error message is displayed. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3794) - -#### Build improvements: {#build-improvements-1} - -- Updated the `jemalloc` library, which fixes a potential memory leak. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3557) -- Profiling with `jemalloc` is enabled by default in order to debug builds. [2cc82f5c](https://github.com/ClickHouse/ClickHouse/commit/2cc82f5cbe266421cd4c1165286c2c47e5ffcb15) -- Added the ability to run integration tests when only `Docker` is installed on the system. [\#3650](https://github.com/ClickHouse/ClickHouse/pull/3650) -- Added the fuzz expression test in SELECT queries. [\#3442](https://github.com/ClickHouse/ClickHouse/pull/3442) -- Added a stress test for commits, which performs functional tests in parallel and in random order to detect more race conditions. [\#3438](https://github.com/ClickHouse/ClickHouse/pull/3438) -- Improved the method for starting clickhouse-server in a Docker image. [Elghazal Ahmed](https://github.com/ClickHouse/ClickHouse/pull/3663) -- For a Docker image, added support for initializing databases using files in the `/docker-entrypoint-initdb.d` directory. [Konstantin Lebedev](https://github.com/ClickHouse/ClickHouse/pull/3695) -- Fixes for builds on ARM. [\#3709](https://github.com/ClickHouse/ClickHouse/pull/3709) - -#### Backward incompatible changes: {#backward-incompatible-changes} - -- Removed the ability to compare the `Date` type with a number. Instead of `toDate('2018-12-18') = 17883`, you must use explicit type conversion `= toDate(17883)` [\#3687](https://github.com/ClickHouse/ClickHouse/pull/3687) - -## ClickHouse release 18.14 {#clickhouse-release-18-14} - -### ClickHouse release 18.14.19, 2018-12-19 {#clickhouse-release-18-14-19-2018-12-19} - -#### Bug fixes: {#bug-fixes-2} - -- Fixed an error that led to problems with updating dictionaries with the ODBC source. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) -- Databases are correctly specified when executing DDL `ON CLUSTER` queries. [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) -- Fixed a segfault if the `max_temporary_non_const_columns` limit was exceeded. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) - -#### Build improvements: {#build-improvements-2} - -- Fixes for builds on ARM. - -### ClickHouse release 18.14.18, 2018-12-04 {#clickhouse-release-18-14-18-2018-12-04} - -#### Bug fixes: {#bug-fixes-3} - -- Fixed error in `dictGet...` function for dictionaries of type `range`, if one of the arguments is constant and other is not. [\#3751](https://github.com/ClickHouse/ClickHouse/pull/3751) -- Fixed error that caused messages `netlink: '...': attribute type 1 has an invalid length` to be printed in Linux kernel log, that was happening only on fresh enough versions of Linux kernel. [\#3749](https://github.com/ClickHouse/ClickHouse/pull/3749) -- Fixed segfault in function `empty` for argument of `FixedString` type. [Daniel, Dao Quang Minh](https://github.com/ClickHouse/ClickHouse/pull/3703) -- Fixed excessive memory allocation when using large value of `max_query_size` setting (a memory chunk of `max_query_size` bytes was preallocated at once). [\#3720](https://github.com/ClickHouse/ClickHouse/pull/3720) - -#### Build changes: {#build-changes} - -- Fixed build with LLVM/Clang libraries of version 7 from the OS packages (these libraries are used for runtime query compilation). [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) - -### ClickHouse release 18.14.17, 2018-11-30 {#clickhouse-release-18-14-17-2018-11-30} - -#### Bug fixes: {#bug-fixes-4} - -- Fixed cases when the ODBC bridge process did not terminate with the main server process. [\#3642](https://github.com/ClickHouse/ClickHouse/pull/3642) -- Fixed synchronous insertion into the `Distributed` table with a columns list that differs from the column list of the remote table. [\#3673](https://github.com/ClickHouse/ClickHouse/pull/3673) -- Fixed a rare race condition that can lead to a crash when dropping a MergeTree table. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) -- Fixed a query deadlock in case when query thread creation fails with the `Resource temporarily unavailable` error. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) -- Fixed parsing of the `ENGINE` clause when the `CREATE AS table` syntax was used and the `ENGINE` clause was specified before the `AS table` (the error resulted in ignoring the specified engine). [\#3692](https://github.com/ClickHouse/ClickHouse/pull/3692) - -### ClickHouse release 18.14.15, 2018-11-21 {#clickhouse-release-18-14-15-2018-11-21} - -#### Bug fixes: {#bug-fixes-5} - -- The size of memory chunk was overestimated while deserializing the column of type `Array(String)` that leads to “Memory limit exceeded” errors. The issue appeared in version 18.12.13. [\#3589](https://github.com/ClickHouse/ClickHouse/issues/3589) - -### ClickHouse release 18.14.14, 2018-11-20 {#clickhouse-release-18-14-14-2018-11-20} - -#### Bug fixes: {#bug-fixes-6} - -- Fixed `ON CLUSTER` queries when cluster configured as secure (flag ``). [\#3599](https://github.com/ClickHouse/ClickHouse/pull/3599) - -#### Build changes: {#build-changes-1} - -- Fixed problems (llvm-7 from system, macos) [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) - -### ClickHouse release 18.14.13, 2018-11-08 {#clickhouse-release-18-14-13-2018-11-08} - -#### Bug fixes: {#bug-fixes-7} - -- Fixed the `Block structure mismatch in MergingSorted stream` error. [\#3162](https://github.com/ClickHouse/ClickHouse/issues/3162) -- Fixed `ON CLUSTER` queries in case when secure connections were turned on in the cluster config (the `` flag). [\#3465](https://github.com/ClickHouse/ClickHouse/pull/3465) -- Fixed an error in queries that used `SAMPLE`, `PREWHERE` and alias columns. [\#3543](https://github.com/ClickHouse/ClickHouse/pull/3543) -- Fixed a rare `unknown compression method` error when the `min_bytes_to_use_direct_io` setting was enabled. [3544](https://github.com/ClickHouse/ClickHouse/pull/3544) - -#### Performance improvements: {#performance-improvements} - -- Fixed performance regression of queries with `GROUP BY` of columns of UInt16 or Date type when executing on AMD EPYC processors. [Igor Lapko](https://github.com/ClickHouse/ClickHouse/pull/3512) -- Fixed performance regression of queries that process long strings. [\#3530](https://github.com/ClickHouse/ClickHouse/pull/3530) - -#### Build improvements: {#build-improvements-3} - -- Improvements for simplifying the Arcadia build. [\#3475](https://github.com/ClickHouse/ClickHouse/pull/3475), [\#3535](https://github.com/ClickHouse/ClickHouse/pull/3535) - -### ClickHouse release 18.14.12, 2018-11-02 {#clickhouse-release-18-14-12-2018-11-02} - -#### Bug fixes: {#bug-fixes-8} - -- Fixed a crash on joining two unnamed subqueries. [\#3505](https://github.com/ClickHouse/ClickHouse/pull/3505) -- Fixed generating incorrect queries (with an empty `WHERE` clause) when querying external databases. [hotid](https://github.com/ClickHouse/ClickHouse/pull/3477) -- Fixed using an incorrect timeout value in ODBC dictionaries. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3511) - -### ClickHouse release 18.14.11, 2018-10-29 {#clickhouse-release-18-14-11-2018-10-29} - -#### Bug fixes: {#bug-fixes-9} - -- Fixed the error `Block structure mismatch in UNION stream: different number of columns` in LIMIT queries. [\#2156](https://github.com/ClickHouse/ClickHouse/issues/2156) -- Fixed errors when merging data in tables containing arrays inside Nested structures. [\#3397](https://github.com/ClickHouse/ClickHouse/pull/3397) -- Fixed incorrect query results if the `merge_tree_uniform_read_distribution` setting is disabled (it is enabled by default). [\#3429](https://github.com/ClickHouse/ClickHouse/pull/3429) -- Fixed an error on inserts to a Distributed table in Native format. [\#3411](https://github.com/ClickHouse/ClickHouse/issues/3411) - -### ClickHouse release 18.14.10, 2018-10-23 {#clickhouse-release-18-14-10-2018-10-23} - -- The `compile_expressions` setting (JIT compilation of expressions) is disabled by default. [\#3410](https://github.com/ClickHouse/ClickHouse/pull/3410) -- The `enable_optimize_predicate_expression` setting is disabled by default. - -### ClickHouse release 18.14.9, 2018-10-16 {#clickhouse-release-18-14-9-2018-10-16} - -#### New features: {#new-features-1} - -- The `WITH CUBE` modifier for `GROUP BY` (the alternative syntax `GROUP BY CUBE(...)` is also available). [\#3172](https://github.com/ClickHouse/ClickHouse/pull/3172) -- Added the `formatDateTime` function. [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/2770) -- Added the `JDBC` table engine and `jdbc` table function (requires installing clickhouse-jdbc-bridge). [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) -- Added functions for working with the ISO week number: `toISOWeek`, `toISOYear`, `toStartOfISOYear`, and `toDayOfYear`. [\#3146](https://github.com/ClickHouse/ClickHouse/pull/3146) -- Now you can use `Nullable` columns for `MySQL` and `ODBC` tables. [\#3362](https://github.com/ClickHouse/ClickHouse/pull/3362) -- Nested data structures can be read as nested objects in `JSONEachRow` format. Added the `input_format_import_nested_json` setting. [Veloman Yunkan](https://github.com/ClickHouse/ClickHouse/pull/3144) -- Parallel processing is available for many `MATERIALIZED VIEW`s when inserting data. See the `parallel_view_processing` setting. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3208) -- Added the `SYSTEM FLUSH LOGS` query (forced log flushes to system tables such as `query_log`) [\#3321](https://github.com/ClickHouse/ClickHouse/pull/3321) -- Now you can use pre-defined `database` and `table` macros when declaring `Replicated` tables. [\#3251](https://github.com/ClickHouse/ClickHouse/pull/3251) -- Added the ability to read `Decimal` type values in engineering notation (indicating powers of ten). [\#3153](https://github.com/ClickHouse/ClickHouse/pull/3153) - -#### Experimental features: {#experimental-features} - -- Optimization of the GROUP BY clause for `LowCardinality data types.` [\#3138](https://github.com/ClickHouse/ClickHouse/pull/3138) -- Optimized calculation of expressions for `LowCardinality data types.` [\#3200](https://github.com/ClickHouse/ClickHouse/pull/3200) - -#### Improvements: {#improvements-2} - -- Significantly reduced memory consumption for queries with `ORDER BY` and `LIMIT`. See the `max_bytes_before_remerge_sort` setting. [\#3205](https://github.com/ClickHouse/ClickHouse/pull/3205) -- In the absence of `JOIN` (`LEFT`, `INNER`, …), `INNER JOIN` is assumed. [\#3147](https://github.com/ClickHouse/ClickHouse/pull/3147) -- Qualified asterisks work correctly in queries with `JOIN`. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3202) -- The `ODBC` table engine correctly chooses the method for quoting identifiers in the SQL dialect of a remote database. [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) -- The `compile_expressions` setting (JIT compilation of expressions) is enabled by default. -- Fixed behavior for simultaneous DROP DATABASE/TABLE IF EXISTS and CREATE DATABASE/TABLE IF NOT EXISTS. Previously, a `CREATE DATABASE ... IF NOT EXISTS` query could return the error message “File … already exists”, and the `CREATE TABLE ... IF NOT EXISTS` and `DROP TABLE IF EXISTS` queries could return `Table ... is creating or attaching right now`. [\#3101](https://github.com/ClickHouse/ClickHouse/pull/3101) -- LIKE and IN expressions with a constant right half are passed to the remote server when querying from MySQL or ODBC tables. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) -- Comparisons with constant expressions in a WHERE clause are passed to the remote server when querying from MySQL and ODBC tables. Previously, only comparisons with constants were passed. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) -- Correct calculation of row width in the terminal for `Pretty` formats, including strings with hieroglyphs. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3257). -- `ON CLUSTER` can be specified for `ALTER UPDATE` queries. -- Improved performance for reading data in `JSONEachRow` format. [\#3332](https://github.com/ClickHouse/ClickHouse/pull/3332) -- Added synonyms for the `LENGTH` and `CHARACTER_LENGTH` functions for compatibility. The `CONCAT` function is no longer case-sensitive. [\#3306](https://github.com/ClickHouse/ClickHouse/pull/3306) -- Added the `TIMESTAMP` synonym for the `DateTime` type. [\#3390](https://github.com/ClickHouse/ClickHouse/pull/3390) -- There is always space reserved for query\_id in the server logs, even if the log line is not related to a query. This makes it easier to parse server text logs with third-party tools. -- Memory consumption by a query is logged when it exceeds the next level of an integer number of gigabytes. [\#3205](https://github.com/ClickHouse/ClickHouse/pull/3205) -- Added compatibility mode for the case when the client library that uses the Native protocol sends fewer columns by mistake than the server expects for the INSERT query. This scenario was possible when using the clickhouse-cpp library. Previously, this scenario caused the server to crash. [\#3171](https://github.com/ClickHouse/ClickHouse/pull/3171) -- In a user-defined WHERE expression in `clickhouse-copier`, you can now use a `partition_key` alias (for additional filtering by source table partition). This is useful if the partitioning scheme changes during copying, but only changes slightly. [\#3166](https://github.com/ClickHouse/ClickHouse/pull/3166) -- The workflow of the `Kafka` engine has been moved to a background thread pool in order to automatically reduce the speed of data reading at high loads. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3215). -- Support for reading `Tuple` and `Nested` values of structures like `struct` in the `Cap'n'Proto format`. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3216) -- The list of top-level domains for the `firstSignificantSubdomain` function now includes the domain `biz`. [decaseal](https://github.com/ClickHouse/ClickHouse/pull/3219) -- In the configuration of external dictionaries, `null_value` is interpreted as the value of the default data type. [\#3330](https://github.com/ClickHouse/ClickHouse/pull/3330) -- Support for the `intDiv` and `intDivOrZero` functions for `Decimal`. [b48402e8](https://github.com/ClickHouse/ClickHouse/commit/b48402e8712e2b9b151e0eef8193811d433a1264) -- Support for the `Date`, `DateTime`, `UUID`, and `Decimal` types as a key for the `sumMap` aggregate function. [\#3281](https://github.com/ClickHouse/ClickHouse/pull/3281) -- Support for the `Decimal` data type in external dictionaries. [\#3324](https://github.com/ClickHouse/ClickHouse/pull/3324) -- Support for the `Decimal` data type in `SummingMergeTree` tables. [\#3348](https://github.com/ClickHouse/ClickHouse/pull/3348) -- Added specializations for `UUID` in `if`. [\#3366](https://github.com/ClickHouse/ClickHouse/pull/3366) -- Reduced the number of `open` and `close` system calls when reading from a `MergeTree table`. [\#3283](https://github.com/ClickHouse/ClickHouse/pull/3283) -- A `TRUNCATE TABLE` query can be executed on any replica (the query is passed to the leader replica). [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3375) - -#### Bug fixes: {#bug-fixes-10} - -- Fixed an issue with `Dictionary` tables for `range_hashed` dictionaries. This error occurred in version 18.12.17. [\#1702](https://github.com/ClickHouse/ClickHouse/pull/1702) -- Fixed an error when loading `range_hashed` dictionaries (the message `Unsupported type Nullable (...)`). This error occurred in version 18.12.17. [\#3362](https://github.com/ClickHouse/ClickHouse/pull/3362) -- Fixed errors in the `pointInPolygon` function due to the accumulation of inaccurate calculations for polygons with a large number of vertices located close to each other. [\#3331](https://github.com/ClickHouse/ClickHouse/pull/3331) [\#3341](https://github.com/ClickHouse/ClickHouse/pull/3341) -- If after merging data parts, the checksum for the resulting part differs from the result of the same merge in another replica, the result of the merge is deleted and the data part is downloaded from the other replica (this is the correct behavior). But after downloading the data part, it couldn’t be added to the working set because of an error that the part already exists (because the data part was deleted with some delay after the merge). This led to cyclical attempts to download the same data. [\#3194](https://github.com/ClickHouse/ClickHouse/pull/3194) -- Fixed incorrect calculation of total memory consumption by queries (because of incorrect calculation, the `max_memory_usage_for_all_queries` setting worked incorrectly and the `MemoryTracking` metric had an incorrect value). This error occurred in version 18.12.13. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3344) -- Fixed the functionality of `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` This error occurred in version 18.12.13. [\#3247](https://github.com/ClickHouse/ClickHouse/pull/3247) -- Fixed unnecessary preparation of data structures for `JOIN`s on the server that initiates the query if the `JOIN` is only performed on remote servers. [\#3340](https://github.com/ClickHouse/ClickHouse/pull/3340) -- Fixed bugs in the `Kafka` engine: deadlocks after exceptions when starting to read data, and locks upon completion [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3215). -- For `Kafka` tables, the optional `schema` parameter was not passed (the schema of the `Cap'n'Proto` format). [Vojtech Splichal](https://github.com/ClickHouse/ClickHouse/pull/3150) -- If the ensemble of ZooKeeper servers has servers that accept the connection but then immediately close it instead of responding to the handshake, ClickHouse chooses to connect another server. Previously, this produced the error `Cannot read all data. Bytes read: 0. Bytes expected: 4.` and the server couldn’t start. [8218cf3a](https://github.com/ClickHouse/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) -- If the ensemble of ZooKeeper servers contains servers for which the DNS query returns an error, these servers are ignored. [17b8e209](https://github.com/ClickHouse/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) -- Fixed type conversion between `Date` and `DateTime` when inserting data in the `VALUES` format (if `input_format_values_interpret_expressions = 1`). Previously, the conversion was performed between the numerical value of the number of days in Unix Epoch time and the Unix timestamp, which led to unexpected results. [\#3229](https://github.com/ClickHouse/ClickHouse/pull/3229) -- Corrected type conversion between `Decimal` and integer numbers. [\#3211](https://github.com/ClickHouse/ClickHouse/pull/3211) -- Fixed errors in the `enable_optimize_predicate_expression` setting. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3231) -- Fixed a parsing error in CSV format with floating-point numbers if a non-default CSV separator is used, such as `;` [\#3155](https://github.com/ClickHouse/ClickHouse/pull/3155) -- Fixed the `arrayCumSumNonNegative` function (it does not accumulate negative values if the accumulator is less than zero). [Aleksey Studnev](https://github.com/ClickHouse/ClickHouse/pull/3163) -- Fixed how `Merge` tables work on top of `Distributed` tables when using `PREWHERE`. [\#3165](https://github.com/ClickHouse/ClickHouse/pull/3165) -- Bug fixes in the `ALTER UPDATE` query. -- Fixed bugs in the `odbc` table function that appeared in version 18.12. [\#3197](https://github.com/ClickHouse/ClickHouse/pull/3197) -- Fixed the operation of aggregate functions with `StateArray` combinators. [\#3188](https://github.com/ClickHouse/ClickHouse/pull/3188) -- Fixed a crash when dividing a `Decimal` value by zero. [69dd6609](https://github.com/ClickHouse/ClickHouse/commit/69dd6609193beb4e7acd3e6ad216eca0ccfb8179) -- Fixed output of types for operations using `Decimal` and integer arguments. [\#3224](https://github.com/ClickHouse/ClickHouse/pull/3224) -- Fixed the segfault during `GROUP BY` on `Decimal128`. [3359ba06](https://github.com/ClickHouse/ClickHouse/commit/3359ba06c39fcd05bfdb87d6c64154819621e13a) -- The `log_query_threads` setting (logging information about each thread of query execution) now takes effect only if the `log_queries` option (logging information about queries) is set to 1. Since the `log_query_threads` option is enabled by default, information about threads was previously logged even if query logging was disabled. [\#3241](https://github.com/ClickHouse/ClickHouse/pull/3241) -- Fixed an error in the distributed operation of the quantiles aggregate function (the error message `Not found column quantile...`). [292a8855](https://github.com/ClickHouse/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) -- Fixed the compatibility problem when working on a cluster of version 18.12.17 servers and older servers at the same time. For distributed queries with GROUP BY keys of both fixed and non-fixed length, if there was a large amount of data to aggregate, the returned data was not always fully aggregated (two different rows contained the same aggregation keys). [\#3254](https://github.com/ClickHouse/ClickHouse/pull/3254) -- Fixed handling of substitutions in `clickhouse-performance-test`, if the query contains only part of the substitutions declared in the test. [\#3263](https://github.com/ClickHouse/ClickHouse/pull/3263) -- Fixed an error when using `FINAL` with `PREWHERE`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) -- Fixed an error when using `PREWHERE` over columns that were added during `ALTER`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) -- Added a check for the absence of `arrayJoin` for `DEFAULT` and `MATERIALIZED` expressions. Previously, `arrayJoin` led to an error when inserting data. [\#3337](https://github.com/ClickHouse/ClickHouse/pull/3337) -- Added a check for the absence of `arrayJoin` in a `PREWHERE` clause. Previously, this led to messages like `Size ... doesn't match` or `Unknown compression method` when executing queries. [\#3357](https://github.com/ClickHouse/ClickHouse/pull/3357) -- Fixed segfault that could occur in rare cases after optimization that replaced AND chains from equality evaluations with the corresponding IN expression. [liuyimin-bytedance](https://github.com/ClickHouse/ClickHouse/pull/3339) -- Minor corrections to `clickhouse-benchmark`: previously, client information was not sent to the server; now the number of queries executed is calculated more accurately when shutting down and for limiting the number of iterations. [\#3351](https://github.com/ClickHouse/ClickHouse/pull/3351) [\#3352](https://github.com/ClickHouse/ClickHouse/pull/3352) - -#### Backward incompatible changes: {#backward-incompatible-changes-1} - -- Removed the `allow_experimental_decimal_type` option. The `Decimal` data type is available for default use. [\#3329](https://github.com/ClickHouse/ClickHouse/pull/3329) - -## ClickHouse release 18.12 {#clickhouse-release-18-12} - -### ClickHouse release 18.12.17, 2018-09-16 {#clickhouse-release-18-12-17-2018-09-16} - -#### New features: {#new-features-2} - -- `invalidate_query` (the ability to specify a query to check whether an external dictionary needs to be updated) is implemented for the `clickhouse` source. [\#3126](https://github.com/ClickHouse/ClickHouse/pull/3126) -- Added the ability to use `UInt*`, `Int*`, and `DateTime` data types (along with the `Date` type) as a `range_hashed` external dictionary key that defines the boundaries of ranges. Now `NULL` can be used to designate an open range. [Vasily Nemkov](https://github.com/ClickHouse/ClickHouse/pull/3123) -- The `Decimal` type now supports `var*` and `stddev*` aggregate functions. [\#3129](https://github.com/ClickHouse/ClickHouse/pull/3129) -- The `Decimal` type now supports mathematical functions (`exp`, `sin` and so on.) [\#3129](https://github.com/ClickHouse/ClickHouse/pull/3129) -- The `system.part_log` table now has the `partition_id` column. [\#3089](https://github.com/ClickHouse/ClickHouse/pull/3089) - -#### Bug fixes: {#bug-fixes-11} - -- `Merge` now works correctly on `Distributed` tables. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3159) -- Fixed incompatibility (unnecessary dependency on the `glibc` version) that made it impossible to run ClickHouse on `Ubuntu Precise` and older versions. The incompatibility arose in version 18.12.13. [\#3130](https://github.com/ClickHouse/ClickHouse/pull/3130) -- Fixed errors in the `enable_optimize_predicate_expression` setting. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) -- Fixed a minor issue with backwards compatibility that appeared when working with a cluster of replicas on versions earlier than 18.12.13 and simultaneously creating a new replica of a table on a server with a newer version (shown in the message `Can not clone replica, because the ... updated to new ClickHouse version`, which is logical, but shouldn’t happen). [\#3122](https://github.com/ClickHouse/ClickHouse/pull/3122) - -#### Backward incompatible changes: {#backward-incompatible-changes-2} - -- The `enable_optimize_predicate_expression` option is enabled by default (which is rather optimistic). If query analysis errors occur that are related to searching for the column names, set `enable_optimize_predicate_expression` to 0. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) - -### ClickHouse release 18.12.14, 2018-09-13 {#clickhouse-release-18-12-14-2018-09-13} - -#### New features: {#new-features-3} - -- Added support for `ALTER UPDATE` queries. [\#3035](https://github.com/ClickHouse/ClickHouse/pull/3035) -- Added the `allow_ddl` option, which restricts the user’s access to DDL queries. [\#3104](https://github.com/ClickHouse/ClickHouse/pull/3104) -- Added the `min_merge_bytes_to_use_direct_io` option for `MergeTree` engines, which allows you to set a threshold for the total size of the merge (when above the threshold, data part files will be handled using O\_DIRECT). [\#3117](https://github.com/ClickHouse/ClickHouse/pull/3117) -- The `system.merges` system table now contains the `partition_id` column. [\#3099](https://github.com/ClickHouse/ClickHouse/pull/3099) - -#### Improvements {#improvements-3} - -- If a data part remains unchanged during mutation, it isn’t downloaded by replicas. [\#3103](https://github.com/ClickHouse/ClickHouse/pull/3103) -- Autocomplete is available for names of settings when working with `clickhouse-client`. [\#3106](https://github.com/ClickHouse/ClickHouse/pull/3106) - -#### Bug fixes: {#bug-fixes-12} - -- Added a check for the sizes of arrays that are elements of `Nested` type fields when inserting. [\#3118](https://github.com/ClickHouse/ClickHouse/pull/3118) -- Fixed an error updating external dictionaries with the `ODBC` source and `hashed` storage. This error occurred in version 18.12.13. -- Fixed a crash when creating a temporary table from a query with an `IN` condition. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3098) -- Fixed an error in aggregate functions for arrays that can have `NULL` elements. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/3097) - -### ClickHouse release 18.12.13, 2018-09-10 {#clickhouse-release-18-12-13-2018-09-10} - -#### New features: {#new-features-4} - -- Added the `DECIMAL(digits, scale)` data type (`Decimal32(scale)`, `Decimal64(scale)`, `Decimal128(scale)`). To enable it, use the setting `allow_experimental_decimal_type`. [\#2846](https://github.com/ClickHouse/ClickHouse/pull/2846) [\#2970](https://github.com/ClickHouse/ClickHouse/pull/2970) [\#3008](https://github.com/ClickHouse/ClickHouse/pull/3008) [\#3047](https://github.com/ClickHouse/ClickHouse/pull/3047) -- New `WITH ROLLUP` modifier for `GROUP BY` (alternative syntax: `GROUP BY ROLLUP(...)`). [\#2948](https://github.com/ClickHouse/ClickHouse/pull/2948) -- In queries with JOIN, the star character expands to a list of columns in all tables, in compliance with the SQL standard. You can restore the old behavior by setting `asterisk_left_columns_only` to 1 on the user configuration level. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2787) -- Added support for JOIN with table functions. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) -- Autocomplete by pressing Tab in clickhouse-client. [Sergey Shcherbin](https://github.com/ClickHouse/ClickHouse/pull/2447) -- Ctrl+C in clickhouse-client clears a query that was entered. [\#2877](https://github.com/ClickHouse/ClickHouse/pull/2877) -- Added the `join_default_strictness` setting (values: `"`, `'any'`, `'all'`). This allows you to not specify `ANY` or `ALL` for `JOIN`. [\#2982](https://github.com/ClickHouse/ClickHouse/pull/2982) -- Each line of the server log related to query processing shows the query ID. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Now you can get query execution logs in clickhouse-client (use the `send_logs_level` setting). With distributed query processing, logs are cascaded from all the servers. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- The `system.query_log` and `system.processes` (`SHOW PROCESSLIST`) tables now have information about all changed settings when you run a query (the nested structure of the `Settings` data). Added the `log_query_settings` setting. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- The `system.query_log` and `system.processes` tables now show information about the number of threads that are participating in query execution (see the `thread_numbers` column). [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Added `ProfileEvents` counters that measure the time spent on reading and writing over the network and reading and writing to disk, the number of network errors, and the time spent waiting when network bandwidth is limited. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Added `ProfileEvents`counters that contain the system metrics from rusage (you can use them to get information about CPU usage in userspace and the kernel, page faults, and context switches), as well as taskstats metrics (use these to obtain information about I/O wait time, CPU wait time, and the amount of data read and recorded, both with and without page cache). [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- The `ProfileEvents` counters are applied globally and for each query, as well as for each query execution thread, which allows you to profile resource consumption by query in detail. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- Added the `system.query_thread_log` table, which contains information about each query execution thread. Added the `log_query_threads` setting. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) -- The `system.metrics` and `system.events` tables now have built-in documentation. [\#3016](https://github.com/ClickHouse/ClickHouse/pull/3016) -- Added the `arrayEnumerateDense` function. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2975) -- Added the `arrayCumSumNonNegative` and `arrayDifference` functions. [Aleksey Studnev](https://github.com/ClickHouse/ClickHouse/pull/2942) -- Added the `retention` aggregate function. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2887) -- Now you can add (merge) states of aggregate functions by using the plus operator, and multiply the states of aggregate functions by a nonnegative constant. [\#3062](https://github.com/ClickHouse/ClickHouse/pull/3062) [\#3034](https://github.com/ClickHouse/ClickHouse/pull/3034) -- Tables in the MergeTree family now have the virtual column `_partition_id`. [\#3089](https://github.com/ClickHouse/ClickHouse/pull/3089) - -#### Experimental features: {#experimental-features-1} - -- Added the `LowCardinality(T)` data type. This data type automatically creates a local dictionary of values and allows data processing without unpacking the dictionary. [\#2830](https://github.com/ClickHouse/ClickHouse/pull/2830) -- Added a cache of JIT-compiled functions and a counter for the number of uses before compiling. To JIT compile expressions, enable the `compile_expressions` setting. [\#2990](https://github.com/ClickHouse/ClickHouse/pull/2990) [\#3077](https://github.com/ClickHouse/ClickHouse/pull/3077) - -#### Improvements: {#improvements-4} - -- Fixed the problem with unlimited accumulation of the replication log when there are abandoned replicas. Added an effective recovery mode for replicas with a long lag. -- Improved performance of `GROUP BY` with multiple aggregation fields when one of them is string and the others are fixed length. -- Improved performance when using `PREWHERE` and with implicit transfer of expressions in `PREWHERE`. -- Improved parsing performance for text formats (`CSV`, `TSV`). [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2977) [\#2980](https://github.com/ClickHouse/ClickHouse/pull/2980) -- Improved performance of reading strings and arrays in binary formats. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2955) -- Increased performance and reduced memory consumption for queries to `system.tables` and `system.columns` when there is a very large number of tables on a single server. [\#2953](https://github.com/ClickHouse/ClickHouse/pull/2953) -- Fixed a performance problem in the case of a large stream of queries that result in an error (the `_dl_addr` function is visible in `perf top`, but the server isn’t using much CPU). [\#2938](https://github.com/ClickHouse/ClickHouse/pull/2938) -- Conditions are cast into the View (when `enable_optimize_predicate_expression` is enabled). [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) -- Improvements to the functionality for the `UUID` data type. [\#3074](https://github.com/ClickHouse/ClickHouse/pull/3074) [\#2985](https://github.com/ClickHouse/ClickHouse/pull/2985) -- The `UUID` data type is supported in The-Alchemist dictionaries. [\#2822](https://github.com/ClickHouse/ClickHouse/pull/2822) -- The `visitParamExtractRaw` function works correctly with nested structures. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2974) -- When the `input_format_skip_unknown_fields` setting is enabled, object fields in `JSONEachRow` format are skipped correctly. [BlahGeek](https://github.com/ClickHouse/ClickHouse/pull/2958) -- For a `CASE` expression with conditions, you can now omit `ELSE`, which is equivalent to `ELSE NULL`. [\#2920](https://github.com/ClickHouse/ClickHouse/pull/2920) -- The operation timeout can now be configured when working with ZooKeeper. [urykhy](https://github.com/ClickHouse/ClickHouse/pull/2971) -- You can specify an offset for `LIMIT n, m` as `LIMIT n OFFSET m`. [\#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) -- You can use the `SELECT TOP n` syntax as an alternative for `LIMIT`. [\#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) -- Increased the size of the queue to write to system tables, so the `SystemLog parameter queue is full` error doesn’t happen as often. -- The `windowFunnel` aggregate function now supports events that meet multiple conditions. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2801) -- Duplicate columns can be used in a `USING` clause for `JOIN`. [\#3006](https://github.com/ClickHouse/ClickHouse/pull/3006) -- `Pretty` formats now have a limit on column alignment by width. Use the `output_format_pretty_max_column_pad_width` setting. If a value is wider, it will still be displayed in its entirety, but the other cells in the table will not be too wide. [\#3003](https://github.com/ClickHouse/ClickHouse/pull/3003) -- The `odbc` table function now allows you to specify the database/schema name. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2885) -- Added the ability to use a username specified in the `clickhouse-client` config file. [Vladimir Kozbin](https://github.com/ClickHouse/ClickHouse/pull/2909) -- The `ZooKeeperExceptions` counter has been split into three counters: `ZooKeeperUserExceptions`, `ZooKeeperHardwareExceptions`, and `ZooKeeperOtherExceptions`. -- `ALTER DELETE` queries work for materialized views. -- Added randomization when running the cleanup thread periodically for `ReplicatedMergeTree` tables in order to avoid periodic load spikes when there are a very large number of `ReplicatedMergeTree` tables. -- Support for `ATTACH TABLE ... ON CLUSTER` queries. [\#3025](https://github.com/ClickHouse/ClickHouse/pull/3025) - -#### Bug fixes: {#bug-fixes-13} - -- Fixed an issue with `Dictionary` tables (throws the `Size of offsets doesn't match size of column` or `Unknown compression method` exception). This bug appeared in version 18.10.3. [\#2913](https://github.com/ClickHouse/ClickHouse/issues/2913) -- Fixed a bug when merging `CollapsingMergeTree` tables if one of the data parts is empty (these parts are formed during merge or `ALTER DELETE` if all data was deleted), and the `vertical` algorithm was used for the merge. [\#3049](https://github.com/ClickHouse/ClickHouse/pull/3049) -- Fixed a race condition during `DROP` or `TRUNCATE` for `Memory` tables with a simultaneous `SELECT`, which could lead to server crashes. This bug appeared in version 1.1.54388. [\#3038](https://github.com/ClickHouse/ClickHouse/pull/3038) -- Fixed the possibility of data loss when inserting in `Replicated` tables if the `Session is expired` error is returned (data loss can be detected by the `ReplicatedDataLoss` metric). This error occurred in version 1.1.54378. [\#2939](https://github.com/ClickHouse/ClickHouse/pull/2939) [\#2949](https://github.com/ClickHouse/ClickHouse/pull/2949) [\#2964](https://github.com/ClickHouse/ClickHouse/pull/2964) -- Fixed a segfault during `JOIN ... ON`. [\#3000](https://github.com/ClickHouse/ClickHouse/pull/3000) -- Fixed the error searching column names when the `WHERE` expression consists entirely of a qualified column name, such as `WHERE table.column`. [\#2994](https://github.com/ClickHouse/ClickHouse/pull/2994) -- Fixed the “Not found column” error that occurred when executing distributed queries if a single column consisting of an IN expression with a subquery is requested from a remote server. [\#3087](https://github.com/ClickHouse/ClickHouse/pull/3087) -- Fixed the `Block structure mismatch in UNION stream: different number of columns` error that occurred for distributed queries if one of the shards is local and the other is not, and optimization of the move to `PREWHERE` is triggered. [\#2226](https://github.com/ClickHouse/ClickHouse/pull/2226) [\#3037](https://github.com/ClickHouse/ClickHouse/pull/3037) [\#3055](https://github.com/ClickHouse/ClickHouse/pull/3055) [\#3065](https://github.com/ClickHouse/ClickHouse/pull/3065) [\#3073](https://github.com/ClickHouse/ClickHouse/pull/3073) [\#3090](https://github.com/ClickHouse/ClickHouse/pull/3090) [\#3093](https://github.com/ClickHouse/ClickHouse/pull/3093) -- Fixed the `pointInPolygon` function for certain cases of non-convex polygons. [\#2910](https://github.com/ClickHouse/ClickHouse/pull/2910) -- Fixed the incorrect result when comparing `nan` with integers. [\#3024](https://github.com/ClickHouse/ClickHouse/pull/3024) -- Fixed an error in the `zlib-ng` library that could lead to segfault in rare cases. [\#2854](https://github.com/ClickHouse/ClickHouse/pull/2854) -- Fixed a memory leak when inserting into a table with `AggregateFunction` columns, if the state of the aggregate function is not simple (allocates memory separately), and if a single insertion request results in multiple small blocks. [\#3084](https://github.com/ClickHouse/ClickHouse/pull/3084) -- Fixed a race condition when creating and deleting the same `Buffer` or `MergeTree` table simultaneously. -- Fixed the possibility of a segfault when comparing tuples made up of certain non-trivial types, such as tuples. [\#2989](https://github.com/ClickHouse/ClickHouse/pull/2989) -- Fixed the possibility of a segfault when running certain `ON CLUSTER` queries. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2960) -- Fixed an error in the `arrayDistinct` function for `Nullable` array elements. [\#2845](https://github.com/ClickHouse/ClickHouse/pull/2845) [\#2937](https://github.com/ClickHouse/ClickHouse/pull/2937) -- The `enable_optimize_predicate_expression` option now correctly supports cases with `SELECT *`. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2929) -- Fixed the segfault when re-initializing the ZooKeeper session. [\#2917](https://github.com/ClickHouse/ClickHouse/pull/2917) -- Fixed potential blocking when working with ZooKeeper. -- Fixed incorrect code for adding nested data structures in a `SummingMergeTree`. -- When allocating memory for states of aggregate functions, alignment is correctly taken into account, which makes it possible to use operations that require alignment when implementing states of aggregate functions. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2808) - -#### Security fix: {#security-fix} - -- Safe use of ODBC data sources. Interaction with ODBC drivers uses a separate `clickhouse-odbc-bridge` process. Errors in third-party ODBC drivers no longer cause problems with server stability or vulnerabilities. [\#2828](https://github.com/ClickHouse/ClickHouse/pull/2828) [\#2879](https://github.com/ClickHouse/ClickHouse/pull/2879) [\#2886](https://github.com/ClickHouse/ClickHouse/pull/2886) [\#2893](https://github.com/ClickHouse/ClickHouse/pull/2893) [\#2921](https://github.com/ClickHouse/ClickHouse/pull/2921) -- Fixed incorrect validation of the file path in the `catBoostPool` table function. [\#2894](https://github.com/ClickHouse/ClickHouse/pull/2894) -- The contents of system tables (`tables`, `databases`, `parts`, `columns`, `parts_columns`, `merges`, `mutations`, `replicas`, and `replication_queue`) are filtered according to the user’s configured access to databases (`allow_databases`). [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2856) - -#### Backward incompatible changes: {#backward-incompatible-changes-3} - -- In queries with JOIN, the star character expands to a list of columns in all tables, in compliance with the SQL standard. You can restore the old behavior by setting `asterisk_left_columns_only` to 1 on the user configuration level. - -#### Build changes: {#build-changes-2} - -- Most integration tests can now be run by commit. -- Code style checks can also be run by commit. -- The `memcpy` implementation is chosen correctly when building on CentOS7/Fedora. [Etienne Champetier](https://github.com/ClickHouse/ClickHouse/pull/2912) -- When using clang to build, some warnings from `-Weverything` have been added, in addition to the regular `-Wall-Wextra -Werror`. [\#2957](https://github.com/ClickHouse/ClickHouse/pull/2957) -- Debugging the build uses the `jemalloc` debug option. -- The interface of the library for interacting with ZooKeeper is declared abstract. [\#2950](https://github.com/ClickHouse/ClickHouse/pull/2950) - -## ClickHouse release 18.10 {#clickhouse-release-18-10} - -### ClickHouse release 18.10.3, 2018-08-13 {#clickhouse-release-18-10-3-2018-08-13} - -#### New features: {#new-features-5} - -- HTTPS can be used for replication. [\#2760](https://github.com/ClickHouse/ClickHouse/pull/2760) -- Added the functions `murmurHash2_64`, `murmurHash3_32`, `murmurHash3_64`, and `murmurHash3_128` in addition to the existing `murmurHash2_32`. [\#2791](https://github.com/ClickHouse/ClickHouse/pull/2791) -- Support for Nullable types in the ClickHouse ODBC driver (`ODBCDriver2` output format). [\#2834](https://github.com/ClickHouse/ClickHouse/pull/2834) -- Support for `UUID` in the key columns. - -#### Improvements: {#improvements-5} - -- Clusters can be removed without restarting the server when they are deleted from the config files. [\#2777](https://github.com/ClickHouse/ClickHouse/pull/2777) -- External dictionaries can be removed without restarting the server when they are removed from config files. [\#2779](https://github.com/ClickHouse/ClickHouse/pull/2779) -- Added `SETTINGS` support for the `Kafka` table engine. [Alexander Marshalov](https://github.com/ClickHouse/ClickHouse/pull/2781) -- Improvements for the `UUID` data type (not yet complete). [\#2618](https://github.com/ClickHouse/ClickHouse/pull/2618) -- Support for empty parts after merges in the `SummingMergeTree`, `CollapsingMergeTree` and `VersionedCollapsingMergeTree` engines. [\#2815](https://github.com/ClickHouse/ClickHouse/pull/2815) -- Old records of completed mutations are deleted (`ALTER DELETE`). [\#2784](https://github.com/ClickHouse/ClickHouse/pull/2784) -- Added the `system.merge_tree_settings` table. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/2841) -- The `system.tables` table now has dependency columns: `dependencies_database` and `dependencies_table`. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2851) -- Added the `max_partition_size_to_drop` config option. [\#2782](https://github.com/ClickHouse/ClickHouse/pull/2782) -- Added the `output_format_json_escape_forward_slashes` option. [Alexander Bocharov](https://github.com/ClickHouse/ClickHouse/pull/2812) -- Added the `max_fetch_partition_retries_count` setting. [\#2831](https://github.com/ClickHouse/ClickHouse/pull/2831) -- Added the `prefer_localhost_replica` setting for disabling the preference for a local replica and going to a local replica without inter-process interaction. [\#2832](https://github.com/ClickHouse/ClickHouse/pull/2832) -- The `quantileExact` aggregate function returns `nan` in the case of aggregation on an empty `Float32` or `Float64` set. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2855) - -#### Bug fixes: {#bug-fixes-14} - -- Removed unnecessary escaping of the connection string parameters for ODBC, which made it impossible to establish a connection. This error occurred in version 18.6.0. -- Fixed the logic for processing `REPLACE PARTITION` commands in the replication queue. If there are two `REPLACE` commands for the same partition, the incorrect logic could cause one of them to remain in the replication queue and not be executed. [\#2814](https://github.com/ClickHouse/ClickHouse/pull/2814) -- Fixed a merge bug when all data parts were empty (parts that were formed from a merge or from `ALTER DELETE` if all data was deleted). This bug appeared in version 18.1.0. [\#2930](https://github.com/ClickHouse/ClickHouse/pull/2930) -- Fixed an error for concurrent `Set` or `Join`. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2823) -- Fixed the `Block structure mismatch in UNION stream: different number of columns` error that occurred for `UNION ALL` queries inside a sub-query if one of the `SELECT` queries contains duplicate column names. [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2094) -- Fixed a memory leak if an exception occurred when connecting to a MySQL server. -- Fixed incorrect clickhouse-client response code in case of a query error. -- Fixed incorrect behavior of materialized views containing DISTINCT. [\#2795](https://github.com/ClickHouse/ClickHouse/issues/2795) - -#### Backward incompatible changes {#backward-incompatible-changes-4} - -- Removed support for CHECK TABLE queries for Distributed tables. - -#### Build changes: {#build-changes-3} - -- The allocator has been replaced: `jemalloc` is now used instead of `tcmalloc`. In some scenarios, this increases speed up to 20%. However, there are queries that have slowed by up to 20%. Memory consumption has been reduced by approximately 10% in some scenarios, with improved stability. With highly competitive loads, CPU usage in userspace and in system shows just a slight increase. [\#2773](https://github.com/ClickHouse/ClickHouse/pull/2773) -- Use of libressl from a submodule. [\#1983](https://github.com/ClickHouse/ClickHouse/pull/1983) [\#2807](https://github.com/ClickHouse/ClickHouse/pull/2807) -- Use of unixodbc from a submodule. [\#2789](https://github.com/ClickHouse/ClickHouse/pull/2789) -- Use of mariadb-connector-c from a submodule. [\#2785](https://github.com/ClickHouse/ClickHouse/pull/2785) -- Added functional test files to the repository that depend on the availability of test data (for the time being, without the test data itself). - -## ClickHouse release 18.6 {#clickhouse-release-18-6} - -### ClickHouse release 18.6.0, 2018-08-02 {#clickhouse-release-18-6-0-2018-08-02} - -#### New features: {#new-features-6} - -- Added support for ON expressions for the JOIN ON syntax: - `JOIN ON Expr([table.]column ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` - The expression must be a chain of equalities joined by the AND operator. Each side of the equality can be an arbitrary expression over the columns of one of the tables. The use of fully qualified column names is supported (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) for the right table. [\#2742](https://github.com/ClickHouse/ClickHouse/pull/2742) -- HTTPS can be enabled for replication. [\#2760](https://github.com/ClickHouse/ClickHouse/pull/2760) - -#### Improvements: {#improvements-6} - -- The server passes the patch component of its version to the client. Data about the patch version component is in `system.processes` and `query_log`. [\#2646](https://github.com/ClickHouse/ClickHouse/pull/2646) - -## ClickHouse release 18.5 {#clickhouse-release-18-5} - -### ClickHouse release 18.5.1, 2018-07-31 {#clickhouse-release-18-5-1-2018-07-31} - -#### New features: {#new-features-7} - -- Added the hash function `murmurHash2_32` [\#2756](https://github.com/ClickHouse/ClickHouse/pull/2756). - -#### Improvements: {#improvements-7} - -- Now you can use the `from_env` [\#2741](https://github.com/ClickHouse/ClickHouse/pull/2741) attribute to set values in config files from environment variables. -- Added case-insensitive versions of the `coalesce`, `ifNull`, and `nullIf functions` [\#2752](https://github.com/ClickHouse/ClickHouse/pull/2752). - -#### Bug fixes: {#bug-fixes-15} - -- Fixed a possible bug when starting a replica [\#2759](https://github.com/ClickHouse/ClickHouse/pull/2759). - -## ClickHouse release 18.4 {#clickhouse-release-18-4} - -### ClickHouse release 18.4.0, 2018-07-28 {#clickhouse-release-18-4-0-2018-07-28} - -#### New features: {#new-features-8} - -- Added system tables: `formats`, `data_type_families`, `aggregate_function_combinators`, `table_functions`, `table_engines`, `collations` [\#2721](https://github.com/ClickHouse/ClickHouse/pull/2721). -- Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster table function` [\#2708](https://github.com/ClickHouse/ClickHouse/pull/2708). -- Support for `HTTP Basic` authentication in the replication protocol [\#2727](https://github.com/ClickHouse/ClickHouse/pull/2727). -- The `has` function now allows searching for a numeric value in an array of `Enum` values [Maxim Khrisanfov](https://github.com/ClickHouse/ClickHouse/pull/2699). -- Support for adding arbitrary message separators when reading from `Kafka` [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2701). - -#### Improvements: {#improvements-8} - -- The `ALTER TABLE t DELETE WHERE` query does not rewrite data parts that were not affected by the WHERE condition [\#2694](https://github.com/ClickHouse/ClickHouse/pull/2694). -- The `use_minimalistic_checksums_in_zookeeper` option for `ReplicatedMergeTree` tables is enabled by default. This setting was added in version 1.1.54378, 2018-04-16. Versions that are older than 1.1.54378 can no longer be installed. -- Support for running `KILL` and `OPTIMIZE` queries that specify `ON CLUSTER` [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2689). - -#### Bug fixes: {#bug-fixes-16} - -- Fixed the error `Column ... is not under an aggregate function and not in GROUP BY` for aggregation with an IN expression. This bug appeared in version 18.1.0. ([bbdd780b](https://github.com/ClickHouse/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) -- Fixed a bug in the `windowFunnel aggregate function` [Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2735). -- Fixed a bug in the `anyHeavy` aggregate function ([a2101df2](https://github.com/ClickHouse/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) -- Fixed server crash when using the `countArray()` aggregate function. - -#### Backward incompatible changes: {#backward-incompatible-changes-5} - -- Parameters for `Kafka` engine was changed from `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` to `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. If your tables use `kafka_schema` or `kafka_num_consumers` parameters, you have to manually edit the metadata files `path/metadata/database/table.sql` and add `kafka_row_delimiter` parameter with `''` value. - -## ClickHouse release 18.1 {#clickhouse-release-18-1} - -### ClickHouse release 18.1.0, 2018-07-23 {#clickhouse-release-18-1-0-2018-07-23} - -#### New features: {#new-features-9} - -- Support for the `ALTER TABLE t DELETE WHERE` query for non-replicated MergeTree tables ([\#2634](https://github.com/ClickHouse/ClickHouse/pull/2634)). -- Support for arbitrary types for the `uniq*` family of aggregate functions ([\#2010](https://github.com/ClickHouse/ClickHouse/issues/2010)). -- Support for arbitrary types in comparison operators ([\#2026](https://github.com/ClickHouse/ClickHouse/issues/2026)). -- The `users.xml` file allows setting a subnet mask in the format `10.0.0.1/255.255.255.0`. This is necessary for using masks for IPv6 networks with zeros in the middle ([\#2637](https://github.com/ClickHouse/ClickHouse/pull/2637)). -- Added the `arrayDistinct` function ([\#2670](https://github.com/ClickHouse/ClickHouse/pull/2670)). -- The SummingMergeTree engine can now work with AggregateFunction type columns ([Constantin S. Pan](https://github.com/ClickHouse/ClickHouse/pull/2566)). - -#### Improvements: {#improvements-9} - -- Changed the numbering scheme for release versions. Now the first part contains the year of release (A.D., Moscow timezone, minus 2000), the second part contains the number for major changes (increases for most releases), and the third part is the patch version. Releases are still backward compatible, unless otherwise stated in the changelog. -- Faster conversions of floating-point numbers to a string ([Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2664)). -- If some rows were skipped during an insert due to parsing errors (this is possible with the `input_allow_errors_num` and `input_allow_errors_ratio` settings enabled), the number of skipped rows is now written to the server log ([Leonardo Cecchi](https://github.com/ClickHouse/ClickHouse/pull/2669)). - -#### Bug fixes: {#bug-fixes-17} - -- Fixed the TRUNCATE command for temporary tables ([Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2624)). -- Fixed a rare deadlock in the ZooKeeper client library that occurred when there was a network error while reading the response ([c315200](https://github.com/ClickHouse/ClickHouse/commit/c315200e64b87e44bdf740707fc857d1fdf7e947)). -- Fixed an error during a CAST to Nullable types ([\#1322](https://github.com/ClickHouse/ClickHouse/issues/1322)). -- Fixed the incorrect result of the `maxIntersection()` function when the boundaries of intervals coincided ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2657)). -- Fixed incorrect transformation of the OR expression chain in a function argument ([chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2663)). -- Fixed performance degradation for queries containing `IN (subquery)` expressions inside another subquery ([\#2571](https://github.com/ClickHouse/ClickHouse/issues/2571)). -- Fixed incompatibility between servers with different versions in distributed queries that use a `CAST` function that isn’t in uppercase letters ([fe8c4d6](https://github.com/ClickHouse/ClickHouse/commit/fe8c4d64e434cacd4ceef34faa9005129f2190a5)). -- Added missing quoting of identifiers for queries to an external DBMS ([\#2635](https://github.com/ClickHouse/ClickHouse/issues/2635)). - -#### Backward incompatible changes: {#backward-incompatible-changes-6} - -- Converting a string containing the number zero to DateTime does not work. Example: `SELECT toDateTime('0')`. This is also the reason that `DateTime DEFAULT '0'` does not work in tables, as well as `0` in dictionaries. Solution: replace `0` with `0000-00-00 00:00:00`. - -## ClickHouse release 1.1 {#clickhouse-release-1-1} - -### ClickHouse release 1.1.54394, 2018-07-12 {#clickhouse-release-1-1-54394-2018-07-12} - -#### New features: {#new-features-10} - -- Added the `histogram` aggregate function ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2521)). -- Now `OPTIMIZE TABLE ... FINAL` can be used without specifying partitions for `ReplicatedMergeTree` ([Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2600)). - -#### Bug fixes: {#bug-fixes-18} - -- Fixed a problem with a very small timeout for sockets (one second) for reading and writing when sending and downloading replicated data, which made it impossible to download larger parts if there is a load on the network or disk (it resulted in cyclical attempts to download parts). This error occurred in version 1.1.54388. -- Fixed issues when using chroot in ZooKeeper if you inserted duplicate data blocks in the table. -- The `has` function now works correctly for an array with Nullable elements ([\#2115](https://github.com/ClickHouse/ClickHouse/issues/2115)). -- The `system.tables` table now works correctly when used in distributed queries. The `metadata_modification_time` and `engine_full` columns are now non-virtual. Fixed an error that occurred if only these columns were queried from the table. -- Fixed how an empty `TinyLog` table works after inserting an empty data block ([\#2563](https://github.com/ClickHouse/ClickHouse/issues/2563)). -- The `system.zookeeper` table works if the value of the node in ZooKeeper is NULL. - -### ClickHouse release 1.1.54390, 2018-07-06 {#clickhouse-release-1-1-54390-2018-07-06} - -#### New features: {#new-features-11} - -- Queries can be sent in `multipart/form-data` format (in the `query` field), which is useful if external data is also sent for query processing ([Olga Hvostikova](https://github.com/ClickHouse/ClickHouse/pull/2490)). -- Added the ability to enable or disable processing single or double quotes when reading data in CSV format. You can configure this in the `format_csv_allow_single_quotes` and `format_csv_allow_double_quotes` settings ([Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2574)). -- Now `OPTIMIZE TABLE ... FINAL` can be used without specifying the partition for non-replicated variants of `MergeTree` ([Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2599)). - -#### Improvements: {#improvements-10} - -- Improved performance, reduced memory consumption, and correct memory consumption tracking with use of the IN operator when a table index could be used ([\#2584](https://github.com/ClickHouse/ClickHouse/pull/2584)). -- Removed redundant checking of checksums when adding a data part. This is important when there are a large number of replicas, because in these cases the total number of checks was equal to N^2. -- Added support for `Array(Tuple(...))` arguments for the `arrayEnumerateUniq` function ([\#2573](https://github.com/ClickHouse/ClickHouse/pull/2573)). -- Added `Nullable` support for the `runningDifference` function ([\#2594](https://github.com/ClickHouse/ClickHouse/pull/2594)). -- Improved query analysis performance when there is a very large number of expressions ([\#2572](https://github.com/ClickHouse/ClickHouse/pull/2572)). -- Faster selection of data parts for merging in `ReplicatedMergeTree` tables. Faster recovery of the ZooKeeper session ([\#2597](https://github.com/ClickHouse/ClickHouse/pull/2597)). -- The `format_version.txt` file for `MergeTree` tables is re-created if it is missing, which makes sense if ClickHouse is launched after copying the directory structure without files ([Ciprian Hacman](https://github.com/ClickHouse/ClickHouse/pull/2593)). - -#### Bug fixes: {#bug-fixes-19} - -- Fixed a bug when working with ZooKeeper that could make it impossible to recover the session and readonly states of tables before restarting the server. -- Fixed a bug when working with ZooKeeper that could result in old nodes not being deleted if the session is interrupted. -- Fixed an error in the `quantileTDigest` function for Float arguments (this bug was introduced in version 1.1.54388) ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2553)). -- Fixed a bug in the index for MergeTree tables if the primary key column is located inside the function for converting types between signed and unsigned integers of the same size ([\#2603](https://github.com/ClickHouse/ClickHouse/pull/2603)). -- Fixed segfault if `macros` are used but they aren’t in the config file ([\#2570](https://github.com/ClickHouse/ClickHouse/pull/2570)). -- Fixed switching to the default database when reconnecting the client ([\#2583](https://github.com/ClickHouse/ClickHouse/pull/2583)). -- Fixed a bug that occurred when the `use_index_for_in_with_subqueries` setting was disabled. - -#### Security fix: {#security-fix-1} - -- Sending files is no longer possible when connected to MySQL (`LOAD DATA LOCAL INFILE`). - -### ClickHouse release 1.1.54388, 2018-06-28 {#clickhouse-release-1-1-54388-2018-06-28} - -#### New features: {#new-features-12} - -- Support for the `ALTER TABLE t DELETE WHERE` query for replicated tables. Added the `system.mutations` table to track progress of this type of queries. -- Support for the `ALTER TABLE t [REPLACE|ATTACH] PARTITION` query for \*MergeTree tables. -- Support for the `TRUNCATE TABLE` query ([Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2260)) -- Several new `SYSTEM` queries for replicated tables (`RESTART REPLICAS`, `SYNC REPLICA`, `[STOP|START] [MERGES|FETCHES|SENDS REPLICATED|REPLICATION QUEUES]`). -- Added the ability to write to a table with the MySQL engine and the corresponding table function ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2294)). -- Added the `url()` table function and the `URL` table engine ([Alexander Sapin](https://github.com/ClickHouse/ClickHouse/pull/2501)). -- Added the `windowFunnel` aggregate function ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2352)). -- New `startsWith` and `endsWith` functions for strings ([Vadim Plakhtinsky](https://github.com/ClickHouse/ClickHouse/pull/2429)). -- The `numbers()` table function now allows you to specify the offset ([Winter Zhang](https://github.com/ClickHouse/ClickHouse/pull/2535)). -- The password to `clickhouse-client` can be entered interactively. -- Server logs can now be sent to syslog ([Alexander Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/2459)). -- Support for logging in dictionaries with a shared library source ([Alexander Sapin](https://github.com/ClickHouse/ClickHouse/pull/2472)). -- Support for custom CSV delimiters ([Ivan Zhukov](https://github.com/ClickHouse/ClickHouse/pull/2263)) -- Added the `date_time_input_format` setting. If you switch this setting to `'best_effort'`, DateTime values will be read in a wide range of formats. -- Added the `clickhouse-obfuscator` utility for data obfuscation. Usage example: publishing data used in performance tests. - -#### Experimental features: {#experimental-features-2} - -- Added the ability to calculate `and` arguments only where they are needed ([Anastasia Tsarkova](https://github.com/ClickHouse/ClickHouse/pull/2272)) -- JIT compilation to native code is now available for some expressions ([pyos](https://github.com/ClickHouse/ClickHouse/pull/2277)). - -#### Bug fixes: {#bug-fixes-20} - -- Duplicates no longer appear for a query with `DISTINCT` and `ORDER BY`. -- Queries with `ARRAY JOIN` and `arrayFilter` no longer return an incorrect result. -- Fixed an error when reading an array column from a Nested structure ([\#2066](https://github.com/ClickHouse/ClickHouse/issues/2066)). -- Fixed an error when analyzing queries with a HAVING clause like `HAVING tuple IN (...)`. -- Fixed an error when analyzing queries with recursive aliases. -- Fixed an error when reading from ReplacingMergeTree with a condition in PREWHERE that filters all rows ([\#2525](https://github.com/ClickHouse/ClickHouse/issues/2525)). -- User profile settings were not applied when using sessions in the HTTP interface. -- Fixed how settings are applied from the command line parameters in clickhouse-local. -- The ZooKeeper client library now uses the session timeout received from the server. -- Fixed a bug in the ZooKeeper client library when the client waited for the server response longer than the timeout. -- Fixed pruning of parts for queries with conditions on partition key columns ([\#2342](https://github.com/ClickHouse/ClickHouse/issues/2342)). -- Merges are now possible after `CLEAR COLUMN IN PARTITION` ([\#2315](https://github.com/ClickHouse/ClickHouse/issues/2315)). -- Type mapping in the ODBC table function has been fixed ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2268)). -- Type comparisons have been fixed for `DateTime` with and without the time zone ([Alexander Bocharov](https://github.com/ClickHouse/ClickHouse/pull/2400)). -- Fixed syntactic parsing and formatting of the `CAST` operator. -- Fixed insertion into a materialized view for the Distributed table engine ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2411)). -- Fixed a race condition when writing data from the `Kafka` engine to materialized views ([Yangkuan Liu](https://github.com/ClickHouse/ClickHouse/pull/2448)). -- Fixed SSRF in the remote() table function. -- Fixed exit behavior of `clickhouse-client` in multiline mode ([\#2510](https://github.com/ClickHouse/ClickHouse/issues/2510)). - -#### Improvements: {#improvements-11} - -- Background tasks in replicated tables are now performed in a thread pool instead of in separate threads ([Silviu Caragea](https://github.com/ClickHouse/ClickHouse/pull/1722)). -- Improved LZ4 compression performance. -- Faster analysis for queries with a large number of JOINs and sub-queries. -- The DNS cache is now updated automatically when there are too many network errors. -- Table inserts no longer occur if the insert into one of the materialized views is not possible because it has too many parts. -- Corrected the discrepancy in the event counters `Query`, `SelectQuery`, and `InsertQuery`. -- Expressions like `tuple IN (SELECT tuple)` are allowed if the tuple types match. -- A server with replicated tables can start even if you haven’t configured ZooKeeper. -- When calculating the number of available CPU cores, limits on cgroups are now taken into account ([Atri Sharma](https://github.com/ClickHouse/ClickHouse/pull/2325)). -- Added chown for config directories in the systemd config file ([Mikhail Shiryaev](https://github.com/ClickHouse/ClickHouse/pull/2421)). - -#### Build changes: {#build-changes-4} - -- The gcc8 compiler can be used for builds. -- Added the ability to build llvm from submodule. -- The version of the librdkafka library has been updated to v0.11.4. -- Added the ability to use the system libcpuid library. The library version has been updated to 0.4.0. -- Fixed the build using the vectorclass library ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2274)). -- Cmake now generates files for ninja by default (like when using `-G Ninja`). -- Added the ability to use the libtinfo library instead of libtermcap ([Georgy Kondratiev](https://github.com/ClickHouse/ClickHouse/pull/2519)). -- Fixed a header file conflict in Fedora Rawhide ([\#2520](https://github.com/ClickHouse/ClickHouse/issues/2520)). - -#### Backward incompatible changes: {#backward-incompatible-changes-7} - -- Removed escaping in `Vertical` and `Pretty*` formats and deleted the `VerticalRaw` format. -- If servers with version 1.1.54388 (or newer) and servers with an older version are used simultaneously in a distributed query and the query has the `cast(x, 'Type')` expression without the `AS` keyword and doesn’t have the word `cast` in uppercase, an exception will be thrown with a message like `Not found column cast(0, 'UInt8') in block`. Solution: Update the server on the entire cluster. - -### ClickHouse release 1.1.54385, 2018-06-01 {#clickhouse-release-1-1-54385-2018-06-01} - -#### Bug fixes: {#bug-fixes-21} - -- Fixed an error that in some cases caused ZooKeeper operations to block. - -### ClickHouse release 1.1.54383, 2018-05-22 {#clickhouse-release-1-1-54383-2018-05-22} - -#### Bug fixes: {#bug-fixes-22} - -- Fixed a slowdown of replication queue if a table has many replicas. - -### ClickHouse release 1.1.54381, 2018-05-14 {#clickhouse-release-1-1-54381-2018-05-14} - -#### Bug fixes: {#bug-fixes-23} - -- Fixed a nodes leak in ZooKeeper when ClickHouse loses connection to ZooKeeper server. - -### ClickHouse release 1.1.54380, 2018-04-21 {#clickhouse-release-1-1-54380-2018-04-21} - -#### New features: {#new-features-13} - -- Added the table function `file(path, format, structure)`. An example reading bytes from `/dev/urandom`: ``` ln -s /dev/urandom /var/lib/clickhouse/user_files/random``clickhouse-client -q "SELECT * FROM file('random', 'RowBinary', 'd UInt8') LIMIT 10" ```. - -#### Improvements: {#improvements-12} - -- Subqueries can be wrapped in `()` brackets to enhance query readability. For example: `(SELECT 1) UNION ALL (SELECT 1)`. -- Simple `SELECT` queries from the `system.processes` table are not included in the `max_concurrent_queries` limit. - -#### Bug fixes: {#bug-fixes-24} - -- Fixed incorrect behavior of the `IN` operator when select from `MATERIALIZED VIEW`. -- Fixed incorrect filtering by partition index in expressions like `partition_key_column IN (...)`. -- Fixed inability to execute `OPTIMIZE` query on non-leader replica if `REANAME` was performed on the table. -- Fixed the authorization error when executing `OPTIMIZE` or `ALTER` queries on a non-leader replica. -- Fixed freezing of `KILL QUERY`. -- Fixed an error in ZooKeeper client library which led to loss of watches, freezing of distributed DDL queue, and slowdowns in the replication queue if a non-empty `chroot` prefix is used in the ZooKeeper configuration. - -#### Backward incompatible changes: {#backward-incompatible-changes-8} - -- Removed support for expressions like `(a, b) IN (SELECT (a, b))` (you can use the equivalent expression `(a, b) IN (SELECT a, b)`). In previous releases, these expressions led to undetermined `WHERE` filtering or caused errors. - -### ClickHouse release 1.1.54378, 2018-04-16 {#clickhouse-release-1-1-54378-2018-04-16} - -#### New features: {#new-features-14} - -- Logging level can be changed without restarting the server. -- Added the `SHOW CREATE DATABASE` query. -- The `query_id` can be passed to `clickhouse-client` (elBroom). -- New setting: `max_network_bandwidth_for_all_users`. -- Added support for `ALTER TABLE ... PARTITION ...` for `MATERIALIZED VIEW`. -- Added information about the size of data parts in uncompressed form in the system table. -- Server-to-server encryption support for distributed tables (`1` in the replica config in ``). -- Configuration of the table level for the `ReplicatedMergeTree` family in order to minimize the amount of data stored in Zookeeper: : `use_minimalistic_checksums_in_zookeeper = 1` -- Configuration of the `clickhouse-client` prompt. By default, server names are now output to the prompt. The server’s display name can be changed. It’s also sent in the `X-ClickHouse-Display-Name` HTTP header (Kirill Shvakov). -- Multiple comma-separated `topics` can be specified for the `Kafka` engine (Tobias Adamson) -- When a query is stopped by `KILL QUERY` or `replace_running_query`, the client receives the `Query was canceled` exception instead of an incomplete result. - -#### Improvements: {#improvements-13} - -- `ALTER TABLE ... DROP/DETACH PARTITION` queries are run at the front of the replication queue. -- `SELECT ... FINAL` and `OPTIMIZE ... FINAL` can be used even when the table has a single data part. -- A `query_log` table is recreated on the fly if it was deleted manually (Kirill Shvakov). -- The `lengthUTF8` function runs faster (zhang2014). -- Improved performance of synchronous inserts in `Distributed` tables (`insert_distributed_sync = 1`) when there is a very large number of shards. -- The server accepts the `send_timeout` and `receive_timeout` settings from the client and applies them when connecting to the client (they are applied in reverse order: the server socket’s `send_timeout` is set to the `receive_timeout` value received from the client, and vice versa). -- More robust crash recovery for asynchronous insertion into `Distributed` tables. -- The return type of the `countEqual` function changed from `UInt32` to `UInt64` (谢磊). - -#### Bug fixes: {#bug-fixes-25} - -- Fixed an error with `IN` when the left side of the expression is `Nullable`. -- Correct results are now returned when using tuples with `IN` when some of the tuple components are in the table index. -- The `max_execution_time` limit now works correctly with distributed queries. -- Fixed errors when calculating the size of composite columns in the `system.columns` table. -- Fixed an error when creating a temporary table `CREATE TEMPORARY TABLE IF NOT EXISTS.` -- Fixed errors in `StorageKafka` (\#\#2075) -- Fixed server crashes from invalid arguments of certain aggregate functions. -- Fixed the error that prevented the `DETACH DATABASE` query from stopping background tasks for `ReplicatedMergeTree` tables. -- `Too many parts` state is less likely to happen when inserting into aggregated materialized views (\#\#2084). -- Corrected recursive handling of substitutions in the config if a substitution must be followed by another substitution on the same level. -- Corrected the syntax in the metadata file when creating a `VIEW` that uses a query with `UNION ALL`. -- `SummingMergeTree` now works correctly for summation of nested data structures with a composite key. -- Fixed the possibility of a race condition when choosing the leader for `ReplicatedMergeTree` tables. - -#### Build changes: {#build-changes-5} - -- The build supports `ninja` instead of `make` and uses `ninja` by default for building releases. -- Renamed packages: `clickhouse-server-base` in `clickhouse-common-static`; `clickhouse-server-common` in `clickhouse-server`; `clickhouse-common-dbg` in `clickhouse-common-static-dbg`. To install, use `clickhouse-server clickhouse-client`. Packages with the old names will still load in the repositories for backward compatibility. - -#### Backward incompatible changes: {#backward-incompatible-changes-9} - -- Removed the special interpretation of an IN expression if an array is specified on the left side. Previously, the expression `arr IN (set)` was interpreted as “at least one `arr` element belongs to the `set`”. To get the same behavior in the new version, write `arrayExists(x -> x IN (set), arr)`. -- Disabled the incorrect use of the socket option `SO_REUSEPORT`, which was incorrectly enabled by default in the Poco library. Note that on Linux there is no longer any reason to simultaneously specify the addresses `::` and `0.0.0.0` for listen – use just `::`, which allows listening to the connection both over IPv4 and IPv6 (with the default kernel config settings). You can also revert to the behavior from previous versions by specifying `1` in the config. - -### ClickHouse release 1.1.54370, 2018-03-16 {#clickhouse-release-1-1-54370-2018-03-16} - -#### New features: {#new-features-15} - -- Added the `system.macros` table and auto updating of macros when the config file is changed. -- Added the `SYSTEM RELOAD CONFIG` query. -- Added the `maxIntersections(left_col, right_col)` aggregate function, which returns the maximum number of simultaneously intersecting intervals `[left; right]`. The `maxIntersectionsPosition(left, right)` function returns the beginning of the “maximum” interval. ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2012)). - -#### Improvements: {#improvements-14} - -- When inserting data in a `Replicated` table, fewer requests are made to `ZooKeeper` (and most of the user-level errors have disappeared from the `ZooKeeper` log). -- Added the ability to create aliases for data sets. Example: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`. - -#### Bug fixes: {#bug-fixes-26} - -- Fixed the `Illegal PREWHERE` error when reading from Merge tables for `Distributed`tables. -- Added fixes that allow you to start clickhouse-server in IPv4-only Docker containers. -- Fixed a race condition when reading from system `system.parts_columns tables.` -- Removed double buffering during a synchronous insert to a `Distributed` table, which could have caused the connection to timeout. -- Fixed a bug that caused excessively long waits for an unavailable replica before beginning a `SELECT` query. -- Fixed incorrect dates in the `system.parts` table. -- Fixed a bug that made it impossible to insert data in a `Replicated` table if `chroot` was non-empty in the configuration of the `ZooKeeper` cluster. -- Fixed the vertical merging algorithm for an empty `ORDER BY` table. -- Restored the ability to use dictionaries in queries to remote tables, even if these dictionaries are not present on the requestor server. This functionality was lost in release 1.1.54362. -- Restored the behavior for queries like `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` when the right side of the `IN` should use a remote `default.table` instead of a local one. This behavior was broken in version 1.1.54358. -- Removed extraneous error-level logging of `Not found column ... in block`. - -### Clickhouse Release 1.1.54362, 2018-03-11 {#clickhouse-release-1-1-54362-2018-03-11} - -#### New features: {#new-features-16} - -- Aggregation without `GROUP BY` for an empty set (such as `SELECT count(*) FROM table WHERE 0`) now returns a result with one row with null values for aggregate functions, in compliance with the SQL standard. To restore the old behavior (return an empty result), set `empty_result_for_aggregation_by_empty_set` to 1. -- Added type conversion for `UNION ALL`. Different alias names are allowed in `SELECT` positions in `UNION ALL`, in compliance with the SQL standard. -- Arbitrary expressions are supported in `LIMIT BY` clauses. Previously, it was only possible to use columns resulting from `SELECT`. -- An index of `MergeTree` tables is used when `IN` is applied to a tuple of expressions from the columns of the primary key. Example: `WHERE (UserID, EventDate) IN ((123, '2000-01-01'), ...)` (Anastasiya Tsarkova). -- Added the `clickhouse-copier` tool for copying between clusters and resharding data (beta). -- Added consistent hashing functions: `yandexConsistentHash`, `jumpConsistentHash`, `sumburConsistentHash`. They can be used as a sharding key in order to reduce the amount of network traffic during subsequent reshardings. -- Added functions: `arrayAny`, `arrayAll`, `hasAny`, `hasAll`, `arrayIntersect`, `arrayResize`. -- Added the `arrayCumSum` function (Javi Santana). -- Added the `parseDateTimeBestEffort`, `parseDateTimeBestEffortOrZero`, and `parseDateTimeBestEffortOrNull` functions to read the DateTime from a string containing text in a wide variety of possible formats. -- Data can be partially reloaded from external dictionaries during updating (load just the records in which the value of the specified field greater than in the previous download) (Arsen Hakobyan). -- Added the `cluster` table function. Example: `cluster(cluster_name, db, table)`. The `remote` table function can accept the cluster name as the first argument, if it is specified as an identifier. -- The `remote` and `cluster` table functions can be used in `INSERT` queries. -- Added the `create_table_query` and `engine_full` virtual columns to the `system.tables`table . The `metadata_modification_time` column is virtual. -- Added the `data_path` and `metadata_path` columns to `system.tables`and`system.databases` tables, and added the `path` column to the `system.parts` and `system.parts_columns` tables. -- Added additional information about merges in the `system.part_log` table. -- An arbitrary partitioning key can be used for the `system.query_log` table (Kirill Shvakov). -- The `SHOW TABLES` query now also shows temporary tables. Added temporary tables and the `is_temporary` column to `system.tables` (zhang2014). -- Added `DROP TEMPORARY TABLE` and `EXISTS TEMPORARY TABLE` queries (zhang2014). -- Support for `SHOW CREATE TABLE` for temporary tables (zhang2014). -- Added the `system_profile` configuration parameter for the settings used by internal processes. -- Support for loading `object_id` as an attribute in `MongoDB` dictionaries (Pavel Litvinenko). -- Reading `null` as the default value when loading data for an external dictionary with the `MongoDB` source (Pavel Litvinenko). -- Reading `DateTime` values in the `Values` format from a Unix timestamp without single quotes. -- Failover is supported in `remote` table functions for cases when some of the replicas are missing the requested table. -- Configuration settings can be overridden in the command line when you run `clickhouse-server`. Example: `clickhouse-server -- --logger.level=information`. -- Implemented the `empty` function from a `FixedString` argument: the function returns 1 if the string consists entirely of null bytes (zhang2014). -- Added the `listen_try`configuration parameter for listening to at least one of the listen addresses without quitting, if some of the addresses can’t be listened to (useful for systems with disabled support for IPv4 or IPv6). -- Added the `VersionedCollapsingMergeTree` table engine. -- Support for rows and arbitrary numeric types for the `library` dictionary source. -- `MergeTree` tables can be used without a primary key (you need to specify `ORDER BY tuple()`). -- A `Nullable` type can be `CAST` to a non-`Nullable` type if the argument is not `NULL`. -- `RENAME TABLE` can be performed for `VIEW`. -- Added the `throwIf` function. -- Added the `odbc_default_field_size` option, which allows you to extend the maximum size of the value loaded from an ODBC source (by default, it is 1024). -- The `system.processes` table and `SHOW PROCESSLIST` now have the `is_cancelled` and `peak_memory_usage` columns. - -#### Improvements: {#improvements-15} - -- Limits and quotas on the result are no longer applied to intermediate data for `INSERT SELECT` queries or for `SELECT` subqueries. -- Fewer false triggers of `force_restore_data` when checking the status of `Replicated` tables when the server starts. -- Added the `allow_distributed_ddl` option. -- Nondeterministic functions are not allowed in expressions for `MergeTree` table keys. -- Files with substitutions from `config.d` directories are loaded in alphabetical order. -- Improved performance of the `arrayElement` function in the case of a constant multidimensional array with an empty array as one of the elements. Example: `[[1], []][x]`. -- The server starts faster now when using configuration files with very large substitutions (for instance, very large lists of IP networks). -- When running a query, table valued functions run once. Previously, `remote` and `mysql` table valued functions performed the same query twice to retrieve the table structure from a remote server. -- The `MkDocs` documentation generator is used. -- When you try to delete a table column that `DEFAULT`/`MATERIALIZED` expressions of other columns depend on, an exception is thrown (zhang2014). -- Added the ability to parse an empty line in text formats as the number 0 for `Float` data types. This feature was previously available but was lost in release 1.1.54342. -- `Enum` values can be used in `min`, `max`, `sum` and some other functions. In these cases, it uses the corresponding numeric values. This feature was previously available but was lost in the release 1.1.54337. -- Added `max_expanded_ast_elements` to restrict the size of the AST after recursively expanding aliases. - -#### Bug fixes: {#bug-fixes-27} - -- Fixed cases when unnecessary columns were removed from subqueries in error, or not removed from subqueries containing `UNION ALL`. -- Fixed a bug in merges for `ReplacingMergeTree` tables. -- Fixed synchronous insertions in `Distributed` tables (`insert_distributed_sync = 1`). -- Fixed segfault for certain uses of `FULL` and `RIGHT JOIN` with duplicate columns in subqueries. -- Fixed segfault for certain uses of `replace_running_query` and `KILL QUERY`. -- Fixed the order of the `source` and `last_exception` columns in the `system.dictionaries` table. -- Fixed a bug when the `DROP DATABASE` query did not delete the file with metadata. -- Fixed the `DROP DATABASE` query for `Dictionary` databases. -- Fixed the low precision of `uniqHLL12` and `uniqCombined` functions for cardinalities greater than 100 million items (Alex Bocharov). -- Fixed the calculation of implicit default values when necessary to simultaneously calculate default explicit expressions in `INSERT` queries (zhang2014). -- Fixed a rare case when a query to a `MergeTree` table couldn’t finish (chenxing-xc). -- Fixed a crash that occurred when running a `CHECK` query for `Distributed` tables if all shards are local (chenxing.xc). -- Fixed a slight performance regression with functions that use regular expressions. -- Fixed a performance regression when creating multidimensional arrays from complex expressions. -- Fixed a bug that could cause an extra `FORMAT` section to appear in an `.sql` file with metadata. -- Fixed a bug that caused the `max_table_size_to_drop` limit to apply when trying to delete a `MATERIALIZED VIEW` looking at an explicitly specified table. -- Fixed incompatibility with old clients (old clients were sometimes sent data with the `DateTime('timezone')` type, which they do not understand). -- Fixed a bug when reading `Nested` column elements of structures that were added using `ALTER` but that are empty for the old partitions, when the conditions for these columns moved to `PREWHERE`. -- Fixed a bug when filtering tables by virtual `_table` columns in queries to `Merge` tables. -- Fixed a bug when using `ALIAS` columns in `Distributed` tables. -- Fixed a bug that made dynamic compilation impossible for queries with aggregate functions from the `quantile` family. -- Fixed a race condition in the query execution pipeline that occurred in very rare cases when using `Merge` tables with a large number of tables, and when using `GLOBAL` subqueries. -- Fixed a crash when passing arrays of different sizes to an `arrayReduce` function when using aggregate functions from multiple arguments. -- Prohibited the use of queries with `UNION ALL` in a `MATERIALIZED VIEW`. -- Fixed an error during initialization of the `part_log` system table when the server starts (by default, `part_log` is disabled). - -#### Backward incompatible changes: {#backward-incompatible-changes-10} - -- Removed the `distributed_ddl_allow_replicated_alter` option. This behavior is enabled by default. -- Removed the `strict_insert_defaults` setting. If you were using this functionality, write to `clickhouse-feedback@yandex-team.com`. -- Removed the `UnsortedMergeTree` engine. - -### Clickhouse Release 1.1.54343, 2018-02-05 {#clickhouse-release-1-1-54343-2018-02-05} - -- Added macros support for defining cluster names in distributed DDL queries and constructors of Distributed tables: `CREATE TABLE distr ON CLUSTER '{cluster}' (...) ENGINE = Distributed('{cluster}', 'db', 'table')`. -- Now queries like `SELECT ... FROM table WHERE expr IN (subquery)` are processed using the `table` index. -- Improved processing of duplicates when inserting to Replicated tables, so they no longer slow down execution of the replication queue. - -### Clickhouse Release 1.1.54342, 2018-01-22 {#clickhouse-release-1-1-54342-2018-01-22} - -This release contains bug fixes for the previous release 1.1.54337: - -- Fixed a regression in 1.1.54337: if the default user has readonly access, then the server refuses to start up with the message `Cannot create database in readonly mode`. -- Fixed a regression in 1.1.54337: on systems with systemd, logs are always written to syslog regardless of the configuration; the watchdog script still uses init.d. -- Fixed a regression in 1.1.54337: wrong default configuration in the Docker image. -- Fixed nondeterministic behavior of GraphiteMergeTree (you can see it in log messages `Data after merge is not byte-identical to the data on another replicas`). -- Fixed a bug that may lead to inconsistent merges after OPTIMIZE query to Replicated tables (you may see it in log messages `Part ... intersects the previous part`). -- Buffer tables now work correctly when MATERIALIZED columns are present in the destination table (by zhang2014). -- Fixed a bug in implementation of NULL. - -### Clickhouse Release 1.1.54337, 2018-01-18 {#clickhouse-release-1-1-54337-2018-01-18} - -#### New features: {#new-features-17} - -- Added support for storage of multi-dimensional arrays and tuples (`Tuple` data type) in tables. -- Support for table functions for `DESCRIBE` and `INSERT` queries. Added support for subqueries in `DESCRIBE`. Examples: `DESC TABLE remote('host', default.hits)`; `DESC TABLE (SELECT 1)`; `INSERT INTO TABLE FUNCTION remote('host', default.hits)`. Support for `INSERT INTO TABLE` in addition to `INSERT INTO`. -- Improved support for time zones. The `DateTime` data type can be annotated with the timezone that is used for parsing and formatting in text formats. Example: `DateTime('Europe/Moscow')`. When timezones are specified in functions for `DateTime` arguments, the return type will track the timezone, and the value will be displayed as expected. -- Added the functions `toTimeZone`, `timeDiff`, `toQuarter`, `toRelativeQuarterNum`. The `toRelativeHour`/`Minute`/`Second` functions can take a value of type `Date` as an argument. The `now` function name is case-sensitive. -- Added the `toStartOfFifteenMinutes` function (Kirill Shvakov). -- Added the `clickhouse format` tool for formatting queries. -- Added the `format_schema_path` configuration parameter (Marek Vavruşa). It is used for specifying a schema in `Cap'n Proto` format. Schema files can be located only in the specified directory. -- Added support for config substitutions (`incl` and `conf.d`) for configuration of external dictionaries and models (Pavel Yakunin). -- Added a column with documentation for the `system.settings` table (Kirill Shvakov). -- Added the `system.parts_columns` table with information about column sizes in each data part of `MergeTree` tables. -- Added the `system.models` table with information about loaded `CatBoost` machine learning models. -- Added the `mysql` and `odbc` table function and corresponding `MySQL` and `ODBC` table engines for accessing remote databases. This functionality is in the beta stage. -- Added the possibility to pass an argument of type `AggregateFunction` for the `groupArray` aggregate function (so you can create an array of states of some aggregate function). -- Removed restrictions on various combinations of aggregate function combinators. For example, you can use `avgForEachIf` as well as `avgIfForEach` aggregate functions, which have different behaviors. -- The `-ForEach` aggregate function combinator is extended for the case of aggregate functions of multiple arguments. -- Added support for aggregate functions of `Nullable` arguments even for cases when the function returns a non-`Nullable` result (added with the contribution of Silviu Caragea). Example: `groupArray`, `groupUniqArray`, `topK`. -- Added the `max_client_network_bandwidth` for `clickhouse-client` (Kirill Shvakov). -- Users with the `readonly = 2` setting are allowed to work with TEMPORARY tables (CREATE, DROP, INSERT…) (Kirill Shvakov). -- Added support for using multiple consumers with the `Kafka` engine. Extended configuration options for `Kafka` (Marek Vavruša). -- Added the `intExp3` and `intExp4` functions. -- Added the `sumKahan` aggregate function. -- Added the to \* Number\* OrNull functions, where \* Number\* is a numeric type. -- Added support for `WITH` clauses for an `INSERT SELECT` query (author: zhang2014). -- Added settings: `http_connection_timeout`, `http_send_timeout`, `http_receive_timeout`. In particular, these settings are used for downloading data parts for replication. Changing these settings allows for faster failover if the network is overloaded. -- Added support for `ALTER` for tables of type `Null` (Anastasiya Tsarkova). -- The `reinterpretAsString` function is extended for all data types that are stored contiguously in memory. -- Added the `--silent` option for the `clickhouse-local` tool. It suppresses printing query execution info in stderr. -- Added support for reading values of type `Date` from text in a format where the month and/or day of the month is specified using a single digit instead of two digits (Amos Bird). - -#### Performance optimizations: {#performance-optimizations} - -- Improved performance of aggregate functions `min`, `max`, `any`, `anyLast`, `anyHeavy`, `argMin`, `argMax` from string arguments. -- Improved performance of the functions `isInfinite`, `isFinite`, `isNaN`, `roundToExp2`. -- Improved performance of parsing and formatting `Date` and `DateTime` type values in text format. -- Improved performance and precision of parsing floating point numbers. -- Lowered memory usage for `JOIN` in the case when the left and right parts have columns with identical names that are not contained in `USING` . -- Improved performance of aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr` by reducing computational stability. The old functions are available under the names `varSampStable`, `varPopStable`, `stddevSampStable`, `stddevPopStable`, `covarSampStable`, `covarPopStable`, `corrStable`. - -#### Bug fixes: {#bug-fixes-28} - -- Fixed data deduplication after running a `DROP` or `DETACH PARTITION` query. In the previous version, dropping a partition and inserting the same data again was not working because inserted blocks were considered duplicates. -- Fixed a bug that could lead to incorrect interpretation of the `WHERE` clause for `CREATE MATERIALIZED VIEW` queries with `POPULATE` . -- Fixed a bug in using the `root_path` parameter in the `zookeeper_servers` configuration. -- Fixed unexpected results of passing the `Date` argument to `toStartOfDay` . -- Fixed the `addMonths` and `subtractMonths` functions and the arithmetic for `INTERVAL n MONTH` in cases when the result has the previous year. -- Added missing support for the `UUID` data type for `DISTINCT` , `JOIN` , and `uniq` aggregate functions and external dictionaries (Evgeniy Ivanov). Support for `UUID` is still incomplete. -- Fixed `SummingMergeTree` behavior in cases when the rows summed to zero. -- Various fixes for the `Kafka` engine (Marek Vavruša). -- Fixed incorrect behavior of the `Join` table engine (Amos Bird). -- Fixed incorrect allocator behavior under FreeBSD and OS X. -- The `extractAll` function now supports empty matches. -- Fixed an error that blocked usage of `libressl` instead of `openssl` . -- Fixed the `CREATE TABLE AS SELECT` query from temporary tables. -- Fixed non-atomicity of updating the replication queue. This could lead to replicas being out of sync until the server restarts. -- Fixed possible overflow in `gcd` , `lcm` and `modulo` (`%` operator) (Maks Skorokhod). -- `-preprocessed` files are now created after changing `umask` (`umask` can be changed in the config). -- Fixed a bug in the background check of parts (`MergeTreePartChecker` ) when using a custom partition key. -- Fixed parsing of tuples (values of the `Tuple` data type) in text formats. -- Improved error messages about incompatible types passed to `multiIf` , `array` and some other functions. -- Redesigned support for `Nullable` types. Fixed bugs that may lead to a server crash. Fixed almost all other bugs related to `NULL` support: incorrect type conversions in INSERT SELECT, insufficient support for Nullable in HAVING and PREWHERE, `join_use_nulls` mode, Nullable types as arguments of `OR` operator, etc. -- Fixed various bugs related to internal semantics of data types. Examples: unnecessary summing of `Enum` type fields in `SummingMergeTree` ; alignment of `Enum` types in `Pretty` formats, etc. -- Stricter checks for allowed combinations of composite columns. -- Fixed the overflow when specifying a very large parameter for the `FixedString` data type. -- Fixed a bug in the `topK` aggregate function in a generic case. -- Added the missing check for equality of array sizes in arguments of n-ary variants of aggregate functions with an `-Array` combinator. -- Fixed a bug in `--pager` for `clickhouse-client` (author: ks1322). -- Fixed the precision of the `exp10` function. -- Fixed the behavior of the `visitParamExtract` function for better compliance with documentation. -- Fixed the crash when incorrect data types are specified. -- Fixed the behavior of `DISTINCT` in the case when all columns are constants. -- Fixed query formatting in the case of using the `tupleElement` function with a complex constant expression as the tuple element index. -- Fixed a bug in `Dictionary` tables for `range_hashed` dictionaries. -- Fixed a bug that leads to excessive rows in the result of `FULL` and `RIGHT JOIN` (Amos Bird). -- Fixed a server crash when creating and removing temporary files in `config.d` directories during config reload. -- Fixed the `SYSTEM DROP DNS CACHE` query: the cache was flushed but addresses of cluster nodes were not updated. -- Fixed the behavior of `MATERIALIZED VIEW` after executing `DETACH TABLE` for the table under the view (Marek Vavruša). - -#### Build improvements: {#build-improvements-4} - -- The `pbuilder` tool is used for builds. The build process is almost completely independent of the build host environment. -- A single build is used for different OS versions. Packages and binaries have been made compatible with a wide range of Linux systems. -- Added the `clickhouse-test` package. It can be used to run functional tests. -- The source tarball can now be published to the repository. It can be used to reproduce the build without using GitHub. -- Added limited integration with Travis CI. Due to limits on build time in Travis, only the debug build is tested and a limited subset of tests are run. -- Added support for `Cap'n'Proto` in the default build. -- Changed the format of documentation sources from `Restricted Text` to `Markdown`. -- Added support for `systemd` (Vladimir Smirnov). It is disabled by default due to incompatibility with some OS images and can be enabled manually. -- For dynamic code generation, `clang` and `lld` are embedded into the `clickhouse` binary. They can also be invoked as `clickhouse clang` and `clickhouse lld` . -- Removed usage of GNU extensions from the code. Enabled the `-Wextra` option. When building with `clang` the default is `libc++` instead of `libstdc++`. -- Extracted `clickhouse_parsers` and `clickhouse_common_io` libraries to speed up builds of various tools. - -#### Backward incompatible changes: {#backward-incompatible-changes-11} - -- The format for marks in `Log` type tables that contain `Nullable` columns was changed in a backward incompatible way. If you have these tables, you should convert them to the `TinyLog` type before starting up the new server version. To do this, replace `ENGINE = Log` with `ENGINE = TinyLog` in the corresponding `.sql` file in the `metadata` directory. If your table doesn’t have `Nullable` columns or if the type of your table is not `Log`, then you don’t need to do anything. -- Removed the `experimental_allow_extended_storage_definition_syntax` setting. Now this feature is enabled by default. -- The `runningIncome` function was renamed to `runningDifferenceStartingWithFirstvalue` to avoid confusion. -- Removed the `FROM ARRAY JOIN arr` syntax when ARRAY JOIN is specified directly after FROM with no table (Amos Bird). -- Removed the `BlockTabSeparated` format that was used solely for demonstration purposes. -- Changed the state format for aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. If you have stored states of these aggregate functions in tables (using the `AggregateFunction` data type or materialized views with corresponding states), please write to clickhouse-feedback@yandex-team.com. -- In previous server versions there was an undocumented feature: if an aggregate function depends on parameters, you can still specify it without parameters in the AggregateFunction data type. Example: `AggregateFunction(quantiles, UInt64)` instead of `AggregateFunction(quantiles(0.5, 0.9), UInt64)`. This feature was lost. Although it was undocumented, we plan to support it again in future releases. -- Enum data types cannot be used in min/max aggregate functions. This ability will be returned in the next release. - -#### Please note when upgrading: {#please-note-when-upgrading} - -- When doing a rolling update on a cluster, at the point when some of the replicas are running the old version of ClickHouse and some are running the new version, replication is temporarily stopped and the message `unknown parameter 'shard'` appears in the log. Replication will continue after all replicas of the cluster are updated. -- If different versions of ClickHouse are running on the cluster servers, it is possible that distributed queries using the following functions will have incorrect results: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. You should update all cluster nodes. - -## [Changelog for 2017](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2017.md) diff --git a/docs/ru/changelog/2019.md b/docs/ru/changelog/2019.md deleted file mode 100644 index 01a0756af14..00000000000 --- a/docs/ru/changelog/2019.md +++ /dev/null @@ -1,2071 +0,0 @@ ---- -en_copy: true ---- - -## ClickHouse release v19.17 {#clickhouse-release-v19-17} - -### ClickHouse release v19.17.6.36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} - -#### Bug Fix {#bug-fix} - -- Fixed potential buffer overflow in decompress. Malicious user can pass fabricated compressed data that could cause read after buffer. This issue was found by Eldar Zaitov from Yandex information security team. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed possible server crash (`std::terminate`) when the server cannot send or write data in JSON or XML format with values of String data type (that require UTF-8 validation) or when compressing result data with Brotli algorithm or in some other rare cases. [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed dictionaries with source from a clickhouse `VIEW`, now reading such dictionaries doesn’t cause the error `There is no query`. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed checking if a client host is allowed by host\_regexp specified in users.xml. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241), [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) -- `RENAME TABLE` for a distributed table now renames the folder containing inserted data before sending to shards. This fixes an issue with successive renames `tableA->tableB`, `tableC->tableA`. [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) -- `range_hashed` external dictionaries created by DDL queries now allow ranges of arbitrary numeric types. [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) -- Fixed `INSERT INTO table SELECT ... FROM mysql(...)` table function. [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -- Fixed segfault in `INSERT INTO TABLE FUNCTION file()` while inserting into a file which doesn’t exist. Now in this case file would be created and then insert would be processed. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fixed bitmapAnd error when intersecting an aggregated bitmap and a scalar bitmap. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) -- Fixed segfault when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier, just like `EXISTS t`. [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed return type for functions `rand` and `randConstant` in case of nullable argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed `DROP DICTIONARY IF EXISTS db.dict`, now it doesn’t throw exception if `db` doesn’t exist. [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) -- If a table wasn’t completely dropped because of server crash, the server will try to restore and load it [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -- Fixed a trivial count query for a distributed table if there are more than two shard local table. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -- Fixed bug that lead to a data race in DB::BlockStreamProfileInfo::calculateRowsBeforeLimit() [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) -- Fixed `ALTER table MOVE part` executed immediately after merging the specified part, which could cause moving a part which the specified part merged into. Now it correctly moves the specified part. [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Expressions for dictionaries can be specified as strings now. This is useful for calculation of attributes while extracting data from non-ClickHouse sources because it allows to use non-ClickHouse syntax for those expressions. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) -- Fixed a very rare race in `clickhouse-copier` because of an overflow in ZXid. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -- Fixed the bug when after the query failed (due to “Too many simultaneous queries” for example) it would not read external tables info, and the - next request would interpret this info as the beginning of the next query causing an error like `Unknown packet from client`. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) -- Avoid null dereference after “Unknown packet X from server” [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -- Restore support of all ICU locales, add the ability to apply collations for constant expressions and add language name to system.collations table. [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) -- Number of streams for read from `StorageFile` and `StorageHDFS` is now limited, to avoid exceeding the memory limit. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) -- Fixed `CHECK TABLE` query for `*MergeTree` tables without key. [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) -- Removed the mutation number from a part name in case there were no mutations. This removing improved the compatibility with older versions. [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) -- Fixed the bug that mutations are skipped for some attached parts due to their data\_version are larger than the table mutation version. [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) -- Allow starting the server with redundant copies of parts after moving them to another device. [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed the error “Sizes of columns doesn’t match” that might appear when using aggregate function columns. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -- Now an exception will be thrown in case of using WITH TIES alongside LIMIT BY. And now it’s possible to use TOP with LIMIT BY. [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) - -### ClickHouse release v19.17.4.11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} - -#### Backward Incompatible Change {#backward-incompatible-change} - -- Using column instead of AST to store scalar subquery results for better performance. Setting `enable_scalar_subquery_optimization` was added in 19.17 and it was enabled by default. It leads to errors like [this](https://github.com/ClickHouse/ClickHouse/issues/7851) during upgrade to 19.17.2 or 19.17.3 from previous versions. This setting was disabled by default in 19.17.4, to make possible upgrading from 19.16 and older versions without errors. [\#7392](https://github.com/ClickHouse/ClickHouse/pull/7392) ([Amos Bird](https://github.com/amosbird)) - -#### New Feature {#new-feature} - -- Add the ability to create dictionaries with DDL queries. [\#7360](https://github.com/ClickHouse/ClickHouse/pull/7360) ([alesapin](https://github.com/alesapin)) -- Make `bloom_filter` type of index supporting `LowCardinality` and `Nullable` [\#7363](https://github.com/ClickHouse/ClickHouse/issues/7363) [\#7561](https://github.com/ClickHouse/ClickHouse/pull/7561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Add function `isValidJSON` to check that passed string is a valid json. [\#5910](https://github.com/ClickHouse/ClickHouse/issues/5910) [\#7293](https://github.com/ClickHouse/ClickHouse/pull/7293) ([Vdimir](https://github.com/Vdimir)) -- Implement `arrayCompact` function [\#7328](https://github.com/ClickHouse/ClickHouse/pull/7328) ([Memo](https://github.com/Joeywzr)) -- Created function `hex` for Decimal numbers. It works like `hex(reinterpretAsString())`, but doesn’t delete last zero bytes. [\#7355](https://github.com/ClickHouse/ClickHouse/pull/7355) ([Mikhail Korotov](https://github.com/millb)) -- Add `arrayFill` and `arrayReverseFill` functions, which replace elements by other elements in front/back of them in the array. [\#7380](https://github.com/ClickHouse/ClickHouse/pull/7380) ([hcz](https://github.com/hczhcz)) -- Add `CRC32IEEE()`/`CRC64()` support [\#7480](https://github.com/ClickHouse/ClickHouse/pull/7480) ([Azat Khuzhin](https://github.com/azat)) -- Implement `char` function similar to one in [mysql](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char) [\#7486](https://github.com/ClickHouse/ClickHouse/pull/7486) ([sundyli](https://github.com/sundy-li)) -- Add `bitmapTransform` function. It transforms an array of values in a bitmap to another array of values, the result is a new bitmap [\#7598](https://github.com/ClickHouse/ClickHouse/pull/7598) ([Zhichang Yu](https://github.com/yuzhichang)) -- Implemented `javaHashUTF16LE()` function [\#7651](https://github.com/ClickHouse/ClickHouse/pull/7651) ([achimbab](https://github.com/achimbab)) -- Add `_shard_num` virtual column for the Distributed engine [\#7624](https://github.com/ClickHouse/ClickHouse/pull/7624) ([Azat Khuzhin](https://github.com/azat)) - -#### Experimental Feature {#experimental-feature} - -- Support for processors (new query execution pipeline) in `MergeTree`. [\#7181](https://github.com/ClickHouse/ClickHouse/pull/7181) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Bug Fix {#bug-fix-1} - -- Fix incorrect float parsing in `Values` [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817) [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -- Fix rare deadlock which can happen when trace\_log is enabled. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) -- Prevent message duplication when producing Kafka table has any MVs selecting from it [\#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([Ivan](https://github.com/abyss7)) -- Support for `Array(LowCardinality(Nullable(String)))` in `IN`. Resolves [\#7364](https://github.com/ClickHouse/ClickHouse/issues/7364) [\#7366](https://github.com/ClickHouse/ClickHouse/pull/7366) ([achimbab](https://github.com/achimbab)) -- Add handling of `SQL_TINYINT` and `SQL_BIGINT`, and fix handling of `SQL_FLOAT` data source types in ODBC Bridge. [\#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([Denis Glazachev](https://github.com/traceon)) -- Fix aggregation (`avg` and quantiles) over empty decimal columns [\#7431](https://github.com/ClickHouse/ClickHouse/pull/7431) ([Andrey Konyaev](https://github.com/akonyaev90)) -- Fix `INSERT` into Distributed with `MATERIALIZED` columns [\#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) -- Make `MOVE PARTITION` work if some parts of partition are already on destination disk or volume [\#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed bug with hardlinks failing to be created during mutations in `ReplicatedMergeTree` in multi-disk configurations. [\#7558](https://github.com/ClickHouse/ClickHouse/pull/7558) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed a bug with a mutation on a MergeTree when whole part remains unchanged and best space is being found on another disk [\#7602](https://github.com/ClickHouse/ClickHouse/pull/7602) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed bug with `keep_free_space_ratio` not being read from disks configuration [\#7645](https://github.com/ClickHouse/ClickHouse/pull/7645) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix bug with table contains only `Tuple` columns or columns with complex paths. Fixes [7541](https://github.com/ClickHouse/ClickHouse/issues/7541). [\#7545](https://github.com/ClickHouse/ClickHouse/pull/7545) ([alesapin](https://github.com/alesapin)) -- Do not account memory for Buffer engine in max\_memory\_usage limit [\#7552](https://github.com/ClickHouse/ClickHouse/pull/7552) ([Azat Khuzhin](https://github.com/azat)) -- Fix final mark usage in `MergeTree` tables ordered by `tuple()`. In rare cases it could lead to `Can't adjust last granule` error while select. [\#7639](https://github.com/ClickHouse/ClickHouse/pull/7639) ([Anton Popov](https://github.com/CurtizJ)) -- Fix bug in mutations that have predicate with actions that require context (for example functions for json), which may lead to crashes or strange exceptions. [\#7664](https://github.com/ClickHouse/ClickHouse/pull/7664) ([alesapin](https://github.com/alesapin)) -- Fix mismatch of database and table names escaping in `data/` and `shadow/` directories [\#7575](https://github.com/ClickHouse/ClickHouse/pull/7575) ([Alexander Burmak](https://github.com/Alex-Burmak)) -- Support duplicated keys in RIGHT\|FULL JOINs, e.g. `ON t.x = u.x AND t.x = u.y`. Fix crash in this case. [\#7586](https://github.com/ClickHouse/ClickHouse/pull/7586) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix `Not found column in block` when joining on expression with RIGHT or FULL JOIN. [\#7641](https://github.com/ClickHouse/ClickHouse/pull/7641) ([Artem Zuikov](https://github.com/4ertus2)) -- One more attempt to fix infinite loop in `PrettySpace` format [\#7591](https://github.com/ClickHouse/ClickHouse/pull/7591) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fix bug in `concat` function when all arguments were `FixedString` of the same size. [\#7635](https://github.com/ClickHouse/ClickHouse/pull/7635) ([alesapin](https://github.com/alesapin)) -- Fixed exception in case of using 1 argument while defining S3, URL and HDFS storages. [\#7618](https://github.com/ClickHouse/ClickHouse/pull/7618) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fix scope of the InterpreterSelectQuery for views with query [\#7601](https://github.com/ClickHouse/ClickHouse/pull/7601) ([Azat Khuzhin](https://github.com/azat)) - -#### Improvement {#improvement} - -- `Nullable` columns recognized and NULL-values handled correctly by ODBC-bridge [\#7402](https://github.com/ClickHouse/ClickHouse/pull/7402) ([Vasily Nemkov](https://github.com/Enmk)) -- Write current batch for distributed send atomically [\#7600](https://github.com/ClickHouse/ClickHouse/pull/7600) ([Azat Khuzhin](https://github.com/azat)) -- Throw an exception if we cannot detect table for column name in query. [\#7358](https://github.com/ClickHouse/ClickHouse/pull/7358) ([Artem Zuikov](https://github.com/4ertus2)) -- Add `merge_max_block_size` setting to `MergeTreeSettings` [\#7412](https://github.com/ClickHouse/ClickHouse/pull/7412) ([Artem Zuikov](https://github.com/4ertus2)) -- Queries with `HAVING` and without `GROUP BY` assume group by constant. So, `SELECT 1 HAVING 1` now returns a result. [\#7496](https://github.com/ClickHouse/ClickHouse/pull/7496) ([Amos Bird](https://github.com/amosbird)) -- Support parsing `(X,)` as tuple similar to python. [\#7501](https://github.com/ClickHouse/ClickHouse/pull/7501), [\#7562](https://github.com/ClickHouse/ClickHouse/pull/7562) ([Amos Bird](https://github.com/amosbird)) -- Make `range` function behaviors almost like pythonic one. [\#7518](https://github.com/ClickHouse/ClickHouse/pull/7518) ([sundyli](https://github.com/sundy-li)) -- Add `constraints` columns to table `system.settings` [\#7553](https://github.com/ClickHouse/ClickHouse/pull/7553) ([Vitaly Baranov](https://github.com/vitlibar)) -- Better Null format for tcp handler, so that it’s possible to use `select ignore() from table format Null` for perf measure via clickhouse-client [\#7606](https://github.com/ClickHouse/ClickHouse/pull/7606) ([Amos Bird](https://github.com/amosbird)) -- Queries like `CREATE TABLE ... AS (SELECT (1, 2))` are parsed correctly [\#7542](https://github.com/ClickHouse/ClickHouse/pull/7542) ([hcz](https://github.com/hczhcz)) - -#### Performance Improvement {#performance-improvement} - -- The performance of aggregation over short string keys is improved. [\#6243](https://github.com/ClickHouse/ClickHouse/pull/6243) ([Alexander Kuzmenkov](https://github.com/akuzm), [Amos Bird](https://github.com/amosbird)) -- Run another pass of syntax/expression analysis to get potential optimizations after constant predicates are folded. [\#7497](https://github.com/ClickHouse/ClickHouse/pull/7497) ([Amos Bird](https://github.com/amosbird)) -- Use storage meta info to evaluate trivial `SELECT count() FROM table;` [\#7510](https://github.com/ClickHouse/ClickHouse/pull/7510) ([Amos Bird](https://github.com/amosbird), [alexey-milovidov](https://github.com/alexey-milovidov)) -- Vectorize processing `arrayReduce` similar to Aggregator `addBatch`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) -- Minor improvements in performance of `Kafka` consumption [\#7475](https://github.com/ClickHouse/ClickHouse/pull/7475) ([Ivan](https://github.com/abyss7)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement} - -- Add support for cross-compiling to the CPU architecture AARCH64. Refactor packager script. [\#7370](https://github.com/ClickHouse/ClickHouse/pull/7370) [\#7539](https://github.com/ClickHouse/ClickHouse/pull/7539) ([Ivan](https://github.com/abyss7)) -- Unpack darwin-x86\_64 and linux-aarch64 toolchains into mounted Docker volume when building packages [\#7534](https://github.com/ClickHouse/ClickHouse/pull/7534) ([Ivan](https://github.com/abyss7)) -- Update Docker Image for Binary Packager [\#7474](https://github.com/ClickHouse/ClickHouse/pull/7474) ([Ivan](https://github.com/abyss7)) -- Fixed compile errors on MacOS Catalina [\#7585](https://github.com/ClickHouse/ClickHouse/pull/7585) ([Ernest Poletaev](https://github.com/ernestp)) -- Some refactoring in query analysis logic: split complex class into several simple ones. [\#7454](https://github.com/ClickHouse/ClickHouse/pull/7454) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix build without submodules [\#7295](https://github.com/ClickHouse/ClickHouse/pull/7295) ([proller](https://github.com/proller)) -- Better `add_globs` in CMake files [\#7418](https://github.com/ClickHouse/ClickHouse/pull/7418) ([Amos Bird](https://github.com/amosbird)) -- Remove hardcoded paths in `unwind` target [\#7460](https://github.com/ClickHouse/ClickHouse/pull/7460) ([Konstantin Podshumok](https://github.com/podshumok)) -- Allow to use mysql format without ssl [\#7524](https://github.com/ClickHouse/ClickHouse/pull/7524) ([proller](https://github.com/proller)) - -#### Other {#other} - -- Added ANTLR4 grammar for ClickHouse SQL dialect [\#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [\#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -## ClickHouse release v19.16 {#clickhouse-release-v19-16} - -#### Clickhouse release v19.16.14.65, 2020-03-25 - -* Fixed up a bug in batched calculations of ternary logical OPs on multiple arguments (more than 10). [#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) This bugfix was backported to version 19.16 by a special request from Altinity. - -#### Clickhouse release v19.16.14.65, 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} - -- Fix distributed subqueries incompatibility with older CH versions. Fixes [\#7851](https://github.com/ClickHouse/ClickHouse/issues/7851) - [(tabplubix)](https://github.com/tavplubix) -- When executing `CREATE` query, fold constant expressions in storage engine arguments. Replace empty database name with current database. Fixes [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Also fix check for local address in `ClickHouseDictionarySource`. - [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) -- Now background merges in `*MergeTree` table engines family preserve storage policy volume order more accurately. - [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Prevent losing data in `Kafka` in rare cases when exception happens after reading suffix but before commit. Fixes [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). Related: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) - [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) -- Fix bug leading to server termination when trying to use / drop `Kafka` table created with wrong parameters. Fixes [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Incorporates [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). - [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) -- Allow using `MaterializedView` with subqueries above `Kafka` tables. - [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) - -#### New Feature {#new-feature-1} - -- Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity. - [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) - -### ClickHouse release v19.16.2.2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} - -#### Backward Incompatible Change {#backward-incompatible-change-1} - -- Add missing arity validation for count/counIf. - [\#7095](https://github.com/ClickHouse/ClickHouse/issues/7095) - [\#7298](https://github.com/ClickHouse/ClickHouse/pull/7298) ([Vdimir](https://github.com/Vdimir)) -- Remove legacy `asterisk_left_columns_only` setting (it was disabled by default). - [\#7335](https://github.com/ClickHouse/ClickHouse/pull/7335) ([Artem - Zuikov](https://github.com/4ertus2)) -- Format strings for Template data format are now specified in files. - [\#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) - ([tavplubix](https://github.com/tavplubix)) - -#### New Feature {#new-feature-2} - -- Introduce uniqCombined64() to calculate cardinality greater than UINT\_MAX. - [\#7213](https://github.com/ClickHouse/ClickHouse/pull/7213), - [\#7222](https://github.com/ClickHouse/ClickHouse/pull/7222) ([Azat - Khuzhin](https://github.com/azat)) -- Support Bloom filter indexes on Array columns. - [\#6984](https://github.com/ClickHouse/ClickHouse/pull/6984) - ([achimbab](https://github.com/achimbab)) -- Add a function `getMacro(name)` that returns String with the value of corresponding `` - from server configuration. [\#7240](https://github.com/ClickHouse/ClickHouse/pull/7240) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Set two configuration options for a dictionary based on an HTTP source: `credentials` and - `http-headers`. [\#7092](https://github.com/ClickHouse/ClickHouse/pull/7092) ([Guillaume - Tassery](https://github.com/YiuRULE)) -- Add a new ProfileEvent `Merge` that counts the number of launched background merges. - [\#7093](https://github.com/ClickHouse/ClickHouse/pull/7093) ([Mikhail - Korotov](https://github.com/millb)) -- Add fullHostName function that returns a fully qualified domain name. - [\#7263](https://github.com/ClickHouse/ClickHouse/issues/7263) - [\#7291](https://github.com/ClickHouse/ClickHouse/pull/7291) ([sundyli](https://github.com/sundy-li)) -- Add function `arraySplit` and `arrayReverseSplit` which split an array by “cut off” - conditions. They are useful in time sequence handling. - [\#7294](https://github.com/ClickHouse/ClickHouse/pull/7294) ([hcz](https://github.com/hczhcz)) -- Add new functions that return the Array of all matched indices in multiMatch family of functions. - [\#7299](https://github.com/ClickHouse/ClickHouse/pull/7299) ([Danila - Kutenin](https://github.com/danlark1)) -- Add a new database engine `Lazy` that is optimized for storing a large number of small -Log - tables. [\#7171](https://github.com/ClickHouse/ClickHouse/pull/7171) ([Nikita - Vasilev](https://github.com/nikvas0)) -- Add aggregate functions groupBitmapAnd, -Or, -Xor for bitmap columns. [\#7109](https://github.com/ClickHouse/ClickHouse/pull/7109) ([Zhichang - Yu](https://github.com/yuzhichang)) -- Add aggregate function combinators -OrNull and -OrDefault, which return null - or default values when there is nothing to aggregate. - [\#7331](https://github.com/ClickHouse/ClickHouse/pull/7331) - ([hcz](https://github.com/hczhcz)) -- Introduce CustomSeparated data format that supports custom escaping and - delimiter rules. [\#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) - ([tavplubix](https://github.com/tavplubix)) -- Support Redis as source of external dictionary. [\#4361](https://github.com/ClickHouse/ClickHouse/pull/4361) [\#6962](https://github.com/ClickHouse/ClickHouse/pull/6962) ([comunodi](https://github.com/comunodi), [Anton - Popov](https://github.com/CurtizJ)) - -#### Bug Fix {#bug-fix-2} - -- Fix wrong query result if it has `WHERE IN (SELECT ...)` section and `optimize_read_in_order` is - used. [\#7371](https://github.com/ClickHouse/ClickHouse/pull/7371) ([Anton - Popov](https://github.com/CurtizJ)) -- Disabled MariaDB authentication plugin, which depends on files outside of project. - [\#7140](https://github.com/ClickHouse/ClickHouse/pull/7140) ([Yuriy - Baranov](https://github.com/yurriy)) -- Fix exception `Cannot convert column ... because it is constant but values of constants are different in source and result` which could rarely happen when functions `now()`, `today()`, - `yesterday()`, `randConstant()` are used. - [\#7156](https://github.com/ClickHouse/ClickHouse/pull/7156) ([Nikolai - Kochetov](https://github.com/KochetovNicolai)) -- Fixed issue of using HTTP keep alive timeout instead of TCP keep alive timeout. - [\#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily - Nemkov](https://github.com/Enmk)) -- Fixed a segmentation fault in groupBitmapOr (issue [\#7109](https://github.com/ClickHouse/ClickHouse/issues/7109)). - [\#7289](https://github.com/ClickHouse/ClickHouse/pull/7289) ([Zhichang - Yu](https://github.com/yuzhichang)) -- For materialized views the commit for Kafka is called after all data were written. - [\#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([Ivan](https://github.com/abyss7)) -- Fixed wrong `duration_ms` value in `system.part_log` table. It was ten times off. - [\#7172](https://github.com/ClickHouse/ClickHouse/pull/7172) ([Vladimir - Chebotarev](https://github.com/excitoon)) -- A quick fix to resolve crash in LIVE VIEW table and re-enabling all LIVE VIEW tests. - [\#7201](https://github.com/ClickHouse/ClickHouse/pull/7201) - ([vzakaznikov](https://github.com/vzakaznikov)) -- Serialize NULL values correctly in min/max indexes of MergeTree parts. - [\#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([Alexander - Kuzmenkov](https://github.com/akuzm)) -- Don’t put virtual columns to .sql metadata when table is created as `CREATE TABLE AS`. - [\#7183](https://github.com/ClickHouse/ClickHouse/pull/7183) ([Ivan](https://github.com/abyss7)) -- Fix segmentation fault in `ATTACH PART` query. - [\#7185](https://github.com/ClickHouse/ClickHouse/pull/7185) - ([alesapin](https://github.com/alesapin)) -- Fix wrong result for some queries given by the optimization of empty IN subqueries and empty - INNER/RIGHT JOIN. [\#7284](https://github.com/ClickHouse/ClickHouse/pull/7284) ([Nikolai - Kochetov](https://github.com/KochetovNicolai)) -- Fixing AddressSanitizer error in the LIVE VIEW getHeader() method. - [\#7271](https://github.com/ClickHouse/ClickHouse/pull/7271) - ([vzakaznikov](https://github.com/vzakaznikov)) - -#### Improvement {#improvement-1} - -- Add a message in case of queue\_wait\_max\_ms wait takes place. - [\#7390](https://github.com/ClickHouse/ClickHouse/pull/7390) ([Azat - Khuzhin](https://github.com/azat)) -- Made setting `s3_min_upload_part_size` table-level. - [\#7059](https://github.com/ClickHouse/ClickHouse/pull/7059) ([Vladimir - Chebotarev](https://github.com/excitoon)) -- Check TTL in StorageFactory. [\#7304](https://github.com/ClickHouse/ClickHouse/pull/7304) - ([sundyli](https://github.com/sundy-li)) -- Squash left-hand blocks in partial merge join (optimization). - [\#7122](https://github.com/ClickHouse/ClickHouse/pull/7122) ([Artem - Zuikov](https://github.com/4ertus2)) -- Do not allow non-deterministic functions in mutations of Replicated table engines, because this - can introduce inconsistencies between replicas. - [\#7247](https://github.com/ClickHouse/ClickHouse/pull/7247) ([Alexander - Kazakov](https://github.com/Akazz)) -- Disable memory tracker while converting exception stack trace to string. It can prevent the loss - of error messages of type `Memory limit exceeded` on server, which caused the `Attempt to read after eof` exception on client. [\#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) - ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Miscellaneous format improvements. Resolves - [\#6033](https://github.com/ClickHouse/ClickHouse/issues/6033), - [\#2633](https://github.com/ClickHouse/ClickHouse/issues/2633), - [\#6611](https://github.com/ClickHouse/ClickHouse/issues/6611), - [\#6742](https://github.com/ClickHouse/ClickHouse/issues/6742) - [\#7215](https://github.com/ClickHouse/ClickHouse/pull/7215) - ([tavplubix](https://github.com/tavplubix)) -- ClickHouse ignores values on the right side of IN operator that are not convertible to the left - side type. Make it work properly for compound types – Array and Tuple. - [\#7283](https://github.com/ClickHouse/ClickHouse/pull/7283) ([Alexander - Kuzmenkov](https://github.com/akuzm)) -- Support missing inequalities for ASOF JOIN. It’s possible to join less-or-equal variant and strict - greater and less variants for ASOF column in ON syntax. - [\#7282](https://github.com/ClickHouse/ClickHouse/pull/7282) ([Artem - Zuikov](https://github.com/4ertus2)) -- Optimize partial merge join. [\#7070](https://github.com/ClickHouse/ClickHouse/pull/7070) - ([Artem Zuikov](https://github.com/4ertus2)) -- Do not use more than 98K of memory in uniqCombined functions. - [\#7236](https://github.com/ClickHouse/ClickHouse/pull/7236), - [\#7270](https://github.com/ClickHouse/ClickHouse/pull/7270) ([Azat - Khuzhin](https://github.com/azat)) -- Flush parts of right-hand joining table on disk in PartialMergeJoin (if there is not enough - memory). Load data back when needed. [\#7186](https://github.com/ClickHouse/ClickHouse/pull/7186) - ([Artem Zuikov](https://github.com/4ertus2)) - -#### Performance Improvement {#performance-improvement-1} - -- Speed up joinGet with const arguments by avoiding data duplication. - [\#7359](https://github.com/ClickHouse/ClickHouse/pull/7359) ([Amos - Bird](https://github.com/amosbird)) -- Return early if the subquery is empty. - [\#7007](https://github.com/ClickHouse/ClickHouse/pull/7007) ([小路](https://github.com/nicelulu)) -- Optimize parsing of SQL expression in Values. - [\#6781](https://github.com/ClickHouse/ClickHouse/pull/6781) - ([tavplubix](https://github.com/tavplubix)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-1} - -- Disable some contribs for cross-compilation to Mac OS. - [\#7101](https://github.com/ClickHouse/ClickHouse/pull/7101) ([Ivan](https://github.com/abyss7)) -- Add missing linking with PocoXML for clickhouse\_common\_io. - [\#7200](https://github.com/ClickHouse/ClickHouse/pull/7200) ([Azat - Khuzhin](https://github.com/azat)) -- Accept multiple test filter arguments in clickhouse-test. - [\#7226](https://github.com/ClickHouse/ClickHouse/pull/7226) ([Alexander - Kuzmenkov](https://github.com/akuzm)) -- Enable musl and jemalloc for ARM. [\#7300](https://github.com/ClickHouse/ClickHouse/pull/7300) - ([Amos Bird](https://github.com/amosbird)) -- Added `--client-option` parameter to `clickhouse-test` to pass additional parameters to client. - [\#7277](https://github.com/ClickHouse/ClickHouse/pull/7277) ([Nikolai - Kochetov](https://github.com/KochetovNicolai)) -- Preserve existing configs on rpm package upgrade. - [\#7103](https://github.com/ClickHouse/ClickHouse/pull/7103) - ([filimonov](https://github.com/filimonov)) -- Fix errors detected by PVS. [\#7153](https://github.com/ClickHouse/ClickHouse/pull/7153) ([Artem - Zuikov](https://github.com/4ertus2)) -- Fix build for Darwin. [\#7149](https://github.com/ClickHouse/ClickHouse/pull/7149) - ([Ivan](https://github.com/abyss7)) -- glibc 2.29 compatibility. [\#7142](https://github.com/ClickHouse/ClickHouse/pull/7142) ([Amos - Bird](https://github.com/amosbird)) -- Make sure dh\_clean does not touch potential source files. - [\#7205](https://github.com/ClickHouse/ClickHouse/pull/7205) ([Amos - Bird](https://github.com/amosbird)) -- Attempt to avoid conflict when updating from altinity rpm - it has config file packaged separately - in clickhouse-server-common. [\#7073](https://github.com/ClickHouse/ClickHouse/pull/7073) - ([filimonov](https://github.com/filimonov)) -- Optimize some header files for faster rebuilds. - [\#7212](https://github.com/ClickHouse/ClickHouse/pull/7212), - [\#7231](https://github.com/ClickHouse/ClickHouse/pull/7231) ([Alexander - Kuzmenkov](https://github.com/akuzm)) -- Add performance tests for Date and DateTime. [\#7332](https://github.com/ClickHouse/ClickHouse/pull/7332) ([Vasily - Nemkov](https://github.com/Enmk)) -- Fix some tests that contained non-deterministic mutations. - [\#7132](https://github.com/ClickHouse/ClickHouse/pull/7132) ([Alexander - Kazakov](https://github.com/Akazz)) -- Add build with MemorySanitizer to CI. [\#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) - ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Avoid use of uninitialized values in MetricsTransmitter. - [\#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat - Khuzhin](https://github.com/azat)) -- Fix some issues in Fields found by MemorySanitizer. - [\#7135](https://github.com/ClickHouse/ClickHouse/pull/7135), - [\#7179](https://github.com/ClickHouse/ClickHouse/pull/7179) ([Alexander - Kuzmenkov](https://github.com/akuzm)), [\#7376](https://github.com/ClickHouse/ClickHouse/pull/7376) - ([Amos Bird](https://github.com/amosbird)) -- Fix undefined behavior in murmurhash32. [\#7388](https://github.com/ClickHouse/ClickHouse/pull/7388) ([Amos - Bird](https://github.com/amosbird)) -- Fix undefined behavior in StoragesInfoStream. [\#7384](https://github.com/ClickHouse/ClickHouse/pull/7384) - ([tavplubix](https://github.com/tavplubix)) -- Fixed constant expressions folding for external database engines (MySQL, ODBC, JDBC). In previous - versions it wasn’t working for multiple constant expressions and was not working at all for Date, - DateTime and UUID. This fixes [\#7245](https://github.com/ClickHouse/ClickHouse/issues/7245) - [\#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixing ThreadSanitizer data race error in the LIVE VIEW when accessing no\_users\_thread variable. - [\#7353](https://github.com/ClickHouse/ClickHouse/pull/7353) - ([vzakaznikov](https://github.com/vzakaznikov)) -- Get rid of malloc symbols in libcommon - [\#7134](https://github.com/ClickHouse/ClickHouse/pull/7134), - [\#7065](https://github.com/ClickHouse/ClickHouse/pull/7065) ([Amos - Bird](https://github.com/amosbird)) -- Add global flag ENABLE\_LIBRARIES for disabling all libraries. - [\#7063](https://github.com/ClickHouse/ClickHouse/pull/7063) - ([proller](https://github.com/proller)) - -#### Code cleanup {#code-cleanup} - -- Generalize configuration repository to prepare for DDL for Dictionaries. [\#7155](https://github.com/ClickHouse/ClickHouse/pull/7155) - ([alesapin](https://github.com/alesapin)) -- Parser for dictionaries DDL without any semantic. - [\#7209](https://github.com/ClickHouse/ClickHouse/pull/7209) - ([alesapin](https://github.com/alesapin)) -- Split ParserCreateQuery into different smaller parsers. - [\#7253](https://github.com/ClickHouse/ClickHouse/pull/7253) - ([alesapin](https://github.com/alesapin)) -- Small refactoring and renaming near external dictionaries. - [\#7111](https://github.com/ClickHouse/ClickHouse/pull/7111) - ([alesapin](https://github.com/alesapin)) -- Refactor some code to prepare for role-based access control. [\#7235](https://github.com/ClickHouse/ClickHouse/pull/7235) ([Vitaly - Baranov](https://github.com/vitlibar)) -- Some improvements in DatabaseOrdinary code. - [\#7086](https://github.com/ClickHouse/ClickHouse/pull/7086) ([Nikita - Vasilev](https://github.com/nikvas0)) -- Do not use iterators in find() and emplace() methods of hash tables. - [\#7026](https://github.com/ClickHouse/ClickHouse/pull/7026) ([Alexander - Kuzmenkov](https://github.com/akuzm)) -- Fix getMultipleValuesFromConfig in case when parameter root is not empty. [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) - ([Mikhail Korotov](https://github.com/millb)) -- Remove some copy-paste (TemporaryFile and TemporaryFileStream) - [\#7166](https://github.com/ClickHouse/ClickHouse/pull/7166) ([Artem - Zuikov](https://github.com/4ertus2)) -- Improved code readability a little bit (`MergeTreeData::getActiveContainingPart`). - [\#7361](https://github.com/ClickHouse/ClickHouse/pull/7361) ([Vladimir - Chebotarev](https://github.com/excitoon)) -- Wait for all scheduled jobs, which are using local objects, if `ThreadPool::schedule(...)` throws - an exception. Rename `ThreadPool::schedule(...)` to `ThreadPool::scheduleOrThrowOnError(...)` and - fix comments to make obvious that it may throw. - [\#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) - ([tavplubix](https://github.com/tavplubix)) - -## ClickHouse release 19.15 {#clickhouse-release-19-15} - -### ClickHouse release 19.15.4.10, 2019-10-31 {#clickhouse-release-19-15-4-10-2019-10-31} - -#### Bug Fix {#bug-fix-3} - -- Added handling of SQL\_TINYINT and SQL\_BIGINT, and fix handling of SQL\_FLOAT data source types in ODBC Bridge. - [\#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([Denis Glazachev](https://github.com/traceon)) -- Allowed to have some parts on destination disk or volume in MOVE PARTITION. - [\#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed NULL-values in nullable columns through ODBC-bridge. - [\#7402](https://github.com/ClickHouse/ClickHouse/pull/7402) ([Vasily Nemkov](https://github.com/Enmk)) -- Fixed INSERT into Distributed non local node with MATERIALIZED columns. - [\#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) -- Fixed function getMultipleValuesFromConfig. - [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) ([Mikhail Korotov](https://github.com/millb)) -- Fixed issue of using HTTP keep alive timeout instead of TCP keep alive timeout. - [\#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily Nemkov](https://github.com/Enmk)) -- Wait for all jobs to finish on exception (fixes rare segfaults). - [\#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) ([tavplubix](https://github.com/tavplubix)) -- Don’t push to MVs when inserting into Kafka table. - [\#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([Ivan](https://github.com/abyss7)) -- Disable memory tracker for exception stack. - [\#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed bad code in transforming query for external database. - [\#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid use of uninitialized values in MetricsTransmitter. - [\#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat Khuzhin](https://github.com/azat)) -- Added example config with macros for tests ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.15.3.6, 2019-10-09 {#clickhouse-release-19-15-3-6-2019-10-09} - -#### Bug Fix {#bug-fix-4} - -- Fixed bad\_variant in hashed dictionary. - ([alesapin](https://github.com/alesapin)) -- Fixed up bug with segmentation fault in ATTACH PART query. - ([alesapin](https://github.com/alesapin)) -- Fixed time calculation in `MergeTreeData`. - ([Vladimir Chebotarev](https://github.com/excitoon)) -- Commit to Kafka explicitly after the writing is finalized. - [\#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([Ivan](https://github.com/abyss7)) -- Serialize NULL values correctly in min/max indexes of MergeTree parts. - [\#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([Alexander Kuzmenkov](https://github.com/akuzm)) - -### ClickHouse release 19.15.2.2, 2019-10-01 {#clickhouse-release-19-15-2-2-2019-10-01} - -#### New Feature {#new-feature-3} - -- Tiered storage: support to use multiple storage volumes for tables with MergeTree engine. It’s possible to store fresh data on SSD and automatically move old data to HDD. ([example](https://clickhouse.github.io/clickhouse-presentations/meetup30/new_features/#12)). [\#4918](https://github.com/ClickHouse/ClickHouse/pull/4918) ([Igr](https://github.com/ObjatieGroba)) [\#6489](https://github.com/ClickHouse/ClickHouse/pull/6489) ([alesapin](https://github.com/alesapin)) -- Add table function `input` for reading incoming data in `INSERT SELECT` query. [\#5450](https://github.com/ClickHouse/ClickHouse/pull/5450) ([palasonic1](https://github.com/palasonic1)) [\#6832](https://github.com/ClickHouse/ClickHouse/pull/6832) ([Anton Popov](https://github.com/CurtizJ)) -- Add a `sparse_hashed` dictionary layout, that is functionally equivalent to the `hashed` layout, but is more memory efficient. It uses about twice as less memory at the cost of slower value retrieval. [\#6894](https://github.com/ClickHouse/ClickHouse/pull/6894) ([Azat Khuzhin](https://github.com/azat)) -- Implement ability to define list of users for access to dictionaries. Only current connected database using. [\#6907](https://github.com/ClickHouse/ClickHouse/pull/6907) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Add `LIMIT` option to `SHOW` query. [\#6944](https://github.com/ClickHouse/ClickHouse/pull/6944) ([Philipp Malkovsky](https://github.com/malkfilipp)) -- Add `bitmapSubsetLimit(bitmap, range_start, limit)` function, that returns subset of the smallest `limit` values in set that is no smaller than `range_start`. [\#6957](https://github.com/ClickHouse/ClickHouse/pull/6957) ([Zhichang Yu](https://github.com/yuzhichang)) -- Add `bitmapMin` and `bitmapMax` functions. [\#6970](https://github.com/ClickHouse/ClickHouse/pull/6970) ([Zhichang Yu](https://github.com/yuzhichang)) -- Add function `repeat` related to [issue-6648](https://github.com/ClickHouse/ClickHouse/issues/6648) [\#6999](https://github.com/ClickHouse/ClickHouse/pull/6999) ([flynn](https://github.com/ucasFL)) - -#### Experimental Feature {#experimental-feature-1} - -- Implement (in memory) Merge Join variant that does not change current pipeline. Result is partially sorted by merge key. Set `partial_merge_join = 1` to use this feature. The Merge Join is still in development. [\#6940](https://github.com/ClickHouse/ClickHouse/pull/6940) ([Artem Zuikov](https://github.com/4ertus2)) -- Add `S3` engine and table function. It is still in development (no authentication support yet). [\#5596](https://github.com/ClickHouse/ClickHouse/pull/5596) ([Vladimir Chebotarev](https://github.com/excitoon)) - -#### Improvement {#improvement-2} - -- Every message read from Kafka is inserted atomically. This resolves almost all known issues with Kafka engine. [\#6950](https://github.com/ClickHouse/ClickHouse/pull/6950) ([Ivan](https://github.com/abyss7)) -- Improvements for failover of Distributed queries. Shorten recovery time, also it is now configurable and can be seen in `system.clusters`. [\#6399](https://github.com/ClickHouse/ClickHouse/pull/6399) ([Vasily Nemkov](https://github.com/Enmk)) -- Support numeric values for Enums directly in `IN` section. \#6766 [\#6941](https://github.com/ClickHouse/ClickHouse/pull/6941) ([dimarub2000](https://github.com/dimarub2000)) -- Support (optional, disabled by default) redirects on URL storage. [\#6914](https://github.com/ClickHouse/ClickHouse/pull/6914) ([maqroll](https://github.com/maqroll)) -- Add information message when client with an older version connects to a server. [\#6893](https://github.com/ClickHouse/ClickHouse/pull/6893) ([Philipp Malkovsky](https://github.com/malkfilipp)) -- Remove maximum backoff sleep time limit for sending data in Distributed tables [\#6895](https://github.com/ClickHouse/ClickHouse/pull/6895) ([Azat Khuzhin](https://github.com/azat)) -- Add ability to send profile events (counters) with cumulative values to graphite. It can be enabled under `` in server `config.xml`. [\#6969](https://github.com/ClickHouse/ClickHouse/pull/6969) ([Azat Khuzhin](https://github.com/azat)) -- Add automatically cast type `T` to `LowCardinality(T)` while inserting data in column of type `LowCardinality(T)` in Native format via HTTP. [\#6891](https://github.com/ClickHouse/ClickHouse/pull/6891) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Add ability to use function `hex` without using `reinterpretAsString` for `Float32`, `Float64`. [\#7024](https://github.com/ClickHouse/ClickHouse/pull/7024) ([Mikhail Korotov](https://github.com/millb)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-2} - -- Add gdb-index to clickhouse binary with debug info. It will speed up startup time of `gdb`. [\#6947](https://github.com/ClickHouse/ClickHouse/pull/6947) ([alesapin](https://github.com/alesapin)) -- Speed up deb packaging with patched dpkg-deb which uses `pigz`. [\#6960](https://github.com/ClickHouse/ClickHouse/pull/6960) ([alesapin](https://github.com/alesapin)) -- Set `enable_fuzzing = 1` to enable libfuzzer instrumentation of all the project code. [\#7042](https://github.com/ClickHouse/ClickHouse/pull/7042) ([kyprizel](https://github.com/kyprizel)) -- Add split build smoke test in CI. [\#7061](https://github.com/ClickHouse/ClickHouse/pull/7061) ([alesapin](https://github.com/alesapin)) -- Add build with MemorySanitizer to CI. [\#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) ([Alexander Kuzmenkov](https://github.com/akuzm)) -- Replace `libsparsehash` with `sparsehash-c11` [\#6965](https://github.com/ClickHouse/ClickHouse/pull/6965) ([Azat Khuzhin](https://github.com/azat)) - -#### Bug Fix {#bug-fix-5} - -- Fixed performance degradation of index analysis on complex keys on large tables. This fixes \#6924. [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix logical error causing segfaults when selecting from Kafka empty topic. [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([Ivan](https://github.com/abyss7)) -- Fix too early MySQL connection close in `MySQLBlockInputStream.cpp`. [\#6882](https://github.com/ClickHouse/ClickHouse/pull/6882) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- Returned support for very old Linux kernels (fix [\#6841](https://github.com/ClickHouse/ClickHouse/issues/6841)) [\#6853](https://github.com/ClickHouse/ClickHouse/pull/6853) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix possible data loss in `insert select` query in case of empty block in input stream. \#6834 \#6862 [\#6911](https://github.com/ClickHouse/ClickHouse/pull/6911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix for function `АrrayEnumerateUniqRanked` with empty arrays in params [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) -- Fix complex queries with array joins and global subqueries. [\#6934](https://github.com/ClickHouse/ClickHouse/pull/6934) ([Ivan](https://github.com/abyss7)) -- Fix `Unknown identifier` error in ORDER BY and GROUP BY with multiple JOINs [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed `MSan` warning while executing function with `LowCardinality` argument. [\#7062](https://github.com/ClickHouse/ClickHouse/pull/7062) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Backward Incompatible Change {#backward-incompatible-change-2} - -- Changed serialization format of bitmap\* aggregate function states to improve performance. Serialized states of bitmap\* from previous versions cannot be read. [\#6908](https://github.com/ClickHouse/ClickHouse/pull/6908) ([Zhichang Yu](https://github.com/yuzhichang)) - -## ClickHouse release 19.14 {#clickhouse-release-19-14} - -### ClickHouse release 19.14.7.15, 2019-10-02 {#clickhouse-release-19-14-7-15-2019-10-02} - -#### Bug Fix {#bug-fix-6} - -- This release also contains all bug fixes from 19.11.12.69. -- Fixed compatibility for distributed queries between 19.14 and earlier versions. This fixes [\#7068](https://github.com/ClickHouse/ClickHouse/issues/7068). [\#7069](https://github.com/ClickHouse/ClickHouse/pull/7069) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.14.6.12, 2019-09-19 {#clickhouse-release-19-14-6-12-2019-09-19} - -#### Bug Fix {#bug-fix-7} - -- Fix for function `АrrayEnumerateUniqRanked` with empty arrays in params. [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) -- Fixed subquery name in queries with `ARRAY JOIN` and `GLOBAL IN subquery` with alias. Use subquery alias for external table name if it is specified. [\#6934](https://github.com/ClickHouse/ClickHouse/pull/6934) ([Ivan](https://github.com/abyss7)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-3} - -- Fix [flapping](https://clickhouse-test-reports.s3.yandex.net/6944/aab95fd5175a513413c7395a73a82044bdafb906/functional_stateless_tests_(debug).html) test `00715_fetch_merged_or_mutated_part_zookeeper` by rewriting it to a shell scripts because it needs to wait for mutations to apply. [\#6977](https://github.com/ClickHouse/ClickHouse/pull/6977) ([Alexander Kazakov](https://github.com/Akazz)) -- Fixed UBSan and MemSan failure in function `groupUniqArray` with emtpy array argument. It was caused by placing of empty `PaddedPODArray` into hash table zero cell because constructor for zero cell value was not called. [\#6937](https://github.com/ClickHouse/ClickHouse/pull/6937) ([Amos Bird](https://github.com/amosbird)) - -### ClickHouse release 19.14.3.3, 2019-09-10 {#clickhouse-release-19-14-3-3-2019-09-10} - -#### New Feature {#new-feature-4} - -- `WITH FILL` modifier for `ORDER BY`. (continuation of [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([Anton Popov](https://github.com/CurtizJ)) -- `WITH TIES` modifier for `LIMIT`. (continuation of [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([Anton Popov](https://github.com/CurtizJ)) -- Parse unquoted `NULL` literal as NULL (if setting `format_csv_unquoted_null_literal_as_null=1`). Initialize null fields with default values if data type of this field is not nullable (if setting `input_format_null_as_default=1`). [\#5990](https://github.com/ClickHouse/ClickHouse/issues/5990) [\#6055](https://github.com/ClickHouse/ClickHouse/pull/6055) ([tavplubix](https://github.com/tavplubix)) -- Support for wildcards in paths of table functions `file` and `hdfs`. If the path contains wildcards, the table will be readonly. Example of usage: `select * from hdfs('hdfs://hdfs1:9000/some_dir/another_dir/*/file{0..9}{0..9}')` and `select * from file('some_dir/{some_file,another_file,yet_another}.tsv', 'TSV', 'value UInt32')`. [\#6092](https://github.com/ClickHouse/ClickHouse/pull/6092) ([Olga Khvostikova](https://github.com/stavrolia)) -- New `system.metric_log` table which stores values of `system.events` and `system.metrics` with specified time interval. [\#6363](https://github.com/ClickHouse/ClickHouse/issues/6363) [\#6467](https://github.com/ClickHouse/ClickHouse/pull/6467) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6530](https://github.com/ClickHouse/ClickHouse/pull/6530) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Allow to write ClickHouse text logs to `system.text_log` table. [\#6037](https://github.com/ClickHouse/ClickHouse/issues/6037) [\#6103](https://github.com/ClickHouse/ClickHouse/pull/6103) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6164](https://github.com/ClickHouse/ClickHouse/pull/6164) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Show private symbols in stack traces (this is done via parsing symbol tables of ELF files). Added information about file and line number in stack traces if debug info is present. Speedup symbol name lookup with indexing symbols present in program. Added new SQL functions for introspection: `demangle` and `addressToLine`. Renamed function `symbolizeAddress` to `addressToSymbol` for consistency. Function `addressToSymbol` will return mangled name for performance reasons and you have to apply `demangle`. Added setting `allow_introspection_functions` which is turned off by default. [\#6201](https://github.com/ClickHouse/ClickHouse/pull/6201) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Table function `values` (the name is case-insensitive). It allows to read from `VALUES` list proposed in [\#5984](https://github.com/ClickHouse/ClickHouse/issues/5984). Example: `SELECT * FROM VALUES('a UInt64, s String', (1, 'one'), (2, 'two'), (3, 'three'))`. [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) -- Added an ability to alter storage settings. Syntax: `ALTER TABLE MODIFY SETTING = `. [\#6366](https://github.com/ClickHouse/ClickHouse/pull/6366) [\#6669](https://github.com/ClickHouse/ClickHouse/pull/6669) [\#6685](https://github.com/ClickHouse/ClickHouse/pull/6685) ([alesapin](https://github.com/alesapin)) -- Support for removing of detached parts. Syntax: `ALTER TABLE DROP DETACHED PART ''`. [\#6158](https://github.com/ClickHouse/ClickHouse/pull/6158) ([tavplubix](https://github.com/tavplubix)) -- Table constraints. Allows to add constraint to table definition which will be checked at insert. [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([Gleb Novikov](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Suppport for cascaded materialized views. [\#6324](https://github.com/ClickHouse/ClickHouse/pull/6324) ([Amos Bird](https://github.com/amosbird)) -- Turn on query profiler by default to sample every query execution thread once a second. [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Input format `ORC`. [\#6454](https://github.com/ClickHouse/ClickHouse/pull/6454) [\#6703](https://github.com/ClickHouse/ClickHouse/pull/6703) ([akonyaev90](https://github.com/akonyaev90)) -- Added two new functions: `sigmoid` and `tanh` (that are useful for machine learning applications). [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Function `hasToken(haystack, token)`, `hasTokenCaseInsensitive(haystack, token)` to check if given token is in haystack. Token is a maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). Token must be a constant string. Supported by tokenbf\_v1 index specialization. [\#6596](https://github.com/ClickHouse/ClickHouse/pull/6596), [\#6662](https://github.com/ClickHouse/ClickHouse/pull/6662) ([Vasily Nemkov](https://github.com/Enmk)) -- New function `neighbor(value, offset[, default_value])`. Allows to reach prev/next value within column in a block of data. [\#5925](https://github.com/ClickHouse/ClickHouse/pull/5925) ([Alex Krash](https://github.com/alex-krash)) [6685365ab8c5b74f9650492c88a012596eb1b0c6](https://github.com/ClickHouse/ClickHouse/commit/6685365ab8c5b74f9650492c88a012596eb1b0c6) [341e2e4587a18065c2da1ca888c73389f48ce36c](https://github.com/ClickHouse/ClickHouse/commit/341e2e4587a18065c2da1ca888c73389f48ce36c) [Alexey Milovidov](https://github.com/alexey-milovidov) -- Created a function `currentUser()`, returning login of authorized user. Added alias `user()` for compatibility with MySQL. [\#6470](https://github.com/ClickHouse/ClickHouse/pull/6470) ([Alex Krash](https://github.com/alex-krash)) -- New aggregate functions `quantilesExactInclusive` and `quantilesExactExclusive` which were proposed in [\#5885](https://github.com/ClickHouse/ClickHouse/issues/5885). [\#6477](https://github.com/ClickHouse/ClickHouse/pull/6477) ([dimarub2000](https://github.com/dimarub2000)) -- Function `bitmapRange(bitmap, range_begin, range_end)` which returns new set with specified range (not include the `range_end`). [\#6314](https://github.com/ClickHouse/ClickHouse/pull/6314) ([Zhichang Yu](https://github.com/yuzhichang)) -- Function `geohashesInBox(longitude_min, latitude_min, longitude_max, latitude_max, precision)` which creates array of precision-long strings of geohash-boxes covering provided area. [\#6127](https://github.com/ClickHouse/ClickHouse/pull/6127) ([Vasily Nemkov](https://github.com/Enmk)) -- Implement support for INSERT query with `Kafka` tables. [\#6012](https://github.com/ClickHouse/ClickHouse/pull/6012) ([Ivan](https://github.com/abyss7)) -- Added support for `_partition` and `_timestamp` virtual columns to Kafka engine. [\#6400](https://github.com/ClickHouse/ClickHouse/pull/6400) ([Ivan](https://github.com/abyss7)) -- Possibility to remove sensitive data from `query_log`, server logs, process list with regexp-based rules. [\#5710](https://github.com/ClickHouse/ClickHouse/pull/5710) ([filimonov](https://github.com/filimonov)) - -#### Experimental Feature {#experimental-feature-2} - -- Input and output data format `Template`. It allows to specify custom format string for input and output. [\#4354](https://github.com/ClickHouse/ClickHouse/issues/4354) [\#6727](https://github.com/ClickHouse/ClickHouse/pull/6727) ([tavplubix](https://github.com/tavplubix)) -- Implementation of `LIVE VIEW` tables that were originally proposed in [\#2898](https://github.com/ClickHouse/ClickHouse/pull/2898), prepared in [\#3925](https://github.com/ClickHouse/ClickHouse/issues/3925), and then updated in [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). See [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) for detailed description. [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) ([vzakaznikov](https://github.com/vzakaznikov)) [\#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#6656](https://github.com/ClickHouse/ClickHouse/pull/6656) ([vzakaznikov](https://github.com/vzakaznikov)) Note that `LIVE VIEW` feature may be removed in next versions. - -#### Bug Fix {#bug-fix-8} - -- This release also contains all bug fixes from 19.13 and 19.11. -- Fix segmentation fault when the table has skip indices and vertical merge happens. [\#6723](https://github.com/ClickHouse/ClickHouse/pull/6723) ([alesapin](https://github.com/alesapin)) -- Fix per-column TTL with non-trivial column defaults. Previously in case of force TTL merge with `OPTIMIZE ... FINAL` query, expired values was replaced by type defaults instead of user-specified column defaults. [\#6796](https://github.com/ClickHouse/ClickHouse/pull/6796) ([Anton Popov](https://github.com/CurtizJ)) -- Fix Kafka messages duplication problem on normal server restart. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([Ivan](https://github.com/abyss7)) -- Fixed infinite loop when reading Kafka messages. Do not pause/resume consumer on subscription at all - otherwise it may get paused indefinitely in some scenarios. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([Ivan](https://github.com/abyss7)) -- Fix `Key expression contains comparison between inconvertible types` exception in `bitmapContains` function. [\#6136](https://github.com/ClickHouse/ClickHouse/issues/6136) [\#6146](https://github.com/ClickHouse/ClickHouse/issues/6146) [\#6156](https://github.com/ClickHouse/ClickHouse/pull/6156) ([dimarub2000](https://github.com/dimarub2000)) -- Fix segfault with enabled `optimize_skip_unused_shards` and missing sharding key. [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Removed extra verbose logging in MySQL interface [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Return the ability to parse boolean settings from ‘true’ and ‘false’ in the configuration file. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) -- Fix crash in `quantile` and `median` function over `Nullable(Decimal128)`. [\#6378](https://github.com/ClickHouse/ClickHouse/pull/6378) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed possible incomplete result returned by `SELECT` query with `WHERE` condition on primary key contained conversion to Float type. It was caused by incorrect checking of monotonicity in `toFloat` function. [\#6248](https://github.com/ClickHouse/ClickHouse/issues/6248) [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) -- Check `max_expanded_ast_elements` setting for mutations. Clear mutations after `TRUNCATE TABLE`. [\#6205](https://github.com/ClickHouse/ClickHouse/pull/6205) ([Winter Zhang](https://github.com/zhang2014)) -- Fix JOIN results for key columns when used with `join_use_nulls`. Attach Nulls instead of columns defaults. [\#6249](https://github.com/ClickHouse/ClickHouse/pull/6249) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix for skip indices with vertical merge and alter. Fix for `Bad size of marks file` exception. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594) [\#6713](https://github.com/ClickHouse/ClickHouse/pull/6713) ([alesapin](https://github.com/alesapin)) -- Fix rare crash in `ALTER MODIFY COLUMN` and vertical merge when one of merged/altered parts is empty (0 rows) [\#6746](https://github.com/ClickHouse/ClickHouse/issues/6746) [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) -- Fixed bug in conversion of `LowCardinality` types in `AggregateFunctionFactory`. This fixes [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix wrong behavior and possible segfaults in `topK` and `topKWeighted` aggregated functions. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed unsafe code around `getIdentifier` function. [\#6401](https://github.com/ClickHouse/ClickHouse/issues/6401) [\#6409](https://github.com/ClickHouse/ClickHouse/pull/6409) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed bug in MySQL wire protocol (is used while connecting to ClickHouse form MySQL client). Caused by heap buffer overflow in `PacketPayloadWriteBuffer`. [\#6212](https://github.com/ClickHouse/ClickHouse/pull/6212) ([Yuriy Baranov](https://github.com/yurriy)) -- Fixed memory leak in `bitmapSubsetInRange` function. [\#6819](https://github.com/ClickHouse/ClickHouse/pull/6819) ([Zhichang Yu](https://github.com/yuzhichang)) -- Fix rare bug when mutation executed after granularity change. [\#6816](https://github.com/ClickHouse/ClickHouse/pull/6816) ([alesapin](https://github.com/alesapin)) -- Allow protobuf message with all fields by default. [\#6132](https://github.com/ClickHouse/ClickHouse/pull/6132) ([Vitaly Baranov](https://github.com/vitlibar)) -- Resolve a bug with `nullIf` function when we send a `NULL` argument on the second argument. [\#6446](https://github.com/ClickHouse/ClickHouse/pull/6446) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Fix rare bug with wrong memory allocation/deallocation in complex key cache dictionaries with string fields which leads to infinite memory consumption (looks like memory leak). Bug reproduces when string size was a power of two starting from eight (8, 16, 32, etc). [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alesapin](https://github.com/alesapin)) -- Fixed Gorilla encoding on small sequences which caused exception `Cannot write after end of buffer`. [\#6398](https://github.com/ClickHouse/ClickHouse/issues/6398) [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Vasily Nemkov](https://github.com/Enmk)) -- Allow to use not nullable types in JOINs with `join_use_nulls` enabled. [\#6705](https://github.com/ClickHouse/ClickHouse/pull/6705) ([Artem Zuikov](https://github.com/4ertus2)) -- Disable `Poco::AbstractConfiguration` substitutions in query in `clickhouse-client`. [\#6706](https://github.com/ClickHouse/ClickHouse/pull/6706) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid deadlock in `REPLACE PARTITION`. [\#6677](https://github.com/ClickHouse/ClickHouse/pull/6677) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Using `arrayReduce` for constant arguments may lead to segfault. [\#6242](https://github.com/ClickHouse/ClickHouse/issues/6242) [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix inconsistent parts which can appear if replica was restored after `DROP PARTITION`. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- Fixed hang in `JSONExtractRaw` function. [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix bug with incorrect skip indices serialization and aggregation with adaptive granularity. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594). [\#6748](https://github.com/ClickHouse/ClickHouse/pull/6748) ([alesapin](https://github.com/alesapin)) -- Fix `WITH ROLLUP` and `WITH CUBE` modifiers of `GROUP BY` with two-level aggregation. [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([Anton Popov](https://github.com/CurtizJ)) -- Fix bug with writing secondary indices marks with adaptive granularity. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alesapin](https://github.com/alesapin)) -- Fix initialization order while server startup. Since `StorageMergeTree::background_task_handle` is initialized in `startup()` the `MergeTreeBlockOutputStream::write()` may try to use it before initialization. Just check if it is initialized. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([Ivan](https://github.com/abyss7)) -- Clearing the data buffer from the previous read operation that was completed with an error. [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) -- Fix bug with enabling adaptive granularity when creating a new replica for Replicated\*MergeTree table. [\#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) -- Fixed possible crash during server startup in case of exception happened in `libunwind` during exception at access to uninitialized `ThreadStatus` structure. [\#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Fix crash in `yandexConsistentHash` function. Found by fuzz test. [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304) [\#6305](https://github.com/ClickHouse/ClickHouse/pull/6305) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed the possibility of hanging queries when server is overloaded and global thread pool becomes near full. This have higher chance to happen on clusters with large number of shards (hundreds), because distributed queries allocate a thread per connection to each shard. For example, this issue may reproduce if a cluster of 330 shards is processing 30 concurrent distributed queries. This issue affects all versions starting from 19.2. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed logic of `arrayEnumerateUniqRanked` function. [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix segfault when decoding symbol table. [\#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([Amos Bird](https://github.com/amosbird)) -- Fixed irrelevant exception in cast of `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Removed extra quoting of description in `system.settings` table. [\#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [\#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid possible deadlock in `TRUNCATE` of Replicated table. [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix reading in order of sorting key. [\#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([Anton Popov](https://github.com/CurtizJ)) -- Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) -- Fix bug opened by [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) (since 19.4.0). Reproduces in queries to Distributed tables over MergeTree tables when we doesn’t query any columns (`SELECT 1`). [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) -- Fixed overflow in integer division of signed type to unsigned type. The behaviour was exactly as in C or C++ language (integer promotion rules) that may be surprising. Please note that the overflow is still possible when dividing large signed number to large unsigned number or vice-versa (but that case is less usual). The issue existed in all server versions. [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214) [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Limit maximum sleep time for throttling when `max_execution_speed` or `max_execution_speed_bytes` is set. Fixed false errors like `Estimated query execution time (inf seconds) is too long`. [\#5547](https://github.com/ClickHouse/ClickHouse/issues/5547) [\#6232](https://github.com/ClickHouse/ClickHouse/pull/6232) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed issues about using `MATERIALIZED` columns and aliases in `MaterializedView`. [\#448](https://github.com/ClickHouse/ClickHouse/issues/448) [\#3484](https://github.com/ClickHouse/ClickHouse/issues/3484) [\#3450](https://github.com/ClickHouse/ClickHouse/issues/3450) [\#2878](https://github.com/ClickHouse/ClickHouse/issues/2878) [\#2285](https://github.com/ClickHouse/ClickHouse/issues/2285) [\#3796](https://github.com/ClickHouse/ClickHouse/pull/3796) ([Amos Bird](https://github.com/amosbird)) [\#6316](https://github.com/ClickHouse/ClickHouse/pull/6316) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix `FormatFactory` behaviour for input streams which are not implemented as processor. [\#6495](https://github.com/ClickHouse/ClickHouse/pull/6495) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed typo. [\#6631](https://github.com/ClickHouse/ClickHouse/pull/6631) ([Alex Ryndin](https://github.com/alexryndin)) -- Typo in the error message ( is -\> are ). [\#6839](https://github.com/ClickHouse/ClickHouse/pull/6839) ([Denis Zhuravlev](https://github.com/den-crane)) -- Fixed error while parsing of columns list from string if type contained a comma (this issue was relevant for `File`, `URL`, `HDFS` storages) [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) - -#### Security Fix {#security-fix} - -- This release also contains all bug security fixes from 19.13 and 19.11. -- Fixed the possibility of a fabricated query to cause server crash due to stack overflow in SQL parser. Fixed the possibility of stack overflow in Merge and Distributed tables, materialized views and conditions for row-level security that involve subqueries. [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Improvement {#improvement-3} - -- Correct implementation of ternary logic for `AND/OR`. [\#6048](https://github.com/ClickHouse/ClickHouse/pull/6048) ([Alexander Kazakov](https://github.com/Akazz)) -- Now values and rows with expired TTL will be removed after `OPTIMIZE ... FINAL` query from old parts without TTL infos or with outdated TTL infos, e.g. after `ALTER ... MODIFY TTL` query. Added queries `SYSTEM STOP/START TTL MERGES` to disallow/allow assign merges with TTL and filter expired values in all merges. [\#6274](https://github.com/ClickHouse/ClickHouse/pull/6274) ([Anton Popov](https://github.com/CurtizJ)) -- Possibility to change the location of ClickHouse history file for client using `CLICKHOUSE_HISTORY_FILE` env. [\#6840](https://github.com/ClickHouse/ClickHouse/pull/6840) ([filimonov](https://github.com/filimonov)) -- Remove `dry_run` flag from `InterpreterSelectQuery`. … [\#6375](https://github.com/ClickHouse/ClickHouse/pull/6375) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Support `ASOF JOIN` with `ON` section. [\#6211](https://github.com/ClickHouse/ClickHouse/pull/6211) ([Artem Zuikov](https://github.com/4ertus2)) -- Better support of skip indexes for mutations and replication. Support for `MATERIALIZE/CLEAR INDEX ... IN PARTITION` query. `UPDATE x = x` recalculates all indices that use column `x`. [\#5053](https://github.com/ClickHouse/ClickHouse/pull/5053) ([Nikita Vasilev](https://github.com/nikvas0)) -- Allow to `ATTACH` live views (for example, at the server startup) regardless to `allow_experimental_live_view` setting. [\#6754](https://github.com/ClickHouse/ClickHouse/pull/6754) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- For stack traces gathered by query profiler, do not include stack frames generated by the query profiler itself. [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now table functions `values`, `file`, `url`, `hdfs` have support for ALIAS columns. [\#6255](https://github.com/ClickHouse/ClickHouse/pull/6255) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Throw an exception if `config.d` file doesn’t have the corresponding root element as the config file. [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) -- Print extra info in exception message for `no space left on device`. [\#6182](https://github.com/ClickHouse/ClickHouse/issues/6182), [\#6252](https://github.com/ClickHouse/ClickHouse/issues/6252) [\#6352](https://github.com/ClickHouse/ClickHouse/pull/6352) ([tavplubix](https://github.com/tavplubix)) -- When determining shards of a `Distributed` table to be covered by a read query (for `optimize_skip_unused_shards` = 1) ClickHouse now checks conditions from both `prewhere` and `where` clauses of select statement. [\#6521](https://github.com/ClickHouse/ClickHouse/pull/6521) ([Alexander Kazakov](https://github.com/Akazz)) -- Enabled `SIMDJSON` for machines without AVX2 but with SSE 4.2 and PCLMUL instruction set. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285) [\#6320](https://github.com/ClickHouse/ClickHouse/pull/6320) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ClickHouse can work on filesystems without `O_DIRECT` support (such as ZFS and BtrFS) without additional tuning. [\#4449](https://github.com/ClickHouse/ClickHouse/issues/4449) [\#6730](https://github.com/ClickHouse/ClickHouse/pull/6730) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Support push down predicate for final subquery. [\#6120](https://github.com/ClickHouse/ClickHouse/pull/6120) ([TCeason](https://github.com/TCeason)) [\#6162](https://github.com/ClickHouse/ClickHouse/pull/6162) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Better `JOIN ON` keys extraction [\#6131](https://github.com/ClickHouse/ClickHouse/pull/6131) ([Artem Zuikov](https://github.com/4ertus2)) -- Upated `SIMDJSON`. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285). [\#6306](https://github.com/ClickHouse/ClickHouse/pull/6306) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Optimize selecting of smallest column for `SELECT count()` query. [\#6344](https://github.com/ClickHouse/ClickHouse/pull/6344) ([Amos Bird](https://github.com/amosbird)) -- Added `strict` parameter in `windowFunnel()`. When the `strict` is set, the `windowFunnel()` applies conditions only for the unique values. [\#6548](https://github.com/ClickHouse/ClickHouse/pull/6548) ([achimbab](https://github.com/achimbab)) -- Safer interface of `mysqlxx::Pool`. [\#6150](https://github.com/ClickHouse/ClickHouse/pull/6150) ([avasiliev](https://github.com/avasiliev)) -- Options line size when executing with `--help` option now corresponds with terminal size. [\#6590](https://github.com/ClickHouse/ClickHouse/pull/6590) ([dimarub2000](https://github.com/dimarub2000)) -- Disable “read in order” optimization for aggregation without keys. [\#6599](https://github.com/ClickHouse/ClickHouse/pull/6599) ([Anton Popov](https://github.com/CurtizJ)) -- HTTP status code for `INCORRECT_DATA` and `TYPE_MISMATCH` error codes was changed from default `500 Internal Server Error` to `400 Bad Request`. [\#6271](https://github.com/ClickHouse/ClickHouse/pull/6271) ([Alexander Rodin](https://github.com/a-rodin)) -- Move Join object from `ExpressionAction` into `AnalyzedJoin`. `ExpressionAnalyzer` and `ExpressionAction` do not know about `Join` class anymore. Its logic is hidden by `AnalyzedJoin` iface. [\#6801](https://github.com/ClickHouse/ClickHouse/pull/6801) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed possible deadlock of distributed queries when one of shards is localhost but the query is sent via network connection. [\#6759](https://github.com/ClickHouse/ClickHouse/pull/6759) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Changed semantic of multiple tables `RENAME` to avoid possible deadlocks. [\#6757](https://github.com/ClickHouse/ClickHouse/issues/6757). [\#6756](https://github.com/ClickHouse/ClickHouse/pull/6756) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Rewritten MySQL compatibility server to prevent loading full packet payload in memory. Decreased memory consumption for each connection to approximately `2 * DBMS_DEFAULT_BUFFER_SIZE` (read/write buffers). [\#5811](https://github.com/ClickHouse/ClickHouse/pull/5811) ([Yuriy Baranov](https://github.com/yurriy)) -- Move AST alias interpreting logic out of parser that doesn’t have to know anything about query semantics. [\#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([Artem Zuikov](https://github.com/4ertus2)) -- Slightly more safe parsing of `NamesAndTypesList`. [\#6408](https://github.com/ClickHouse/ClickHouse/issues/6408). [\#6410](https://github.com/ClickHouse/ClickHouse/pull/6410) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `clickhouse-copier`: Allow use `where_condition` from config with `partition_key` alias in query for checking partition existence (Earlier it was used only in reading data queries). [\#6577](https://github.com/ClickHouse/ClickHouse/pull/6577) ([proller](https://github.com/proller)) -- Added optional message argument in `throwIf`. ([\#5772](https://github.com/ClickHouse/ClickHouse/issues/5772)) [\#6329](https://github.com/ClickHouse/ClickHouse/pull/6329) ([Vdimir](https://github.com/Vdimir)) -- Server exception got while sending insertion data is now being processed in client as well. [\#5891](https://github.com/ClickHouse/ClickHouse/issues/5891) [\#6711](https://github.com/ClickHouse/ClickHouse/pull/6711) ([dimarub2000](https://github.com/dimarub2000)) -- Added a metric `DistributedFilesToInsert` that shows the total number of files in filesystem that are selected to send to remote servers by Distributed tables. The number is summed across all shards. [\#6600](https://github.com/ClickHouse/ClickHouse/pull/6600) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Move most of JOINs prepare logic from `ExpressionAction/ExpressionAnalyzer` to `AnalyzedJoin`. [\#6785](https://github.com/ClickHouse/ClickHouse/pull/6785) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix TSan [warning](https://clickhouse-test-reports.s3.yandex.net/6399/c1c1d1daa98e199e620766f1bd06a5921050a00d/functional_stateful_tests_(thread).html) ‘lock-order-inversion’. [\#6740](https://github.com/ClickHouse/ClickHouse/pull/6740) ([Vasily Nemkov](https://github.com/Enmk)) -- Better information messages about lack of Linux capabilities. Logging fatal errors with “fatal” level, that will make it easier to find in `system.text_log`. [\#6441](https://github.com/ClickHouse/ClickHouse/pull/6441) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- When enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`, `ORDER BY`, it didn’t check the free disk space. The fix add a new setting `min_free_disk_space`, when the free disk space it smaller then the threshold, the query will stop and throw `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([Weiqing Xu](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Removed recursive rwlock by thread. It makes no sense, because threads are reused between queries. `SELECT` query may acquire a lock in one thread, hold a lock from another thread and exit from first thread. In the same time, first thread can be reused by `DROP` query. This will lead to false “Attempt to acquire exclusive lock recursively” messages. [\#6771](https://github.com/ClickHouse/ClickHouse/pull/6771) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Split `ExpressionAnalyzer.appendJoin()`. Prepare a place in `ExpressionAnalyzer` for `MergeJoin`. [\#6524](https://github.com/ClickHouse/ClickHouse/pull/6524) ([Artem Zuikov](https://github.com/4ertus2)) -- Added `mysql_native_password` authentication plugin to MySQL compatibility server. [\#6194](https://github.com/ClickHouse/ClickHouse/pull/6194) ([Yuriy Baranov](https://github.com/yurriy)) -- Less number of `clock_gettime` calls; fixed ABI compatibility between debug/release in `Allocator` (insignificant issue). [\#6197](https://github.com/ClickHouse/ClickHouse/pull/6197) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Move `collectUsedColumns` from `ExpressionAnalyzer` to `SyntaxAnalyzer`. `SyntaxAnalyzer` makes `required_source_columns` itself now. [\#6416](https://github.com/ClickHouse/ClickHouse/pull/6416) ([Artem Zuikov](https://github.com/4ertus2)) -- Add setting `joined_subquery_requires_alias` to require aliases for subselects and table functions in `FROM` that more than one table is present (i.e. queries with JOINs). [\#6733](https://github.com/ClickHouse/ClickHouse/pull/6733) ([Artem Zuikov](https://github.com/4ertus2)) -- Extract `GetAggregatesVisitor` class from `ExpressionAnalyzer`. [\#6458](https://github.com/ClickHouse/ClickHouse/pull/6458) ([Artem Zuikov](https://github.com/4ertus2)) -- `system.query_log`: change data type of `type` column to `Enum`. [\#6265](https://github.com/ClickHouse/ClickHouse/pull/6265) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Static linking of `sha256_password` authentication plugin. [\#6512](https://github.com/ClickHouse/ClickHouse/pull/6512) ([Yuriy Baranov](https://github.com/yurriy)) -- Avoid extra dependency for the setting `compile` to work. In previous versions, the user may get error like `cannot open crti.o`, `unable to find library -lc` etc. [\#6309](https://github.com/ClickHouse/ClickHouse/pull/6309) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- More validation of the input that may come from malicious replica. [\#6303](https://github.com/ClickHouse/ClickHouse/pull/6303) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now `clickhouse-obfuscator` file is available in `clickhouse-client` package. In previous versions it was available as `clickhouse obfuscator` (with whitespace). [\#5816](https://github.com/ClickHouse/ClickHouse/issues/5816) [\#6609](https://github.com/ClickHouse/ClickHouse/pull/6609) ([dimarub2000](https://github.com/dimarub2000)) -- Fixed deadlock when we have at least two queries that read at least two tables in different order and another query that performs DDL operation on one of tables. Fixed another very rare deadlock. [\#6764](https://github.com/ClickHouse/ClickHouse/pull/6764) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added `os_thread_ids` column to `system.processes` and `system.query_log` for better debugging possibilities. [\#6763](https://github.com/ClickHouse/ClickHouse/pull/6763) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- A workaround for PHP mysqlnd extension bugs which occur when `sha256_password` is used as a default authentication plugin (described in [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([Yuriy Baranov](https://github.com/yurriy)) -- Remove unneeded place with changed nullability columns. [\#6693](https://github.com/ClickHouse/ClickHouse/pull/6693) ([Artem Zuikov](https://github.com/4ertus2)) -- Set default value of `queue_max_wait_ms` to zero, because current value (five seconds) makes no sense. There are rare circumstances when this settings has any use. Added settings `replace_running_query_max_wait_ms`, `kafka_max_wait_ms` and `connection_pool_max_wait_ms` for disambiguation. [\#6692](https://github.com/ClickHouse/ClickHouse/pull/6692) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Extract `SelectQueryExpressionAnalyzer` from `ExpressionAnalyzer`. Keep the last one for non-select queries. [\#6499](https://github.com/ClickHouse/ClickHouse/pull/6499) ([Artem Zuikov](https://github.com/4ertus2)) -- Removed duplicating input and output formats. [\#6239](https://github.com/ClickHouse/ClickHouse/pull/6239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Allow user to override `poll_interval` and `idle_connection_timeout` settings on connection. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `MergeTree` now has an additional option `ttl_only_drop_parts` (disabled by default) to avoid partial pruning of parts, so that they dropped completely when all the rows in a part are expired. [\#6191](https://github.com/ClickHouse/ClickHouse/pull/6191) ([Sergi Vladykin](https://github.com/svladykin)) -- Type checks for set index functions. Throw exception if function got a wrong type. This fixes fuzz test with UBSan. [\#6511](https://github.com/ClickHouse/ClickHouse/pull/6511) ([Nikita Vasilev](https://github.com/nikvas0)) - -#### Performance Improvement {#performance-improvement-2} - -- Optimize queries with `ORDER BY expressions` clause, where `expressions` have coinciding prefix with sorting key in `MergeTree` tables. This optimization is controlled by `optimize_read_in_order` setting. [\#6054](https://github.com/ClickHouse/ClickHouse/pull/6054) [\#6629](https://github.com/ClickHouse/ClickHouse/pull/6629) ([Anton Popov](https://github.com/CurtizJ)) -- Allow to use multiple threads during parts loading and removal. [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Implemented batch variant of updating aggregate function states. It may lead to performance benefits. [\#6435](https://github.com/ClickHouse/ClickHouse/pull/6435) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Using `FastOps` library for functions `exp`, `log`, `sigmoid`, `tanh`. FastOps is a fast vector math library from Michael Parakhin (Yandex CTO). Improved performance of `exp` and `log` functions more than 6 times. The functions `exp` and `log` from `Float32` argument will return `Float32` (in previous versions they always return `Float64`). Now `exp(nan)` may return `inf`. The result of `exp` and `log` functions may be not the nearest machine representable number to the true answer. [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexey-milovidov](https://github.com/alexey-milovidov)) Using Danila Kutenin variant to make fastops working [\#6317](https://github.com/ClickHouse/ClickHouse/pull/6317) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Disable consecutive key optimization for `UInt8/16`. [\#6298](https://github.com/ClickHouse/ClickHouse/pull/6298) [\#6701](https://github.com/ClickHouse/ClickHouse/pull/6701) ([akuzm](https://github.com/akuzm)) -- Improved performance of `simdjson` library by getting rid of dynamic allocation in `ParsedJson::Iterator`. [\#6479](https://github.com/ClickHouse/ClickHouse/pull/6479) ([Vitaly Baranov](https://github.com/vitlibar)) -- Pre-fault pages when allocating memory with `mmap()`. [\#6667](https://github.com/ClickHouse/ClickHouse/pull/6667) ([akuzm](https://github.com/akuzm)) -- Fix performance bug in `Decimal` comparison. [\#6380](https://github.com/ClickHouse/ClickHouse/pull/6380) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-4} - -- Remove Compiler (runtime template instantiation) because we’ve win over it’s performance. [\#6646](https://github.com/ClickHouse/ClickHouse/pull/6646) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added performance test to show degradation of performance in gcc-9 in more isolated way. [\#6302](https://github.com/ClickHouse/ClickHouse/pull/6302) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added table function `numbers_mt`, which is multithreaded version of `numbers`. Updated performance tests with hash functions. [\#6554](https://github.com/ClickHouse/ClickHouse/pull/6554) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Comparison mode in `clickhouse-benchmark` [\#6220](https://github.com/ClickHouse/ClickHouse/issues/6220) [\#6343](https://github.com/ClickHouse/ClickHouse/pull/6343) ([dimarub2000](https://github.com/dimarub2000)) -- Best effort for printing stack traces. Also added `SIGPROF` as a debugging signal to print stack trace of a running thread. [\#6529](https://github.com/ClickHouse/ClickHouse/pull/6529) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Every function in its own file, part 10. [\#6321](https://github.com/ClickHouse/ClickHouse/pull/6321) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Remove doubled const `TABLE_IS_READ_ONLY`. [\#6566](https://github.com/ClickHouse/ClickHouse/pull/6566) ([filimonov](https://github.com/filimonov)) -- Formatting changes for `StringHashMap` PR [\#5417](https://github.com/ClickHouse/ClickHouse/issues/5417). [\#6700](https://github.com/ClickHouse/ClickHouse/pull/6700) ([akuzm](https://github.com/akuzm)) -- Better subquery for join creation in `ExpressionAnalyzer`. [\#6824](https://github.com/ClickHouse/ClickHouse/pull/6824) ([Artem Zuikov](https://github.com/4ertus2)) -- Remove a redundant condition (found by PVS Studio). [\#6775](https://github.com/ClickHouse/ClickHouse/pull/6775) ([akuzm](https://github.com/akuzm)) -- Separate the hash table interface for `ReverseIndex`. [\#6672](https://github.com/ClickHouse/ClickHouse/pull/6672) ([akuzm](https://github.com/akuzm)) -- Refactoring of settings. [\#6689](https://github.com/ClickHouse/ClickHouse/pull/6689) ([alesapin](https://github.com/alesapin)) -- Add comments for `set` index functions. [\#6319](https://github.com/ClickHouse/ClickHouse/pull/6319) ([Nikita Vasilev](https://github.com/nikvas0)) -- Increase OOM score in debug version on Linux. [\#6152](https://github.com/ClickHouse/ClickHouse/pull/6152) ([akuzm](https://github.com/akuzm)) -- HDFS HA now work in debug build. [\#6650](https://github.com/ClickHouse/ClickHouse/pull/6650) ([Weiqing Xu](https://github.com/weiqxu)) -- Added a test to `transform_query_for_external_database`. [\#6388](https://github.com/ClickHouse/ClickHouse/pull/6388) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add test for multiple materialized views for Kafka table. [\#6509](https://github.com/ClickHouse/ClickHouse/pull/6509) ([Ivan](https://github.com/abyss7)) -- Make a better build scheme. [\#6500](https://github.com/ClickHouse/ClickHouse/pull/6500) ([Ivan](https://github.com/abyss7)) -- Fixed `test_external_dictionaries` integration in case it was executed under non root user. [\#6507](https://github.com/ClickHouse/ClickHouse/pull/6507) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- The bug reproduces when total size of written packets exceeds `DBMS_DEFAULT_BUFFER_SIZE`. [\#6204](https://github.com/ClickHouse/ClickHouse/pull/6204) ([Yuriy Baranov](https://github.com/yurriy)) -- Added a test for `RENAME` table race condition [\#6752](https://github.com/ClickHouse/ClickHouse/pull/6752) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid data race on Settings in `KILL QUERY`. [\#6753](https://github.com/ClickHouse/ClickHouse/pull/6753) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add integration test for handling errors by a cache dictionary. [\#6755](https://github.com/ClickHouse/ClickHouse/pull/6755) ([Vitaly Baranov](https://github.com/vitlibar)) -- Disable parsing of ELF object files on Mac OS, because it makes no sense. [\#6578](https://github.com/ClickHouse/ClickHouse/pull/6578) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Attempt to make changelog generator better. [\#6327](https://github.com/ClickHouse/ClickHouse/pull/6327) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Adding `-Wshadow` switch to the GCC. [\#6325](https://github.com/ClickHouse/ClickHouse/pull/6325) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) -- Removed obsolete code for `mimalloc` support. [\#6715](https://github.com/ClickHouse/ClickHouse/pull/6715) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `zlib-ng` determines x86 capabilities and saves this info to global variables. This is done in defalteInit call, which may be made by different threads simultaneously. To avoid multithreaded writes, do it on library startup. [\#6141](https://github.com/ClickHouse/ClickHouse/pull/6141) ([akuzm](https://github.com/akuzm)) -- Regression test for a bug which in join which was fixed in [\#5192](https://github.com/ClickHouse/ClickHouse/issues/5192). [\#6147](https://github.com/ClickHouse/ClickHouse/pull/6147) ([Bakhtiyor Ruziev](https://github.com/theruziev)) -- Fixed MSan report. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix flapping TTL test. [\#6782](https://github.com/ClickHouse/ClickHouse/pull/6782) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed false data race in `MergeTreeDataPart::is_frozen` field. [\#6583](https://github.com/ClickHouse/ClickHouse/pull/6583) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed timeouts in fuzz test. In previous version, it managed to find false hangup in query `SELECT * FROM numbers_mt(gccMurmurHash(''))`. [\#6582](https://github.com/ClickHouse/ClickHouse/pull/6582) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added debug checks to `static_cast` of columns. [\#6581](https://github.com/ClickHouse/ClickHouse/pull/6581) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Support for Oracle Linux in official RPM packages. [\#6356](https://github.com/ClickHouse/ClickHouse/issues/6356) [\#6585](https://github.com/ClickHouse/ClickHouse/pull/6585) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Changed json perftests from `once` to `loop` type. [\#6536](https://github.com/ClickHouse/ClickHouse/pull/6536) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- `odbc-bridge.cpp` defines `main()` so it should not be included in `clickhouse-lib`. [\#6538](https://github.com/ClickHouse/ClickHouse/pull/6538) ([Orivej Desh](https://github.com/orivej)) -- Test for crash in `FULL|RIGHT JOIN` with nulls in right table’s keys. [\#6362](https://github.com/ClickHouse/ClickHouse/pull/6362) ([Artem Zuikov](https://github.com/4ertus2)) -- Added a test for the limit on expansion of aliases just in case. [\#6442](https://github.com/ClickHouse/ClickHouse/pull/6442) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Switched from `boost::filesystem` to `std::filesystem` where appropriate. [\#6253](https://github.com/ClickHouse/ClickHouse/pull/6253) [\#6385](https://github.com/ClickHouse/ClickHouse/pull/6385) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added RPM packages to website. [\#6251](https://github.com/ClickHouse/ClickHouse/pull/6251) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add a test for fixed `Unknown identifier` exception in `IN` section. [\#6708](https://github.com/ClickHouse/ClickHouse/pull/6708) ([Artem Zuikov](https://github.com/4ertus2)) -- Simplify `shared_ptr_helper` because people facing difficulties understanding it. [\#6675](https://github.com/ClickHouse/ClickHouse/pull/6675) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added performance tests for fixed Gorilla and DoubleDelta codec. [\#6179](https://github.com/ClickHouse/ClickHouse/pull/6179) ([Vasily Nemkov](https://github.com/Enmk)) -- Split the integration test `test_dictionaries` into 4 separate tests. [\#6776](https://github.com/ClickHouse/ClickHouse/pull/6776) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix PVS-Studio warning in `PipelineExecutor`. [\#6777](https://github.com/ClickHouse/ClickHouse/pull/6777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Allow to use `library` dictionary source with ASan. [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added option to generate changelog from a list of PRs. [\#6350](https://github.com/ClickHouse/ClickHouse/pull/6350) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Lock the `TinyLog` storage when reading. [\#6226](https://github.com/ClickHouse/ClickHouse/pull/6226) ([akuzm](https://github.com/akuzm)) -- Check for broken symlinks in CI. [\#6634](https://github.com/ClickHouse/ClickHouse/pull/6634) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Increase timeout for “stack overflow” test because it may take a long time in debug build. [\#6637](https://github.com/ClickHouse/ClickHouse/pull/6637) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added a check for double whitespaces. [\#6643](https://github.com/ClickHouse/ClickHouse/pull/6643) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix `new/delete` memory tracking when build with sanitizers. Tracking is not clear. It only prevents memory limit exceptions in tests. [\#6450](https://github.com/ClickHouse/ClickHouse/pull/6450) ([Artem Zuikov](https://github.com/4ertus2)) -- Enable back the check of undefined symbols while linking. [\#6453](https://github.com/ClickHouse/ClickHouse/pull/6453) ([Ivan](https://github.com/abyss7)) -- Avoid rebuilding `hyperscan` every day. [\#6307](https://github.com/ClickHouse/ClickHouse/pull/6307) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed UBSan report in `ProtobufWriter`. [\#6163](https://github.com/ClickHouse/ClickHouse/pull/6163) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Don’t allow to use query profiler with sanitizers because it is not compatible. [\#6769](https://github.com/ClickHouse/ClickHouse/pull/6769) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add test for reloading a dictionary after fail by timer. [\#6114](https://github.com/ClickHouse/ClickHouse/pull/6114) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix inconsistency in `PipelineExecutor::prepareProcessor` argument type. [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Added a test for bad URIs. [\#6493](https://github.com/ClickHouse/ClickHouse/pull/6493) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added more checks to `CAST` function. This should get more information about segmentation fault in fuzzy test. [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Added `gcc-9` support to `docker/builder` container that builds image locally. [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([Gleb Novikov](https://github.com/NanoBjorn)) -- Test for primary key with `LowCardinality(String)`. [\#5044](https://github.com/ClickHouse/ClickHouse/issues/5044) [\#6219](https://github.com/ClickHouse/ClickHouse/pull/6219) ([dimarub2000](https://github.com/dimarub2000)) -- Fixed tests affected by slow stack traces printing. [\#6315](https://github.com/ClickHouse/ClickHouse/pull/6315) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add a test case for crash in `groupUniqArray` fixed in [\#6029](https://github.com/ClickHouse/ClickHouse/pull/6029). [\#4402](https://github.com/ClickHouse/ClickHouse/issues/4402) [\#6129](https://github.com/ClickHouse/ClickHouse/pull/6129) ([akuzm](https://github.com/akuzm)) -- Fixed indices mutations tests. [\#6645](https://github.com/ClickHouse/ClickHouse/pull/6645) ([Nikita Vasilev](https://github.com/nikvas0)) -- In performance test, do not read query log for queries we didn’t run. [\#6427](https://github.com/ClickHouse/ClickHouse/pull/6427) ([akuzm](https://github.com/akuzm)) -- Materialized view now could be created with any low cardinality types regardless to the setting about suspicious low cardinality types. [\#6428](https://github.com/ClickHouse/ClickHouse/pull/6428) ([Olga Khvostikova](https://github.com/stavrolia)) -- Updated tests for `send_logs_level` setting. [\#6207](https://github.com/ClickHouse/ClickHouse/pull/6207) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix build under gcc-8.2. [\#6196](https://github.com/ClickHouse/ClickHouse/pull/6196) ([Max Akhmedov](https://github.com/zlobober)) -- Fix build with internal libc++. [\#6724](https://github.com/ClickHouse/ClickHouse/pull/6724) ([Ivan](https://github.com/abyss7)) -- Fix shared build with `rdkafka` library [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([Ivan](https://github.com/abyss7)) -- Fixes for Mac OS build (incomplete). [\#6390](https://github.com/ClickHouse/ClickHouse/pull/6390) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#6429](https://github.com/ClickHouse/ClickHouse/pull/6429) ([alex-zaitsev](https://github.com/alex-zaitsev)) -- Fix “splitted” build. [\#6618](https://github.com/ClickHouse/ClickHouse/pull/6618) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Other build fixes: [\#6186](https://github.com/ClickHouse/ClickHouse/pull/6186) ([Amos Bird](https://github.com/amosbird)) [\#6486](https://github.com/ClickHouse/ClickHouse/pull/6486) [\#6348](https://github.com/ClickHouse/ClickHouse/pull/6348) ([vxider](https://github.com/Vxider)) [\#6744](https://github.com/ClickHouse/ClickHouse/pull/6744) ([Ivan](https://github.com/abyss7)) [\#6016](https://github.com/ClickHouse/ClickHouse/pull/6016) [\#6421](https://github.com/ClickHouse/ClickHouse/pull/6421) [\#6491](https://github.com/ClickHouse/ClickHouse/pull/6491) ([proller](https://github.com/proller)) - -#### Backward Incompatible Change {#backward-incompatible-change-3} - -- Removed rarely used table function `catBoostPool` and storage `CatBoostPool`. If you have used this table function, please write email to `clickhouse-feedback@yandex-team.com`. Note that CatBoost integration remains and will be supported. [\#6279](https://github.com/ClickHouse/ClickHouse/pull/6279) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Disable `ANY RIGHT JOIN` and `ANY FULL JOIN` by default. Set `any_join_distinct_right_table_keys` setting to enable them. [\#5126](https://github.com/ClickHouse/ClickHouse/issues/5126) [\#6351](https://github.com/ClickHouse/ClickHouse/pull/6351) ([Artem Zuikov](https://github.com/4ertus2)) - -## ClickHouse release 19.13 {#clickhouse-release-19-13} - -### ClickHouse release 19.13.6.51, 2019-10-02 {#clickhouse-release-19-13-6-51-2019-10-02} - -#### Bug Fix {#bug-fix-9} - -- This release also contains all bug fixes from 19.11.12.69. - -### ClickHouse release 19.13.5.44, 2019-09-20 {#clickhouse-release-19-13-5-44-2019-09-20} - -#### Bug Fix {#bug-fix-10} - -- This release also contains all bug fixes from 19.14.6.12. -- Fixed possible inconsistent state of table while executing `DROP` query for replicated table while zookeeper is not accessible. [\#6045](https://github.com/ClickHouse/ClickHouse/issues/6045) [\#6413](https://github.com/ClickHouse/ClickHouse/pull/6413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Fix for data race in StorageMerge [\#6717](https://github.com/ClickHouse/ClickHouse/pull/6717) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix bug introduced in query profiler which leads to endless recv from socket. [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) ([alesapin](https://github.com/alesapin)) -- Fix excessive CPU usage while executing `JSONExtractRaw` function over a boolean value. [\#6208](https://github.com/ClickHouse/ClickHouse/pull/6208) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fixes the regression while pushing to materialized view. [\#6415](https://github.com/ClickHouse/ClickHouse/pull/6415) ([Ivan](https://github.com/abyss7)) -- Table function `url` had the vulnerability allowed the attacker to inject arbitrary HTTP headers in the request. This issue was found by [Nikita Tikhomirov](https://github.com/NSTikhomirov). [\#6466](https://github.com/ClickHouse/ClickHouse/pull/6466) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix useless `AST` check in Set index. [\#6510](https://github.com/ClickHouse/ClickHouse/issues/6510) [\#6651](https://github.com/ClickHouse/ClickHouse/pull/6651) ([Nikita Vasilev](https://github.com/nikvas0)) -- Fixed parsing of `AggregateFunction` values embedded in query. [\#6575](https://github.com/ClickHouse/ClickHouse/issues/6575) [\#6773](https://github.com/ClickHouse/ClickHouse/pull/6773) ([Zhichang Yu](https://github.com/yuzhichang)) -- Fixed wrong behaviour of `trim` functions family. [\#6647](https://github.com/ClickHouse/ClickHouse/pull/6647) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.13.4.32, 2019-09-10 {#clickhouse-release-19-13-4-32-2019-09-10} - -#### Bug Fix {#bug-fix-11} - -- This release also contains all bug security fixes from 19.11.9.52 and 19.11.10.54. -- Fixed data race in `system.parts` table and `ALTER` query. [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245) [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed mismatched header in streams happened in case of reading from empty distributed table with sample and prewhere. [\#6167](https://github.com/ClickHouse/ClickHouse/issues/6167) ([Lixiang Qian](https://github.com/fancyqlx)) [\#6823](https://github.com/ClickHouse/ClickHouse/pull/6823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed crash when using `IN` clause with a subquery with a tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- Fix case with same column names in `GLOBAL JOIN ON` section. [\#6181](https://github.com/ClickHouse/ClickHouse/pull/6181) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix crash when casting types to `Decimal` that do not support it. Throw exception instead. [\#6297](https://github.com/ClickHouse/ClickHouse/pull/6297) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed crash in `extractAll()` function. [\#6644](https://github.com/ClickHouse/ClickHouse/pull/6644) ([Artem Zuikov](https://github.com/4ertus2)) -- Query transformation for `MySQL`, `ODBC`, `JDBC` table functions now works properly for `SELECT WHERE` queries with multiple `AND` expressions. [\#6381](https://github.com/ClickHouse/ClickHouse/issues/6381) [\#6676](https://github.com/ClickHouse/ClickHouse/pull/6676) ([dimarub2000](https://github.com/dimarub2000)) -- Added previous declaration checks for MySQL 8 integration. [\#6569](https://github.com/ClickHouse/ClickHouse/pull/6569) ([Rafael David Tinoco](https://github.com/rafaeldtinoco)) - -#### Security Fix {#security-fix-1} - -- Fix two vulnerabilities in codecs in decompression phase (malicious user can fabricate compressed data that will lead to buffer overflow in decompression). [\#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([Artem Zuikov](https://github.com/4ertus2)) - -### ClickHouse release 19.13.3.26, 2019-08-22 {#clickhouse-release-19-13-3-26-2019-08-22} - -#### Bug Fix {#bug-fix-12} - -- Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) -- Fix NPE when using IN clause with a subquery with a tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- Fixed an issue that if a stale replica becomes alive, it may still have data parts that were removed by DROP PARTITION. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- Fixed issue with parsing CSV [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) -- Fixed data race in system.parts table and ALTER query. This fixes [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed possible data loss after `ALTER DELETE` query on table with skipping index. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) - -#### Security Fix {#security-fix-2} - -- If the attacker has write access to ZooKeeper and is able to run custom server available from the network where ClickHouse run, it can create custom-built malicious server that will act as ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. Found by Eldar Zaitov, information security team at Yandex. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.13.2.19, 2019-08-14 {#clickhouse-release-19-13-2-19-2019-08-14} - -#### New Feature {#new-feature-5} - -- Sampling profiler on query level. [Example](https://gist.github.com/alexey-milovidov/92758583dd41c24c360fdb8d6a4da194). [\#4247](https://github.com/ClickHouse/ClickHouse/issues/4247) ([laplab](https://github.com/laplab)) [\#6124](https://github.com/ClickHouse/ClickHouse/pull/6124) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) -- Allow to specify a list of columns with `COLUMNS('regexp')` expression that works like a more sophisticated variant of `*` asterisk. [\#5951](https://github.com/ClickHouse/ClickHouse/pull/5951) ([mfridental](https://github.com/mfridental)), ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `CREATE TABLE AS table_function()` is now possible [\#6057](https://github.com/ClickHouse/ClickHouse/pull/6057) ([dimarub2000](https://github.com/dimarub2000)) -- Adam optimizer for stochastic gradient descent is used by default in `stochasticLinearRegression()` and `stochasticLogisticRegression()` aggregate functions, because it shows good quality without almost any tuning. [\#6000](https://github.com/ClickHouse/ClickHouse/pull/6000) ([Quid37](https://github.com/Quid37)) -- Added functions for working with the сustom week number [\#5212](https://github.com/ClickHouse/ClickHouse/pull/5212) ([Andy Yang](https://github.com/andyyzh)) -- `RENAME` queries now work with all storages. [\#5953](https://github.com/ClickHouse/ClickHouse/pull/5953) ([Ivan](https://github.com/abyss7)) -- Now client receive logs from server with any desired level by setting `send_logs_level` regardless to the log level specified in server settings. [\#5964](https://github.com/ClickHouse/ClickHouse/pull/5964) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) - -#### Backward Incompatible Change {#backward-incompatible-change-4} - -- The setting `input_format_defaults_for_omitted_fields` is enabled by default. Inserts in Distributed tables need this setting to be the same on cluster (you need to set it before rolling update). It enables calculation of complex default expressions for omitted fields in `JSONEachRow` and `CSV*` formats. It should be the expected behavior but may lead to negligible performance difference. [\#6043](https://github.com/ClickHouse/ClickHouse/pull/6043) ([Artem Zuikov](https://github.com/4ertus2)), [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) - -#### Experimental features {#experimental-features} - -- New query processing pipeline. Use `experimental_use_processors=1` option to enable it. Use for your own trouble. [\#4914](https://github.com/ClickHouse/ClickHouse/pull/4914) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Bug Fix {#bug-fix-13} - -- Kafka integration has been fixed in this version. -- Fixed `DoubleDelta` encoding of `Int64` for large `DoubleDelta` values, improved `DoubleDelta` encoding for random data for `Int32`. [\#5998](https://github.com/ClickHouse/ClickHouse/pull/5998) ([Vasily Nemkov](https://github.com/Enmk)) -- Fixed overestimation of `max_rows_to_read` if the setting `merge_tree_uniform_read_distribution` is set to 0. [\#6019](https://github.com/ClickHouse/ClickHouse/pull/6019) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Improvement {#improvement-4} - -- Throws an exception if `config.d` file doesn’t have the corresponding root element as the config file [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) - -#### Performance Improvement {#performance-improvement-3} - -- Optimize `count()`. Now it uses the smallest column (if possible). [\#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([Amos Bird](https://github.com/amosbird)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-5} - -- Report memory usage in performance tests. [\#5899](https://github.com/ClickHouse/ClickHouse/pull/5899) ([akuzm](https://github.com/akuzm)) -- Fix build with external `libcxx` [\#6010](https://github.com/ClickHouse/ClickHouse/pull/6010) ([Ivan](https://github.com/abyss7)) -- Fix shared build with `rdkafka` library [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([Ivan](https://github.com/abyss7)) - -## ClickHouse release 19.11 {#clickhouse-release-19-11} - -### ClickHouse release 19.11.13.74, 2019-11-01 {#clickhouse-release-19-11-13-74-2019-11-01} - -#### Bug Fix {#bug-fix-14} - -- Fixed rare crash in `ALTER MODIFY COLUMN` and vertical merge when one of merged/altered parts is empty (0 rows). [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) -- Manual update of `SIMDJSON`. This fixes possible flooding of stderr files with bogus json diagnostic messages. [\#7548](https://github.com/ClickHouse/ClickHouse/pull/7548) ([Alexander Kazakov](https://github.com/Akazz)) -- Fixed bug with `mrk` file extension for mutations ([alesapin](https://github.com/alesapin)) - -### ClickHouse release 19.11.12.69, 2019-10-02 {#clickhouse-release-19-11-12-69-2019-10-02} - -#### Bug Fix {#bug-fix-15} - -- Fixed performance degradation of index analysis on complex keys on large tables. This fixes [\#6924](https://github.com/ClickHouse/ClickHouse/issues/6924). [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid rare SIGSEGV while sending data in tables with Distributed engine (`Failed to send batch: file with index XXXXX is absent`). [\#7032](https://github.com/ClickHouse/ClickHouse/pull/7032) ([Azat Khuzhin](https://github.com/azat)) -- Fix `Unknown identifier` with multiple joins. This fixes [\#5254](https://github.com/ClickHouse/ClickHouse/issues/5254). [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([Artem Zuikov](https://github.com/4ertus2)) - -### ClickHouse release 19.11.11.57, 2019-09-13 {#clickhouse-release-19-11-11-57-2019-09-13} - -- Fix logical error causing segfaults when selecting from Kafka empty topic. [\#6902](https://github.com/ClickHouse/ClickHouse/issues/6902) [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([Ivan](https://github.com/abyss7)) -- Fix for function `АrrayEnumerateUniqRanked` with empty arrays in params. [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) - -### ClickHouse release 19.11.10.54, 2019-09-10 {#clickhouse-release-19-11-10-54-2019-09-10} - -#### Bug Fix {#bug-fix-16} - -- Do store offsets for Kafka messages manually to be able to commit them all at once for all partitions. Fixes potential duplication in “one consumer - many partitions” scenario. [\#6872](https://github.com/ClickHouse/ClickHouse/pull/6872) ([Ivan](https://github.com/abyss7)) - -### ClickHouse release 19.11.9.52, 2019-09-6 {#clickhouse-release-19-11-9-52-2019-09-6} - -- Improve error handling in cache dictionaries. [\#6737](https://github.com/ClickHouse/ClickHouse/pull/6737) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fixed bug in function `arrayEnumerateUniqRanked`. [\#6779](https://github.com/ClickHouse/ClickHouse/pull/6779) ([proller](https://github.com/proller)) -- Fix `JSONExtract` function while extracting a `Tuple` from JSON. [\#6718](https://github.com/ClickHouse/ClickHouse/pull/6718) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fixed possible data loss after `ALTER DELETE` query on table with skipping index. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) -- Fixed performance test. [\#6392](https://github.com/ClickHouse/ClickHouse/pull/6392) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Parquet: Fix reading boolean columns. [\#6579](https://github.com/ClickHouse/ClickHouse/pull/6579) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed wrong behaviour of `nullIf` function for constant arguments. [\#6518](https://github.com/ClickHouse/ClickHouse/pull/6518) ([Guillaume Tassery](https://github.com/YiuRULE)) [\#6580](https://github.com/ClickHouse/ClickHouse/pull/6580) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix Kafka messages duplication problem on normal server restart. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([Ivan](https://github.com/abyss7)) -- Fixed an issue when long `ALTER UPDATE` or `ALTER DELETE` may prevent regular merges to run. Prevent mutations from executing if there is no enough free threads available. [\#6502](https://github.com/ClickHouse/ClickHouse/issues/6502) [\#6617](https://github.com/ClickHouse/ClickHouse/pull/6617) ([tavplubix](https://github.com/tavplubix)) -- Fixed error with processing “timezone” in server configuration file. [\#6709](https://github.com/ClickHouse/ClickHouse/pull/6709) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix kafka tests. [\#6805](https://github.com/ClickHouse/ClickHouse/pull/6805) ([Ivan](https://github.com/abyss7)) - -#### Security Fix {#security-fix-3} - -- If the attacker has write access to ZooKeeper and is able to run custom server available from the network where ClickHouse runs, it can create custom-built malicious server that will act as ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. Found by Eldar Zaitov, information security team at Yandex. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.11.8.46, 2019-08-22 {#clickhouse-release-19-11-8-46-2019-08-22} - -#### Bug Fix {#bug-fix-17} - -- Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) -- Fix NPE when using IN clause with a subquery with a tuple. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -- Fixed an issue that if a stale replica becomes alive, it may still have data parts that were removed by DROP PARTITION. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) -- Fixed issue with parsing CSV [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) -- Fixed data race in system.parts table and ALTER query. This fixes [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.11.7.40, 2019-08-14 {#clickhouse-release-19-11-7-40-2019-08-14} - -#### Bug fix {#bug-fix-18} - -- Kafka integration has been fixed in this version. -- Fix segfault when using `arrayReduce` for constant arguments. [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed `toFloat()` monotonicity. [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) -- Fix segfault with enabled `optimize_skip_unused_shards` and missing sharding key. [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([CurtizJ](https://github.com/CurtizJ)) -- Fixed logic of `arrayEnumerateUniqRanked` function. [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Removed extra verbose logging from MySQL handler. [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix wrong behavior and possible segfaults in `topK` and `topKWeighted` aggregated functions. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([CurtizJ](https://github.com/CurtizJ)) -- Do not expose virtual columns in `system.columns` table. This is required for backward compatibility. [\#6406](https://github.com/ClickHouse/ClickHouse/pull/6406) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix bug with memory allocation for string fields in complex key cache dictionary. [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alesapin](https://github.com/alesapin)) -- Fix bug with enabling adaptive granularity when creating new replica for `Replicated*MergeTree` table. [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) -- Fix infinite loop when reading Kafka messages. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([abyss7](https://github.com/abyss7)) -- Fixed the possibility of a fabricated query to cause server crash due to stack overflow in SQL parser and possibility of stack overflow in `Merge` and `Distributed` tables [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed Gorilla encoding error on small sequences. [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Enmk](https://github.com/Enmk)) - -#### Improvement {#improvement-5} - -- Allow user to override `poll_interval` and `idle_connection_timeout` settings on connection. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.11.5.28, 2019-08-05 {#clickhouse-release-19-11-5-28-2019-08-05} - -#### Bug fix {#bug-fix-19} - -- Fixed the possibility of hanging queries when server is overloaded. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix FPE in yandexConsistentHash function. This fixes [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304). [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed bug in conversion of `LowCardinality` types in `AggregateFunctionFactory`. This fixes [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix parsing of `bool` settings from `true` and `false` strings in configuration files. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) -- Fix rare bug with incompatible stream headers in queries to `Distributed` table over `MergeTree` table when part of `WHERE` moves to `PREWHERE`. [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) -- Fixed overflow in integer division of signed type to unsigned type. This fixes [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214). [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Backward Incompatible Change {#backward-incompatible-change-5} - -- `Kafka` still broken. - -### ClickHouse release 19.11.4.24, 2019-08-01 {#clickhouse-release-19-11-4-24-2019-08-01} - -#### Bug Fix {#bug-fix-20} - -- Fix bug with writing secondary indices marks with adaptive granularity. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alesapin](https://github.com/alesapin)) -- Fix `WITH ROLLUP` and `WITH CUBE` modifiers of `GROUP BY` with two-level aggregation. [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed hang in `JSONExtractRaw` function. Fixed [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix segfault in ExternalLoader::reloadOutdated(). [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fixed the case when server may close listening sockets but not shutdown and continue serving remaining queries. You may end up with two running clickhouse-server processes. Sometimes, the server may return an error `bad_function_call` for remaining queries. [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed useless and incorrect condition on update field for initial loading of external dictionaries via ODBC, MySQL, ClickHouse and HTTP. This fixes [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed irrelevant exception in cast of `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix non-deterministic result of “uniq” aggregate function in extreme rare cases. The bug was present in all ClickHouse versions. [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Segfault when we set a little bit too high CIDR on the function `IPv6CIDRToRange`. [\#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Fixed small memory leak when server throw many exceptions from many different contexts. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix the situation when consumer got paused before subscription and not resumed afterwards. [\#6075](https://github.com/ClickHouse/ClickHouse/pull/6075) ([Ivan](https://github.com/abyss7)) Note that Kafka is broken in this version. -- Clearing the Kafka data buffer from the previous read operation that was completed with an error [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) Note that Kafka is broken in this version. -- Since `StorageMergeTree::background_task_handle` is initialized in `startup()` the `MergeTreeBlockOutputStream::write()` may try to use it before initialization. Just check if it is initialized. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([Ivan](https://github.com/abyss7)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-6} - -- Added official `rpm` packages. [\#5740](https://github.com/ClickHouse/ClickHouse/pull/5740) ([proller](https://github.com/proller)) ([alesapin](https://github.com/alesapin)) -- Add an ability to build `.rpm` and `.tgz` packages with `packager` script. [\#5769](https://github.com/ClickHouse/ClickHouse/pull/5769) ([alesapin](https://github.com/alesapin)) -- Fixes for “Arcadia” build system. [\#6223](https://github.com/ClickHouse/ClickHouse/pull/6223) ([proller](https://github.com/proller)) - -#### Backward Incompatible Change {#backward-incompatible-change-6} - -- `Kafka` is broken in this version. - -### ClickHouse release 19.11.3.11, 2019-07-18 {#clickhouse-release-19-11-3-11-2019-07-18} - -#### New Feature {#new-feature-6} - -- Added support for prepared statements. [\#5331](https://github.com/ClickHouse/ClickHouse/pull/5331/) ([Alexander](https://github.com/sanych73)) [\#5630](https://github.com/ClickHouse/ClickHouse/pull/5630) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `DoubleDelta` and `Gorilla` column codecs [\#5600](https://github.com/ClickHouse/ClickHouse/pull/5600) ([Vasily Nemkov](https://github.com/Enmk)) -- Added `os_thread_priority` setting that allows to control the “nice” value of query processing threads that is used by OS to adjust dynamic scheduling priority. It requires `CAP_SYS_NICE` capabilities to work. This implements [\#5858](https://github.com/ClickHouse/ClickHouse/issues/5858) [\#5909](https://github.com/ClickHouse/ClickHouse/pull/5909) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Implement `_topic`, `_offset`, `_key` columns for Kafka engine [\#5382](https://github.com/ClickHouse/ClickHouse/pull/5382) ([Ivan](https://github.com/abyss7)) Note that Kafka is broken in this version. -- Add aggregate function combinator `-Resample` [\#5590](https://github.com/ClickHouse/ClickHouse/pull/5590) ([hcz](https://github.com/hczhcz)) -- Aggregate functions `groupArrayMovingSum(win_size)(x)` and `groupArrayMovingAvg(win_size)(x)`, which calculate moving sum/avg with or without window-size limitation. [\#5595](https://github.com/ClickHouse/ClickHouse/pull/5595) ([inv2004](https://github.com/inv2004)) -- Add synonim `arrayFlatten` \<-\> `flatten` [\#5764](https://github.com/ClickHouse/ClickHouse/pull/5764) ([hcz](https://github.com/hczhcz)) -- Intergate H3 function `geoToH3` from Uber. [\#4724](https://github.com/ClickHouse/ClickHouse/pull/4724) ([Remen Ivan](https://github.com/BHYCHIK)) [\#5805](https://github.com/ClickHouse/ClickHouse/pull/5805) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Bug Fix {#bug-fix-21} - -- Implement DNS cache with asynchronous update. Separate thread resolves all hosts and updates DNS cache with period (setting `dns_cache_update_period`). It should help, when ip of hosts changes frequently. [\#5857](https://github.com/ClickHouse/ClickHouse/pull/5857) ([Anton Popov](https://github.com/CurtizJ)) -- Fix segfault in `Delta` codec which affects columns with values less than 32 bits size. The bug led to random memory corruption. [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) -- Fix segfault in TTL merge with non-physical columns in block. [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) -- Fix rare bug in checking of part with `LowCardinality` column. Previously `checkDataPart` always fails for part with `LowCardinality` column. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) -- Avoid hanging connections when server thread pool is full. It is important for connections from `remote` table function or connections to a shard without replicas when there is long connection timeout. This fixes [\#5878](https://github.com/ClickHouse/ClickHouse/issues/5878) [\#5881](https://github.com/ClickHouse/ClickHouse/pull/5881) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Support for constant arguments to `evalMLModel` function. This fixes [\#5817](https://github.com/ClickHouse/ClickHouse/issues/5817) [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed the issue when ClickHouse determines default time zone as `UCT` instead of `UTC`. This fixes [\#5804](https://github.com/ClickHouse/ClickHouse/issues/5804). [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed buffer underflow in `visitParamExtractRaw`. This fixes [\#5901](https://github.com/ClickHouse/ClickHouse/issues/5901) [\#5902](https://github.com/ClickHouse/ClickHouse/pull/5902) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now distributed `DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER` queries will be executed directly on leader replica. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) -- Fix `coalesce` for `ColumnConst` with `ColumnNullable` + related changes. [\#5755](https://github.com/ClickHouse/ClickHouse/pull/5755) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix the `ReadBufferFromKafkaConsumer` so that it keeps reading new messages after `commit()` even if it was stalled before [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([Ivan](https://github.com/abyss7)) -- Fix `FULL` and `RIGHT` JOIN results when joining on `Nullable` keys in right table. [\#5859](https://github.com/ClickHouse/ClickHouse/pull/5859) ([Artem Zuikov](https://github.com/4ertus2)) -- Possible fix of infinite sleeping of low-priority queries. [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix race condition, which cause that some queries may not appear in query\_log after `SYSTEM FLUSH LOGS` query. [\#5456](https://github.com/ClickHouse/ClickHouse/issues/5456) [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) -- Fixed `heap-use-after-free` ASan warning in ClusterCopier caused by watch which try to use already removed copier object. [\#5871](https://github.com/ClickHouse/ClickHouse/pull/5871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed wrong `StringRef` pointer returned by some implementations of `IColumn::deserializeAndInsertFromArena`. This bug affected only unit-tests. [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Prevent source and intermediate array join columns of masking same name columns. [\#5941](https://github.com/ClickHouse/ClickHouse/pull/5941) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix insert and select query to MySQL engine with MySQL style identifier quoting. [\#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([Winter Zhang](https://github.com/zhang2014)) -- Now `CHECK TABLE` query can work with MergeTree engine family. It returns check status and message if any for each part (or file in case of simplier engines). Also, fix bug in fetch of a broken part. [\#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([alesapin](https://github.com/alesapin)) -- Fix SPLIT\_SHARED\_LIBRARIES runtime [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([Danila Kutenin](https://github.com/danlark1)) -- Fixed time zone initialization when `/etc/localtime` is a relative symlink like `../usr/share/zoneinfo/Europe/Moscow` [\#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- clickhouse-copier: Fix use-after free on shutdown [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([proller](https://github.com/proller)) -- Updated `simdjson`. Fixed the issue that some invalid JSONs with zero bytes successfully parse. [\#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix shutdown of SystemLogs [\#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([Anton Popov](https://github.com/CurtizJ)) -- Fix hanging when condition in invalidate\_query depends on a dictionary. [\#6011](https://github.com/ClickHouse/ClickHouse/pull/6011) ([Vitaly Baranov](https://github.com/vitlibar)) - -#### Improvement {#improvement-6} - -- Allow unresolvable addresses in cluster configuration. They will be considered unavailable and tried to resolve at every connection attempt. This is especially useful for Kubernetes. This fixes [\#5714](https://github.com/ClickHouse/ClickHouse/issues/5714) [\#5924](https://github.com/ClickHouse/ClickHouse/pull/5924) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Close idle TCP connections (with one hour timeout by default). This is especially important for large clusters with multiple distributed tables on every server, because every server can possibly keep a connection pool to every other server, and after peak query concurrency, connections will stall. This fixes [\#5879](https://github.com/ClickHouse/ClickHouse/issues/5879) [\#5880](https://github.com/ClickHouse/ClickHouse/pull/5880) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Better quality of `topK` function. Changed the SavingSpace set behavior to remove the last element if the new element have a bigger weight. [\#5833](https://github.com/ClickHouse/ClickHouse/issues/5833) [\#5850](https://github.com/ClickHouse/ClickHouse/pull/5850) ([Guillaume Tassery](https://github.com/YiuRULE)) -- URL functions to work with domains now can work for incomplete URLs without scheme [\#5725](https://github.com/ClickHouse/ClickHouse/pull/5725) ([alesapin](https://github.com/alesapin)) -- Checksums added to the `system.parts_columns` table. [\#5874](https://github.com/ClickHouse/ClickHouse/pull/5874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -- Added `Enum` data type as a synonim for `Enum8` or `Enum16`. [\#5886](https://github.com/ClickHouse/ClickHouse/pull/5886) ([dimarub2000](https://github.com/dimarub2000)) -- Full bit transpose variant for `T64` codec. Could lead to better compression with `zstd`. [\#5742](https://github.com/ClickHouse/ClickHouse/pull/5742) ([Artem Zuikov](https://github.com/4ertus2)) -- Condition on `startsWith` function now can uses primary key. This fixes [\#5310](https://github.com/ClickHouse/ClickHouse/issues/5310) and [\#5882](https://github.com/ClickHouse/ClickHouse/issues/5882) [\#5919](https://github.com/ClickHouse/ClickHouse/pull/5919) ([dimarub2000](https://github.com/dimarub2000)) -- Allow to use `clickhouse-copier` with cross-replication cluster topology by permitting empty database name. [\#5745](https://github.com/ClickHouse/ClickHouse/pull/5745) ([nvartolomei](https://github.com/nvartolomei)) -- Use `UTC` as default timezone on a system without `tzdata` (e.g. bare Docker container). Before this patch, error message `Could not determine local time zone` was printed and server or client refused to start. [\#5827](https://github.com/ClickHouse/ClickHouse/pull/5827) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Returned back support for floating point argument in function `quantileTiming` for backward compatibility. [\#5911](https://github.com/ClickHouse/ClickHouse/pull/5911) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Show which table is missing column in error messages. [\#5768](https://github.com/ClickHouse/ClickHouse/pull/5768) ([Ivan](https://github.com/abyss7)) -- Disallow run query with same query\_id by various users [\#5430](https://github.com/ClickHouse/ClickHouse/pull/5430) ([proller](https://github.com/proller)) -- More robust code for sending metrics to Graphite. It will work even during long multiple `RENAME TABLE` operation. [\#5875](https://github.com/ClickHouse/ClickHouse/pull/5875) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- More informative error messages will be displayed when ThreadPool cannot schedule a task for execution. This fixes [\#5305](https://github.com/ClickHouse/ClickHouse/issues/5305) [\#5801](https://github.com/ClickHouse/ClickHouse/pull/5801) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Inverting ngramSearch to be more intuitive [\#5807](https://github.com/ClickHouse/ClickHouse/pull/5807) ([Danila Kutenin](https://github.com/danlark1)) -- Add user parsing in HDFS engine builder [\#5946](https://github.com/ClickHouse/ClickHouse/pull/5946) ([akonyaev90](https://github.com/akonyaev90)) -- Update default value of `max_ast_elements parameter` [\#5933](https://github.com/ClickHouse/ClickHouse/pull/5933) ([Artem Konovalov](https://github.com/izebit)) -- Added a notion of obsolete settings. The obsolete setting `allow_experimental_low_cardinality_type` can be used with no effect. [0f15c01c6802f7ce1a1494c12c846be8c98944cd](https://github.com/ClickHouse/ClickHouse/commit/0f15c01c6802f7ce1a1494c12c846be8c98944cd) [Alexey Milovidov](https://github.com/alexey-milovidov) - -#### Performance Improvement {#performance-improvement-4} - -- Increase number of streams to SELECT from Merge table for more uniform distribution of threads. Added setting `max_streams_multiplier_for_merge_tables`. This fixes [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5915](https://github.com/ClickHouse/ClickHouse/pull/5915) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-7} - -- Add a backward compatibility test for client-server interaction with different versions of clickhouse. [\#5868](https://github.com/ClickHouse/ClickHouse/pull/5868) ([alesapin](https://github.com/alesapin)) -- Test coverage information in every commit and pull request. [\#5896](https://github.com/ClickHouse/ClickHouse/pull/5896) ([alesapin](https://github.com/alesapin)) -- Cooperate with address sanitizer to support our custom allocators (`Arena` and `ArenaWithFreeLists`) for better debugging of “use-after-free” errors. [\#5728](https://github.com/ClickHouse/ClickHouse/pull/5728) ([akuzm](https://github.com/akuzm)) -- Switch to [LLVM libunwind implementation](https://github.com/llvm-mirror/libunwind) for C++ exception handling and for stack traces printing [\#4828](https://github.com/ClickHouse/ClickHouse/pull/4828) ([Nikita Lapkov](https://github.com/laplab)) -- Add two more warnings from -Weverything [\#5923](https://github.com/ClickHouse/ClickHouse/pull/5923) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Allow to build ClickHouse with Memory Sanitizer. [\#3949](https://github.com/ClickHouse/ClickHouse/pull/3949) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed ubsan report about `bitTest` function in fuzz test. [\#5943](https://github.com/ClickHouse/ClickHouse/pull/5943) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Docker: added possibility to init a ClickHouse instance which requires authentication. [\#5727](https://github.com/ClickHouse/ClickHouse/pull/5727) ([Korviakov Andrey](https://github.com/shurshun)) -- Update librdkafka to version 1.1.0 [\#5872](https://github.com/ClickHouse/ClickHouse/pull/5872) ([Ivan](https://github.com/abyss7)) -- Add global timeout for integration tests and disable some of them in tests code. [\#5741](https://github.com/ClickHouse/ClickHouse/pull/5741) ([alesapin](https://github.com/alesapin)) -- Fix some ThreadSanitizer failures. [\#5854](https://github.com/ClickHouse/ClickHouse/pull/5854) ([akuzm](https://github.com/akuzm)) -- The `--no-undefined` option forces the linker to check all external names for existence while linking. It’s very useful to track real dependencies between libraries in the split build mode. [\#5855](https://github.com/ClickHouse/ClickHouse/pull/5855) ([Ivan](https://github.com/abyss7)) -- Added performance test for [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5914](https://github.com/ClickHouse/ClickHouse/pull/5914) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed compatibility with gcc-7. [\#5840](https://github.com/ClickHouse/ClickHouse/pull/5840) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added support for gcc-9. This fixes [\#5717](https://github.com/ClickHouse/ClickHouse/issues/5717) [\#5774](https://github.com/ClickHouse/ClickHouse/pull/5774) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed error when libunwind can be linked incorrectly. [\#5948](https://github.com/ClickHouse/ClickHouse/pull/5948) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed a few warnings found by PVS-Studio. [\#5921](https://github.com/ClickHouse/ClickHouse/pull/5921) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added initial support for `clang-tidy` static analyzer. [\#5806](https://github.com/ClickHouse/ClickHouse/pull/5806) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Convert BSD/Linux endian macros( ‘be64toh’ and ‘htobe64’) to the Mac OS X equivalents [\#5785](https://github.com/ClickHouse/ClickHouse/pull/5785) ([Fu Chen](https://github.com/fredchenbj)) -- Improved integration tests guide. [\#5796](https://github.com/ClickHouse/ClickHouse/pull/5796) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixing build at macosx + gcc9 [\#5822](https://github.com/ClickHouse/ClickHouse/pull/5822) ([filimonov](https://github.com/filimonov)) -- Fix a hard-to-spot typo: aggreAGte -\> aggregate. [\#5753](https://github.com/ClickHouse/ClickHouse/pull/5753) ([akuzm](https://github.com/akuzm)) -- Fix freebsd build [\#5760](https://github.com/ClickHouse/ClickHouse/pull/5760) ([proller](https://github.com/proller)) -- Add link to experimental YouTube channel to website [\#5845](https://github.com/ClickHouse/ClickHouse/pull/5845) ([Ivan Blinkov](https://github.com/blinkov)) -- CMake: add option for coverage flags: WITH\_COVERAGE [\#5776](https://github.com/ClickHouse/ClickHouse/pull/5776) ([proller](https://github.com/proller)) -- Fix initial size of some inline PODArray’s. [\#5787](https://github.com/ClickHouse/ClickHouse/pull/5787) ([akuzm](https://github.com/akuzm)) -- clickhouse-server.postinst: fix os detection for centos 6 [\#5788](https://github.com/ClickHouse/ClickHouse/pull/5788) ([proller](https://github.com/proller)) -- Added Arch linux package generation. [\#5719](https://github.com/ClickHouse/ClickHouse/pull/5719) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Split Common/config.h by libs (dbms) [\#5715](https://github.com/ClickHouse/ClickHouse/pull/5715) ([proller](https://github.com/proller)) -- Fixes for “Arcadia” build platform [\#5795](https://github.com/ClickHouse/ClickHouse/pull/5795) ([proller](https://github.com/proller)) -- Fixes for unconventional build (gcc9, no submodules) [\#5792](https://github.com/ClickHouse/ClickHouse/pull/5792) ([proller](https://github.com/proller)) -- Require explicit type in unalignedStore because it was proven to be bug-prone [\#5791](https://github.com/ClickHouse/ClickHouse/pull/5791) ([akuzm](https://github.com/akuzm)) -- Fixes MacOS build [\#5830](https://github.com/ClickHouse/ClickHouse/pull/5830) ([filimonov](https://github.com/filimonov)) -- Performance test concerning the new JIT feature with bigger dataset, as requested here [\#5263](https://github.com/ClickHouse/ClickHouse/issues/5263) [\#5887](https://github.com/ClickHouse/ClickHouse/pull/5887) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Run stateful tests in stress test [12693e568722f11e19859742f56428455501fd2a](https://github.com/ClickHouse/ClickHouse/commit/12693e568722f11e19859742f56428455501fd2a) ([alesapin](https://github.com/alesapin)) - -#### Backward Incompatible Change {#backward-incompatible-change-7} - -- `Kafka` is broken in this version. -- Enable `adaptive_index_granularity` = 10MB by default for new `MergeTree` tables. If you created new MergeTree tables on version 19.11+, downgrade to versions prior to 19.6 will be impossible. [\#5628](https://github.com/ClickHouse/ClickHouse/pull/5628) ([alesapin](https://github.com/alesapin)) -- Removed obsolete undocumented embedded dictionaries that were used by Yandex.Metrica. The functions `OSIn`, `SEIn`, `OSToRoot`, `SEToRoot`, `OSHierarchy`, `SEHierarchy` are no longer available. If you are using these functions, write email to clickhouse-feedback@yandex-team.com. Note: at the last moment we decided to keep these functions for a while. [\#5780](https://github.com/ClickHouse/ClickHouse/pull/5780) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -## ClickHouse release 19.10 {#clickhouse-release-19-10} - -### ClickHouse release 19.10.1.5, 2019-07-12 {#clickhouse-release-19-10-1-5-2019-07-12} - -#### New Feature {#new-feature-7} - -- Add new column codec: `T64`. Made for (U)IntX/EnumX/Data(Time)/DecimalX columns. It should be good for columns with constant or small range values. Codec itself allows enlarge or shrink data type without re-compression. [\#5557](https://github.com/ClickHouse/ClickHouse/pull/5557) ([Artem Zuikov](https://github.com/4ertus2)) -- Add database engine `MySQL` that allow to view all the tables in remote MySQL server [\#5599](https://github.com/ClickHouse/ClickHouse/pull/5599) ([Winter Zhang](https://github.com/zhang2014)) -- `bitmapContains` implementation. It’s 2x faster than `bitmapHasAny` if the second bitmap contains one element. [\#5535](https://github.com/ClickHouse/ClickHouse/pull/5535) ([Zhichang Yu](https://github.com/yuzhichang)) -- Support for `crc32` function (with behaviour exactly as in MySQL or PHP). Do not use it if you need a hash function. [\#5661](https://github.com/ClickHouse/ClickHouse/pull/5661) ([Remen Ivan](https://github.com/BHYCHIK)) -- Implemented `SYSTEM START/STOP DISTRIBUTED SENDS` queries to control asynchronous inserts into `Distributed` tables. [\#4935](https://github.com/ClickHouse/ClickHouse/pull/4935) ([Winter Zhang](https://github.com/zhang2014)) - -#### Bug Fix {#bug-fix-22} - -- Ignore query execution limits and max parts size for merge limits while executing mutations. [\#5659](https://github.com/ClickHouse/ClickHouse/pull/5659) ([Anton Popov](https://github.com/CurtizJ)) -- Fix bug which may lead to deduplication of normal blocks (extremely rare) and insertion of duplicate blocks (more often). [\#5549](https://github.com/ClickHouse/ClickHouse/pull/5549) ([alesapin](https://github.com/alesapin)) -- Fix of function `arrayEnumerateUniqRanked` for arguments with empty arrays [\#5559](https://github.com/ClickHouse/ClickHouse/pull/5559) ([proller](https://github.com/proller)) -- Don’t subscribe to Kafka topics without intent to poll any messages. [\#5698](https://github.com/ClickHouse/ClickHouse/pull/5698) ([Ivan](https://github.com/abyss7)) -- Make setting `join_use_nulls` get no effect for types that cannot be inside Nullable [\#5700](https://github.com/ClickHouse/ClickHouse/pull/5700) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fixed `Incorrect size of index granularity` errors [\#5720](https://github.com/ClickHouse/ClickHouse/pull/5720) ([coraxster](https://github.com/coraxster)) -- Fix Float to Decimal convert overflow [\#5607](https://github.com/ClickHouse/ClickHouse/pull/5607) ([coraxster](https://github.com/coraxster)) -- Flush buffer when `WriteBufferFromHDFS`’s destructor is called. This fixes writing into `HDFS`. [\#5684](https://github.com/ClickHouse/ClickHouse/pull/5684) ([Xindong Peng](https://github.com/eejoin)) - -#### Improvement {#improvement-7} - -- Treat empty cells in `CSV` as default values when the setting `input_format_defaults_for_omitted_fields` is enabled. [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) -- Non-blocking loading of external dictionaries. [\#5567](https://github.com/ClickHouse/ClickHouse/pull/5567) ([Vitaly Baranov](https://github.com/vitlibar)) -- Network timeouts can be dynamically changed for already established connections according to the settings. [\#4558](https://github.com/ClickHouse/ClickHouse/pull/4558) ([Konstantin Podshumok](https://github.com/podshumok)) -- Using “public\_suffix\_list” for functions `firstSignificantSubdomain`, `cutToFirstSignificantSubdomain`. It’s using a perfect hash table generated by `gperf` with a list generated from the file: https://publicsuffix.org/list/public\_suffix\_list.dat. (for example, now we recognize the domain `ac.uk` as non-significant). [\#5030](https://github.com/ClickHouse/ClickHouse/pull/5030) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Adopted `IPv6` data type in system tables; unified client info columns in `system.processes` and `system.query_log` [\#5640](https://github.com/ClickHouse/ClickHouse/pull/5640) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Using sessions for connections with MySQL compatibility protocol. \#5476 [\#5646](https://github.com/ClickHouse/ClickHouse/pull/5646) ([Yuriy Baranov](https://github.com/yurriy)) -- Support more `ALTER` queries `ON CLUSTER`. [\#5593](https://github.com/ClickHouse/ClickHouse/pull/5593) [\#5613](https://github.com/ClickHouse/ClickHouse/pull/5613) ([sundyli](https://github.com/sundy-li)) -- Support `` section in `clickhouse-local` config file. [\#5540](https://github.com/ClickHouse/ClickHouse/pull/5540) ([proller](https://github.com/proller)) -- Allow run query with `remote` table function in `clickhouse-local` [\#5627](https://github.com/ClickHouse/ClickHouse/pull/5627) ([proller](https://github.com/proller)) - -#### Performance Improvement {#performance-improvement-5} - -- Add the possibility to write the final mark at the end of MergeTree columns. It allows to avoid useless reads for keys that are out of table data range. It is enabled only if adaptive index granularity is in use. [\#5624](https://github.com/ClickHouse/ClickHouse/pull/5624) ([alesapin](https://github.com/alesapin)) -- Improved performance of MergeTree tables on very slow filesystems by reducing number of `stat` syscalls. [\#5648](https://github.com/ClickHouse/ClickHouse/pull/5648) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed performance degradation in reading from MergeTree tables that was introduced in version 19.6. Fixes \#5631. [\#5633](https://github.com/ClickHouse/ClickHouse/pull/5633) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-8} - -- Implemented `TestKeeper` as an implementation of ZooKeeper interface used for testing [\#5643](https://github.com/ClickHouse/ClickHouse/pull/5643) ([alexey-milovidov](https://github.com/alexey-milovidov)) ([levushkin aleksej](https://github.com/alexey-milovidov)) -- From now on `.sql` tests can be run isolated by server, in parallel, with random database. It allows to run them faster, add new tests with custom server configurations, and be sure that different tests doesn’t affect each other. [\#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([Ivan](https://github.com/abyss7)) -- Remove `` and `` from performance tests [\#5672](https://github.com/ClickHouse/ClickHouse/pull/5672) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fixed “select\_format” performance test for `Pretty` formats [\#5642](https://github.com/ClickHouse/ClickHouse/pull/5642) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -## ClickHouse release 19.9 {#clickhouse-release-19-9} - -### ClickHouse release 19.9.3.31, 2019-07-05 {#clickhouse-release-19-9-3-31-2019-07-05} - -#### Bug Fix {#bug-fix-23} - -- Fix segfault in Delta codec which affects columns with values less than 32 bits size. The bug led to random memory corruption. [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) -- Fix rare bug in checking of part with LowCardinality column. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) -- Fix segfault in TTL merge with non-physical columns in block. [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) -- Fix potential infinite sleeping of low-priority queries. [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix how ClickHouse determines default time zone as UCT instead of UTC. [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix bug about executing distributed DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER queries on follower replica before leader replica. Now they will be executed directly on leader replica. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) -- Fix race condition, which cause that some queries may not appear in query\_log instantly after SYSTEM FLUSH LOGS query. [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) -- Added missing support for constant arguments to `evalMLModel` function. [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.9.2.4, 2019-06-24 {#clickhouse-release-19-9-2-4-2019-06-24} - -#### New Feature {#new-feature-8} - -- Print information about frozen parts in `system.parts` table. [\#5471](https://github.com/ClickHouse/ClickHouse/pull/5471) ([proller](https://github.com/proller)) -- Ask client password on clickhouse-client start on tty if not set in arguments [\#5092](https://github.com/ClickHouse/ClickHouse/pull/5092) ([proller](https://github.com/proller)) -- Implement `dictGet` and `dictGetOrDefault` functions for Decimal types. [\#5394](https://github.com/ClickHouse/ClickHouse/pull/5394) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Improvement {#improvement-8} - -- Debian init: Add service stop timeout [\#5522](https://github.com/ClickHouse/ClickHouse/pull/5522) ([proller](https://github.com/proller)) -- Add setting forbidden by default to create table with suspicious types for LowCardinality [\#5448](https://github.com/ClickHouse/ClickHouse/pull/5448) ([Olga Khvostikova](https://github.com/stavrolia)) -- Regression functions return model weights when not used as State in function `evalMLMethod`. [\#5411](https://github.com/ClickHouse/ClickHouse/pull/5411) ([Quid37](https://github.com/Quid37)) -- Rename and improve regression methods. [\#5492](https://github.com/ClickHouse/ClickHouse/pull/5492) ([Quid37](https://github.com/Quid37)) -- Clearer interfaces of string searchers. [\#5586](https://github.com/ClickHouse/ClickHouse/pull/5586) ([Danila Kutenin](https://github.com/danlark1)) - -#### Bug Fix {#bug-fix-24} - -- Fix potential data loss in Kafka [\#5445](https://github.com/ClickHouse/ClickHouse/pull/5445) ([Ivan](https://github.com/abyss7)) -- Fix potential infinite loop in `PrettySpace` format when called with zero columns [\#5560](https://github.com/ClickHouse/ClickHouse/pull/5560) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fixed UInt32 overflow bug in linear models. Allow eval ML model for non-const model argument. [\#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- `ALTER TABLE ... DROP INDEX IF EXISTS ...` should not raise an exception if provided index does not exist [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) -- Fix segfault with `bitmapHasAny` in scalar subquery [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- Fixed error when replication connection pool doesn’t retry to resolve host, even when DNS cache was dropped. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) -- Fixed `ALTER ... MODIFY TTL` on ReplicatedMergeTree. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539) ([Anton Popov](https://github.com/CurtizJ)) -- Fix INSERT into Distributed table with MATERIALIZED column [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) -- Fix bad alloc when truncate Join storage [\#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([TCeason](https://github.com/TCeason)) -- In recent versions of package tzdata some of files are symlinks now. The current mechanism for detecting default timezone gets broken and gives wrong names for some timezones. Now at least we force the timezone name to the contents of TZ if provided. [\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([Ivan](https://github.com/abyss7)) -- Fix some extremely rare cases with MultiVolnitsky searcher when the constant needles in sum are at least 16KB long. The algorithm missed or overwrote the previous results which can lead to the incorrect result of `multiSearchAny`. [\#5588](https://github.com/ClickHouse/ClickHouse/pull/5588) ([Danila Kutenin](https://github.com/danlark1)) -- Fix the issue when settings for ExternalData requests couldn’t use ClickHouse settings. Also, for now, settings `date_time_input_format` and `low_cardinality_allow_in_native_format` cannot be used because of the ambiguity of names (in external data it can be interpreted as table format and in the query it can be a setting). [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila Kutenin](https://github.com/danlark1)) -- Fix bug when parts were removed only from FS without dropping them from Zookeeper. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alesapin](https://github.com/alesapin)) -- Remove debug logging from MySQL protocol [\#5478](https://github.com/ClickHouse/ClickHouse/pull/5478) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Skip ZNONODE during DDL query processing [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) -- Fix mix `UNION ALL` result column type. There were cases with inconsistent data and column types of resulting columns. [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) -- Throw an exception on wrong integers in `dictGetT` functions instead of crash. [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix wrong element\_count and load\_factor for hashed dictionary in `system.dictionaries` table. [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-9} - -- Fixed build without `Brotli` HTTP compression support (`ENABLE_BROTLI=OFF` cmake variable). [\#5521](https://github.com/ClickHouse/ClickHouse/pull/5521) ([Anton Yuzhaninov](https://github.com/citrin)) -- Include roaring.h as roaring/roaring.h [\#5523](https://github.com/ClickHouse/ClickHouse/pull/5523) ([Orivej Desh](https://github.com/orivej)) -- Fix gcc9 warnings in hyperscan (\#line directive is evil!) [\#5546](https://github.com/ClickHouse/ClickHouse/pull/5546) ([Danila Kutenin](https://github.com/danlark1)) -- Fix all warnings when compiling with gcc-9. Fix some contrib issues. Fix gcc9 ICE and submit it to bugzilla. [\#5498](https://github.com/ClickHouse/ClickHouse/pull/5498) ([Danila Kutenin](https://github.com/danlark1)) -- Fixed linking with lld [\#5477](https://github.com/ClickHouse/ClickHouse/pull/5477) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Remove unused specializations in dictionaries [\#5452](https://github.com/ClickHouse/ClickHouse/pull/5452) ([Artem Zuikov](https://github.com/4ertus2)) -- Improvement performance tests for formatting and parsing tables for different types of files [\#5497](https://github.com/ClickHouse/ClickHouse/pull/5497) ([Olga Khvostikova](https://github.com/stavrolia)) -- Fixes for parallel test run [\#5506](https://github.com/ClickHouse/ClickHouse/pull/5506) ([proller](https://github.com/proller)) -- Docker: use configs from clickhouse-test [\#5531](https://github.com/ClickHouse/ClickHouse/pull/5531) ([proller](https://github.com/proller)) -- Fix compile for FreeBSD [\#5447](https://github.com/ClickHouse/ClickHouse/pull/5447) ([proller](https://github.com/proller)) -- Upgrade boost to 1.70 [\#5570](https://github.com/ClickHouse/ClickHouse/pull/5570) ([proller](https://github.com/proller)) -- Fix build clickhouse as submodule [\#5574](https://github.com/ClickHouse/ClickHouse/pull/5574) ([proller](https://github.com/proller)) -- Improve JSONExtract performance tests [\#5444](https://github.com/ClickHouse/ClickHouse/pull/5444) ([Vitaly Baranov](https://github.com/vitlibar)) - -## ClickHouse release 19.8 {#clickhouse-release-19-8} - -### ClickHouse release 19.8.3.8, 2019-06-11 {#clickhouse-release-19-8-3-8-2019-06-11} - -#### New Features {#new-features} - -- Added functions to work with JSON [\#4686](https://github.com/ClickHouse/ClickHouse/pull/4686) ([hcz](https://github.com/hczhcz)) [\#5124](https://github.com/ClickHouse/ClickHouse/pull/5124). ([Vitaly Baranov](https://github.com/vitlibar)) -- Add a function basename, with a similar behaviour to a basename function, which exists in a lot of languages (`os.path.basename` in python, `basename` in PHP, etc…). Work with both an UNIX-like path or a Windows path. [\#5136](https://github.com/ClickHouse/ClickHouse/pull/5136) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Added `LIMIT n, m BY` or `LIMIT m OFFSET n BY` syntax to set offset of n for LIMIT BY clause. [\#5138](https://github.com/ClickHouse/ClickHouse/pull/5138) ([Anton Popov](https://github.com/CurtizJ)) -- Added new data type `SimpleAggregateFunction`, which allows to have columns with light aggregation in an `AggregatingMergeTree`. This can only be used with simple functions like `any`, `anyLast`, `sum`, `min`, `max`. [\#4629](https://github.com/ClickHouse/ClickHouse/pull/4629) ([Boris Granveaud](https://github.com/bgranvea)) -- Added support for non-constant arguments in function `ngramDistance` [\#5198](https://github.com/ClickHouse/ClickHouse/pull/5198) ([Danila Kutenin](https://github.com/danlark1)) -- Added functions `skewPop`, `skewSamp`, `kurtPop` and `kurtSamp` to compute for sequence skewness, sample skewness, kurtosis and sample kurtosis respectively. [\#5200](https://github.com/ClickHouse/ClickHouse/pull/5200) ([hcz](https://github.com/hczhcz)) -- Support rename operation for `MaterializeView` storage. [\#5209](https://github.com/ClickHouse/ClickHouse/pull/5209) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Added server which allows connecting to ClickHouse using MySQL client. [\#4715](https://github.com/ClickHouse/ClickHouse/pull/4715) ([Yuriy Baranov](https://github.com/yurriy)) -- Add `toDecimal*OrZero` and `toDecimal*OrNull` functions. [\#5291](https://github.com/ClickHouse/ClickHouse/pull/5291) ([Artem Zuikov](https://github.com/4ertus2)) -- Support Decimal types in functions: `quantile`, `quantiles`, `median`, `quantileExactWeighted`, `quantilesExactWeighted`, medianExactWeighted. [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([Artem Zuikov](https://github.com/4ertus2)) -- Added `toValidUTF8` function, which replaces all invalid UTF-8 characters by replacement character � (U+FFFD). [\#5322](https://github.com/ClickHouse/ClickHouse/pull/5322) ([Danila Kutenin](https://github.com/danlark1)) -- Added `format` function. Formatting constant pattern (simplified Python format pattern) with the strings listed in the arguments. [\#5330](https://github.com/ClickHouse/ClickHouse/pull/5330) ([Danila Kutenin](https://github.com/danlark1)) -- Added `system.detached_parts` table containing information about detached parts of `MergeTree` tables. [\#5353](https://github.com/ClickHouse/ClickHouse/pull/5353) ([akuzm](https://github.com/akuzm)) -- Added `ngramSearch` function to calculate the non-symmetric difference between needle and haystack. [\#5418](https://github.com/ClickHouse/ClickHouse/pull/5418)[\#5422](https://github.com/ClickHouse/ClickHouse/pull/5422) ([Danila Kutenin](https://github.com/danlark1)) -- Implementation of basic machine learning methods (stochastic linear regression and logistic regression) using aggregate functions interface. Has different strategies for updating model weights (simple gradient descent, momentum method, Nesterov method). Also supports mini-batches of custom size. [\#4943](https://github.com/ClickHouse/ClickHouse/pull/4943) ([Quid37](https://github.com/Quid37)) -- Implementation of `geohashEncode` and `geohashDecode` functions. [\#5003](https://github.com/ClickHouse/ClickHouse/pull/5003) ([Vasily Nemkov](https://github.com/Enmk)) -- Added aggregate function `timeSeriesGroupSum`, which can aggregate different time series that sample timestamp not alignment. It will use linear interpolation between two sample timestamp and then sum time-series together. Added aggregate function `timeSeriesGroupRateSum`, which calculates the rate of time-series and then sum rates together. [\#4542](https://github.com/ClickHouse/ClickHouse/pull/4542) ([Yangkuan Liu](https://github.com/LiuYangkuan)) -- Added functions `IPv4CIDRtoIPv4Range` and `IPv6CIDRtoIPv6Range` to calculate the lower and higher bounds for an IP in the subnet using a CIDR. [\#5095](https://github.com/ClickHouse/ClickHouse/pull/5095) ([Guillaume Tassery](https://github.com/YiuRULE)) -- Add a X-ClickHouse-Summary header when we send a query using HTTP with enabled setting `send_progress_in_http_headers`. Return the usual information of X-ClickHouse-Progress, with additional information like how many rows and bytes were inserted in the query. [\#5116](https://github.com/ClickHouse/ClickHouse/pull/5116) ([Guillaume Tassery](https://github.com/YiuRULE)) - -#### Improvements {#improvements} - -- Added `max_parts_in_total` setting for MergeTree family of tables (default: 100 000) that prevents unsafe specification of partition key \#5166. [\#5171](https://github.com/ClickHouse/ClickHouse/pull/5171) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `clickhouse-obfuscator`: derive seed for individual columns by combining initial seed with column name, not column position. This is intended to transform datasets with multiple related tables, so that tables will remain JOINable after transformation. [\#5178](https://github.com/ClickHouse/ClickHouse/pull/5178) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added functions `JSONExtractRaw`, `JSONExtractKeyAndValues`. Renamed functions `jsonExtract` to `JSONExtract`. When something goes wrong these functions return the correspondent values, not `NULL`. Modified function `JSONExtract`, now it gets the return type from its last parameter and doesn’t inject nullables. Implemented fallback to RapidJSON in case AVX2 instructions are not available. Simdjson library updated to a new version. [\#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([Vitaly Baranov](https://github.com/vitlibar)) -- Now `if` and `multiIf` functions don’t rely on the condition’s `Nullable`, but rely on the branches for sql compatibility. [\#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([Jian Wu](https://github.com/janplus)) -- `In` predicate now generates `Null` result from `Null` input like the `Equal` function. [\#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([Jian Wu](https://github.com/janplus)) -- Check the time limit every (flush\_interval / poll\_timeout) number of rows from Kafka. This allows to break the reading from Kafka consumer more frequently and to check the time limits for the top-level streams [\#5249](https://github.com/ClickHouse/ClickHouse/pull/5249) ([Ivan](https://github.com/abyss7)) -- Link rdkafka with bundled SASL. It should allow to use SASL SCRAM authentication [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([Ivan](https://github.com/abyss7)) -- Batched version of RowRefList for ALL JOINS. [\#5267](https://github.com/ClickHouse/ClickHouse/pull/5267) ([Artem Zuikov](https://github.com/4ertus2)) -- clickhouse-server: more informative listen error messages. [\#5268](https://github.com/ClickHouse/ClickHouse/pull/5268) ([proller](https://github.com/proller)) -- Support dictionaries in clickhouse-copier for functions in `` [\#5270](https://github.com/ClickHouse/ClickHouse/pull/5270) ([proller](https://github.com/proller)) -- Add new setting `kafka_commit_every_batch` to regulate Kafka committing policy. - It allows to set commit mode: after every batch of messages is handled, or after the whole block is written to the storage. It’s a trade-off between losing some messages or reading them twice in some extreme situations. [\#5308](https://github.com/ClickHouse/ClickHouse/pull/5308) ([Ivan](https://github.com/abyss7)) -- Make `windowFunnel` support other Unsigned Integer Types. [\#5320](https://github.com/ClickHouse/ClickHouse/pull/5320) ([sundyli](https://github.com/sundy-li)) -- Allow to shadow virtual column `_table` in Merge engine. [\#5325](https://github.com/ClickHouse/ClickHouse/pull/5325) ([Ivan](https://github.com/abyss7)) -- Make `sequenceMatch` aggregate functions support other unsigned Integer types [\#5339](https://github.com/ClickHouse/ClickHouse/pull/5339) ([sundyli](https://github.com/sundy-li)) -- Better error messages if checksum mismatch is most likely caused by hardware failures. [\#5355](https://github.com/ClickHouse/ClickHouse/pull/5355) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Check that underlying tables support sampling for `StorageMerge` [\#5366](https://github.com/ClickHouse/ClickHouse/pull/5366) ([Ivan](https://github.com/abyss7)) -- Сlose MySQL connections after their usage in external dictionaries. It is related to issue \#893. [\#5395](https://github.com/ClickHouse/ClickHouse/pull/5395) ([Clément Rodriguez](https://github.com/clemrodriguez)) -- Improvements of MySQL Wire Protocol. Changed name of format to MySQLWire. Using RAII for calling RSA\_free. Disabling SSL if context cannot be created. [\#5419](https://github.com/ClickHouse/ClickHouse/pull/5419) ([Yuriy Baranov](https://github.com/yurriy)) -- clickhouse-client: allow to run with unaccessable history file (read-only, no disk space, file is directory, …). [\#5431](https://github.com/ClickHouse/ClickHouse/pull/5431) ([proller](https://github.com/proller)) -- Respect query settings in asynchronous INSERTs into Distributed tables. [\#4936](https://github.com/ClickHouse/ClickHouse/pull/4936) ([TCeason](https://github.com/TCeason)) -- Renamed functions `leastSqr` to `simpleLinearRegression`, `LinearRegression` to `linearRegression`, `LogisticRegression` to `logisticRegression`. [\#5391](https://github.com/ClickHouse/ClickHouse/pull/5391) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Performance Improvements {#performance-improvements} - -- Parallelize processing of parts of non-replicated MergeTree tables in ALTER MODIFY query. [\#4639](https://github.com/ClickHouse/ClickHouse/pull/4639) ([Ivan Kush](https://github.com/IvanKush)) -- Optimizations in regular expressions extraction. [\#5193](https://github.com/ClickHouse/ClickHouse/pull/5193) [\#5191](https://github.com/ClickHouse/ClickHouse/pull/5191) ([Danila Kutenin](https://github.com/danlark1)) -- Do not add right join key column to join result if it’s used only in join on section. [\#5260](https://github.com/ClickHouse/ClickHouse/pull/5260) ([Artem Zuikov](https://github.com/4ertus2)) -- Freeze the Kafka buffer after first empty response. It avoids multiple invokations of `ReadBuffer::next()` for empty result in some row-parsing streams. [\#5283](https://github.com/ClickHouse/ClickHouse/pull/5283) ([Ivan](https://github.com/abyss7)) -- `concat` function optimization for multiple arguments. [\#5357](https://github.com/ClickHouse/ClickHouse/pull/5357) ([Danila Kutenin](https://github.com/danlark1)) -- Query optimisation. Allow push down IN statement while rewriting commа/cross join into inner one. [\#5396](https://github.com/ClickHouse/ClickHouse/pull/5396) ([Artem Zuikov](https://github.com/4ertus2)) -- Upgrade our LZ4 implementation with reference one to have faster decompression. [\#5070](https://github.com/ClickHouse/ClickHouse/pull/5070) ([Danila Kutenin](https://github.com/danlark1)) -- Implemented MSD radix sort (based on kxsort), and partial sorting. [\#5129](https://github.com/ClickHouse/ClickHouse/pull/5129) ([Evgenii Pravda](https://github.com/kvinty)) - -#### Bug Fixes {#bug-fixes} - -- Fix push require columns with join [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([Winter Zhang](https://github.com/zhang2014)) -- Fixed bug, when ClickHouse is run by systemd, the command `sudo service clickhouse-server forcerestart` was not working as expected. [\#5204](https://github.com/ClickHouse/ClickHouse/pull/5204) ([proller](https://github.com/proller)) -- Fix http error codes in DataPartsExchange (interserver http server on 9009 port always returned code 200, even on errors). [\#5216](https://github.com/ClickHouse/ClickHouse/pull/5216) ([proller](https://github.com/proller)) -- Fix SimpleAggregateFunction for String longer than MAX\_SMALL\_STRING\_SIZE [\#5311](https://github.com/ClickHouse/ClickHouse/pull/5311) ([Azat Khuzhin](https://github.com/azat)) -- Fix error for `Decimal` to `Nullable(Decimal)` conversion in IN. Support other Decimal to Decimal conversions (including different scales). [\#5350](https://github.com/ClickHouse/ClickHouse/pull/5350) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed FPU clobbering in simdjson library that lead to wrong calculation of `uniqHLL` and `uniqCombined` aggregate function and math functions such as `log`. [\#5354](https://github.com/ClickHouse/ClickHouse/pull/5354) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed handling mixed const/nonconst cases in JSON functions. [\#5435](https://github.com/ClickHouse/ClickHouse/pull/5435) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix `retention` function. Now all conditions that satisfy in a row of data are added to the data state. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) -- Fix result type for `quantileExact` with Decimals. [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Documentation {#documentation} - -- Translate documentation for `CollapsingMergeTree` to chinese. [\#5168](https://github.com/ClickHouse/ClickHouse/pull/5168) ([张风啸](https://github.com/AlexZFX)) -- Translate some documentation about table engines to chinese. - [\#5134](https://github.com/ClickHouse/ClickHouse/pull/5134) - [\#5328](https://github.com/ClickHouse/ClickHouse/pull/5328) - ([never lee](https://github.com/neverlee)) - -#### Build/Testing/Packaging Improvements {#buildtestingpackaging-improvements} - -- Fix some sanitizer reports that show probable use-after-free.[\#5139](https://github.com/ClickHouse/ClickHouse/pull/5139) [\#5143](https://github.com/ClickHouse/ClickHouse/pull/5143) [\#5393](https://github.com/ClickHouse/ClickHouse/pull/5393) ([Ivan](https://github.com/abyss7)) -- Move performance tests out of separate directories for convenience. [\#5158](https://github.com/ClickHouse/ClickHouse/pull/5158) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix incorrect performance tests. [\#5255](https://github.com/ClickHouse/ClickHouse/pull/5255) ([alesapin](https://github.com/alesapin)) -- Added a tool to calculate checksums caused by bit flips to debug hardware issues. [\#5334](https://github.com/ClickHouse/ClickHouse/pull/5334) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Make runner script more usable. [\#5340](https://github.com/ClickHouse/ClickHouse/pull/5340)[\#5360](https://github.com/ClickHouse/ClickHouse/pull/5360) ([filimonov](https://github.com/filimonov)) -- Add small instruction how to write performance tests. [\#5408](https://github.com/ClickHouse/ClickHouse/pull/5408) ([alesapin](https://github.com/alesapin)) -- Add ability to make substitutions in create, fill and drop query in performance tests [\#5367](https://github.com/ClickHouse/ClickHouse/pull/5367) ([Olga Khvostikova](https://github.com/stavrolia)) - -## ClickHouse release 19.7 {#clickhouse-release-19-7} - -### ClickHouse release 19.7.5.29, 2019-07-05 {#clickhouse-release-19-7-5-29-2019-07-05} - -#### Bug Fix {#bug-fix-25} - -- Fix performance regression in some queries with JOIN. [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([Winter Zhang](https://github.com/zhang2014)) - -### ClickHouse release 19.7.5.27, 2019-06-09 {#clickhouse-release-19-7-5-27-2019-06-09} - -#### New features {#new-features-1} - -- Added bitmap related functions `bitmapHasAny` and `bitmapHasAll` analogous to `hasAny` and `hasAll` functions for arrays. [\#5279](https://github.com/ClickHouse/ClickHouse/pull/5279) ([Sergi Vladykin](https://github.com/svladykin)) - -#### Bug Fixes {#bug-fixes-1} - -- Fix segfault on `minmax` INDEX with Null value. [\#5246](https://github.com/ClickHouse/ClickHouse/pull/5246) ([Nikita Vasilev](https://github.com/nikvas0)) -- Mark all input columns in LIMIT BY as required output. It fixes ‘Not found column’ error in some distributed queries. [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([Constantin S. Pan](https://github.com/kvap)) -- Fix “Column ‘0’ already exists” error in `SELECT .. PREWHERE` on column with DEFAULT [\#5397](https://github.com/ClickHouse/ClickHouse/pull/5397) ([proller](https://github.com/proller)) -- Fix `ALTER MODIFY TTL` query on `ReplicatedMergeTree`. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539/commits) ([Anton Popov](https://github.com/CurtizJ)) -- Don’t crash the server when Kafka consumers have failed to start. [\#5285](https://github.com/ClickHouse/ClickHouse/pull/5285) ([Ivan](https://github.com/abyss7)) -- Fixed bitmap functions produce wrong result. [\#5359](https://github.com/ClickHouse/ClickHouse/pull/5359) ([Andy Yang](https://github.com/andyyzh)) -- Fix element\_count for hashed dictionary (do not include duplicates) [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) -- Use contents of environment variable TZ as the name for timezone. It helps to correctly detect default timezone in some cases.[\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([Ivan](https://github.com/abyss7)) -- Do not try to convert integers in `dictGetT` functions, because it doesn’t work correctly. Throw an exception instead. [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix settings in ExternalData HTTP request. [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila - Kutenin](https://github.com/danlark1)) -- Fix bug when parts were removed only from FS without dropping them from Zookeeper. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alesapin](https://github.com/alesapin)) -- Fix segmentation fault in `bitmapHasAny` function. [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- Fixed error when replication connection pool doesn’t retry to resolve host, even when DNS cache was dropped. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) -- Fixed `DROP INDEX IF EXISTS` query. Now `ALTER TABLE ... DROP INDEX IF EXISTS ...` query doesn’t raise an exception if provided index does not exist. [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) -- Fix union all supertype column. There were cases with inconsistent data and column types of resulting columns. [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) -- Skip ZNONODE during DDL query processing. Before if another node removes the znode in task queue, the one that - did not process it, but already get list of children, will terminate the DDLWorker thread. [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) -- Fix INSERT into Distributed() table with MATERIALIZED column. [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) - -### ClickHouse release 19.7.3.9, 2019-05-30 {#clickhouse-release-19-7-3-9-2019-05-30} - -#### New Features {#new-features-2} - -- Allow to limit the range of a setting that can be specified by user. - These constraints can be set up in user settings profile. - [\#4931](https://github.com/ClickHouse/ClickHouse/pull/4931) ([Vitaly - Baranov](https://github.com/vitlibar)) -- Add a second version of the function `groupUniqArray` with an optional - `max_size` parameter that limits the size of the resulting array. This - behavior is similar to `groupArray(max_size)(x)` function. - [\#5026](https://github.com/ClickHouse/ClickHouse/pull/5026) ([Guillaume - Tassery](https://github.com/YiuRULE)) -- For TSVWithNames/CSVWithNames input file formats, column order can now be - determined from file header. This is controlled by - `input_format_with_names_use_header` parameter. - [\#5081](https://github.com/ClickHouse/ClickHouse/pull/5081) - ([Alexander](https://github.com/Akazz)) - -#### Bug Fixes {#bug-fixes-2} - -- Crash with uncompressed\_cache + JOIN during merge (\#5197) - [\#5133](https://github.com/ClickHouse/ClickHouse/pull/5133) ([Danila - Kutenin](https://github.com/danlark1)) -- Segmentation fault on a clickhouse-client query to system tables. \#5066 - [\#5127](https://github.com/ClickHouse/ClickHouse/pull/5127) - ([Ivan](https://github.com/abyss7)) -- Data loss on heavy load via KafkaEngine (\#4736) - [\#5080](https://github.com/ClickHouse/ClickHouse/pull/5080) - ([Ivan](https://github.com/abyss7)) -- Fixed very rare data race condition that could happen when executing a query with UNION ALL involving at least two SELECTs from system.columns, system.tables, system.parts, system.parts\_tables or tables of Merge family and performing ALTER of columns of the related tables concurrently. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Performance Improvements {#performance-improvements-1} - -- Use radix sort for sorting by single numeric column in `ORDER BY` without - `LIMIT`. [\#5106](https://github.com/ClickHouse/ClickHouse/pull/5106), - [\#4439](https://github.com/ClickHouse/ClickHouse/pull/4439) - ([Evgenii Pravda](https://github.com/kvinty), - [alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Documentation {#documentation-1} - -- Translate documentation for some table engines to Chinese. - [\#5107](https://github.com/ClickHouse/ClickHouse/pull/5107), - [\#5094](https://github.com/ClickHouse/ClickHouse/pull/5094), - [\#5087](https://github.com/ClickHouse/ClickHouse/pull/5087) - ([张风啸](https://github.com/AlexZFX)), - [\#5068](https://github.com/ClickHouse/ClickHouse/pull/5068) ([never - lee](https://github.com/neverlee)) - -#### Build/Testing/Packaging Improvements {#buildtestingpackaging-improvements-1} - -- Print UTF-8 characters properly in `clickhouse-test`. - [\#5084](https://github.com/ClickHouse/ClickHouse/pull/5084) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add command line parameter for clickhouse-client to always load suggestion - data. [\#5102](https://github.com/ClickHouse/ClickHouse/pull/5102) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Resolve some of PVS-Studio warnings. - [\#5082](https://github.com/ClickHouse/ClickHouse/pull/5082) - ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Update LZ4 [\#5040](https://github.com/ClickHouse/ClickHouse/pull/5040) ([Danila - Kutenin](https://github.com/danlark1)) -- Add gperf to build requirements for upcoming pull request \#5030. - [\#5110](https://github.com/ClickHouse/ClickHouse/pull/5110) - ([proller](https://github.com/proller)) - -## ClickHouse release 19.6 {#clickhouse-release-19-6} - -### ClickHouse release 19.6.3.18, 2019-06-13 {#clickhouse-release-19-6-3-18-2019-06-13} - -#### Bug Fixes {#bug-fixes-3} - -- Fixed IN condition pushdown for queries from table functions `mysql` and `odbc` and corresponding table engines. This fixes \#3540 and \#2384. [\#5313](https://github.com/ClickHouse/ClickHouse/pull/5313) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix deadlock in Zookeeper. [\#5297](https://github.com/ClickHouse/ClickHouse/pull/5297) ([github1youlc](https://github.com/github1youlc)) -- Allow quoted decimals in CSV. [\#5284](https://github.com/ClickHouse/ClickHouse/pull/5284) ([Artem Zuikov](https://github.com/4ertus2) -- Disallow conversion from float Inf/NaN into Decimals (throw exception). [\#5282](https://github.com/ClickHouse/ClickHouse/pull/5282) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix data race in rename query. [\#5247](https://github.com/ClickHouse/ClickHouse/pull/5247) ([Winter Zhang](https://github.com/zhang2014)) -- Temporarily disable LFAlloc. Usage of LFAlloc might lead to a lot of MAP\_FAILED in allocating UncompressedCache and in a result to crashes of queries at high loaded servers. [cfdba93](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([Danila Kutenin](https://github.com/danlark1)) - -### ClickHouse release 19.6.2.11, 2019-05-13 {#clickhouse-release-19-6-2-11-2019-05-13} - -#### New Features {#new-features-3} - -- TTL expressions for columns and tables. [\#4212](https://github.com/ClickHouse/ClickHouse/pull/4212) ([Anton Popov](https://github.com/CurtizJ)) -- Added support for `brotli` compression for HTTP responses (Accept-Encoding: br) [\#4388](https://github.com/ClickHouse/ClickHouse/pull/4388) ([Mikhail](https://github.com/fandyushin)) -- Added new function `isValidUTF8` for checking whether a set of bytes is correctly utf-8 encoded. [\#4934](https://github.com/ClickHouse/ClickHouse/pull/4934) ([Danila Kutenin](https://github.com/danlark1)) -- Add new load balancing policy `first_or_random` which sends queries to the first specified host and if it’s inaccessible send queries to random hosts of shard. Useful for cross-replication topology setups. [\#5012](https://github.com/ClickHouse/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei)) - -#### Experimental Features {#experimental-features-1} - -- Add setting `index_granularity_bytes` (adaptive index granularity) for MergeTree\* tables family. [\#4826](https://github.com/ClickHouse/ClickHouse/pull/4826) ([alesapin](https://github.com/alesapin)) - -#### Improvements {#improvements-1} - -- Added support for non-constant and negative size and length arguments for function `substringUTF8`. [\#4989](https://github.com/ClickHouse/ClickHouse/pull/4989) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Disable push-down to right table in left join, left table in right join, and both tables in full join. This fixes wrong JOIN results in some cases. [\#4846](https://github.com/ClickHouse/ClickHouse/pull/4846) ([Ivan](https://github.com/abyss7)) -- `clickhouse-copier`: auto upload task configuration from `--task-file` option [\#4876](https://github.com/ClickHouse/ClickHouse/pull/4876) ([proller](https://github.com/proller)) -- Added typos handler for storage factory and table functions factory. [\#4891](https://github.com/ClickHouse/ClickHouse/pull/4891) ([Danila Kutenin](https://github.com/danlark1)) -- Support asterisks and qualified asterisks for multiple joins without subqueries [\#4898](https://github.com/ClickHouse/ClickHouse/pull/4898) ([Artem Zuikov](https://github.com/4ertus2)) -- Make missing column error message more user friendly. [\#4915](https://github.com/ClickHouse/ClickHouse/pull/4915) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Performance Improvements {#performance-improvements-2} - -- Significant speedup of ASOF JOIN [\#4924](https://github.com/ClickHouse/ClickHouse/pull/4924) ([Martijn Bakker](https://github.com/Gladdy)) - -#### Backward Incompatible Changes {#backward-incompatible-changes} - -- HTTP header `Query-Id` was renamed to `X-ClickHouse-Query-Id` for consistency. [\#4972](https://github.com/ClickHouse/ClickHouse/pull/4972) ([Mikhail](https://github.com/fandyushin)) - -#### Bug Fixes {#bug-fixes-4} - -- Fixed potential null pointer dereference in `clickhouse-copier`. [\#4900](https://github.com/ClickHouse/ClickHouse/pull/4900) ([proller](https://github.com/proller)) -- Fixed error on query with JOIN + ARRAY JOIN [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed hanging on start of the server when a dictionary depends on another dictionary via a database with engine=Dictionary. [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([Vitaly Baranov](https://github.com/vitlibar)) -- Partially fix distributed\_product\_mode = local. It’s possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There’s not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix potentially wrong result for `SELECT DISTINCT` with `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed very rare data race condition that could happen when executing a query with UNION ALL involving at least two SELECTs from system.columns, system.tables, system.parts, system.parts\_tables or tables of Merge family and performing ALTER of columns of the related tables concurrently. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Build/Testing/Packaging Improvements {#buildtestingpackaging-improvements-2} - -- Fixed test failures when running clickhouse-server on different host [\#4713](https://github.com/ClickHouse/ClickHouse/pull/4713) ([Vasily Nemkov](https://github.com/Enmk)) -- clickhouse-test: Disable color control sequences in non tty environment. [\#4937](https://github.com/ClickHouse/ClickHouse/pull/4937) ([alesapin](https://github.com/alesapin)) -- clickhouse-test: Allow use any test database (remove `test.` qualification where it possible) [\#5008](https://github.com/ClickHouse/ClickHouse/pull/5008) ([proller](https://github.com/proller)) -- Fix ubsan errors [\#5037](https://github.com/ClickHouse/ClickHouse/pull/5037) ([Vitaly Baranov](https://github.com/vitlibar)) -- Yandex LFAlloc was added to ClickHouse to allocate MarkCache and UncompressedCache data in different ways to catch segfaults more reliable [\#4995](https://github.com/ClickHouse/ClickHouse/pull/4995) ([Danila Kutenin](https://github.com/danlark1)) -- Python util to help with backports and changelogs. [\#4949](https://github.com/ClickHouse/ClickHouse/pull/4949) ([Ivan](https://github.com/abyss7)) - -## ClickHouse release 19.5 {#clickhouse-release-19-5} - -### ClickHouse release 19.5.4.22, 2019-05-13 {#clickhouse-release-19-5-4-22-2019-05-13} - -#### Bug fixes {#bug-fixes-5} - -- Fixed possible crash in bitmap\* functions [\#5220](https://github.com/ClickHouse/ClickHouse/pull/5220) [\#5228](https://github.com/ClickHouse/ClickHouse/pull/5228) ([Andy Yang](https://github.com/andyyzh)) -- Fixed very rare data race condition that could happen when executing a query with UNION ALL involving at least two SELECTs from system.columns, system.tables, system.parts, system.parts\_tables or tables of Merge family and performing ALTER of columns of the related tables concurrently. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed error `Set for IN is not created yet in case of using single LowCardinality column in the left part of IN`. This error happened if LowCardinality column was the part of primary key. \#5031 [\#5154](https://github.com/ClickHouse/ClickHouse/pull/5154) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Modification of retention function: If a row satisfies both the first and NTH condition, only the first satisfied condition is added to the data state. Now all conditions that satisfy in a row of data are added to the data state. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) - -### ClickHouse release 19.5.3.8, 2019-04-18 {#clickhouse-release-19-5-3-8-2019-04-18} - -#### Bug fixes {#bug-fixes-6} - -- Fixed type of setting `max_partitions_per_insert_block` from boolean to UInt64. [\#5028](https://github.com/ClickHouse/ClickHouse/pull/5028) ([Mohammad Hossein Sekhavat](https://github.com/mhsekhavat)) - -### ClickHouse release 19.5.2.6, 2019-04-15 {#clickhouse-release-19-5-2-6-2019-04-15} - -#### New Features {#new-features-4} - -- [Hyperscan](https://github.com/intel/hyperscan) multiple regular expression matching was added (functions `multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780), [\#4841](https://github.com/ClickHouse/ClickHouse/pull/4841) ([Danila Kutenin](https://github.com/danlark1)) -- `multiSearchFirstPosition` function was added. [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) -- Implement the predefined expression filter per row for tables. [\#4792](https://github.com/ClickHouse/ClickHouse/pull/4792) ([Ivan](https://github.com/abyss7)) -- A new type of data skipping indices based on bloom filters (can be used for `equal`, `in` and `like` functions). [\#4499](https://github.com/ClickHouse/ClickHouse/pull/4499) ([Nikita Vasilev](https://github.com/nikvas0)) -- Added `ASOF JOIN` which allows to run queries that join to the most recent value known. [\#4774](https://github.com/ClickHouse/ClickHouse/pull/4774) [\#4867](https://github.com/ClickHouse/ClickHouse/pull/4867) [\#4863](https://github.com/ClickHouse/ClickHouse/pull/4863) [\#4875](https://github.com/ClickHouse/ClickHouse/pull/4875) ([Martijn Bakker](https://github.com/Gladdy), [Artem Zuikov](https://github.com/4ertus2)) -- Rewrite multiple `COMMA JOIN` to `CROSS JOIN`. Then rewrite them to `INNER JOIN` if possible. [\#4661](https://github.com/ClickHouse/ClickHouse/pull/4661) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Improvement {#improvement-9} - -- `topK` and `topKWeighted` now supports custom `loadFactor` (fixes issue [\#4252](https://github.com/ClickHouse/ClickHouse/issues/4252)). [\#4634](https://github.com/ClickHouse/ClickHouse/pull/4634) ([Kirill Danshin](https://github.com/kirillDanshin)) -- Allow to use `parallel_replicas_count > 1` even for tables without sampling (the setting is simply ignored for them). In previous versions it was lead to exception. [\#4637](https://github.com/ClickHouse/ClickHouse/pull/4637) ([Alexey Elymanov](https://github.com/digitalist)) -- Support for `CREATE OR REPLACE VIEW`. Allow to create a view or set a new definition in a single statement. [\#4654](https://github.com/ClickHouse/ClickHouse/pull/4654) ([Boris Granveaud](https://github.com/bgranvea)) -- `Buffer` table engine now supports `PREWHERE`. [\#4671](https://github.com/ClickHouse/ClickHouse/pull/4671) ([Yangkuan Liu](https://github.com/LiuYangkuan)) -- Add ability to start replicated table without metadata in zookeeper in `readonly` mode. [\#4691](https://github.com/ClickHouse/ClickHouse/pull/4691) ([alesapin](https://github.com/alesapin)) -- Fixed flicker of progress bar in clickhouse-client. The issue was most noticeable when using `FORMAT Null` with streaming queries. [\#4811](https://github.com/ClickHouse/ClickHouse/pull/4811) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Allow to disable functions with `hyperscan` library on per user basis to limit potentially excessive and uncontrolled resource usage. [\#4816](https://github.com/ClickHouse/ClickHouse/pull/4816) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add version number logging in all errors. [\#4824](https://github.com/ClickHouse/ClickHouse/pull/4824) ([proller](https://github.com/proller)) -- Added restriction to the `multiMatch` functions which requires string size to fit into `unsigned int`. Also added the number of arguments limit to the `multiSearch` functions. [\#4834](https://github.com/ClickHouse/ClickHouse/pull/4834) ([Danila Kutenin](https://github.com/danlark1)) -- Improved usage of scratch space and error handling in Hyperscan. [\#4866](https://github.com/ClickHouse/ClickHouse/pull/4866) ([Danila Kutenin](https://github.com/danlark1)) -- Fill `system.graphite_detentions` from a table config of `*GraphiteMergeTree` engine tables. [\#4584](https://github.com/ClickHouse/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Rename `trigramDistance` function to `ngramDistance` and add more functions with `CaseInsensitive` and `UTF`. [\#4602](https://github.com/ClickHouse/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1)) -- Improved data skipping indices calculation. [\#4640](https://github.com/ClickHouse/ClickHouse/pull/4640) ([Nikita Vasilev](https://github.com/nikvas0)) -- Keep ordinary, `DEFAULT`, `MATERIALIZED` and `ALIAS` columns in a single list (fixes issue [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn)) - -#### Bug Fix {#bug-fix-26} - -- Avoid `std::terminate` in case of memory allocation failure. Now `std::bad_alloc` exception is thrown as expected. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixes capnproto reading from buffer. Sometimes files wasn’t loaded successfully by HTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) -- Fix error `Unknown log entry type: 0` after `OPTIMIZE TABLE FINAL` query. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Bird](https://github.com/amosbird)) -- Wrong arguments to `hasAny` or `hasAll` functions may lead to segfault. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Deadlock may happen while executing `DROP DATABASE dictionary` query. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix undefined behavior in `median` and `quantile` functions. [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) -- Fix compression level detection when `network_compression_method` in lowercase. Broken in v19.1. [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) -- Fixed ignorance of `UTC` setting (fixes issue [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) -- Fix `histogram` function behaviour with `Distributed` tables. [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) -- Fixed tsan report `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed TSan report on shutdown due to race condition in system logs usage. Fixed potential use-after-free on shutdown when part\_log is enabled. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix recheck parts in `ReplicatedMergeTreeAlterThread` in case of error. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Arithmetic operations on intermediate aggregate function states were not working for constant arguments (such as subquery results). [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Always backquote column names in metadata. Otherwise it’s impossible to create a table with column named `index` (server won’t restart due to malformed `ATTACH` query in metadata). [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix crash in `ALTER ... MODIFY ORDER BY` on `Distributed` table. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) -- Fix segfault in `JOIN ON` with enabled `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([Winter Zhang](https://github.com/zhang2014)) -- Fix bug with adding an extraneous row after consuming a protobuf message from Kafka. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix crash of `JOIN` on not-nullable vs nullable column. Fix `NULLs` in right keys in `ANY JOIN` + `join_use_nulls`. [\#4815](https://github.com/ClickHouse/ClickHouse/pull/4815) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix segmentation fault in `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) -- Fixed race condition in `SELECT` from `system.tables` if the table is renamed or altered concurrently. [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed data race when fetching data part that is already obsolete. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed rare data race that can happen during `RENAME` table of MergeTree family. [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed segmentation fault in function `arrayIntersect`. Segmentation fault could happen if function was called with mixed constant and ordinary arguments. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) -- Fixed reading from `Array(LowCardinality)` column in rare case when column contained a long sequence of empty arrays. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix crash in `FULL/RIGHT JOIN` when we joining on nullable vs not nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix `No message received` exception while fetching parts between replicas. [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin)) -- Fixed `arrayIntersect` function wrong result in case of several repeated values in single array. [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix a race condition during concurrent `ALTER COLUMN` queries that could lead to a server crash (fixes issue [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fix incorrect result in `FULL/RIGHT JOIN` with const column. [\#4723](https://github.com/ClickHouse/ClickHouse/pull/4723) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix duplicates in `GLOBAL JOIN` with asterisk. [\#4705](https://github.com/ClickHouse/ClickHouse/pull/4705) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix parameter deduction in `ALTER MODIFY` of column `CODEC` when column type is not specified. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin)) -- Functions `cutQueryStringAndFragment()` and `queryStringAndFragment()` now works correctly when `URL` contains a fragment and no query. [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix rare bug when setting `min_bytes_to_use_direct_io` is greater than zero, which occures when thread have to seek backward in column file. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin)) -- Fix wrong argument types for aggregate functions with `LowCardinality` arguments (fixes issue [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix wrong name qualification in `GLOBAL JOIN`. [\#4969](https://github.com/ClickHouse/ClickHouse/pull/4969) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix function `toISOWeek` result for year 1970. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix `DROP`, `TRUNCATE` and `OPTIMIZE` queries duplication, when executed on `ON CLUSTER` for `ReplicatedMergeTree*` tables family. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin)) - -#### Backward Incompatible Change {#backward-incompatible-change-8} - -- Rename setting `insert_sample_with_metadata` to setting `input_format_defaults_for_omitted_fields`. [\#4771](https://github.com/ClickHouse/ClickHouse/pull/4771) ([Artem Zuikov](https://github.com/4ertus2)) -- Added setting `max_partitions_per_insert_block` (with value 100 by default). If inserted block contains larger number of partitions, an exception is thrown. Set it to 0 if you want to remove the limit (not recommended). [\#4845](https://github.com/ClickHouse/ClickHouse/pull/4845) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Multi-search functions were renamed (`multiPosition` to `multiSearchAllPositions`, `multiSearch` to `multiSearchAny`, `firstMatch` to `multiSearchFirstIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) - -#### Performance Improvement {#performance-improvement-6} - -- Optimize Volnitsky searcher by inlining, giving about 5-10% search improvement for queries with many needles or many similar bigrams. [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) -- Fix performance issue when setting `use_uncompressed_cache` is greater than zero, which appeared when all read data contained in cache. [\#4913](https://github.com/ClickHouse/ClickHouse/pull/4913) ([alesapin](https://github.com/alesapin)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-10} - -- Hardening debug build: more granular memory mappings and ASLR; add memory protection for mark cache and index. This allows to find more memory stomping bugs in case when ASan and MSan cannot do it. [\#4632](https://github.com/ClickHouse/ClickHouse/pull/4632) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add support for cmake variables `ENABLE_PROTOBUF`, `ENABLE_PARQUET` and `ENABLE_BROTLI` which allows to enable/disable the above features (same as we can do for librdkafka, mysql, etc). [\#4669](https://github.com/ClickHouse/ClickHouse/pull/4669) ([Silviu Caragea](https://github.com/silviucpp)) -- Add ability to print process list and stacktraces of all threads if some queries are hung after test run. [\#4675](https://github.com/ClickHouse/ClickHouse/pull/4675) ([alesapin](https://github.com/alesapin)) -- Add retries on `Connection loss` error in `clickhouse-test`. [\#4682](https://github.com/ClickHouse/ClickHouse/pull/4682) ([alesapin](https://github.com/alesapin)) -- Add freebsd build with vagrant and build with thread sanitizer to packager script. [\#4712](https://github.com/ClickHouse/ClickHouse/pull/4712) [\#4748](https://github.com/ClickHouse/ClickHouse/pull/4748) ([alesapin](https://github.com/alesapin)) -- Now user asked for password for user `'default'` during installation. [\#4725](https://github.com/ClickHouse/ClickHouse/pull/4725) ([proller](https://github.com/proller)) -- Suppress warning in `rdkafka` library. [\#4740](https://github.com/ClickHouse/ClickHouse/pull/4740) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Allow ability to build without ssl. [\#4750](https://github.com/ClickHouse/ClickHouse/pull/4750) ([proller](https://github.com/proller)) -- Add a way to launch clickhouse-server image from a custom user. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Upgrade contrib boost to 1.69. [\#4793](https://github.com/ClickHouse/ClickHouse/pull/4793) ([proller](https://github.com/proller)) -- Disable usage of `mremap` when compiled with Thread Sanitizer. Surprisingly enough, TSan does not intercept `mremap` (though it does intercept `mmap`, `munmap`) that leads to false positives. Fixed TSan report in stateful tests. [\#4859](https://github.com/ClickHouse/ClickHouse/pull/4859) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add test checking using format schema via HTTP interface. [\#4864](https://github.com/ClickHouse/ClickHouse/pull/4864) ([Vitaly Baranov](https://github.com/vitlibar)) - -## ClickHouse release 19.4 {#clickhouse-release-19-4} - -### ClickHouse release 19.4.4.33, 2019-04-17 {#clickhouse-release-19-4-4-33-2019-04-17} - -#### Bug Fixes {#bug-fixes-7} - -- Avoid `std::terminate` in case of memory allocation failure. Now `std::bad_alloc` exception is thrown as expected. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixes capnproto reading from buffer. Sometimes files wasn’t loaded successfully by HTTP. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) -- Fix error `Unknown log entry type: 0` after `OPTIMIZE TABLE FINAL` query. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Bird](https://github.com/amosbird)) -- Wrong arguments to `hasAny` or `hasAll` functions may lead to segfault. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Deadlock may happen while executing `DROP DATABASE dictionary` query. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix undefined behavior in `median` and `quantile` functions. [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) -- Fix compression level detection when `network_compression_method` in lowercase. Broken in v19.1. [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) -- Fixed ignorance of `UTC` setting (fixes issue [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) -- Fix `histogram` function behaviour with `Distributed` tables. [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) -- Fixed tsan report `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed TSan report on shutdown due to race condition in system logs usage. Fixed potential use-after-free on shutdown when part\_log is enabled. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix recheck parts in `ReplicatedMergeTreeAlterThread` in case of error. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Arithmetic operations on intermediate aggregate function states were not working for constant arguments (such as subquery results). [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Always backquote column names in metadata. Otherwise it’s impossible to create a table with column named `index` (server won’t restart due to malformed `ATTACH` query in metadata). [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix crash in `ALTER ... MODIFY ORDER BY` on `Distributed` table. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) -- Fix segfault in `JOIN ON` with enabled `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([Winter Zhang](https://github.com/zhang2014)) -- Fix bug with adding an extraneous row after consuming a protobuf message from Kafka. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix segmentation fault in `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) -- Fixed race condition in `SELECT` from `system.tables` if the table is renamed or altered concurrently. [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed data race when fetching data part that is already obsolete. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed rare data race that can happen during `RENAME` table of MergeTree family. [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed segmentation fault in function `arrayIntersect`. Segmentation fault could happen if function was called with mixed constant and ordinary arguments. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) -- Fixed reading from `Array(LowCardinality)` column in rare case when column contained a long sequence of empty arrays. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix `No message received` exception while fetching parts between replicas. [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin)) -- Fixed `arrayIntersect` function wrong result in case of several repeated values in single array. [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix a race condition during concurrent `ALTER COLUMN` queries that could lead to a server crash (fixes issue [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fix parameter deduction in `ALTER MODIFY` of column `CODEC` when column type is not specified. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin)) -- Functions `cutQueryStringAndFragment()` and `queryStringAndFragment()` now works correctly when `URL` contains a fragment and no query. [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) -- Fix rare bug when setting `min_bytes_to_use_direct_io` is greater than zero, which occures when thread have to seek backward in column file. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin)) -- Fix wrong argument types for aggregate functions with `LowCardinality` arguments (fixes issue [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fix function `toISOWeek` result for year 1970. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix `DROP`, `TRUNCATE` and `OPTIMIZE` queries duplication, when executed on `ON CLUSTER` for `ReplicatedMergeTree*` tables family. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin)) - -#### Improvements {#improvements-2} - -- Keep ordinary, `DEFAULT`, `MATERIALIZED` and `ALIAS` columns in a single list (fixes issue [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn)) - -### ClickHouse release 19.4.3.11, 2019-04-02 {#clickhouse-release-19-4-3-11-2019-04-02} - -#### Bug Fixes {#bug-fixes-8} - -- Fix crash in `FULL/RIGHT JOIN` when we joining on nullable vs not nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix segmentation fault in `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-11} - -- Add a way to launch clickhouse-server image from a custom user. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) - -### ClickHouse release 19.4.2.7, 2019-03-30 {#clickhouse-release-19-4-2-7-2019-03-30} - -#### Bug Fixes {#bug-fixes-9} - -- Fixed reading from `Array(LowCardinality)` column in rare case when column contained a long sequence of empty arrays. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -### ClickHouse release 19.4.1.3, 2019-03-19 {#clickhouse-release-19-4-1-3-2019-03-19} - -#### Bug Fixes {#bug-fixes-10} - -- Fixed remote queries which contain both `LIMIT BY` and `LIMIT`. Previously, if `LIMIT BY` and `LIMIT` were used for remote query, `LIMIT` could happen before `LIMIT BY`, which led to too filtered result. [\#4708](https://github.com/ClickHouse/ClickHouse/pull/4708) ([Constantin S. Pan](https://github.com/kvap)) - -### ClickHouse release 19.4.0.49, 2019-03-09 {#clickhouse-release-19-4-0-49-2019-03-09} - -#### New Features {#new-features-5} - -- Added full support for `Protobuf` format (input and output, nested data structures). [\#4174](https://github.com/ClickHouse/ClickHouse/pull/4174) [\#4493](https://github.com/ClickHouse/ClickHouse/pull/4493) ([Vitaly Baranov](https://github.com/vitlibar)) -- Added bitmap functions with Roaring Bitmaps. [\#4207](https://github.com/ClickHouse/ClickHouse/pull/4207) ([Andy Yang](https://github.com/andyyzh)) [\#4568](https://github.com/ClickHouse/ClickHouse/pull/4568) ([Vitaly Baranov](https://github.com/vitlibar)) -- Parquet format support. [\#4448](https://github.com/ClickHouse/ClickHouse/pull/4448) ([proller](https://github.com/proller)) -- N-gram distance was added for fuzzy string comparison. It is similar to q-gram metrics in R language. [\#4466](https://github.com/ClickHouse/ClickHouse/pull/4466) ([Danila Kutenin](https://github.com/danlark1)) -- Combine rules for graphite rollup from dedicated aggregation and retention patterns. [\#4426](https://github.com/ClickHouse/ClickHouse/pull/4426) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -- Added `max_execution_speed` and `max_execution_speed_bytes` to limit resource usage. Added `min_execution_speed_bytes` setting to complement the `min_execution_speed`. [\#4430](https://github.com/ClickHouse/ClickHouse/pull/4430) ([Winter Zhang](https://github.com/zhang2014)) -- Implemented function `flatten`. [\#4555](https://github.com/ClickHouse/ClickHouse/pull/4555) [\#4409](https://github.com/ClickHouse/ClickHouse/pull/4409) ([alexey-milovidov](https://github.com/alexey-milovidov), [kzon](https://github.com/kzon)) -- Added functions `arrayEnumerateDenseRanked` and `arrayEnumerateUniqRanked` (it’s like `arrayEnumerateUniq` but allows to fine tune array depth to look inside multidimensional arrays). [\#4475](https://github.com/ClickHouse/ClickHouse/pull/4475) ([proller](https://github.com/proller)) [\#4601](https://github.com/ClickHouse/ClickHouse/pull/4601) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Multiple JOINS with some restrictions: no asterisks, no complex aliases in ON/WHERE/GROUP BY/… [\#4462](https://github.com/ClickHouse/ClickHouse/pull/4462) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Bug Fixes {#bug-fixes-11} - -- This release also contains all bug fixes from 19.3 and 19.1. -- Fixed bug in data skipping indices: order of granules after INSERT was incorrect. [\#4407](https://github.com/ClickHouse/ClickHouse/pull/4407) ([Nikita Vasilev](https://github.com/nikvas0)) -- Fixed `set` index for `Nullable` and `LowCardinality` columns. Before it, `set` index with `Nullable` or `LowCardinality` column led to error `Data type must be deserialized with multiple streams` while selecting. [\#4594](https://github.com/ClickHouse/ClickHouse/pull/4594) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Correctly set update\_time on full `executable` dictionary update. [\#4551](https://github.com/ClickHouse/ClickHouse/pull/4551) ([Tema Novikov](https://github.com/temoon)) -- Fix broken progress bar in 19.3. [\#4627](https://github.com/ClickHouse/ClickHouse/pull/4627) ([filimonov](https://github.com/filimonov)) -- Fixed inconsistent values of MemoryTracker when memory region was shrinked, in certain cases. [\#4619](https://github.com/ClickHouse/ClickHouse/pull/4619) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed undefined behaviour in ThreadPool. [\#4612](https://github.com/ClickHouse/ClickHouse/pull/4612) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed a very rare crash with the message `mutex lock failed: Invalid argument` that could happen when a MergeTree table was dropped concurrently with a SELECT. [\#4608](https://github.com/ClickHouse/ClickHouse/pull/4608) ([Alex Zatelepin](https://github.com/ztlpn)) -- ODBC driver compatibility with `LowCardinality` data type. [\#4381](https://github.com/ClickHouse/ClickHouse/pull/4381) ([proller](https://github.com/proller)) -- FreeBSD: Fixup for `AIOcontextPool: Found io_event with unknown id 0` error. [\#4438](https://github.com/ClickHouse/ClickHouse/pull/4438) ([urgordeadbeef](https://github.com/urgordeadbeef)) -- `system.part_log` table was created regardless to configuration. [\#4483](https://github.com/ClickHouse/ClickHouse/pull/4483) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix undefined behaviour in `dictIsIn` function for cache dictionaries. [\#4515](https://github.com/ClickHouse/ClickHouse/pull/4515) ([alesapin](https://github.com/alesapin)) -- Fixed a deadlock when a SELECT query locks the same table multiple times (e.g. from different threads or when executing multiple subqueries) and there is a concurrent DDL query. [\#4535](https://github.com/ClickHouse/ClickHouse/pull/4535) ([Alex Zatelepin](https://github.com/ztlpn)) -- Disable compile\_expressions by default until we get own `llvm` contrib and can test it with `clang` and `asan`. [\#4579](https://github.com/ClickHouse/ClickHouse/pull/4579) ([alesapin](https://github.com/alesapin)) -- Prevent `std::terminate` when `invalidate_query` for `clickhouse` external dictionary source has returned wrong resultset (empty or more than one row or more than one column). Fixed issue when the `invalidate_query` was performed every five seconds regardless to the `lifetime`. [\#4583](https://github.com/ClickHouse/ClickHouse/pull/4583) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Avoid deadlock when the `invalidate_query` for a dictionary with `clickhouse` source was involving `system.dictionaries` table or `Dictionaries` database (rare case). [\#4599](https://github.com/ClickHouse/ClickHouse/pull/4599) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixes for CROSS JOIN with empty WHERE. [\#4598](https://github.com/ClickHouse/ClickHouse/pull/4598) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed segfault in function “replicate” when constant argument is passed. [\#4603](https://github.com/ClickHouse/ClickHouse/pull/4603) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix lambda function with predicate optimizer. [\#4408](https://github.com/ClickHouse/ClickHouse/pull/4408) ([Winter Zhang](https://github.com/zhang2014)) -- Multiple JOINs multiple fixes. [\#4595](https://github.com/ClickHouse/ClickHouse/pull/4595) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Improvements {#improvements-3} - -- Support aliases in JOIN ON section for right table columns. [\#4412](https://github.com/ClickHouse/ClickHouse/pull/4412) ([Artem Zuikov](https://github.com/4ertus2)) -- Result of multiple JOINs need correct result names to be used in subselects. Replace flat aliases with source names in result. [\#4474](https://github.com/ClickHouse/ClickHouse/pull/4474) ([Artem Zuikov](https://github.com/4ertus2)) -- Improve push-down logic for joined statements. [\#4387](https://github.com/ClickHouse/ClickHouse/pull/4387) ([Ivan](https://github.com/abyss7)) - -#### Performance Improvements {#performance-improvements-3} - -- Improved heuristics of “move to PREWHERE” optimization. [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Use proper lookup tables that uses HashTable’s API for 8-bit and 16-bit keys. [\#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([Amos Bird](https://github.com/amosbird)) -- Improved performance of string comparison. [\#4564](https://github.com/ClickHouse/ClickHouse/pull/4564) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Cleanup distributed DDL queue in a separate thread so that it doesn’t slow down the main loop that processes distributed DDL tasks. [\#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([Alex Zatelepin](https://github.com/ztlpn)) -- When `min_bytes_to_use_direct_io` is set to 1, not every file was opened with O\_DIRECT mode because the data size to read was sometimes underestimated by the size of one compressed block. [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-12} - -- Added support for clang-9 [\#4604](https://github.com/ClickHouse/ClickHouse/pull/4604) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix wrong `__asm__` instructions (again) [\#4621](https://github.com/ClickHouse/ClickHouse/pull/4621) ([Konstantin Podshumok](https://github.com/podshumok)) -- Add ability to specify settings for `clickhouse-performance-test` from command line. [\#4437](https://github.com/ClickHouse/ClickHouse/pull/4437) ([alesapin](https://github.com/alesapin)) -- Add dictionaries tests to integration tests. [\#4477](https://github.com/ClickHouse/ClickHouse/pull/4477) ([alesapin](https://github.com/alesapin)) -- Added queries from the benchmark on the website to automated performance tests. [\#4496](https://github.com/ClickHouse/ClickHouse/pull/4496) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `xxhash.h` does not exist in external lz4 because it is an implementation detail and its symbols are namespaced with `XXH_NAMESPACE` macro. When lz4 is external, xxHash has to be external too, and the dependents have to link to it. [\#4495](https://github.com/ClickHouse/ClickHouse/pull/4495) ([Orivej Desh](https://github.com/orivej)) -- Fixed a case when `quantileTiming` aggregate function can be called with negative or floating point argument (this fixes fuzz test with undefined behaviour sanitizer). [\#4506](https://github.com/ClickHouse/ClickHouse/pull/4506) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Spelling error correction. [\#4531](https://github.com/ClickHouse/ClickHouse/pull/4531) ([sdk2](https://github.com/sdk2)) -- Fix compilation on Mac. [\#4371](https://github.com/ClickHouse/ClickHouse/pull/4371) ([Vitaly Baranov](https://github.com/vitlibar)) -- Build fixes for FreeBSD and various unusual build configurations. [\#4444](https://github.com/ClickHouse/ClickHouse/pull/4444) ([proller](https://github.com/proller)) - -## ClickHouse release 19.3 {#clickhouse-release-19-3} - -### ClickHouse release 19.3.9.1, 2019-04-02 {#clickhouse-release-19-3-9-1-2019-04-02} - -#### Bug Fixes {#bug-fixes-12} - -- Fix crash in `FULL/RIGHT JOIN` when we joining on nullable vs not nullable. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) -- Fix segmentation fault in `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) -- Fixed reading from `Array(LowCardinality)` column in rare case when column contained a long sequence of empty arrays. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - -#### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-13} - -- Add a way to launch clickhouse-server image from a custom user [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) - -### ClickHouse release 19.3.7, 2019-03-12 {#clickhouse-release-19-3-7-2019-03-12} - -#### Bug fixes {#bug-fixes-13} - -- Fixed error in \#3920. This error manifests itself as random cache corruption (messages `Unknown codec family code`, `Cannot seek through file`) and segfaults. This bug first appeared in version 19.1 and is present in versions up to 19.1.10 and 19.3.6. [\#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.3.6, 2019-03-02 {#clickhouse-release-19-3-6-2019-03-02} - -#### Bug fixes {#bug-fixes-14} - -- When there are more than 1000 threads in a thread pool, `std::terminate` may happen on thread exit. [Azat Khuzhin](https://github.com/azat) [\#4485](https://github.com/ClickHouse/ClickHouse/pull/4485) [\#4505](https://github.com/ClickHouse/ClickHouse/pull/4505) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Now it’s possible to create `ReplicatedMergeTree*` tables with comments on columns without defaults and tables with columns codecs without comments and defaults. Also fix comparison of codecs. [\#4523](https://github.com/ClickHouse/ClickHouse/pull/4523) ([alesapin](https://github.com/alesapin)) -- Fixed crash on JOIN with array or tuple. [\#4552](https://github.com/ClickHouse/ClickHouse/pull/4552) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed crash in clickhouse-copier with the message `ThreadStatus not created`. [\#4540](https://github.com/ClickHouse/ClickHouse/pull/4540) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed hangup on server shutdown if distributed DDLs were used. [\#4472](https://github.com/ClickHouse/ClickHouse/pull/4472) ([Alex Zatelepin](https://github.com/ztlpn)) -- Incorrect column numbers were printed in error message about text format parsing for columns with number greater than 10. [\#4484](https://github.com/ClickHouse/ClickHouse/pull/4484) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Build/Testing/Packaging Improvements {#buildtestingpackaging-improvements-3} - -- Fixed build with AVX enabled. [\#4527](https://github.com/ClickHouse/ClickHouse/pull/4527) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Enable extended accounting and IO accounting based on good known version instead of kernel under which it is compiled. [\#4541](https://github.com/ClickHouse/ClickHouse/pull/4541) ([nvartolomei](https://github.com/nvartolomei)) -- Allow to skip setting of core\_dump.size\_limit, warning instead of throw if limit set fail. [\#4473](https://github.com/ClickHouse/ClickHouse/pull/4473) ([proller](https://github.com/proller)) -- Removed the `inline` tags of `void readBinary(...)` in `Field.cpp`. Also merged redundant `namespace DB` blocks. [\#4530](https://github.com/ClickHouse/ClickHouse/pull/4530) ([hcz](https://github.com/hczhcz)) - -### ClickHouse release 19.3.5, 2019-02-21 {#clickhouse-release-19-3-5-2019-02-21} - -#### Bug fixes {#bug-fixes-15} - -- Fixed bug with large http insert queries processing. [\#4454](https://github.com/ClickHouse/ClickHouse/pull/4454) ([alesapin](https://github.com/alesapin)) -- Fixed backward incompatibility with old versions due to wrong implementation of `send_logs_level` setting. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed backward incompatibility of table function `remote` introduced with column comments. [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.3.4, 2019-02-16 {#clickhouse-release-19-3-4-2019-02-16} - -#### Improvements {#improvements-4} - -- Table index size is not accounted for memory limits when doing `ATTACH TABLE` query. Avoided the possibility that a table cannot be attached after being detached. [\#4396](https://github.com/ClickHouse/ClickHouse/pull/4396) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Slightly raised up the limit on max string and array size received from ZooKeeper. It allows to continue to work with increased size of `CLIENT_JVMFLAGS=-Djute.maxbuffer=...` on ZooKeeper. [\#4398](https://github.com/ClickHouse/ClickHouse/pull/4398) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Allow to repair abandoned replica even if it already has huge number of nodes in its queue. [\#4399](https://github.com/ClickHouse/ClickHouse/pull/4399) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Add one required argument to `SET` index (max stored rows number). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) - -#### Bug Fixes {#bug-fixes-16} - -- Fixed `WITH ROLLUP` result for group by single `LowCardinality` key. [\#4384](https://github.com/ClickHouse/ClickHouse/pull/4384) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed bug in the set index (dropping a granule if it contains more than `max_rows` rows). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) -- A lot of FreeBSD build fixes. [\#4397](https://github.com/ClickHouse/ClickHouse/pull/4397) ([proller](https://github.com/proller)) -- Fixed aliases substitution in queries with subquery containing same alias (issue [\#4110](https://github.com/ClickHouse/ClickHouse/issues/4110)). [\#4351](https://github.com/ClickHouse/ClickHouse/pull/4351) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Build/Testing/Packaging Improvements {#buildtestingpackaging-improvements-4} - -- Add ability to run `clickhouse-server` for stateless tests in docker image. [\#4347](https://github.com/ClickHouse/ClickHouse/pull/4347) ([Vasily Nemkov](https://github.com/Enmk)) - -### ClickHouse release 19.3.3, 2019-02-13 {#clickhouse-release-19-3-3-2019-02-13} - -#### New Features {#new-features-6} - -- Added the `KILL MUTATION` statement that allows removing mutations that are for some reasons stuck. Added `latest_failed_part`, `latest_fail_time`, `latest_fail_reason` fields to the `system.mutations` table for easier troubleshooting. [\#4287](https://github.com/ClickHouse/ClickHouse/pull/4287) ([Alex Zatelepin](https://github.com/ztlpn)) -- Added aggregate function `entropy` which computes Shannon entropy. [\#4238](https://github.com/ClickHouse/ClickHouse/pull/4238) ([Quid37](https://github.com/Quid37)) -- Added ability to send queries `INSERT INTO tbl VALUES (....` to server without splitting on `query` and `data` parts. [\#4301](https://github.com/ClickHouse/ClickHouse/pull/4301) ([alesapin](https://github.com/alesapin)) -- Generic implementation of `arrayWithConstant` function was added. [\#4322](https://github.com/ClickHouse/ClickHouse/pull/4322) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Implemented `NOT BETWEEN` comparison operator. [\#4228](https://github.com/ClickHouse/ClickHouse/pull/4228) ([Dmitry Naumov](https://github.com/nezed)) -- Implement `sumMapFiltered` in order to be able to limit the number of keys for which values will be summed by `sumMap`. [\#4129](https://github.com/ClickHouse/ClickHouse/pull/4129) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) -- Added support of `Nullable` types in `mysql` table function. [\#4198](https://github.com/ClickHouse/ClickHouse/pull/4198) ([Emmanuel Donin de Rosière](https://github.com/edonin)) -- Support for arbitrary constant expressions in `LIMIT` clause. [\#4246](https://github.com/ClickHouse/ClickHouse/pull/4246) ([k3box](https://github.com/k3box)) -- Added `topKWeighted` aggregate function that takes additional argument with (unsigned integer) weight. [\#4245](https://github.com/ClickHouse/ClickHouse/pull/4245) ([Andrew Golman](https://github.com/andrewgolman)) -- `StorageJoin` now supports `join_any_take_last_row` setting that allows overwriting existing values of the same key. [\#3973](https://github.com/ClickHouse/ClickHouse/pull/3973) ([Amos Bird](https://github.com/amosbird) -- Added function `toStartOfInterval`. [\#4304](https://github.com/ClickHouse/ClickHouse/pull/4304) ([Vitaly Baranov](https://github.com/vitlibar)) -- Added `RowBinaryWithNamesAndTypes` format. [\#4200](https://github.com/ClickHouse/ClickHouse/pull/4200) ([Oleg V. Kozlyuk](https://github.com/DarkWanderer)) -- Added `IPv4` and `IPv6` data types. More effective implementations of `IPv*` functions. [\#3669](https://github.com/ClickHouse/ClickHouse/pull/3669) ([Vasily Nemkov](https://github.com/Enmk)) -- Added function `toStartOfTenMinutes()`. [\#4298](https://github.com/ClickHouse/ClickHouse/pull/4298) ([Vitaly Baranov](https://github.com/vitlibar)) -- Added `Protobuf` output format. [\#4005](https://github.com/ClickHouse/ClickHouse/pull/4005) [\#4158](https://github.com/ClickHouse/ClickHouse/pull/4158) ([Vitaly Baranov](https://github.com/vitlibar)) -- Added brotli support for HTTP interface for data import (INSERTs). [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([Mikhail](https://github.com/fandyushin)) -- Added hints while user make typo in function name or type in command line client. [\#4239](https://github.com/ClickHouse/ClickHouse/pull/4239) ([Danila Kutenin](https://github.com/danlark1)) -- Added `Query-Id` to Server’s HTTP Response header. [\#4231](https://github.com/ClickHouse/ClickHouse/pull/4231) ([Mikhail](https://github.com/fandyushin)) - -#### Experimental features {#experimental-features-2} - -- Added `minmax` and `set` data skipping indices for MergeTree table engines family. [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([Nikita Vasilev](https://github.com/nikvas0)) -- Added conversion of `CROSS JOIN` to `INNER JOIN` if possible. [\#4221](https://github.com/ClickHouse/ClickHouse/pull/4221) [\#4266](https://github.com/ClickHouse/ClickHouse/pull/4266) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Bug Fixes {#bug-fixes-17} - -- Fixed `Not found column` for duplicate columns in `JOIN ON` section. [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) -- Make `START REPLICATED SENDS` command start replicated sends. [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) -- Fixed aggregate functions execution with `Array(LowCardinality)` arguments. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Fixed wrong behaviour when doing `INSERT ... SELECT ... FROM file(...)` query and file has `CSVWithNames` or `TSVWIthNames` format and the first data row is missing. [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed crash on dictionary reload if dictionary not available. This bug was appeared in 19.1.6. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) -- Fixed `ALL JOIN` with duplicates in right table. [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed segmentation fault with `use_uncompressed_cache=1` and exception with wrong uncompressed size. This bug was appeared in 19.1.6. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alesapin](https://github.com/alesapin)) -- Fixed `compile_expressions` bug with comparison of big (more than int16) dates. [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alesapin](https://github.com/alesapin)) -- Fixed infinite loop when selecting from table function `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Temporarily disable predicate optimization for `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([Winter Zhang](https://github.com/zhang2014)) -- Fixed `Illegal instruction` error when using base64 functions on old CPUs. This error has been reproduced only when ClickHouse was compiled with gcc-8. [\#4275](https://github.com/ClickHouse/ClickHouse/pull/4275) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed `No message received` error when interacting with PostgreSQL ODBC Driver through TLS connection. Also fixes segfault when using MySQL ODBC Driver. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed incorrect result when `Date` and `DateTime` arguments are used in branches of conditional operator (function `if`). Added generic case for function `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- ClickHouse dictionaries now load within `clickhouse` process. [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed deadlock when `SELECT` from a table with `File` engine was retried after `No such file or directory` error. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed race condition when selecting from `system.tables` may give `table doesn't exist` error. [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `clickhouse-client` can segfault on exit while loading data for command line suggestions if it was run in interactive mode. [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed a bug when the execution of mutations containing `IN` operators was producing incorrect results. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fixed error: if there is a database with `Dictionary` engine, all dictionaries forced to load at server startup, and if there is a dictionary with ClickHouse source from localhost, the dictionary cannot load. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed error when system logs are tried to create again at server shutdown. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Correctly return the right type and properly handle locks in `joinGet` function. [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([Amos Bird](https://github.com/amosbird)) -- Added `sumMapWithOverflow` function. [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) -- Fixed segfault with `allow_experimental_multiple_joins_emulation`. [52de2c](https://github.com/ClickHouse/ClickHouse/commit/52de2cd927f7b5257dd67e175f0a5560a48840d0) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed bug with incorrect `Date` and `DateTime` comparison. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) -- Fixed fuzz test under undefined behavior sanitizer: added parameter type check for `quantile*Weighted` family of functions. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed rare race condition when removing of old data parts can fail with `File not found` error. [\#4378](https://github.com/ClickHouse/ClickHouse/pull/4378) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix install package with missing /etc/clickhouse-server/config.xml. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) - -#### Build/Testing/Packaging Improvements {#buildtestingpackaging-improvements-5} - -- Debian package: correct /etc/clickhouse-server/preprocessed link according to config. [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) -- Various build fixes for FreeBSD. [\#4225](https://github.com/ClickHouse/ClickHouse/pull/4225) ([proller](https://github.com/proller)) -- Added ability to create, fill and drop tables in perftest. [\#4220](https://github.com/ClickHouse/ClickHouse/pull/4220) ([alesapin](https://github.com/alesapin)) -- Added a script to check for duplicate includes. [\#4326](https://github.com/ClickHouse/ClickHouse/pull/4326) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added ability to run queries by index in performance test. [\#4264](https://github.com/ClickHouse/ClickHouse/pull/4264) ([alesapin](https://github.com/alesapin)) -- Package with debug symbols is suggested to be installed. [\#4274](https://github.com/ClickHouse/ClickHouse/pull/4274) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Refactoring of performance-test. Better logging and signals handling. [\#4171](https://github.com/ClickHouse/ClickHouse/pull/4171) ([alesapin](https://github.com/alesapin)) -- Added docs to anonymized Yandex.Metrika datasets. [\#4164](https://github.com/ClickHouse/ClickHouse/pull/4164) ([alesapin](https://github.com/alesapin)) -- Аdded tool for converting an old month-partitioned part to the custom-partitioned format. [\#4195](https://github.com/ClickHouse/ClickHouse/pull/4195) ([Alex Zatelepin](https://github.com/ztlpn)) -- Added docs about two datasets in s3. [\#4144](https://github.com/ClickHouse/ClickHouse/pull/4144) ([alesapin](https://github.com/alesapin)) -- Added script which creates changelog from pull requests description. [\#4169](https://github.com/ClickHouse/ClickHouse/pull/4169) [\#4173](https://github.com/ClickHouse/ClickHouse/pull/4173) ([KochetovNicolai](https://github.com/KochetovNicolai)) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Added puppet module for Clickhouse. [\#4182](https://github.com/ClickHouse/ClickHouse/pull/4182) ([Maxim Fedotov](https://github.com/MaxFedotov)) -- Added docs for a group of undocumented functions. [\#4168](https://github.com/ClickHouse/ClickHouse/pull/4168) ([Winter Zhang](https://github.com/zhang2014)) -- ARM build fixes. [\#4210](https://github.com/ClickHouse/ClickHouse/pull/4210)[\#4306](https://github.com/ClickHouse/ClickHouse/pull/4306) [\#4291](https://github.com/ClickHouse/ClickHouse/pull/4291) ([proller](https://github.com/proller)) ([proller](https://github.com/proller)) -- Dictionary tests now able to run from `ctest`. [\#4189](https://github.com/ClickHouse/ClickHouse/pull/4189) ([proller](https://github.com/proller)) -- Now `/etc/ssl` is used as default directory with SSL certificates. [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added checking SSE and AVX instruction at start. [\#4234](https://github.com/ClickHouse/ClickHouse/pull/4234) ([Igr](https://github.com/igron99)) -- Init script will wait server until start. [\#4281](https://github.com/ClickHouse/ClickHouse/pull/4281) ([proller](https://github.com/proller)) - -#### Backward Incompatible Changes {#backward-incompatible-changes-1} - -- Removed `allow_experimental_low_cardinality_type` setting. `LowCardinality` data types are production ready. [\#4323](https://github.com/ClickHouse/ClickHouse/pull/4323) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Reduce mark cache size and uncompressed cache size accordingly to available memory amount. [\#4240](https://github.com/ClickHouse/ClickHouse/pull/4240) ([Lopatin Konstantin](https://github.com/k-lopatin) -- Added keyword `INDEX` in `CREATE TABLE` query. A column with name `index` must be quoted with backticks or double quotes: `` `index` ``. [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([Nikita Vasilev](https://github.com/nikvas0)) -- `sumMap` now promote result type instead of overflow. The old `sumMap` behavior can be obtained by using `sumMapWithOverflow` function. [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) - -#### Performance Improvements {#performance-improvements-4} - -- `std::sort` replaced by `pdqsort` for queries without `LIMIT`. [\#4236](https://github.com/ClickHouse/ClickHouse/pull/4236) ([Evgenii Pravda](https://github.com/kvinty)) -- Now server reuse threads from global thread pool. This affects performance in some corner cases. [\#4150](https://github.com/ClickHouse/ClickHouse/pull/4150) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Improvements {#improvements-5} - -- Implemented AIO support for FreeBSD. [\#4305](https://github.com/ClickHouse/ClickHouse/pull/4305) ([urgordeadbeef](https://github.com/urgordeadbeef)) -- `SELECT * FROM a JOIN b USING a, b` now return `a` and `b` columns only from the left table. [\#4141](https://github.com/ClickHouse/ClickHouse/pull/4141) ([Artem Zuikov](https://github.com/4ertus2)) -- Allow `-C` option of client to work as `-c` option. [\#4232](https://github.com/ClickHouse/ClickHouse/pull/4232) ([syominsergey](https://github.com/syominsergey)) -- Now option `--password` used without value requires password from stdin. [\#4230](https://github.com/ClickHouse/ClickHouse/pull/4230) ([BSD\_Conqueror](https://github.com/bsd-conqueror)) -- Added highlighting of unescaped metacharacters in string literals that contain `LIKE` expressions or regexps. [\#4327](https://github.com/ClickHouse/ClickHouse/pull/4327) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added cancelling of HTTP read only queries if client socket goes away. [\#4213](https://github.com/ClickHouse/ClickHouse/pull/4213) ([nvartolomei](https://github.com/nvartolomei)) -- Now server reports progress to keep client connections alive. [\#4215](https://github.com/ClickHouse/ClickHouse/pull/4215) ([Ivan](https://github.com/abyss7)) -- Slightly better message with reason for OPTIMIZE query with `optimize_throw_if_noop` setting enabled. [\#4294](https://github.com/ClickHouse/ClickHouse/pull/4294) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added support of `--version` option for clickhouse server. [\#4251](https://github.com/ClickHouse/ClickHouse/pull/4251) ([Lopatin Konstantin](https://github.com/k-lopatin)) -- Added `--help/-h` option to `clickhouse-server`. [\#4233](https://github.com/ClickHouse/ClickHouse/pull/4233) ([Yuriy Baranov](https://github.com/yurriy)) -- Added support for scalar subqueries with aggregate function state result. [\#4348](https://github.com/ClickHouse/ClickHouse/pull/4348) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Improved server shutdown time and ALTERs waiting time. [\#4372](https://github.com/ClickHouse/ClickHouse/pull/4372) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added info about the replicated\_can\_become\_leader setting to system.replicas and add logging if the replica won’t try to become leader. [\#4379](https://github.com/ClickHouse/ClickHouse/pull/4379) ([Alex Zatelepin](https://github.com/ztlpn)) - -## ClickHouse release 19.1 {#clickhouse-release-19-1} - -### ClickHouse release 19.1.14, 2019-03-14 {#clickhouse-release-19-1-14-2019-03-14} - -- Fixed error `Column ... queried more than once` that may happen if the setting `asterisk_left_columns_only` is set to 1 in case of using `GLOBAL JOIN` with `SELECT *` (rare case). The issue does not exist in 19.3 and newer. [6bac7d8d](https://github.com/ClickHouse/ClickHouse/pull/4692/commits/6bac7d8d11a9b0d6de0b32b53c47eb2f6f8e7062) ([Artem Zuikov](https://github.com/4ertus2)) - -### ClickHouse release 19.1.13, 2019-03-12 {#clickhouse-release-19-1-13-2019-03-12} - -This release contains exactly the same set of patches as 19.3.7. - -### ClickHouse release 19.1.10, 2019-03-03 {#clickhouse-release-19-1-10-2019-03-03} - -This release contains exactly the same set of patches as 19.3.6. - -## ClickHouse release 19.1 {#clickhouse-release-19-1-1} - -### ClickHouse release 19.1.9, 2019-02-21 {#clickhouse-release-19-1-9-2019-02-21} - -#### Bug fixes {#bug-fixes-18} - -- Fixed backward incompatibility with old versions due to wrong implementation of `send_logs_level` setting. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed backward incompatibility of table function `remote` introduced with column comments. [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.1.8, 2019-02-16 {#clickhouse-release-19-1-8-2019-02-16} - -#### Bug Fixes {#bug-fixes-19} - -- Fix install package with missing /etc/clickhouse-server/config.xml. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) - -## ClickHouse release 19.1 {#clickhouse-release-19-1-2} - -### ClickHouse release 19.1.7, 2019-02-15 {#clickhouse-release-19-1-7-2019-02-15} - -#### Bug Fixes {#bug-fixes-20} - -- Correctly return the right type and properly handle locks in `joinGet` function. [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([Amos Bird](https://github.com/amosbird)) -- Fixed error when system logs are tried to create again at server shutdown. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed error: if there is a database with `Dictionary` engine, all dictionaries forced to load at server startup, and if there is a dictionary with ClickHouse source from localhost, the dictionary cannot load. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed a bug when the execution of mutations containing `IN` operators was producing incorrect results. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) -- `clickhouse-client` can segfault on exit while loading data for command line suggestions if it was run in interactive mode. [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed race condition when selecting from `system.tables` may give `table doesn't exist` error. [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed deadlock when `SELECT` from a table with `File` engine was retried after `No such file or directory` error. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed an issue: local ClickHouse dictionaries are loaded via TCP, but should load within process. [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed `No message received` error when interacting with PostgreSQL ODBC Driver through TLS connection. Also fixes segfault when using MySQL ODBC Driver. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Temporarily disable predicate optimization for `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([Winter Zhang](https://github.com/zhang2014)) -- Fixed infinite loop when selecting from table function `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed `compile_expressions` bug with comparison of big (more than int16) dates. [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alesapin](https://github.com/alesapin)) -- Fixed segmentation fault with `uncompressed_cache=1` and exception with wrong uncompressed size. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alesapin](https://github.com/alesapin)) -- Fixed `ALL JOIN` with duplicates in right table. [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) -- Fixed wrong behaviour when doing `INSERT ... SELECT ... FROM file(...)` query and file has `CSVWithNames` or `TSVWIthNames` format and the first data row is missing. [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed aggregate functions execution with `Array(LowCardinality)` arguments. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Debian package: correct /etc/clickhouse-server/preprocessed link according to config. [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) -- Fixed fuzz test under undefined behavior sanitizer: added parameter type check for `quantile*Weighted` family of functions. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Make `START REPLICATED SENDS` command start replicated sends. [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) -- Fixed `Not found column` for duplicate columns in JOIN ON section. [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) -- Now `/etc/ssl` is used as default directory with SSL certificates. [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed crash on dictionary reload if dictionary not available. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) -- Fixed bug with incorrect `Date` and `DateTime` comparison. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) -- Fixed incorrect result when `Date` and `DateTime` arguments are used in branches of conditional operator (function `if`). Added generic case for function `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### ClickHouse release 19.1.6, 2019-01-24 {#clickhouse-release-19-1-6-2019-01-24} - -#### New Features {#new-features-7} - -- Custom per column compression codecs for tables. [\#3899](https://github.com/ClickHouse/ClickHouse/pull/3899) [\#4111](https://github.com/ClickHouse/ClickHouse/pull/4111) ([alesapin](https://github.com/alesapin), [Winter Zhang](https://github.com/zhang2014), [Anatoly](https://github.com/Sindbag)) -- Added compression codec `Delta`. [\#4052](https://github.com/ClickHouse/ClickHouse/pull/4052) ([alesapin](https://github.com/alesapin)) -- Allow to `ALTER` compression codecs. [\#4054](https://github.com/ClickHouse/ClickHouse/pull/4054) ([alesapin](https://github.com/alesapin)) -- Added functions `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` for SQL standard compatibility. [\#3826](https://github.com/ClickHouse/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov)) -- Support for write in `HDFS` tables and `hdfs` table function. [\#4084](https://github.com/ClickHouse/ClickHouse/pull/4084) ([alesapin](https://github.com/alesapin)) -- Added functions to search for multiple constant strings from big haystack: `multiPosition`, `multiSearch` ,`firstMatch` also with `-UTF8`, `-CaseInsensitive`, and `-CaseInsensitiveUTF8` variants. [\#4053](https://github.com/ClickHouse/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1)) -- Pruning of unused shards if `SELECT` query filters by sharding key (setting `optimize_skip_unused_shards`). [\#3851](https://github.com/ClickHouse/ClickHouse/pull/3851) ([Gleb Kanterov](https://github.com/kanterov), [Ivan](https://github.com/abyss7)) -- Allow `Kafka` engine to ignore some number of parsing errors per block. [\#4094](https://github.com/ClickHouse/ClickHouse/pull/4094) ([Ivan](https://github.com/abyss7)) -- Added support for `CatBoost` multiclass models evaluation. Function `modelEvaluate` returns tuple with per-class raw predictions for multiclass models. `libcatboostmodel.so` should be built with [\#607](https://github.com/catboost/catboost/pull/607). [\#3959](https://github.com/ClickHouse/ClickHouse/pull/3959) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Added functions `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [\#4097](https://github.com/ClickHouse/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea)) -- Added hashing functions `xxHash64` and `xxHash32`. [\#3905](https://github.com/ClickHouse/ClickHouse/pull/3905) ([filimonov](https://github.com/filimonov)) -- Added `gccMurmurHash` hashing function (GCC flavoured Murmur hash) which uses the same hash seed as [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191) [\#4000](https://github.com/ClickHouse/ClickHouse/pull/4000) ([sundyli](https://github.com/sundy-li)) -- Added hashing functions `javaHash`, `hiveHash`. [\#3811](https://github.com/ClickHouse/ClickHouse/pull/3811) ([shangshujie365](https://github.com/shangshujie365)) -- Added table function `remoteSecure`. Function works as `remote`, but uses secure connection. [\#4088](https://github.com/ClickHouse/ClickHouse/pull/4088) ([proller](https://github.com/proller)) - -#### Experimental features {#experimental-features-3} - -- Added multiple JOINs emulation (`allow_experimental_multiple_joins_emulation` setting). [\#3946](https://github.com/ClickHouse/ClickHouse/pull/3946) ([Artem Zuikov](https://github.com/4ertus2)) - -#### Bug Fixes {#bug-fixes-21} - -- Make `compiled_expression_cache_size` setting limited by default to lower memory consumption. [\#4041](https://github.com/ClickHouse/ClickHouse/pull/4041) ([alesapin](https://github.com/alesapin)) -- Fix a bug that led to hangups in threads that perform ALTERs of Replicated tables and in the thread that updates configuration from ZooKeeper. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3891](https://github.com/ClickHouse/ClickHouse/issues/3891) [\#3934](https://github.com/ClickHouse/ClickHouse/pull/3934) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fixed a race condition when executing a distributed ALTER task. The race condition led to more than one replica trying to execute the task and all replicas except one failing with a ZooKeeper error. [\#3904](https://github.com/ClickHouse/ClickHouse/pull/3904) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fix a bug when `from_zk` config elements weren’t refreshed after a request to ZooKeeper timed out. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3947](https://github.com/ClickHouse/ClickHouse/pull/3947) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fix bug with wrong prefix for IPv4 subnet masks. [\#3945](https://github.com/ClickHouse/ClickHouse/pull/3945) ([alesapin](https://github.com/alesapin)) -- Fixed crash (`std::terminate`) in rare cases when a new thread cannot be created due to exhausted resources. [\#3956](https://github.com/ClickHouse/ClickHouse/pull/3956) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix bug when in `remote` table function execution when wrong restrictions were used for in `getStructureOfRemoteTable`. [\#4009](https://github.com/ClickHouse/ClickHouse/pull/4009) ([alesapin](https://github.com/alesapin)) -- Fix a leak of netlink sockets. They were placed in a pool where they were never deleted and new sockets were created at the start of a new thread when all current sockets were in use. [\#4017](https://github.com/ClickHouse/ClickHouse/pull/4017) ([Alex Zatelepin](https://github.com/ztlpn)) -- Fix bug with closing `/proc/self/fd` directory earlier than all fds were read from `/proc` after forking `odbc-bridge` subprocess. [\#4120](https://github.com/ClickHouse/ClickHouse/pull/4120) ([alesapin](https://github.com/alesapin)) -- Fixed String to UInt monotonic conversion in case of usage String in primary key. [\#3870](https://github.com/ClickHouse/ClickHouse/pull/3870) ([Winter Zhang](https://github.com/zhang2014)) -- Fixed error in calculation of integer conversion function monotonicity. [\#3921](https://github.com/ClickHouse/ClickHouse/pull/3921) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed segfault in `arrayEnumerateUniq`, `arrayEnumerateDense` functions in case of some invalid arguments. [\#3909](https://github.com/ClickHouse/ClickHouse/pull/3909) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fix UB in StorageMerge. [\#3910](https://github.com/ClickHouse/ClickHouse/pull/3910) ([Amos Bird](https://github.com/amosbird)) -- Fixed segfault in functions `addDays`, `subtractDays`. [\#3913](https://github.com/ClickHouse/ClickHouse/pull/3913) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed error: functions `round`, `floor`, `trunc`, `ceil` may return bogus result when executed on integer argument and large negative scale. [\#3914](https://github.com/ClickHouse/ClickHouse/pull/3914) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed a bug induced by ‘kill query sync’ which leads to a core dump. [\#3916](https://github.com/ClickHouse/ClickHouse/pull/3916) ([muVulDeePecker](https://github.com/fancyqlx)) -- Fix bug with long delay after empty replication queue. [\#3928](https://github.com/ClickHouse/ClickHouse/pull/3928) [\#3932](https://github.com/ClickHouse/ClickHouse/pull/3932) ([alesapin](https://github.com/alesapin)) -- Fixed excessive memory usage in case of inserting into table with `LowCardinality` primary key. [\#3955](https://github.com/ClickHouse/ClickHouse/pull/3955) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Fixed `LowCardinality` serialization for `Native` format in case of empty arrays. [\#3907](https://github.com/ClickHouse/ClickHouse/issues/3907) [\#4011](https://github.com/ClickHouse/ClickHouse/pull/4011) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Fixed incorrect result while using distinct by single LowCardinality numeric column. [\#3895](https://github.com/ClickHouse/ClickHouse/issues/3895) [\#4012](https://github.com/ClickHouse/ClickHouse/pull/4012) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Fixed specialized aggregation with LowCardinality key (in case when `compile` setting is enabled). [\#3886](https://github.com/ClickHouse/ClickHouse/pull/3886) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Fix user and password forwarding for replicated tables queries. [\#3957](https://github.com/ClickHouse/ClickHouse/pull/3957) ([alesapin](https://github.com/alesapin)) ([小路](https://github.com/nicelulu)) -- Fixed very rare race condition that can happen when listing tables in Dictionary database while reloading dictionaries. [\#3970](https://github.com/ClickHouse/ClickHouse/pull/3970) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed incorrect result when HAVING was used with ROLLUP or CUBE. [\#3756](https://github.com/ClickHouse/ClickHouse/issues/3756) [\#3837](https://github.com/ClickHouse/ClickHouse/pull/3837) ([Sam Chou](https://github.com/reflection)) -- Fixed column aliases for query with `JOIN ON` syntax and distributed tables. [\#3980](https://github.com/ClickHouse/ClickHouse/pull/3980) ([Winter Zhang](https://github.com/zhang2014)) -- Fixed error in internal implementation of `quantileTDigest` (found by Artem Vakhrushev). This error never happens in ClickHouse and was relevant only for those who use ClickHouse codebase as a library directly. [\#3935](https://github.com/ClickHouse/ClickHouse/pull/3935) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Improvements {#improvements-6} - -- Support for `IF NOT EXISTS` in `ALTER TABLE ADD COLUMN` statements along with `IF EXISTS` in `DROP/MODIFY/CLEAR/COMMENT COLUMN`. [\#3900](https://github.com/ClickHouse/ClickHouse/pull/3900) ([Boris Granveaud](https://github.com/bgranvea)) -- Function `parseDateTimeBestEffort`: support for formats `DD.MM.YYYY`, `DD.MM.YY`, `DD-MM-YYYY`, `DD-Mon-YYYY`, `DD/Month/YYYY` and similar. [\#3922](https://github.com/ClickHouse/ClickHouse/pull/3922) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- `CapnProtoInputStream` now support jagged structures. [\#4063](https://github.com/ClickHouse/ClickHouse/pull/4063) ([Odin Hultgren Van Der Horst](https://github.com/Miniwoffer)) -- Usability improvement: added a check that server process is started from the data directory’s owner. Do not allow to start server from root if the data belongs to non-root user. [\#3785](https://github.com/ClickHouse/ClickHouse/pull/3785) ([sergey-v-galtsev](https://github.com/sergey-v-galtsev)) -- Better logic of checking required columns during analysis of queries with JOINs. [\#3930](https://github.com/ClickHouse/ClickHouse/pull/3930) ([Artem Zuikov](https://github.com/4ertus2)) -- Decreased the number of connections in case of large number of Distributed tables in a single server. [\#3726](https://github.com/ClickHouse/ClickHouse/pull/3726) ([Winter Zhang](https://github.com/zhang2014)) -- Supported totals row for `WITH TOTALS` query for ODBC driver. [\#3836](https://github.com/ClickHouse/ClickHouse/pull/3836) ([Maksim Koritckiy](https://github.com/nightweb)) -- Allowed to use `Enum`s as integers inside if function. [\#3875](https://github.com/ClickHouse/ClickHouse/pull/3875) ([Ivan](https://github.com/abyss7)) -- Added `low_cardinality_allow_in_native_format` setting. If disabled, do not use `LowCadrinality` type in `Native` format. [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) ([KochetovNicolai](https://github.com/KochetovNicolai)) -- Removed some redundant objects from compiled expressions cache to lower memory usage. [\#4042](https://github.com/ClickHouse/ClickHouse/pull/4042) ([alesapin](https://github.com/alesapin)) -- Add check that `SET send_logs_level = 'value'` query accept appropriate value. [\#3873](https://github.com/ClickHouse/ClickHouse/pull/3873) ([Sabyanin Maxim](https://github.com/s-mx)) -- Fixed data type check in type conversion functions. [\#3896](https://github.com/ClickHouse/ClickHouse/pull/3896) ([Winter Zhang](https://github.com/zhang2014)) - -#### Performance Improvements {#performance-improvements-5} - -- Add a MergeTree setting `use_minimalistic_part_header_in_zookeeper`. If enabled, Replicated tables will store compact part metadata in a single part znode. This can dramatically reduce ZooKeeper snapshot size (especially if the tables have a lot of columns). Note that after enabling this setting you will not be able to downgrade to a version that doesn’t support it. [\#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) -- Add an DFA-based implementation for functions `sequenceMatch` and `sequenceCount` in case pattern doesn’t contain time. [\#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) -- Performance improvement for integer numbers serialization. [\#3968](https://github.com/ClickHouse/ClickHouse/pull/3968) ([Amos Bird](https://github.com/amosbird)) -- Zero left padding PODArray so that -1 element is always valid and zeroed. It’s used for branchless calculation of offsets. [\#3920](https://github.com/ClickHouse/ClickHouse/pull/3920) ([Amos Bird](https://github.com/amosbird)) -- Reverted `jemalloc` version which lead to performance degradation. [\#4018](https://github.com/ClickHouse/ClickHouse/pull/4018) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -#### Backward Incompatible Changes {#backward-incompatible-changes-2} - -- Removed undocumented feature `ALTER MODIFY PRIMARY KEY` because it was superseded by the `ALTER MODIFY ORDER BY` command. [\#3887](https://github.com/ClickHouse/ClickHouse/pull/3887) ([Alex Zatelepin](https://github.com/ztlpn)) -- Removed function `shardByHash`. [\#3833](https://github.com/ClickHouse/ClickHouse/pull/3833) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Forbid using scalar subqueries with result of type `AggregateFunction`. [\#3865](https://github.com/ClickHouse/ClickHouse/pull/3865) ([Ivan](https://github.com/abyss7)) - -#### Build/Testing/Packaging Improvements {#buildtestingpackaging-improvements-6} - -- Added support for PowerPC (`ppc64le`) build. [\#4132](https://github.com/ClickHouse/ClickHouse/pull/4132) ([Danila Kutenin](https://github.com/danlark1)) -- Stateful functional tests are run on public available dataset. [\#3969](https://github.com/ClickHouse/ClickHouse/pull/3969) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed error when the server cannot start with the `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` message within Docker or systemd-nspawn. [\#4136](https://github.com/ClickHouse/ClickHouse/pull/4136) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Updated `rdkafka` library to v1.0.0-RC5. Used cppkafka instead of raw C interface. [\#4025](https://github.com/ClickHouse/ClickHouse/pull/4025) ([Ivan](https://github.com/abyss7)) -- Updated `mariadb-client` library. Fixed one of issues found by UBSan. [\#3924](https://github.com/ClickHouse/ClickHouse/pull/3924) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Some fixes for UBSan builds. [\#3926](https://github.com/ClickHouse/ClickHouse/pull/3926) [\#3021](https://github.com/ClickHouse/ClickHouse/pull/3021) [\#3948](https://github.com/ClickHouse/ClickHouse/pull/3948) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added per-commit runs of tests with UBSan build. -- Added per-commit runs of PVS-Studio static analyzer. -- Fixed bugs found by PVS-Studio. [\#4013](https://github.com/ClickHouse/ClickHouse/pull/4013) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed glibc compatibility issues. [\#4100](https://github.com/ClickHouse/ClickHouse/pull/4100) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Move Docker images to 18.10 and add compatibility file for glibc \>= 2.28 [\#3965](https://github.com/ClickHouse/ClickHouse/pull/3965) ([alesapin](https://github.com/alesapin)) -- Add env variable if user don’t want to chown directories in server Docker image. [\#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([alesapin](https://github.com/alesapin)) -- Enabled most of the warnings from `-Weverything` in clang. Enabled `-Wpedantic`. [\#3986](https://github.com/ClickHouse/ClickHouse/pull/3986) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added a few more warnings that are available only in clang 8. [\#3993](https://github.com/ClickHouse/ClickHouse/pull/3993) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Link to `libLLVM` rather than to individual LLVM libs when using shared linking. [\#3989](https://github.com/ClickHouse/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej)) -- Added sanitizer variables for test images. [\#4072](https://github.com/ClickHouse/ClickHouse/pull/4072) ([alesapin](https://github.com/alesapin)) -- `clickhouse-server` debian package will recommend `libcap2-bin` package to use `setcap` tool for setting capabilities. This is optional. [\#4093](https://github.com/ClickHouse/ClickHouse/pull/4093) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Improved compilation time, fixed includes. [\#3898](https://github.com/ClickHouse/ClickHouse/pull/3898) ([proller](https://github.com/proller)) -- Added performance tests for hash functions. [\#3918](https://github.com/ClickHouse/ClickHouse/pull/3918) ([filimonov](https://github.com/filimonov)) -- Fixed cyclic library dependences. [\#3958](https://github.com/ClickHouse/ClickHouse/pull/3958) ([proller](https://github.com/proller)) -- Improved compilation with low available memory. [\#4030](https://github.com/ClickHouse/ClickHouse/pull/4030) ([proller](https://github.com/proller)) -- Added test script to reproduce performance degradation in `jemalloc`. [\#4036](https://github.com/ClickHouse/ClickHouse/pull/4036) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed misspells in comments and string literals under `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([maiha](https://github.com/maiha)) -- Fixed typos in comments. [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty)) - -## [Changelog for 2018](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2018.md) diff --git a/docs/ru/changelog/index.md b/docs/ru/changelog/index.md deleted file mode 120000 index 79b747aee1b..00000000000 --- a/docs/ru/changelog/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../CHANGELOG.md \ No newline at end of file diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index f096bdb92cf..9716f4f1cd2 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -1,20 +1,16 @@ ---- -en_copy: true ---- +# Поставщики облачных услуг ClickHouse {#clickhouse-cloud-service-providers} -# ClickHouse Cloud Service Providers {#clickhouse-cloud-service-providers} - -!!! info "Info" - If you have launched a public cloud with managed ClickHouse service, feel free to [open a pull-request](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) adding it to the following list. +!!! info "Инфо" + Если вы запустили публичный облачный сервис с управляемым ClickHouse, не стесняйтесь [открыть pull request](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) c добавлением его в последующий список. ## Yandex Cloud {#yandex-cloud} -[Yandex Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) provides the following key features: +[Yandex Managed Service for ClickHouse](https://cloud.yandex.ru/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) предоставляет следующие ключевые возможности: -- Fully managed ZooKeeper service for [ClickHouse replication](../operations/table_engines/replication.md) -- Multiple storage type choices -- Replicas in different availability zones -- Encryption and isolation -- Automated maintenance +- Полностью управляемый сервис ZooKeeper для [репликации ClickHouse](../engines/table_engines/mergetree_family/replication.md) +- Выбор типа хранилища +- Реплики в разных зонах доступности +- Шифрование и изоляция +- Автоматизированное техническое обслуживание -{## [Original article](https://clickhouse.tech/docs/en/commercial/cloud/) ##} +{## [Оригинальная статья](https://clickhouse.tech/docs/ru/commercial/cloud/) ##} diff --git a/docs/ru/commercial/index.md b/docs/ru/commercial/index.md new file mode 100644 index 00000000000..dc78f6f4899 --- /dev/null +++ b/docs/ru/commercial/index.md @@ -0,0 +1,7 @@ +--- +toc_folder_title: Коммерческие услуги +toc_priority: 70 +toc_title: Коммерческие услуги +--- + + diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md deleted file mode 100644 index dc92d425d37..00000000000 --- a/docs/ru/development/architecture.md +++ /dev/null @@ -1,200 +0,0 @@ ---- -en_copy: true ---- - -# Overview of ClickHouse Architecture {#overview-of-clickhouse-architecture} - -ClickHouse is a true column-oriented DBMS. Data is stored by columns and during the execution of arrays (vectors or chunks of columns). Whenever possible, operations are dispatched on arrays, rather than on individual values. It is called “vectorized query execution,” and it helps lower the cost of actual data processing. - -> This idea is nothing new. It dates back to the `APL` programming language and its descendants: `A +`, `J`, `K`, and `Q`. Array programming is used in scientific data processing. Neither is this idea something new in relational databases: for example, it is used in the `Vectorwise` system. - -There are two different approaches for speeding up query processing: vectorized query execution and runtime code generation. The latter removes all indirection and dynamic dispatch. Neither of these approaches is strictly better than the other. Runtime code generation can be better when it fuses many operations, thus fully utilizing CPU execution units and the pipeline. Vectorized query execution can be less practical because it involves temporary vectors that must be written to the cache and read back. If the temporary data does not fit in the L2 cache, this becomes an issue. But vectorized query execution more easily utilizes the SIMD capabilities of the CPU. A [research paper](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) written by our friends shows that it is better to combine both approaches. ClickHouse uses vectorized query execution and has limited initial support for runtime code generation. - -## Columns {#columns} - -`IColumn` interface is used to represent columns in memory (actually, chunks of columns). This interface provides helper methods for the implementation of various relational operators. Almost all operations are immutable: they do not modify the original column, but create a new modified one. For example, the `IColumn :: filter` method accepts a filter byte mask. It is used for the `WHERE` and `HAVING` relational operators. Additional examples: the `IColumn :: permute` method to support `ORDER BY`, the `IColumn :: cut` method to support `LIMIT`. - -Various `IColumn` implementations (`ColumnUInt8`, `ColumnString`, and so on) are responsible for the memory layout of columns. The memory layout is usually a contiguous array. For the integer type of columns, it is just one contiguous array, like `std :: vector`. For `String` and `Array` columns, it is two vectors: one for all array elements, placed contiguously, and a second one for offsets to the beginning of each array. There is also `ColumnConst` that stores just one value in memory, but looks like a column. - -## Field {#field} - -Nevertheless, it is possible to work with individual values as well. To represent an individual value, the `Field` is used. `Field` is just a discriminated union of `UInt64`, `Int64`, `Float64`, `String` and `Array`. `IColumn` has the `operator[]` method to get the n-th value as a `Field` and the `insert` method to append a `Field` to the end of a column. These methods are not very efficient, because they require dealing with temporary `Field` objects representing an individual value. There are more efficient methods, such as `insertFrom`, `insertRangeFrom`, and so on. - -`Field` doesn’t have enough information about a specific data type for a table. For example, `UInt8`, `UInt16`, `UInt32`, and `UInt64` are all represented as `UInt64` in a `Field`. - -## Leaky Abstractions {#leaky-abstractions} - -`IColumn` has methods for common relational transformations of data, but they don’t meet all needs. For example, `ColumnUInt64` doesn’t have a method to calculate the sum of two columns, and `ColumnString` doesn’t have a method to run a substring search. These countless routines are implemented outside of `IColumn`. - -Various functions on columns can be implemented in a generic, non-efficient way using `IColumn` methods to extract `Field` values, or in a specialized way using knowledge of inner memory layout of data in a specific `IColumn` implementation. It is implemented by casting functions to a specific `IColumn` type and deal with internal representation directly. For example, `ColumnUInt64` has the `getData` method that returns a reference to an internal array, then a separate routine reads or fills that array directly. We have “leaky abstractions” to allow efficient specializations of various routines. - -## Data Types {#data_types} - -`IDataType` is responsible for serialization and deserialization: for reading and writing chunks of columns or individual values in binary or text form. `IDataType` directly corresponds to data types in tables. For example, there are `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` and so on. - -`IDataType` and `IColumn` are only loosely related to each other. Different data types can be represented in memory by the same `IColumn` implementations. For example, `DataTypeUInt32` and `DataTypeDateTime` are both represented by `ColumnUInt32` or `ColumnConstUInt32`. In addition, the same data type can be represented by different `IColumn` implementations. For example, `DataTypeUInt8` can be represented by `ColumnUInt8` or `ColumnConstUInt8`. - -`IDataType` only stores metadata. For instance, `DataTypeUInt8` doesn’t store anything at all (except vptr) and `DataTypeFixedString` stores just `N` (the size of fixed-size strings). - -`IDataType` has helper methods for various data formats. Examples are methods to serialize a value with possible quoting, to serialize a value for JSON, and to serialize a value as part of the XML format. There is no direct correspondence to data formats. For example, the different data formats `Pretty` and `TabSeparated` can use the same `serializeTextEscaped` helper method from the `IDataType` interface. - -## Block {#block} - -A `Block` is a container that represents a subset (chunk) of a table in memory. It is just a set of triples: `(IColumn, IDataType, column name)`. During query execution, data is processed by `Block`s. If we have a `Block`, we have data (in the `IColumn` object), we have information about its type (in `IDataType`) that tells us how to deal with that column, and we have the column name. It could be either the original column name from the table or some artificial name assigned for getting temporary results of calculations. - -When we calculate some function over columns in a block, we add another column with its result to the block, and we don’t touch columns for arguments of the function because operations are immutable. Later, unneeded columns can be removed from the block, but not modified. It is convenient for the elimination of common subexpressions. - -Blocks are created for every processed chunk of data. Note that for the same type of calculation, the column names and types remain the same for different blocks, and only column data changes. It is better to split block data from the block header because small block sizes have a high overhead of temporary strings for copying shared\_ptrs and column names. - -## Block Streams {#block-streams} - -Block streams are for processing data. We use streams of blocks to read data from somewhere, perform data transformations, or write data to somewhere. `IBlockInputStream` has the `read` method to fetch the next block while available. `IBlockOutputStream` has the `write` method to push the block somewhere. - -Streams are responsible for: - -1. Reading or writing to a table. The table just returns a stream for reading or writing blocks. -2. Implementing data formats. For example, if you want to output data to a terminal in `Pretty` format, you create a block output stream where you push blocks, and it formats them. -3. Performing data transformations. Let’s say you have `IBlockInputStream` and want to create a filtered stream. You create `FilterBlockInputStream` and initialize it with your stream. Then when you pull a block from `FilterBlockInputStream`, it pulls a block from your stream, filters it, and returns the filtered block to you. Query execution pipelines are represented this way. - -There are more sophisticated transformations. For example, when you pull from `AggregatingBlockInputStream`, it reads all data from its source, aggregates it, and then returns a stream of aggregated data for you. Another example: `UnionBlockInputStream` accepts many input sources in the constructor and also a number of threads. It launches multiple threads and reads from multiple sources in parallel. - -> Block streams use the “pull” approach to control flow: when you pull a block from the first stream, it consequently pulls the required blocks from nested streams, and the entire execution pipeline will work. Neither “pull” nor “push” is the best solution, because control flow is implicit, and that limits the implementation of various features like simultaneous execution of multiple queries (merging many pipelines together). This limitation could be overcome with coroutines or just running extra threads that wait for each other. We may have more possibilities if we make control flow explicit: if we locate the logic for passing data from one calculation unit to another outside of those calculation units. Read this [article](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) for more thoughts. - -We should note that the query execution pipeline creates temporary data at each step. We try to keep block size small enough so that temporary data fits in the CPU cache. With that assumption, writing and reading temporary data is almost free in comparison with other calculations. We could consider an alternative, which is to fuse many operations in the pipeline together. It could make the pipeline as short as possible and remove much of the temporary data, which could be an advantage, but it also has drawbacks. For example, a split pipeline makes it easy to implement caching intermediate data, stealing intermediate data from similar queries running at the same time, and merging pipelines for similar queries. - -## Formats {#formats} - -Data formats are implemented with block streams. There are “presentational” formats only suitable for the output of data to the client, such as `Pretty` format, which provides only `IBlockOutputStream`. And there are input/output formats, such as `TabSeparated` or `JSONEachRow`. - -There are also row streams: `IRowInputStream` and `IRowOutputStream`. They allow you to pull/push data by individual rows, not by blocks. And they are only needed to simplify the implementation of row-oriented formats. The wrappers `BlockInputStreamFromRowInputStream` and `BlockOutputStreamFromRowOutputStream` allow you to convert row-oriented streams to regular block-oriented streams. - -## I/O {#io} - -For byte-oriented input/output, there are `ReadBuffer` and `WriteBuffer` abstract classes. They are used instead of C++ `iostream`s. Don’t worry: every mature C++ project is using something other than `iostream`s for good reasons. - -`ReadBuffer` and `WriteBuffer` are just a contiguous buffer and a cursor pointing to the position in that buffer. Implementations may own or not own the memory for the buffer. There is a virtual method to fill the buffer with the following data (for `ReadBuffer`) or to flush the buffer somewhere (for `WriteBuffer`). The virtual methods are rarely called. - -Implementations of `ReadBuffer`/`WriteBuffer` are used for working with files and file descriptors and network sockets, for implementing compression (`CompressedWriteBuffer` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names `ConcatReadBuffer`, `LimitReadBuffer`, and `HashingWriteBuffer` speak for themselves. - -Read/WriteBuffers only deal with bytes. There are functions from `ReadHelpers` and `WriteHelpers` header files to help with formatting input/output. For example, there are helpers to write a number in decimal format. - -Let’s look at what happens when you want to write a result set in `JSON` format to stdout. You have a result set ready to be fetched from `IBlockInputStream`. You create `WriteBufferFromFileDescriptor(STDOUT_FILENO)` to write bytes to stdout. You create `JSONRowOutputStream`, initialized with that `WriteBuffer`, to write rows in `JSON` to stdout. You create `BlockOutputStreamFromRowOutputStream` on top of it, to represent it as `IBlockOutputStream`. Then you call `copyData` to transfer data from `IBlockInputStream` to `IBlockOutputStream`, and everything works. Internally, `JSONRowOutputStream` will write various JSON delimiters and call the `IDataType::serializeTextJSON` method with a reference to `IColumn` and the row number as arguments. Consequently, `IDataType::serializeTextJSON` will call a method from `WriteHelpers.h`: for example, `writeText` for numeric types and `writeJSONString` for `DataTypeString`. - -## Tables {#tables} - -The `IStorage` interface represents tables. Different implementations of that interface are different table engines. Examples are `StorageMergeTree`, `StorageMemory`, and so on. Instances of these classes are just tables. - -The key `IStorage` methods are `read` and `write`. There are also `alter`, `rename`, `drop`, and so on. The `read` method accepts the following arguments: the set of columns to read from a table, the `AST` query to consider, and the desired number of streams to return. It returns one or multiple `IBlockInputStream` objects and information about the stage of data processing that was completed inside a table engine during query execution. - -In most cases, the read method is only responsible for reading the specified columns from a table, not for any further data processing. All further data processing is done by the query interpreter and is outside the responsibility of `IStorage`. - -But there are notable exceptions: - -- The AST query is passed to the `read` method, and the table engine can use it to derive index usage and to read fewer data from a table. -- Sometimes the table engine can process data itself to a specific stage. For example, `StorageDistributed` can send a query to remote servers, ask them to process data to a stage where data from different remote servers can be merged, and return that preprocessed data. The query interpreter then finishes processing the data. - -The table’s `read` method can return multiple `IBlockInputStream` objects to allow parallel data processing. These multiple block input streams can read from a table in parallel. Then you can wrap these streams with various transformations (such as expression evaluation or filtering) that can be calculated independently and create a `UnionBlockInputStream` on top of them, to read from multiple streams in parallel. - -There are also `TableFunction`s. These are functions that return a temporary `IStorage` object to use in the `FROM` clause of a query. - -To get a quick idea of how to implement your table engine, look at something simple, like `StorageMemory` or `StorageTinyLog`. - -> As the result of the `read` method, `IStorage` returns `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. - -## Parsers {#parsers} - -A hand-written recursive descent parser parses a query. For example, `ParserSelectQuery` just recursively calls the underlying parsers for various parts of the query. Parsers create an `AST`. The `AST` is represented by nodes, which are instances of `IAST`. - -> Parser generators are not used for historical reasons. - -## Interpreters {#interpreters} - -Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery` and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the INSERT query is the `IBlockOutputStream` to write data for insertion to, and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time. - -`InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations or query. - -## Functions {#functions} - -There are ordinary functions and aggregate functions. For aggregate functions, see the next section. - -Ordinary functions don’t change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`’s of data to implement vectorized query execution. - -There are some miscellaneous functions, like [blockSize](../query_language/functions/other_functions.md#function-blocksize), [rowNumberInBlock](../query_language/functions/other_functions.md#function-rownumberinblock), and [runningAccumulate](../query_language/functions/other_functions.md#function-runningaccumulate), that exploit block processing and violate the independence of rows. - -ClickHouse has strong typing, so there’s no implicit type conversion. If a function doesn’t support a specific combination of types, it throws an exception. But functions can work (be overloaded) for many different combinations of types. For example, the `plus` function (to implement the `+` operator) works for any combination of numeric types: `UInt8` + `Float32`, `UInt16` + `Int8`, and so on. Also, some variadic functions can accept any number of arguments, such as the `concat` function. - -Implementing a function may be slightly inconvenient because a function explicitly dispatches supported data types and supported `IColumns`. For example, the `plus` function has code generated by instantiation of a C++ template for each combination of numeric types, and constant or non-constant left and right arguments. - -It is an excellent place to implement runtime code generation to avoid template code bloat. Also, it makes it possible to add fused functions like fused multiply-add or to make multiple comparisons in one loop iteration. - -Due to vectorized query execution, functions are not short-circuited. For example, if you write `WHERE f(x) AND g(y)`, both sides are calculated, even for rows, when `f(x)` is zero (except when `f(x)` is a zero constant expression). But if the selectivity of the `f(x)` condition is high, and calculation of `f(x)` is much cheaper than `g(y)`, it’s better to implement multi-pass calculation. It would first calculate `f(x)`, then filter columns by the result, and then calculate `g(y)` only for smaller, filtered chunks of data. - -## Aggregate Functions {#aggregate-functions} - -Aggregate functions are stateful functions. They accumulate passed values into some state and allow you to get results from that state. They are managed with the `IAggregateFunction` interface. States can be rather simple (the state for `AggregateFunctionCount` is just a single `UInt64` value) or quite complex (the state of `AggregateFunctionUniqCombined` is a combination of a linear array, a hash table, and a `HyperLogLog` probabilistic data structure). - -States are allocated in `Arena` (a memory pool) to deal with multiple states while executing a high-cardinality `GROUP BY` query. States can have a non-trivial constructor and destructor: for example, complicated aggregation states can allocate additional memory themselves. It requires some attention to creating and destroying states and properly passing their ownership and destruction order. - -Aggregation states can be serialized and deserialized to pass over the network during distributed query execution or to write them on the disk where there is not enough RAM. They can even be stored in a table with the `DataTypeAggregateFunction` to allow incremental aggregation of data. - -> The serialized data format for aggregate function states is not versioned right now. It is ok if aggregate states are only stored temporarily. But we have the `AggregatingMergeTree` table engine for incremental aggregation, and people are already using it in production. It is the reason why backward compatibility is required when changing the serialized format for any aggregate function in the future. - -## Server {#server} - -The server implements several different interfaces: - -- An HTTP interface for any foreign clients. -- A TCP interface for the native ClickHouse client and for cross-server communication during distributed query execution. -- An interface for transferring data for replication. - -Internally, it is just a primitive multithreaded server without coroutines or fibers. Since the server is not designed to process a high rate of simple queries but to process a relatively low rate of complex queries, each of them can process a vast amount of data for analytics. - -The server initializes the `Context` class with the necessary environment for query execution: the list of available databases, users and access rights, settings, clusters, the process list, the query log, and so on. Interpreters use this environment. - -We maintain full backward and forward compatibility for the server TCP protocol: old clients can talk to new servers, and new clients can talk to old servers. But we don’t want to maintain it eternally, and we are removing support for old versions after about one year. - -!!! note "Note" - For most external applications, we recommend using the HTTP interface because it is simple and easy to use. The TCP protocol is more tightly linked to internal data structures: it uses an internal format for passing blocks of data, and it uses custom framing for compressed data. We haven’t released a C library for that protocol because it requires linking most of the ClickHouse codebase, which is not practical. - -## Distributed Query Execution {#distributed-query-execution} - -Servers in a cluster setup are mostly independent. You can create a `Distributed` table on one or all servers in a cluster. The `Distributed` table does not store data itself – it only provides a “view” to all local tables on multiple nodes of a cluster. When you SELECT from a `Distributed` table, it rewrites that query, chooses remote nodes according to load balancing settings, and sends the query to them. The `Distributed` table requests remote servers to process a query just up to a stage where intermediate results from different servers can be merged. Then it receives the intermediate results and merges them. The distributed table tries to distribute as much work as possible to remote servers and does not send much intermediate data over the network. - -Things become more complicated when you have subqueries in IN or JOIN clauses, and each of them uses a `Distributed` table. We have different strategies for the execution of these queries. - -There is no global query plan for distributed query execution. Each node has its local query plan for its part of the job. We only have simple one-pass distributed query execution: we send queries for remote nodes and then merge the results. But this is not feasible for complicated queries with high cardinality GROUP BYs or with a large amount of temporary data for JOIN. In such cases, we need to “reshuffle” data between servers, which requires additional coordination. ClickHouse does not support that kind of query execution, and we need to work on it. - -## Merge Tree {#merge-tree} - -`MergeTree` is a family of storage engines that supports indexing by primary key. The primary key can be an arbitrary tuple of columns or expressions. Data in a `MergeTree` table is stored in “parts”. Each part stores data in the primary key order, so data is ordered lexicographically by the primary key tuple. All the table columns are stored in separate `column.bin` files in these parts. The files consist of compressed blocks. Each block is usually from 64 KB to 1 MB of uncompressed data, depending on the average value size. The blocks consist of column values placed contiguously one after the other. Column values are in the same order for each column (the primary key defines the order), so when you iterate by many columns, you get values for the corresponding rows. - -The primary key itself is “sparse”. It doesn’t address every single row, but only some ranges of data. A separate `primary.idx` file has the value of the primary key for each N-th row, where N is called `index_granularity` (usually, N = 8192). Also, for each column, we have `column.mrk` files with “marks,” which are offsets to each N-th row in the data file. Each mark is a pair: the offset in the file to the beginning of the compressed block, and the offset in the decompressed block to the beginning of data. Usually, compressed blocks are aligned by marks, and the offset in the decompressed block is zero. Data for `primary.idx` always resides in memory, and data for `column.mrk` files is cached. - -When we are going to read something from a part in `MergeTree`, we look at `primary.idx` data and locate ranges that could contain requested data, then look at `column.mrk` data and calculate offsets for where to start reading those ranges. Because of sparseness, excess data may be read. ClickHouse is not suitable for a high load of simple point queries, because the entire range with `index_granularity` rows must be read for each key, and the entire compressed block must be decompressed for each column. We made the index sparse because we must be able to maintain trillions of rows per single server without noticeable memory consumption for the index. Also, because the primary key is sparse, it is not unique: it cannot check the existence of the key in the table at INSERT time. You could have many rows with the same key in a table. - -When you `INSERT` a bunch of data into `MergeTree`, that bunch is sorted by primary key order and forms a new part. There are background threads that periodically select some parts and merge them into a single sorted part to keep the number of parts relatively low. That’s why it is called `MergeTree`. Of course, merging leads to “write amplification”. All parts are immutable: they are only created and deleted, but not modified. When SELECT is executed, it holds a snapshot of the table (a set of parts). After merging, we also keep old parts for some time to make a recovery after failure easier, so if we see that some merged part is probably broken, we can replace it with its source parts. - -`MergeTree` is not an LSM tree because it doesn’t contain “memtable” and “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity’s sake, and because we are already inserting data in batches in our applications. - -> MergeTree tables can only have one (primary) index: there aren’t any secondary indices. It would be nice to allow multiple physical representations under one logical table, for example, to store data in more than one physical order or even to allow representations with pre-aggregated data along with original data. - -There are MergeTree engines that are doing additional work during background merges. Examples are `CollapsingMergeTree` and `AggregatingMergeTree`. This could be treated as special support for updates. Keep in mind that these are not real updates because users usually have no control over the time when background merges are executed, and data in a `MergeTree` table is almost always stored in more than one part, not in completely merged form. - -## Replication {#replication} - -Replication in ClickHouse can be configured on a per-table basis. You could have some replicated and some non-replicated tables on the same server. You could also have tables replicated in different ways, such as one table with two-factor replication and another with three-factor. - -Replication is implemented in the `ReplicatedMergeTree` storage engine. The path in `ZooKeeper` is specified as a parameter for the storage engine. All tables with the same path in `ZooKeeper` become replicas of each other: they synchronize their data and maintain consistency. Replicas can be added and removed dynamically simply by creating or dropping a table. - -Replication uses an asynchronous multi-master scheme. You can insert data into any replica that has a session with `ZooKeeper`, and data is replicated to all other replicas asynchronously. Because ClickHouse doesn’t support UPDATEs, replication is conflict-free. As there is no quorum acknowledgment of inserts, just-inserted data might be lost if one node fails. - -Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop a partition, and so on. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the “get the part” action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. It is achieved by electing one replica as the leader, and that replica initiates merges and writes “merge parts” actions to the log. - -Replication is physical: only compressed parts are transferred between nodes, not queries. Merges are processed on each replica independently in most cases to lower the network costs by avoiding network amplification. Large merged parts are sent over the network only in cases of significant replication lag. - -Besides, each replica stores its state in ZooKeeper as the set of parts and its checksums. When the state on the local filesystem diverges from the reference state in ZooKeeper, the replica restores its consistency by downloading missing and broken parts from other replicas. When there is some unexpected or broken data in the local filesystem, ClickHouse does not remove it, but moves it to a separate directory and forgets it. - -!!! note "Note" - The ClickHouse cluster consists of independent shards, and each shard consists of replicas. The cluster is **not elastic**, so after adding a new shard, data is not rebalanced between shards automatically. Instead, the cluster load is supposed to be adjusted to be uneven. This implementation gives you more control, and it is ok for relatively small clusters, such as tens of nodes. But for clusters with hundreds of nodes that we are using in production, this approach becomes a significant drawback. We should implement a table engine that spans across the cluster with dynamically replicated regions that could be split and balanced between clusters automatically. - -{## [Original article](https://clickhouse.tech/docs/en/development/architecture/) ##} diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md new file mode 120000 index 00000000000..abda4dd48a8 --- /dev/null +++ b/docs/ru/development/architecture.md @@ -0,0 +1 @@ +../../en/development/architecture.md \ No newline at end of file diff --git a/docs/ru/development/browse_code.md b/docs/ru/development/browse_code.md index f87e3be7f4b..814b213a6a7 100644 --- a/docs/ru/development/browse_code.md +++ b/docs/ru/development/browse_code.md @@ -1,6 +1,6 @@ # Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse} -Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. +Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). diff --git a/docs/ru/development/build.md b/docs/ru/development/build.md deleted file mode 100644 index 32042a4128e..00000000000 --- a/docs/ru/development/build.md +++ /dev/null @@ -1,138 +0,0 @@ ---- -en_copy: true ---- - -# How to Build ClickHouse for Development {#how-to-build-clickhouse-for-development} - -The following tutorial is based on the Ubuntu Linux system. -With appropriate changes, it should also work on any other Linux distribution. -Supported platforms: x86\_64 and AArch64. Support for Power9 is experimental. - -## Install Git, CMake, Python and Ninja {#install-git-cmake-python-and-ninja} - -``` bash -$ sudo apt-get install git cmake python ninja-build -``` - -Or cmake3 instead of cmake on older systems. - -## Install GCC 9 {#install-gcc-9} - -There are several ways to do this. - -### Install from a PPA Package {#install-from-a-ppa-package} - -``` bash -$ sudo apt-get install software-properties-common -$ sudo apt-add-repository ppa:ubuntu-toolchain-r/test -$ sudo apt-get update -$ sudo apt-get install gcc-9 g++-9 -``` - -### Install from Sources {#install-from-sources} - -Look at [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) - -## Use GCC 9 for Builds {#use-gcc-9-for-builds} - -``` bash -$ export CC=gcc-9 -$ export CXX=g++-9 -``` - -## Checkout ClickHouse Sources {#checkout-clickhouse-sources} - -``` bash -$ git clone --recursive git@github.com:ClickHouse/ClickHouse.git -``` - -or - -``` bash -$ git clone --recursive https://github.com/ClickHouse/ClickHouse.git -``` - -## Build ClickHouse {#build-clickhouse} - -``` bash -$ cd ClickHouse -$ mkdir build -$ cd build -$ cmake .. -$ ninja -$ cd .. -``` - -To create an executable, run `ninja clickhouse`. -This will create the `programs/clickhouse` executable, which can be used with `client` or `server` arguments. - -# How to Build ClickHouse on Any Linux {#how-to-build-clickhouse-on-any-linux} - -The build requires the following components: - -- Git (is used only to checkout the sources, it’s not needed for the build) -- CMake 3.10 or newer -- Ninja (recommended) or Make -- C++ compiler: gcc 9 or clang 8 or newer -- Linker: lld or gold (the classic GNU ld won’t work) -- Python (is only used inside LLVM build and it is optional) - -If all the components are installed, you may build in the same way as the steps above. - -Example for Ubuntu Eoan: - - sudo apt update - sudo apt install git cmake ninja-build g++ python - git clone --recursive https://github.com/ClickHouse/ClickHouse.git - mkdir build && cd build - cmake ../ClickHouse - ninja - -Example for OpenSUSE Tumbleweed: - - sudo zypper install git cmake ninja gcc-c++ python lld - git clone --recursive https://github.com/ClickHouse/ClickHouse.git - mkdir build && cd build - cmake ../ClickHouse - ninja - -Example for Fedora Rawhide: - - sudo yum update - yum --nogpg install git cmake make gcc-c++ python2 - git clone --recursive https://github.com/ClickHouse/ClickHouse.git - mkdir build && cd build - cmake ../ClickHouse - make -j $(nproc) - -# You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse} - -ClickHouse is available in pre-built binaries and packages. Binaries are portable and can be run on any Linux flavour. - -They are built for stable, prestable and testing releases as long as for every commit to master and for every pull request. - -To find the freshest build from `master`, go to [commits page](https://github.com/ClickHouse/ClickHouse/commits/master), click on the first green checkmark or red cross near commit, and click to the “Details” link right after “ClickHouse Build Check”. - -# How to Build ClickHouse Debian Package {#how-to-build-clickhouse-debian-package} - -## Install Git and Pbuilder {#install-git-and-pbuilder} - -``` bash -$ sudo apt-get update -$ sudo apt-get install git python pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring -``` - -## Checkout ClickHouse Sources {#checkout-clickhouse-sources-1} - -``` bash -$ git clone --recursive --branch master https://github.com/ClickHouse/ClickHouse.git -$ cd ClickHouse -``` - -## Run Release Script {#run-release-script} - -``` bash -$ ./release -``` - -[Original article](https://clickhouse.tech/docs/en/development/build/) diff --git a/docs/ru/development/build.md b/docs/ru/development/build.md new file mode 120000 index 00000000000..480dbc2e9f5 --- /dev/null +++ b/docs/ru/development/build.md @@ -0,0 +1 @@ +../../en/development/build.md \ No newline at end of file diff --git a/docs/ru/development/build_cross_arm.md b/docs/ru/development/build_cross_arm.md deleted file mode 100644 index 0936a3133b2..00000000000 --- a/docs/ru/development/build_cross_arm.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -en_copy: true ---- - -# How to Build ClickHouse on Linux for AARCH64 (ARM64) architecture {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} - -This is for the case when you have Linux machine and want to use it to build `clickhouse` binary that will run on another Linux machine with AARCH64 CPU architecture. This is intended for continuous integration checks that run on Linux servers. - -The cross-build for AARCH64 is based on the [Build instructions](build.md), follow them first. - -# Install Clang-8 {#install-clang-8} - -Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup. -For example, in Ubuntu Bionic you can use the following commands: - -``` bash -echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee /etc/apt/sources.list.d/llvm.list -sudo apt-get update -sudo apt-get install clang-8 -``` - -# Install Cross-Compilation Toolset {#install-cross-compilation-toolset} - -``` bash -cd ClickHouse -mkdir -p build-aarch64/cmake/toolchain/linux-aarch64 -wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en' -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cmake/toolchain/linux-aarch64 --strip-components=1 -``` - -# Build ClickHouse {#build-clickhouse} - -``` bash -cd ClickHouse -mkdir build-arm64 -CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake -ninja -C build-arm64 -``` - -The resulting binary will run only on Linux with the AARCH64 CPU architecture. diff --git a/docs/ru/development/build_cross_arm.md b/docs/ru/development/build_cross_arm.md new file mode 120000 index 00000000000..983a9872dc1 --- /dev/null +++ b/docs/ru/development/build_cross_arm.md @@ -0,0 +1 @@ +../../en/development/build_cross_arm.md \ No newline at end of file diff --git a/docs/ru/development/build_cross_osx.md b/docs/ru/development/build_cross_osx.md deleted file mode 100644 index a708dc4d4f3..00000000000 --- a/docs/ru/development/build_cross_osx.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -en_copy: true ---- - -# How to Build ClickHouse on Linux for Mac OS X {#how-to-build-clickhouse-on-linux-for-mac-os-x} - -This is for the case when you have Linux machine and want to use it to build `clickhouse` binary that will run on OS X. This is intended for continuous integration checks that run on Linux servers. If you want to build ClickHouse directly on Mac OS X, then proceed with [another instruction](build_osx.md). - -The cross-build for Mac OS X is based on the [Build instructions](build.md), follow them first. - -# Install Clang-8 {#install-clang-8} - -Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup. -For example the commands for Bionic are like: - -``` bash -sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list -sudo apt-get install clang-8 -``` - -# Install Cross-Compilation Toolset {#install-cross-compilation-toolset} - -Let’s remember the path where we install `cctools` as ${CCTOOLS} - -``` bash -mkdir ${CCTOOLS} - -git clone https://github.com/tpoechtrager/apple-libtapi.git -cd apple-libtapi -INSTALLPREFIX=${CCTOOLS} ./build.sh -./install.sh -cd .. - -git clone https://github.com/tpoechtrager/cctools-port.git -cd cctools-port/cctools -./configure --prefix=${CCTOOLS} --with-libtapi=${CCTOOLS} --target=x86_64-apple-darwin -make install -``` - -Also, we need to download macOS X SDK into the working tree. - -``` bash -cd ClickHouse -wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' -mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 -tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 -``` - -# Build ClickHouse {#build-clickhouse} - -``` bash -cd ClickHouse -mkdir build-osx -CC=clang-8 CXX=clang++-8 cmake . -Bbuild-osx -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake \ - -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar \ - -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib \ - -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -ninja -C build-osx -``` - -The resulting binary will have a Mach-O executable format and can’t be run on Linux. diff --git a/docs/ru/development/build_cross_osx.md b/docs/ru/development/build_cross_osx.md new file mode 120000 index 00000000000..72e64e8631f --- /dev/null +++ b/docs/ru/development/build_cross_osx.md @@ -0,0 +1 @@ +../../en/development/build_cross_osx.md \ No newline at end of file diff --git a/docs/ru/development/build_osx.md b/docs/ru/development/build_osx.md deleted file mode 100644 index 6b1839aaf7f..00000000000 --- a/docs/ru/development/build_osx.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -en_copy: true ---- - -# How to Build ClickHouse on Mac OS X {#how-to-build-clickhouse-on-mac-os-x} - -Build should work on Mac OS X 10.15 (Catalina) - -## Install Homebrew {#install-homebrew} - -``` bash -$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -``` - -## Install Required Compilers, Tools, and Libraries {#install-required-compilers-tools-and-libraries} - -``` bash -$ brew install cmake ninja libtool gettext -``` - -## Checkout ClickHouse Sources {#checkout-clickhouse-sources} - -``` bash -$ git clone --recursive git@github.com:ClickHouse/ClickHouse.git -``` - -or - -``` bash -$ git clone --recursive https://github.com/ClickHouse/ClickHouse.git - -$ cd ClickHouse -``` - -## Build ClickHouse {#build-clickhouse} - -``` bash -$ mkdir build -$ cd build -$ cmake .. -DCMAKE_CXX_COMPILER=`which clang++` -DCMAKE_C_COMPILER=`which clang` -$ ninja -$ cd .. -``` - -## Caveats {#caveats} - -If you intend to run clickhouse-server, make sure to increase the system’s maxfiles variable. - -!!! info "Note" - You’ll need to use sudo. - -To do so, create the following file: - -/Library/LaunchDaemons/limit.maxfiles.plist: - -``` xml - - - - - Label - limit.maxfiles - ProgramArguments - - launchctl - limit - maxfiles - 524288 - 524288 - - RunAtLoad - - ServiceIPC - - - -``` - -Execute the following command: - -``` bash -$ sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist -``` - -Reboot. - -To check if it’s working, you can use `ulimit -n` command. - -[Original article](https://clickhouse.tech/docs/en/development/build_osx/) diff --git a/docs/ru/development/build_osx.md b/docs/ru/development/build_osx.md new file mode 120000 index 00000000000..f9adaf24584 --- /dev/null +++ b/docs/ru/development/build_osx.md @@ -0,0 +1 @@ +../../en/development/build_osx.md \ No newline at end of file diff --git a/docs/ru/development/developer_instruction.md b/docs/ru/development/developer_instruction.md index 4bc2ada8c1e..11ac3a73f6e 100644 --- a/docs/ru/development/developer_instruction.md +++ b/docs/ru/development/developer_instruction.md @@ -71,7 +71,7 @@ ClickHouse не работает и не собирается на 32-битны После этого, вы сможете добавлять в свой репозиторий обновления из репозитория Яндекса с помощью команды `git pull upstream master`. -## Работа с сабмодулями git {#rabota-s-sabmoduliami-git} +## Работа с сабмодулями Git {#rabota-s-sabmoduliami-git} Работа с сабмодулями git может быть достаточно болезненной. Следующие команды позволят содержать их в порядке: @@ -267,7 +267,7 @@ Mac OS X: clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv -# Создание pull request {#sozdanie-pull-request} +# Создание Pull Request {#sozdanie-pull-request} Откройте свой форк репозитория в интерфейсе GitHub. Если вы вели разработку в бранче, выберите этот бранч. На странице будет доступна кнопка «Pull request». По сути, это означает «создать заявку на принятие моих изменений в основной репозиторий». diff --git a/docs/ru/development/index.md b/docs/ru/development/index.md deleted file mode 100644 index 727e89ca891..00000000000 --- a/docs/ru/development/index.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -en_copy: true ---- - -# ClickHouse Development {#clickhouse-development} - -[Original article](https://clickhouse.tech/docs/en/development/) diff --git a/docs/ru/development/index.md b/docs/ru/development/index.md new file mode 120000 index 00000000000..1e2ad97dcc5 --- /dev/null +++ b/docs/ru/development/index.md @@ -0,0 +1 @@ +../../en/development/index.md \ No newline at end of file diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md index 091419394f7..a282ec6ec5c 100644 --- a/docs/ru/development/style.md +++ b/docs/ru/development/style.md @@ -431,9 +431,9 @@ enum class CompressionMethod Примеры: -- проще всего разместить объект на стеке, или сделать его членом другого класса. -- для большого количества маленьких объектов используйте контейнеры. -- для автоматического освобождения маленького количества объектов, выделенных на куче, используйте `shared_ptr/unique_ptr`. +- проще всего разместить объект на стеке, или сделать его членом другого класса. +- для большого количества маленьких объектов используйте контейнеры. +- для автоматического освобождения маленького количества объектов, выделенных на куче, используйте `shared_ptr/unique_ptr`. **2.** Управление ресурсами. diff --git a/docs/ru/development/tests.md b/docs/ru/development/tests.md deleted file mode 100644 index c703d6cd5b3..00000000000 --- a/docs/ru/development/tests.md +++ /dev/null @@ -1,249 +0,0 @@ ---- -en_copy: true ---- - -# ClickHouse Testing {#clickhouse-testing} - -## Functional Tests {#functional-tests} - -Functional tests are the most simple and convenient to use. Most of ClickHouse features can be tested with functional tests and they are mandatory to use for every change in ClickHouse code that can be tested that way. - -Each functional test sends one or multiple queries to the running ClickHouse server and compares the result with reference. - -Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and not available to general public. We tend to use only `stateless` tests and avoid adding new `stateful` tests. - -Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery --testmode`. `.sh` test is a script that is run by itself. - -To run all tests, use `testskhouse-test` tool. Look `--help` for the list of possible options. You can simply run all tests or run subset of tests filtered by substring in test name: `./clickhouse-test substring`. - -The most simple way to invoke functional tests is to copy `clickhouse-client` to `/usr/bin/`, run `clickhouse-server` and then run `./clickhouse-test` from its own directory. - -To add new test, create a `.sql` or `.sh` file in `queries/0_stateless` directory, check it manually and then generate `.reference` file in the following way: `clickhouse-client -n --testmode < 00000_test.sql > 00000_test.reference` or `./00000_test.sh > ./00000_test.reference`. - -Tests should use (create, drop, etc) only tables in `test` database that is assumed to be created beforehand; also tests can use temporary tables. - -If you want to use distributed queries in functional tests, you can leverage `remote` table function with `127.0.0.{1..2}` addresses for the server to query itself; or you can use predefined test clusters in server configuration file like `test_shard_localhost`. - -Some tests are marked with `zookeeper`, `shard` or `long` in their names. -`zookeeper` is for tests that are using ZooKeeper. `shard` is for tests that -requires server to listen `127.0.0.*`; `distributed` or `global` have the same -meaning. `long` is for tests that run slightly longer that one second. You can -disable these groups of tests using `--no-zookeeper`, `--no-shard` and -`--no-long` options, respectively. - -## Known bugs {#known-bugs} - -If we know some bugs that can be easily reproduced by functional tests, we place prepared functional tests in `queries/bugs` directory. These tests will be moved to `teststests_stateless` when bugs are fixed. - -## Integration Tests {#integration-tests} - -Integration tests allow to test ClickHouse in clustered configuration and ClickHouse interaction with other servers like MySQL, Postgres, MongoDB. They are useful to emulate network splits, packet drops, etc. These tests are run under Docker and create multiple containers with various software. - -See `testsgration/README.md` on how to run these tests. - -Note that integration of ClickHouse with third-party drivers is not tested. Also we currently don’t have integration tests with our JDBC and ODBC drivers. - -## Unit Tests {#unit-tests} - -Unit tests are useful when you want to test not the ClickHouse as a whole, but a single isolated library or class. You can enable or disable build of tests with `ENABLE_TESTS` CMake option. Unit tests (and other test programs) are located in `tests` subdirectories across the code. To run unit tests, type `ninja test`. Some tests use `gtest`, but some are just programs that return non-zero exit code on test failure. - -It’s not necessarily to have unit tests if the code is already covered by functional tests (and functional tests are usually much more simple to use). - -## Performance Tests {#performance-tests} - -Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `clickhouse performance-test` tool (that is embedded in `clickhouse` binary). See `--help` for invocation. - -Each test run one or miltiple queries (possibly with combinations of parameters) in a loop with some conditions for stop (like “maximum execution speed is not changing in three seconds”) and measure some metrics about query performance (like “maximum execution speed”). Some tests can contain preconditions on preloaded test dataset. - -If you want to improve performance of ClickHouse in some scenario, and if improvements can be observed on simple queries, it is highly recommended to write a performance test. It always makes sense to use `perf top` or other perf tools during your tests. - -## Test Tools And Scripts {#test-tools-and-scripts} - -Some programs in `tests` directory are not prepared tests, but are test tools. For example, for `Lexer` there is a tool `dbms/Parsers/tests/lexer` that just do tokenization of stdin and writes colorized result to stdout. You can use these kind of tools as a code examples and for exploration and manual testing. - -You can also place pair of files `.sh` and `.reference` along with the tool to run it on some predefined input - then script result can be compared to `.reference` file. These kind of tests are not automated. - -## Miscellanous Tests {#miscellanous-tests} - -There are tests for external dictionaries located at `tests/external_dictionaries` and for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests. - -There is separate test for quorum inserts. This test run ClickHouse cluster on separate servers and emulate various failure cases: network split, packet drop (between ClickHouse nodes, between ClickHouse and ZooKeeper, between ClickHouse server and client, etc.), `kill -9`, `kill -STOP` and `kill -CONT` , like [Jepsen](https://aphyr.com/tags/Jepsen). Then the test checks that all acknowledged inserts was written and all rejected inserts was not. - -Quorum test was written by separate team before ClickHouse was open-sourced. This team no longer work with ClickHouse. Test was accidentially written in Java. For these reasons, quorum test must be rewritten and moved to integration tests. - -## Manual Testing {#manual-testing} - -When you develop a new feature, it is reasonable to also test it manually. You can do it with the following steps: - -Build ClickHouse. Run ClickHouse from the terminal: change directory to `programs/clickhouse-server` and run it with `./clickhouse-server`. It will use configuration (`config.xml`, `users.xml` and files within `config.d` and `users.d` directories) from the current directory by default. To connect to ClickHouse server, run `programs/clickhouse-client/clickhouse-client`. - -Note that all clickhouse tools (server, client, etc) are just symlinks to a single binary named `clickhouse`. You can find this binary at `programs/clickhouse`. All tools can also be invoked as `clickhouse tool` instead of `clickhouse-tool`. - -Alternatively you can install ClickHouse package: either stable release from Yandex repository or you can build package for yourself with `./release` in ClickHouse sources root. Then start the server with `sudo service clickhouse-server start` (or stop to stop the server). Look for logs at `/etc/clickhouse-server/clickhouse-server.log`. - -When ClickHouse is already installed on your system, you can build a new `clickhouse` binary and replace the existing binary: - -``` bash -$ sudo service clickhouse-server stop -$ sudo cp ./clickhouse /usr/bin/ -$ sudo service clickhouse-server start -``` - -Also you can stop system clickhouse-server and run your own with the same configuration but with logging to terminal: - -``` bash -$ sudo service clickhouse-server stop -$ sudo -u clickhouse /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml -``` - -Example with gdb: - -``` bash -$ sudo -u clickhouse gdb --args /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml -``` - -If the system clickhouse-server is already running and you don’t want to stop it, you can change port numbers in your `config.xml` (or override them in a file in `config.d` directory), provide appropriate data path, and run it. - -`clickhouse` binary has almost no dependencies and works across wide range of Linux distributions. To quick and dirty test your changes on a server, you can simply `scp` your fresh built `clickhouse` binary to your server and then run it as in examples above. - -## Testing Environment {#testing-environment} - -Before publishing release as stable we deploy it on testing environment. Testing environment is a cluster that process 1/39 part of [Yandex.Metrica](https://metrica.yandex.com/) data. We share our testing environment with Yandex.Metrica team. ClickHouse is upgraded without downtime on top of existing data. We look at first that data is processed successfully without lagging from realtime, the replication continue to work and there is no issues visible to Yandex.Metrica team. First check can be done in the following way: - -``` sql -SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h; -``` - -In some cases we also deploy to testing environment of our friend teams in Yandex: Market, Cloud, etc. Also we have some hardware servers that are used for development purposes. - -## Load Testing {#load-testing} - -After deploying to testing environment we run load testing with queries from production cluster. This is done manually. - -Make sure you have enabled `query_log` on your production cluster. - -Collect query log for a day or more: - -``` bash -$ clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv -``` - -This is a way complicated example. `type = 2` will filter queries that are executed successfully. `query LIKE '%ym:%'` is to select relevant queries from Yandex.Metrica. `is_initial_query` is to select only queries that are initiated by client, not by ClickHouse itself (as parts of distributed query processing). - -`scp` this log to your testing cluster and run it as following: - -``` bash -$ clickhouse benchmark --concurrency 16 < queries.tsv -``` - -(probably you also want to specify a `--user`) - -Then leave it for a night or weekend and go take a rest. - -You should check that `clickhouse-server` doesn’t crash, memory footprint is bounded and performance not degrading over time. - -Precise query execution timings are not recorded and not compared due to high variability of queries and environment. - -## Build Tests {#build-tests} - -Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. Tests are located at `ci` directory. They run build from source inside Docker, Vagrant, and sometimes with `qemu-user-static` inside Docker. These tests are under development and test runs are not automated. - -Motivation: - -Normally we release and run all tests on a single variant of ClickHouse build. But there are alternative build variants that are not thoroughly tested. Examples: - -- build on FreeBSD; -- build on Debian with libraries from system packages; -- build with shared linking of libraries; -- build on AArch64 platform; -- build on PowerPc platform. - -For example, build with system packages is bad practice, because we cannot guarantee what exact version of packages a system will have. But this is really needed by Debian maintainers. For this reason we at least have to support this variant of build. Another example: shared linking is a common source of trouble, but it is needed for some enthusiasts. - -Though we cannot run all tests on all variant of builds, we want to check at least that various build variants are not broken. For this purpose we use build tests. - -## Testing For Protocol Compatibility {#testing-for-protocol-compatibility} - -When we extend ClickHouse network protocol, we test manually that old clickhouse-client works with new clickhouse-server and new clickhouse-client works with old clickhouse-server (simply by running binaries from corresponding packages). - -## Help From The Compiler {#help-from-the-compiler} - -Main ClickHouse code (that is located in `dbms` directory) is built with `-Wall -Wextra -Werror` and with some additional enabled warnings. Although these options are not enabled for third-party libraries. - -Clang has even more useful warnings - you can look for them with `-Weverything` and pick something to default build. - -For production builds, gcc is used (it still generates slightly more efficient code than clang). For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang, `libc++` is used instead of `libstdc++` and when building with debug mode, debug version of `libc++` is used that allows to catch more errors at runtime. - -## Sanitizers {#sanitizers} - -**Address sanitizer**. -We run functional and integration tests under ASan on per-commit basis. - -**Valgrind (Memcheck)**. -We run functional tests under Valgrind overnight. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse). - -**Undefined behaviour sanitizer.** -We run functional and integration tests under ASan on per-commit basis. - -**Thread sanitizer**. -We run functional tests under TSan on per-commit basis. We still don’t run integration tests under TSan on per-commit basis. - -**Memory sanitizer**. -Currently we still don’t use MSan. - -**Debug allocator.** -Debug version of `jemalloc` is used for debug build. - -## Fuzzing {#fuzzing} - -We use simple fuzz test to generate random SQL queries and to check that the server doesn’t die. Fuzz testing is performed with Address sanitizer. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). - -As of December 2018, we still don’t use isolated fuzz testing of library code. - -## Security Audit {#security-audit} - -People from Yandex Cloud department do some basic overview of ClickHouse capabilities from the security standpoint. - -## Static Analyzers {#static-analyzers} - -We run `PVS-Studio` on per-commit basis. We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/). - -If you use `CLion` as an IDE, you can leverage some `clang-tidy` checks out of the box. - -## Hardening {#hardening} - -`FORTIFY_SOURCE` is used by default. It is almost useless, but still makes sense in rare cases and we don’t disable it. - -## Code Style {#code-style} - -Code style rules are described [here](https://clickhouse.tech/docs/en/development/style/). - -To check for some common style violations, you can use `utils/check-style` script. - -To force proper style of your code, you can use `clang-format`. File `.clang-format` is located at the sources root. It mostly corresponding with our actual code style. But it’s not recommended to apply `clang-format` to existing files because it makes formatting worse. You can use `clang-format-diff` tool that you can find in clang source repository. - -Alternatively you can try `uncrustify` tool to reformat your code. Configuration is in `uncrustify.cfg` in the sources root. It is less tested than `clang-format`. - -`CLion` has its own code formatter that has to be tuned for our code style. - -## Metrica B2B Tests {#metrica-b2b-tests} - -Each ClickHouse release is tested with Yandex Metrica and AppMetrica engines. Testing and stable versions of ClickHouse are deployed on VMs and run with a small copy of Metrica engine that is processing fixed sample of input data. Then results of two instances of Metrica engine are compared together. - -These tests are automated by separate team. Due to high number of moving parts, tests are fail most of the time by completely unrelated reasons, that are very difficult to figure out. Most likely these tests have negative value for us. Nevertheless these tests was proved to be useful in about one or two times out of hundreds. - -## Test Coverage {#test-coverage} - -As of July 2018 we don’t track test coverage. - -## Test Automation {#test-automation} - -We run tests with Yandex internal CI and job automation system named “Sandbox”. - -Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored eternally. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you. - -We don’t use Travis CI due to the limit on time and computational power. -We don’t use Jenkins. It was used before and now we are happy we are not using Jenkins. - -[Original article](https://clickhouse.tech/docs/en/development/tests/) -velopment/tests/) diff --git a/docs/ru/development/tests.md b/docs/ru/development/tests.md new file mode 120000 index 00000000000..c03d36c3916 --- /dev/null +++ b/docs/ru/development/tests.md @@ -0,0 +1 @@ +../../en/development/tests.md \ No newline at end of file diff --git a/docs/ru/database_engines/index.md b/docs/ru/engines/database_engines/index.md similarity index 78% rename from docs/ru/database_engines/index.md rename to docs/ru/engines/database_engines/index.md index 982324e0408..75086eaf678 100644 --- a/docs/ru/database_engines/index.md +++ b/docs/ru/engines/database_engines/index.md @@ -2,7 +2,7 @@ Движки баз данных обеспечивают работу с таблицами. -По умолчанию ClickHouse использует собственный движок баз данных, который поддерживает конфигурируемые [движки таблиц](../operations/table_engines/index.md) и [диалект SQL](../query_language/syntax.md). +По умолчанию ClickHouse использует собственный движок баз данных, который поддерживает конфигурируемые [движки таблиц](../../engines/database_engines/index.md) и [диалект SQL](../../engines/database_engines/index.md). Также можно использовать следующие движки баз данных: diff --git a/docs/ru/database_engines/lazy.md b/docs/ru/engines/database_engines/lazy.md similarity index 100% rename from docs/ru/database_engines/lazy.md rename to docs/ru/engines/database_engines/lazy.md diff --git a/docs/ru/database_engines/mysql.md b/docs/ru/engines/database_engines/mysql.md similarity index 66% rename from docs/ru/database_engines/mysql.md rename to docs/ru/engines/database_engines/mysql.md index 420ca370297..1dbcb67e8f1 100644 --- a/docs/ru/database_engines/mysql.md +++ b/docs/ru/engines/database_engines/mysql.md @@ -6,8 +6,6 @@ Не поддерживаемые виды запросов: -- `ATTACH`/`DETACH` -- `DROP` - `RENAME` - `CREATE TABLE` - `ALTER` @@ -16,7 +14,7 @@ ``` sql CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', 'database', 'user', 'password') +ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') ``` **Параметры движка** @@ -30,23 +28,23 @@ ENGINE = MySQL('host:port', 'database', 'user', 'password') | MySQL | ClickHouse | |----------------------------------|---------------------------------------------| -| UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) | -| TINYINT | [Int8](../data_types/int_uint.md) | -| UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) | -| SMALLINT | [Int16](../data_types/int_uint.md) | -| UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) | -| INT, MEDIUMINT | [Int32](../data_types/int_uint.md) | -| UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) | -| BIGINT | [Int64](../data_types/int_uint.md) | -| FLOAT | [Float32](../data_types/float.md) | -| DOUBLE | [Float64](../data_types/float.md) | -| DATE | [Date](../data_types/date.md) | -| DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) | -| BINARY | [FixedString](../data_types/fixedstring.md) | +| UNSIGNED TINYINT | [UInt8](../../engines/database_engines/mysql.md) | +| TINYINT | [Int8](../../engines/database_engines/mysql.md) | +| UNSIGNED SMALLINT | [UInt16](../../engines/database_engines/mysql.md) | +| SMALLINT | [Int16](../../engines/database_engines/mysql.md) | +| UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../../engines/database_engines/mysql.md) | +| INT, MEDIUMINT | [Int32](../../engines/database_engines/mysql.md) | +| UNSIGNED BIGINT | [UInt64](../../engines/database_engines/mysql.md) | +| BIGINT | [Int64](../../engines/database_engines/mysql.md) | +| FLOAT | [Float32](../../engines/database_engines/mysql.md) | +| DOUBLE | [Float64](../../engines/database_engines/mysql.md) | +| DATE | [Date](../../engines/database_engines/mysql.md) | +| DATETIME, TIMESTAMP | [DateTime](../../engines/database_engines/mysql.md) | +| BINARY | [FixedString](../../engines/database_engines/mysql.md) | -Все прочие типы данных преобразуются в [String](../data_types/string.md). +Все прочие типы данных преобразуются в [String](../../engines/database_engines/mysql.md). -[Nullable](../data_types/nullable.md) поддержан. +[Nullable](../../engines/database_engines/mysql.md) поддержан. ## Примеры использования {#primery-ispolzovaniia} diff --git a/docs/ru/engines/index.md b/docs/ru/engines/index.md new file mode 100644 index 00000000000..a3d85155cbb --- /dev/null +++ b/docs/ru/engines/index.md @@ -0,0 +1,6 @@ +--- +toc_folder_title: Движки +toc_priority: 25 +--- + + diff --git a/docs/ru/operations/table_engines/index.md b/docs/ru/engines/table_engines/index.md similarity index 64% rename from docs/ru/operations/table_engines/index.md rename to docs/ru/engines/table_engines/index.md index 775164ccb52..fdf9cd50bf6 100644 --- a/docs/ru/operations/table_engines/index.md +++ b/docs/ru/engines/table_engines/index.md @@ -13,27 +13,27 @@ ### MergeTree {#mergetree} -Наиболее универсальные и функциональные движки таблиц для задач с высокой загрузкой. Общим свойством этих движков является быстрая вставка данных с последующей фоновой обработкой данных. Движки `*MergeTree` поддерживают репликацию данных (в [Replicated\*](replication.md) версиях движков), партиционирование, и другие возможности не поддержанные для других движков. +Наиболее универсальные и функциональные движки таблиц для задач с высокой загрузкой. Общим свойством этих движков является быстрая вставка данных с последующей фоновой обработкой данных. Движки `*MergeTree` поддерживают репликацию данных (в [Replicated\*](mergetree_family/replication.md) версиях движков), партиционирование, и другие возможности не поддержанные для других движков. Движки семейства: -- [MergeTree](mergetree.md) -- [ReplacingMergeTree](replacingmergetree.md) -- [SummingMergeTree](summingmergetree.md) -- [AggregatingMergeTree](aggregatingmergetree.md) -- [CollapsingMergeTree](collapsingmergetree.md) -- [VersionedCollapsingMergeTree](versionedcollapsingmergetree.md) -- [GraphiteMergeTree](graphitemergetree.md) +- [MergeTree](mergetree_family/mergetree.md) +- [ReplacingMergeTree](mergetree_family/replacingmergetree.md) +- [SummingMergeTree](mergetree_family/summingmergetree.md) +- [AggregatingMergeTree](mergetree_family/aggregatingmergetree.md) +- [CollapsingMergeTree](mergetree_family/collapsingmergetree.md) +- [VersionedCollapsingMergeTree](mergetree_family/versionedcollapsingmergetree.md) +- [GraphiteMergeTree](mergetree_family/graphitemergetree.md) ### Log {#log} -Простые [движки](log_family.md) с минимальной функциональностью. Они наиболее эффективны, когда вам нужно быстро записать много небольших таблиц (до примерно 1 миллиона строк) и прочитать их позже целиком. +Простые [движки](log_family/index.md) с минимальной функциональностью. Они наиболее эффективны, когда вам нужно быстро записать много небольших таблиц (до примерно 1 миллиона строк) и прочитать их позже целиком. Движки семейства: -- [TinyLog](tinylog.md) -- [StripeLog](stripelog.md) -- [Log](log.md) +- [TinyLog](log_family/tinylog.md) +- [StripeLog](log_family/stripelog.md) +- [Log](log_family/log.md) ### Движки для интеграции {#dvizhki-dlia-integratsii} @@ -41,27 +41,27 @@ Движки семейства: -- [Kafka](kafka.md) -- [MySQL](mysql.md) -- [ODBC](odbc.md) -- [JDBC](jdbc.md) +- [Kafka](integrations/kafka.md) +- [MySQL](integrations/mysql.md) +- [ODBC](integrations/odbc.md) +- [JDBC](integrations/jdbc.md) ### Специальные движки {#spetsialnye-dvizhki} Движки семейства: -- [Distributed](distributed.md) -- [MaterializedView](materializedview.md) -- [Dictionary](dictionary.md) -- [Merge](merge.md) -- [File](file.md) -- [Null](null.md) -- [Set](set.md) -- [Join](join.md) -- [URL](url.md) -- [View](view.md) -- [Memory](memory.md) -- [Buffer](buffer.md) +- [Distributed](special/distributed.md) +- [MaterializedView](special/materializedview.md) +- [Dictionary](special/dictionary.md) +- [Merge](special/merge.md) +- [File](special/file.md) +- [Null](special/null.md) +- [Set](special/set.md) +- [Join](special/join.md) +- [URL](special/url.md) +- [View](special/view.md) +- [Memory](special/memory.md) +- [Buffer](special/buffer.md) ## Виртуальные столбцы {#table_engines-virtual-columns} diff --git a/docs/ru/operations/table_engines/hdfs.md b/docs/ru/engines/table_engines/integrations/hdfs.md similarity index 94% rename from docs/ru/operations/table_engines/hdfs.md rename to docs/ru/engines/table_engines/integrations/hdfs.md index 4f892b1e492..26b97a99f77 100644 --- a/docs/ru/operations/table_engines/hdfs.md +++ b/docs/ru/engines/table_engines/integrations/hdfs.md @@ -1,6 +1,6 @@ # HDFS {#table_engines-hdfs} -Управляет данными в HDFS. Данный движок похож на движки [File](file.md) и [URL](url.md). +Управляет данными в HDFS. Данный движок похож на движки [File](../special/file.md) и [URL](../special/url.md). ## Использование движка {#ispolzovanie-dvizhka} @@ -9,7 +9,7 @@ ENGINE = HDFS(URI, format) ``` В параметр `URI` нужно передавать полный URI файла в HDFS. -Параметр `format` должен быть таким, который ClickHouse может использовать и в запросах `INSERT`, и в запросах `SELECT`. Полный список поддерживаемых форматов смотрите в разделе [Форматы](../../interfaces/formats.md#formats). +Параметр `format` должен быть таким, который ClickHouse может использовать и в запросах `INSERT`, и в запросах `SELECT`. Полный список поддерживаемых форматов смотрите в разделе [Форматы](../../../interfaces/formats.md#formats). Часть URI с путем файла может содержать шаблоны. В этом случае таблица может использоваться только для чтения. **Пример:** @@ -56,7 +56,7 @@ SELECT * FROM hdfs_engine_table LIMIT 2 - `{some_string,another_string,yet_another_one}` — Заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`. - `{N..M}` — Заменяет любое число в интервале от `N` до `M` включительно (может содержать ведущие нули). -Конструкция с `{}` аналогична табличной функции [remote](../../query_language/table_functions/remote.md). +Конструкция с `{}` аналогична табличной функции [remote](../../../engines/table_engines/integrations/hdfs.md). **Пример** diff --git a/docs/ru/engines/table_engines/integrations/index.md b/docs/ru/engines/table_engines/integrations/index.md new file mode 100644 index 00000000000..716d00cdd98 --- /dev/null +++ b/docs/ru/engines/table_engines/integrations/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: Integrations +toc_priority: 30 +--- + diff --git a/docs/ru/operations/table_engines/jdbc.md b/docs/ru/engines/table_engines/integrations/jdbc.md similarity index 95% rename from docs/ru/operations/table_engines/jdbc.md rename to docs/ru/engines/table_engines/integrations/jdbc.md index d9a66244849..ae461a539be 100644 --- a/docs/ru/operations/table_engines/jdbc.md +++ b/docs/ru/engines/table_engines/integrations/jdbc.md @@ -4,7 +4,7 @@ Для реализации соединения по JDBC ClickHouse использует отдельную программу [clickhouse-jdbc-bridge](https://github.com/alex-krash/clickhouse-jdbc-bridge), которая должна запускаться как демон. -Движок поддерживает тип данных [Nullable](../../data_types/nullable.md). +Движок поддерживает тип данных [Nullable](../../../engines/table_engines/integrations/jdbc.md). ## Создание таблицы {#sozdanie-tablitsy} @@ -82,6 +82,6 @@ FROM jdbc_table ## Смотрите также {#smotrite-takzhe} -- [Табличная функция JDBC](../../query_language/table_functions/jdbc.md). +- [Табличная функция JDBC](../../../engines/table_engines/integrations/jdbc.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/jdbc/) diff --git a/docs/ru/operations/table_engines/kafka.md b/docs/ru/engines/table_engines/integrations/kafka.md similarity index 95% rename from docs/ru/operations/table_engines/kafka.md rename to docs/ru/engines/table_engines/integrations/kafka.md index 960eecf49d0..c43a7b5d37d 100644 --- a/docs/ru/operations/table_engines/kafka.md +++ b/docs/ru/engines/table_engines/integrations/kafka.md @@ -33,7 +33,7 @@ SETTINGS - `kafka_broker_list` – перечень брокеров, разделенный запятыми (`localhost:9092`). - `kafka_topic_list` – перечень необходимых топиков Kafka. - `kafka_group_name` – группа потребителя Kafka. Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы. -- `kafka_format` – формат сообщений. Названия форматов должны быть теми же, что можно использовать в секции `FORMAT`, например, `JSONEachRow`. Подробнее читайте в разделе [Форматы](../../interfaces/formats.md). +- `kafka_format` – формат сообщений. Названия форматов должны быть теми же, что можно использовать в секции `FORMAT`, например, `JSONEachRow`. Подробнее читайте в разделе [Форматы](../../../interfaces/formats.md). Опциональные параметры: @@ -123,7 +123,7 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format SELECT level, sum(total) FROM daily GROUP BY level; ``` -Для улучшения производительности полученные сообщения группируются в блоки размера [max\_insert\_block\_size](../settings/settings.md#settings-max_insert_block_size). Если блок не удалось сформировать за [stream\_flush\_interval\_ms](../settings/settings.md) миллисекунд, то данные будут сброшены в таблицу независимо от полноты блока. +Для улучшения производительности полученные сообщения группируются в блоки размера [max\_insert\_block\_size](../../../operations/settings/settings.md#settings-max_insert_block_size). Если блок не удалось сформировать за [stream\_flush\_interval\_ms](../../../operations/settings/settings.md) миллисекунд, то данные будут сброшены в таблицу независимо от полноты блока. Чтобы остановить получение данных топика или изменить логику преобразования, отсоедините материализованное представление: diff --git a/docs/ru/operations/table_engines/mysql.md b/docs/ru/engines/table_engines/integrations/mysql.md similarity index 89% rename from docs/ru/operations/table_engines/mysql.md rename to docs/ru/engines/table_engines/integrations/mysql.md index 09ca9077c2c..7260b182c6d 100644 --- a/docs/ru/operations/table_engines/mysql.md +++ b/docs/ru/engines/table_engines/integrations/mysql.md @@ -13,12 +13,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); ``` -Смотрите подробное описание запроса [CREATE TABLE](../../query_language/create.md#create-table-query). +Смотрите подробное описание запроса [CREATE TABLE](../../../engines/table_engines/integrations/mysql.md#create-table-query). Структура таблицы может отличаться от исходной структуры таблицы MySQL: - Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке. -- Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [приводить](../../query_language/functions/type_conversion_functions.md#type_conversion_function-cast) значения к типам данных ClickHouse. +- Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [приводить](../../../engines/table_engines/integrations/mysql.md#type_conversion_function-cast) значения к типам данных ClickHouse. **Параметры движка** @@ -92,7 +92,7 @@ SELECT * FROM mysql_table ## Смотрите также {#smotrite-takzhe} -- [Табличная функция ‘mysql’](../../query_language/table_functions/mysql.md) -- [Использование MySQL в качестве источника для внешнего словаря](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql) +- [Табличная функция ‘mysql’](../../../engines/table_engines/integrations/mysql.md) +- [Использование MySQL в качестве источника для внешнего словаря](../../../engines/table_engines/integrations/mysql.md#dicts-external_dicts_dict_sources-mysql) [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/mysql/) diff --git a/docs/ru/operations/table_engines/odbc.md b/docs/ru/engines/table_engines/integrations/odbc.md similarity index 90% rename from docs/ru/operations/table_engines/odbc.md rename to docs/ru/engines/table_engines/integrations/odbc.md index b3dde77890c..6124a928315 100644 --- a/docs/ru/operations/table_engines/odbc.md +++ b/docs/ru/engines/table_engines/integrations/odbc.md @@ -4,7 +4,7 @@ Чтобы использование ODBC было безопасным, ClickHouse использует отдельную программу `clickhouse-odbc-bridge`. Если драйвер ODBC подгружать непосредственно из `clickhouse-server`, то проблемы с драйвером могут привести к аварийной остановке сервера ClickHouse. ClickHouse автоматически запускает `clickhouse-odbc-bridge` по мере необходимости. Программа устанавливается из того же пакета, что и `clickhouse-server`. -Движок поддерживает тип данных [Nullable](../../data_types/nullable.md). +Движок поддерживает тип данных [Nullable](../../../engines/table_engines/integrations/odbc.md). ## Создание таблицы {#sozdanie-tablitsy} @@ -18,12 +18,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ENGINE = ODBC(connection_settings, external_database, external_table) ``` -Смотрите подробное описание запроса [CREATE TABLE](../../query_language/create.md#create-table-query). +Смотрите подробное описание запроса [CREATE TABLE](../../../engines/table_engines/integrations/odbc.md#create-table-query). Структура таблицы может отличаться от структуры исходной таблицы в удалённой СУБД: - Имена столбцов должны быть такими же, как в исходной таблице, но вы можете использовать только некоторые из этих столбцов и в любом порядке. -- Типы столбцов могут отличаться от типов аналогичных столбцов в исходной таблице. ClickHouse пытается [приводить](../../query_language/functions/type_conversion_functions.md#type_conversion_function-cast) значения к типам данных ClickHouse. +- Типы столбцов могут отличаться от типов аналогичных столбцов в исходной таблице. ClickHouse пытается [приводить](../../../engines/table_engines/integrations/odbc.md#type_conversion_function-cast) значения к типам данных ClickHouse. **Параметры движка** @@ -119,7 +119,7 @@ SELECT * FROM odbc_t ## Смотрите также {#smotrite-takzhe} -- [Внешние словари ODBC](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc) -- [Табличная функция odbc](../../query_language/table_functions/odbc.md) +- [Внешние словари ODBC](../../../engines/table_engines/integrations/odbc.md#dicts-external_dicts_dict_sources-odbc) +- [Табличная функция odbc](../../../engines/table_engines/integrations/odbc.md) [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/odbc/) diff --git a/docs/ru/engines/table_engines/log_family/index.md b/docs/ru/engines/table_engines/log_family/index.md new file mode 100644 index 00000000000..a64371200a6 --- /dev/null +++ b/docs/ru/engines/table_engines/log_family/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: Log Family +toc_priority: 29 +--- + diff --git a/docs/ru/operations/table_engines/log.md b/docs/ru/engines/table_engines/log_family/log.md similarity index 100% rename from docs/ru/operations/table_engines/log.md rename to docs/ru/engines/table_engines/log_family/log.md diff --git a/docs/ru/operations/table_engines/log_family.md b/docs/ru/engines/table_engines/log_family/log_family.md similarity index 97% rename from docs/ru/operations/table_engines/log_family.md rename to docs/ru/engines/table_engines/log_family/log_family.md index 597d331981c..f132c2e8d33 100644 --- a/docs/ru/operations/table_engines/log_family.md +++ b/docs/ru/engines/table_engines/log_family/log_family.md @@ -20,7 +20,7 @@ Во время запросов `INSERT` таблица блокируется, а другие запросы на чтение и запись ожидают разблокировки таблицы. Если запросов на запись данных нет, то можно выполнять любое количество конкуретных запросов на чтение. -- Не поддерживают операции [мутации](../../query_language/alter.md#alter-mutations). +- Не поддерживают операции [мутации](../../../engines/table_engines/log_family/log_family.md#alter-mutations). - Не поддерживают индексы. diff --git a/docs/ru/operations/table_engines/stripelog.md b/docs/ru/engines/table_engines/log_family/stripelog.md similarity index 98% rename from docs/ru/operations/table_engines/stripelog.md rename to docs/ru/engines/table_engines/log_family/stripelog.md index 3f69e1bdd73..4eb4d4620de 100644 --- a/docs/ru/operations/table_engines/stripelog.md +++ b/docs/ru/engines/table_engines/log_family/stripelog.md @@ -15,7 +15,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = StripeLog ``` -Смотрите подробное описание запроса [CREATE TABLE](../../query_language/create.md#create-table-query). +Смотрите подробное описание запроса [CREATE TABLE](../../../engines/table_engines/log_family/stripelog.md#create-table-query). ## Запись данных {#table_engines-stripelog-writing-the-data} diff --git a/docs/ru/operations/table_engines/tinylog.md b/docs/ru/engines/table_engines/log_family/tinylog.md similarity index 100% rename from docs/ru/operations/table_engines/tinylog.md rename to docs/ru/engines/table_engines/log_family/tinylog.md diff --git a/docs/ru/operations/table_engines/aggregatingmergetree.md b/docs/ru/engines/table_engines/mergetree_family/aggregatingmergetree.md similarity index 95% rename from docs/ru/operations/table_engines/aggregatingmergetree.md rename to docs/ru/engines/table_engines/mergetree_family/aggregatingmergetree.md index 64ae3aa037c..8fdf063f569 100644 --- a/docs/ru/operations/table_engines/aggregatingmergetree.md +++ b/docs/ru/engines/table_engines/mergetree_family/aggregatingmergetree.md @@ -4,7 +4,7 @@ Таблицы типа `AggregatingMergeTree` могут использоваться для инкрементальной агрегации данных, в том числе, для агрегирующих материализованных представлений. -Движок обрабатывает все столбцы типа [AggregateFunction](../../data_types/nested_data_structures/aggregatefunction.md). +Движок обрабатывает все столбцы типа [AggregateFunction](../../../engines/table_engines/mergetree_family/aggregatingmergetree.md). Использование `AggregatingMergeTree` оправдано только в том случае, когда это уменьшает количество строк на порядки. @@ -23,7 +23,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -Описание параметров запроса смотрите в [описании запроса](../../query_language/create.md). +Описание параметров запроса смотрите в [описании запроса](../../../engines/table_engines/mergetree_family/aggregatingmergetree.md). **Секции запроса** diff --git a/docs/ru/operations/table_engines/collapsingmergetree.md b/docs/ru/engines/table_engines/mergetree_family/collapsingmergetree.md similarity index 99% rename from docs/ru/operations/table_engines/collapsingmergetree.md rename to docs/ru/engines/table_engines/mergetree_family/collapsingmergetree.md index 38d4d475e07..5179ac06fa5 100644 --- a/docs/ru/operations/table_engines/collapsingmergetree.md +++ b/docs/ru/engines/table_engines/mergetree_family/collapsingmergetree.md @@ -21,7 +21,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -Подробности про `CREATE TABLE` смотрите в [описании запроса](../../query_language/create.md). +Подробности про `CREATE TABLE` смотрите в [описании запроса](../../../engines/table_engines/mergetree_family/collapsingmergetree.md). **Параметры CollapsingMergeTree** diff --git a/docs/ru/operations/table_engines/custom_partitioning_key.md b/docs/ru/engines/table_engines/mergetree_family/custom_partitioning_key.md similarity index 91% rename from docs/ru/operations/table_engines/custom_partitioning_key.md rename to docs/ru/engines/table_engines/mergetree_family/custom_partitioning_key.md index c2b846ef3c1..039ab1ba0cf 100644 --- a/docs/ru/operations/table_engines/custom_partitioning_key.md +++ b/docs/ru/engines/table_engines/mergetree_family/custom_partitioning_key.md @@ -1,6 +1,6 @@ # Произвольный ключ партиционирования {#proizvolnyi-kliuch-partitsionirovaniia} -Партиционирование данных доступно для таблиц семейства [MergeTree](mergetree.md) (включая [реплицированные таблицы](replication.md)). Таблицы [MaterializedView](materializedview.md), созданные на основе таблиц MergeTree, также поддерживают партиционирование. +Партиционирование данных доступно для таблиц семейства [MergeTree](mergetree.md) (включая [реплицированные таблицы](replication.md)). Таблицы [MaterializedView](../special/materializedview.md), созданные на основе таблиц MergeTree, также поддерживают партиционирование. Партиция – это набор записей в таблице, объединенных по какому-либо критерию. Например, партиция может быть по месяцу, по дню или по типу события. Данные для разных партиций хранятся отдельно. Это позволяет оптимизировать работу с данными, так как при обработке запросов будет использоваться только необходимое подмножество из всевозможных данных. Например, при получении данных за определенный месяц, ClickHouse будет считывать данные только за этот месяц. @@ -33,7 +33,7 @@ ORDER BY (CounterID, StartDate, intHash32(UserID)); !!! info "Info" Не рекомендуется делать слишком гранулированное партиционирование – то есть задавать партиции по столбцу, в котором будет слишком большой разброс значений (речь идет о порядке более тысячи партиций). Это приведет к скоплению большого числа файлов и файловых дескрипторов в системе, что может значительно снизить производительность запросов `SELECT`. -Чтобы получить набор кусков и партиций таблицы, можно воспользоваться системной таблицей [system.parts](../system_tables.md#system_tables-parts). В качестве примера рассмотрим таблицу `visits`, в которой задано партиционирование по месяцам. Выполним `SELECT` для таблицы `system.parts`: +Чтобы получить набор кусков и партиций таблицы, можно воспользоваться системной таблицей [system.parts](../../../engines/table_engines/mergetree_family/custom_partitioning_key.md#system_tables-parts). В качестве примера рассмотрим таблицу `visits`, в которой задано партиционирование по месяцам. Выполним `SELECT` для таблицы `system.parts`: ``` sql SELECT @@ -74,7 +74,7 @@ WHERE table = 'visits' Как видно из примера выше, таблица содержит несколько отдельных кусков для одной и той же партиции (например, куски `201901_1_3_1` и `201901_1_9_2` принадлежат партиции `201901`). Это означает, что эти куски еще не были объединены – в файловой системе они хранятся отдельно. После того как будет выполнено автоматическое слияние данных (выполняется примерно спустя 10 минут после вставки данных), исходные куски будут объединены в один более крупный кусок и помечены как неактивные. -Вы можете запустить внеочередное слияние данных с помощью запроса [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize). Пример: +Вы можете запустить внеочередное слияние данных с помощью запроса [OPTIMIZE](../../../engines/table_engines/mergetree_family/custom_partitioning_key.md#misc_operations-optimize). Пример: ``` sql OPTIMIZE TABLE visits PARTITION 201902; @@ -115,12 +115,12 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached Директория `detached` содержит куски, отсоединенные от таблицы с помощью запроса [DETACH](#alter_detach-partition). Поврежденные куски также попадают в эту директорию – они не удаляются с сервера. -Сервер не использует куски из директории `detached`. Вы можете в любое время добавлять, удалять, модифицировать данные в директории detached - сервер не будет об этом знать, пока вы не сделаете запрос [ATTACH](../../query_language/alter.md#alter_attach-partition). +Сервер не использует куски из директории `detached`. Вы можете в любое время добавлять, удалять, модифицировать данные в директории detached - сервер не будет об этом знать, пока вы не сделаете запрос [ATTACH](../../../engines/table_engines/mergetree_family/custom_partitioning_key.md#alter_attach-partition). Следует иметь в виду, что при работающем сервере нельзя вручную изменять набор кусков на файловой системе, так как сервер не будет знать об этом. Для нереплицируемых таблиц, вы можете это делать при остановленном сервере, однако это не рекомендуется. Для реплицируемых таблиц, набор кусков нельзя менять в любом случае. -ClickHouse позволяет производить различные манипуляции с кусками: удалять, копировать из одной таблицы в другую или создавать их резервные копии. Подробнее см. в разделе [Манипуляции с партициями и кусками](../../query_language/alter.md#alter_manipulations-with-partitions). +ClickHouse позволяет производить различные манипуляции с кусками: удалять, копировать из одной таблицы в другую или создавать их резервные копии. Подробнее см. в разделе [Манипуляции с партициями и кусками](../../../engines/table_engines/mergetree_family/custom_partitioning_key.md#alter_manipulations-with-partitions). [Оригинальная статья:](https://clickhouse.tech/docs/ru/operations/table_engines/custom_partitioning_key/) diff --git a/docs/ru/operations/table_engines/graphitemergetree.md b/docs/ru/engines/table_engines/mergetree_family/graphitemergetree.md similarity index 93% rename from docs/ru/operations/table_engines/graphitemergetree.md rename to docs/ru/engines/table_engines/mergetree_family/graphitemergetree.md index c128da7ac02..305300fc9a5 100644 --- a/docs/ru/operations/table_engines/graphitemergetree.md +++ b/docs/ru/engines/table_engines/mergetree_family/graphitemergetree.md @@ -23,7 +23,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -Смотрите описание запроса [CREATE TABLE](../../query_language/create.md#create-table-query). +Смотрите описание запроса [CREATE TABLE](../../../engines/table_engines/mergetree_family/graphitemergetree.md#create-table-query). В таблице должны быть столбцы для следующих данных: @@ -74,7 +74,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ## Конфигурация rollup {#rollup-configuration} -Настройки прореживания данных задаются параметром [graphite\_rollup](../server_settings/settings.md#server_settings-graphite_rollup) в конфигурации сервера . Имя параметра может быть любым. Можно создать несколько конфигураций и использовать их для разных таблиц. +Настройки прореживания данных задаются параметром [graphite\_rollup](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-graphite_rollup) в конфигурации сервера . Имя параметра может быть любым. Можно создать несколько конфигураций и использовать их для разных таблиц. Структура конфигурации rollup: diff --git a/docs/ru/engines/table_engines/mergetree_family/index.md b/docs/ru/engines/table_engines/mergetree_family/index.md new file mode 100644 index 00000000000..6a23ef23a8e --- /dev/null +++ b/docs/ru/engines/table_engines/mergetree_family/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: MergeTree Family +toc_priority: 28 +--- + diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/engines/table_engines/mergetree_family/mergetree.md similarity index 87% rename from docs/ru/operations/table_engines/mergetree.md rename to docs/ru/engines/table_engines/mergetree_family/mergetree.md index f8bbc983b74..e0ab59c8a05 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/engines/table_engines/mergetree_family/mergetree.md @@ -23,7 +23,7 @@ При необходимости можно задать способ сэмплирования данных в таблице. !!! info "Info" - Движок [Merge](merge.md) не относится к семейству `*MergeTree`. + Движок [Merge](../special/merge.md) не относится к семейству `*MergeTree`. ## Создание таблицы {#table_engine-mergetree-creating-a-table} @@ -44,7 +44,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -Описание параметров смотрите в [описании запроса CREATE](../../query_language/create.md). +Описание параметров смотрите в [описании запроса CREATE](../../../engines/table_engines/mergetree_family/mergetree.md). !!! note "Note" `INDEX` — экспериментальная возможность, смотрите [Индексы пропуска данных](#table_engine-mergetree-data_skipping-indexes). @@ -55,7 +55,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `PARTITION BY` — [ключ партиционирования](custom_partitioning_key.md). - Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../data_types/date.md). В этом случае имена партиций имеют формат `"YYYYMM"`. + Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table_engines/mergetree_family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`. - `ORDER BY` — ключ сортировки. @@ -84,7 +84,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage). - `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage). - `enable_mixed_granularity_parts` — включает или выключает переход к ограничению размера гранул с помощью настройки `index_granularity_bytes`. До версии 19.11, размер гранул ограничивался только настройкой `index_granularity`. Настройка `index_granularity_bytes` улучшает производительность ClickHouse при выборке данных из таблиц с большими (десятки и сотни мегабайтов) строками. Если у вас есть таблицы с большими строками, можно включить эту настройку, чтобы повысить эффективность запросов `SELECT`. - - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../server_settings/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". + - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../../../operations/server_configuration_parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". - `min_merge_bytes_to_use_direct_io` — минимальный объём данных при слиянии, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объём хранения всех данных, подлежащих слиянию. Если общий объём хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байтов. - `merge_with_ttl_timeout` — минимальное время в секундах перед повторным слиянием с TTL. По умолчанию — 86400 (1 день). @@ -100,7 +100,7 @@ ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDa В примере мы устанавливаем партиционирование по месяцам. -Также мы задаем выражение для сэмплирования в виде хэша по идентификатору посетителя. Это позволяет псевдослучайным образом перемешать данные в таблице для каждого `CounterID` и `EventDate`. Если при выборке данных задать секцию [SAMPLE](../../query_language/select.md#select-sample-clause), то ClickHouse вернёт равномерно-псевдослучайную выборку данных для подмножества посетителей. +Также мы задаем выражение для сэмплирования в виде хэша по идентификатору посетителя. Это позволяет псевдослучайным образом перемешать данные в таблице для каждого `CounterID` и `EventDate`. Если при выборке данных задать секцию [SAMPLE](../../../engines/table_engines/mergetree_family/mergetree.md#select-sample-clause), то ClickHouse вернёт равномерно-псевдослучайную выборку данных для подмножества посетителей. `index_granularity` можно было не указывать, поскольку 8192 — это значение по умолчанию. @@ -122,9 +122,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **Параметры MergeTree()** -- `date-column` — имя столбца с типом [Date](../../data_types/date.md). На основе этого столбца ClickHouse автоматически создаёт партиции по месяцам. Имена партиций имеют формат `"YYYYMM"`. +- `date-column` — имя столбца с типом [Date](../../../engines/table_engines/mergetree_family/mergetree.md). На основе этого столбца ClickHouse автоматически создаёт партиции по месяцам. Имена партиций имеют формат `"YYYYMM"`. - `sampling_expression` — выражение для сэмплирования. -- `(primary, key)` — первичный ключ. Тип — [Tuple()](../../data_types/tuple.md) +- `(primary, key)` — первичный ключ. Тип — [Tuple()](../../../engines/table_engines/mergetree_family/mergetree.md) - `index_granularity` — гранулярность индекса. Число строк данных между «засечками» индекса. Для большинства задач подходит значение 8192. **Пример** @@ -214,7 +214,7 @@ ClickHouse не требует уникального первичного кл В этом сценарии имеет смысл оставить в первичном ключе всего несколько столбцов, которые обеспечат эффективную фильтрацию по индексу, а остальные столбцы-измерения добавить в выражение ключа сортировки. -[ALTER ключа сортировки](../../query_language/alter.md) — лёгкая операция, так как при одновременном добавлении нового столбца в таблицу и ключ сортировки не нужно изменять данные кусков (они остаются упорядоченными и по новому выражению ключа). +[ALTER ключа сортировки](../../../engines/table_engines/mergetree_family/mergetree.md) — лёгкая операция, так как при одновременном добавлении нового столбца в таблицу и ключ сортировки не нужно изменять данные кусков (они остаются упорядоченными и по новому выражению ключа). ### Использование индексов и партиций в запросах {#ispolzovanie-indeksov-i-partitsii-v-zaprosakh} @@ -246,7 +246,7 @@ ClickHouse будет использовать индекс по первичн SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' ``` -Чтобы проверить, сможет ли ClickHouse использовать индекс при выполнении запроса, используйте настройки [force\_index\_by\_date](../settings/settings.md#settings-force_index_by_date) и [force\_primary\_key](../settings/settings.md#settings-force_primary_key). +Чтобы проверить, сможет ли ClickHouse использовать индекс при выполнении запроса, используйте настройки [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) и [force\_primary\_key](../../../operations/settings/settings.md#settings-force_primary_key). Ключ партиционирования по месяцам обеспечивает чтение только тех блоков данных, которые содержат даты из нужного диапазона. При этом блок данных может содержать данные за многие даты (до целого месяца). В пределах одного блока данные упорядочены по первичному ключу, который может не содержать дату в качестве первого столбца. В связи с этим, при использовании запроса с указанием условия только на дату, но не на префикс первичного ключа, будет читаться данных больше, чем за одну дату. @@ -304,7 +304,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Поддержанные типы данных: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`. - Фильтром могут пользоваться функции: [equals](../../query_language/functions/comparison_functions.md), [notEquals](../../query_language/functions/comparison_functions.md), [in](../../query_language/functions/in_functions.md), [notIn](../../query_language/functions/in_functions.md). + Фильтром могут пользоваться функции: [equals](../../../engines/table_engines/mergetree_family/mergetree.md), [notEquals](../../../engines/table_engines/mergetree_family/mergetree.md), [in](../../../engines/table_engines/mergetree_family/mergetree.md), [notIn](../../../engines/table_engines/mergetree_family/mergetree.md). **Примеры** @@ -321,21 +321,21 @@ INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARIT | Function (operator) / Index | primary key | minmax | ngrambf\_v1 | tokenbf\_v1 | bloom\_filter | |----------------------------------------------------------------------------------------------------------|-------------|--------|-------------|-------------|---------------| -| [equals (=, ==)](../../query_language/functions/comparison_functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notEquals(!=, \<\>)](../../query_language/functions/comparison_functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [like](../../query_language/functions/string_search_functions.md#function-like) | ✔ | ✔ | ✔ | ✗ | ✗ | -| [notLike](../../query_language/functions/string_search_functions.md#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [startsWith](../../query_language/functions/string_functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [endsWith](../../query_language/functions/string_functions.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | -| [multiSearchAny](../../query_language/functions/string_search_functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✔ | ✗ | -| [in](../../query_language/functions/in_functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notIn](../../query_language/functions/in_functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [less (\<)](../../query_language/functions/comparison_functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greater (\>)](../../query_language/functions/comparison_functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [lessOrEquals (\<=)](../../query_language/functions/comparison_functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greaterOrEquals (\>=)](../../query_language/functions/comparison_functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [empty](../../query_language/functions/array_functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [notEmpty](../../query_language/functions/array_functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [equals (=, ==)](../../../engines/table_engines/mergetree_family/mergetree.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, \<\>)](../../../engines/table_engines/mergetree_family/mergetree.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [like](../../../engines/table_engines/mergetree_family/mergetree.md#function-like) | ✔ | ✔ | ✔ | ✗ | ✗ | +| [notLike](../../../engines/table_engines/mergetree_family/mergetree.md#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [startsWith](../../../engines/table_engines/mergetree_family/mergetree.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [endsWith](../../../engines/table_engines/mergetree_family/mergetree.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | +| [multiSearchAny](../../../engines/table_engines/mergetree_family/mergetree.md#function-multisearchany) | ✗ | ✗ | ✔ | ✔ | ✗ | +| [in](../../../engines/table_engines/mergetree_family/mergetree.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notIn](../../../engines/table_engines/mergetree_family/mergetree.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [less (\<)](../../../engines/table_engines/mergetree_family/mergetree.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greater (\>)](../../../engines/table_engines/mergetree_family/mergetree.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [lessOrEquals (\<=)](../../../engines/table_engines/mergetree_family/mergetree.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greaterOrEquals (\>=)](../../../engines/table_engines/mergetree_family/mergetree.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [empty](../../../engines/table_engines/mergetree_family/mergetree.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [notEmpty](../../../engines/table_engines/mergetree_family/mergetree.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | | hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | Функции с постоянным агрументом, который меньше, чем размер ngram не могут использовать индекс `ngrambf_v1` для оптимизации запроса. @@ -367,7 +367,7 @@ INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARIT Секция `TTL` может быть установлена как для всей таблицы, так и для каждого отдельного столбца. Правила `TTL` для таблицы позволяют указать целевые диски или тома для фонового перемещения на них частей данных. -Выражения должны возвращать тип [Date](../../data_types/date.md) или [DateTime](../../data_types/datetime.md). +Выражения должны возвращать тип [Date](../../../engines/table_engines/mergetree_family/mergetree.md) или [DateTime](../../../engines/table_engines/mergetree_family/mergetree.md). Для задания времени жизни столбца, например: @@ -376,7 +376,7 @@ TTL time_column TTL time_column + interval ``` -Чтобы задать `interval`, используйте операторы [интервала времени](../../query_language/operators.md#operators-datetime). +Чтобы задать `interval`, используйте операторы [интервала времени](../../../engines/table_engines/mergetree_family/mergetree.md#operators-datetime). ``` sql TTL date_time + INTERVAL 1 MONTH @@ -465,7 +465,7 @@ ALTER TABLE example_table Когда ClickHouse видит, что некоторые данные устарели, он выполняет внеплановые мёржи. Для управление частотой подобных мёржей, можно задать настройку [merge\_with\_ttl\_timeout](#mergetree_setting-merge_with_ttl_timeout). Если её значение слишком низкое, придется выполнять много внеплановых мёржей, которые могут начать потреблять значительную долю ресурсов сервера. -Если вы выполните запрос `SELECT` между слияниями вы можете получить устаревшие данные. Чтобы избежать этого используйте запрос [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) перед `SELECT`. +Если вы выполните запрос `SELECT` между слияниями вы можете получить устаревшие данные. Чтобы избежать этого используйте запрос [OPTIMIZE](../../../engines/table_engines/mergetree_family/mergetree.md#misc_operations-optimize) перед `SELECT`. ## Хранение данных таблицы на нескольких блочных устройствах {#table_engine-mergetree-multiple-volumes} @@ -473,16 +473,16 @@ ALTER TABLE example_table Движки таблиц семейства `MergeTree` могут хранить данные на нескольких блочных устройствах. Это может оказаться полезным, например, при неявном разделении данных одной таблицы на «горячие» и «холодные». Наиболее свежая часть занимает малый объём и запрашивается регулярно, а большой хвост исторических данных запрашивается редко. При наличии в системе нескольких дисков, «горячая» часть данных может быть размещена на быстрых дисках (например, на NVMe SSD или в памяти), а холодная на более медленных (например, HDD). -Минимальной перемещаемой единицей для `MergeTree` является кусок данных (data part). Данные одного куска могут находится только на одном диске. Куски могут перемещаться между дисками в фоне, согласно пользовательским настройкам, а также с помощью запросов [ALTER](../../query_language/alter.md#alter_move-partition). +Минимальной перемещаемой единицей для `MergeTree` является кусок данных (data part). Данные одного куска могут находится только на одном диске. Куски могут перемещаться между дисками в фоне, согласно пользовательским настройкам, а также с помощью запросов [ALTER](../../../engines/table_engines/mergetree_family/mergetree.md#alter_move-partition). ### Термины {#terminy} - Диск — примонтированное в файловой системе блочное устройство. -- Диск по умолчанию — диск, на котором находится путь, указанный в конфигурационной настройке сервера [path](../server_settings/settings.md#server_settings-path). +- Диск по умолчанию — диск, на котором находится путь, указанный в конфигурационной настройке сервера [path](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-path). - Том (Volume) — упорядоченный набор равноценных дисков (схоже с [JBOD](https://ru.wikipedia.org/wiki/JBOD)) - Политика хранения (StoragePolicy) — множество томов с правилами перемещения данных между ними. -У всех описанных сущностей при создании указываются имена, можно найти в системных таблицах [system.storage\_policies](../system_tables.md#system_tables-storage_policies) и [system.disks](../system_tables.md#system_tables-disks). Имя политики хранения можно указать в настройке `storage_policy` движков таблиц семейства `MergeTree`. +У всех описанных сущностей при создании указываются имена, можно найти в системных таблицах [system.storage\_policies](../../../engines/table_engines/mergetree_family/mergetree.md#system_tables-storage_policies) и [system.disks](../../../engines/table_engines/mergetree_family/mergetree.md#system_tables-disks). Имя политики хранения можно указать в настройке `storage_policy` движков таблиц семейства `MergeTree`. ### Конфигурация {#table_engine-mergetree-multiple-volumes-configure} @@ -616,9 +616,9 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' В таблицах `MergeTree` данные попадают на диск несколькими способами: - В результате вставки (запрос `INSERT`). -- В фоновых операциях слияний и [мутаций](../../query_language/alter.md#alter-mutations). +- В фоновых операциях слияний и [мутаций](../../../engines/table_engines/mergetree_family/mergetree.md#alter-mutations). - При скачивании данных с другой реплики. -- В результате заморозки партиций [ALTER TABLE … FREEZE PARTITION](../../query_language/alter.md#alter_freeze-partition). +- В результате заморозки партиций [ALTER TABLE … FREEZE PARTITION](../../../engines/table_engines/mergetree_family/mergetree.md#alter_freeze-partition). Во всех случаях, кроме мутаций и заморозки партиций, при записи куска выбирается том и диск в соответствии с указанной конфигурацией хранилища: @@ -627,9 +627,8 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Мутации и запросы заморозки партиций в реализации используют [жесткие ссылки](https://ru.wikipedia.org/wiki/%D0%96%D1%91%D1%81%D1%82%D0%BA%D0%B0%D1%8F_%D1%81%D1%81%D1%8B%D0%BB%D0%BA%D0%B0). Жесткие ссылки между различными дисками не поддерживаются, поэтому в случае таких операций куски размещаются на тех же дисках, что и исходные. -В фоне куски перемещаются между томами на основе информации о занятом месте (настройка `move_factor`) по порядку, в котором указаны тома в конфигурации. Данные никогда не перемещаются с последнего тома и на первый том. Следить за фоновыми перемещениями можно с помощью системных таблиц [system.part\_log](../system_tables.md#system_tables-part-log) (поле `type = MOVE_PART`) и [system.parts](../system_tables.md#system_tables-parts) (поля `path` и `disk`). Также подробная информация о перемещениях доступна в логах сервера. - -С помощью запроса [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../query_language/alter.md#alter_move-partition) пользователь может принудительно перенести кусок или партицию с одного раздела на другой. При этом учитываются все ограничения, указанные для фоновых операций. Запрос самостоятельно инициирует процесс перемещения не дожидаясь фоновых операций. В случае недостатка места или неудовлетворения ограничениям пользователь получит сообщение об ошибке. +В фоне куски перемещаются между томами на основе информации о занятом месте (настройка `move_factor`) по порядку, в котором указаны тома в конфигурации. Данные никогда не перемещаются с последнего тома и на первый том. Следить за фоновыми перемещениями можно с помощью системных таблиц [system.part\_log](../../../engines/table_engines/mergetree_family/mergetree.md#system_tables-part-log) (поле `type = MOVE_PART`) и [system.parts](../../../engines/table_engines/mergetree_family/mergetree.md#system_tables-parts) (поля `path` и `disk`). Также подробная информация о перемещениях доступна в логах сервера. +С помощью запроса [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../engines/table_engines/mergetree_family/mergetree.md#alter_move-partition) пользователь может принудительно перенести кусок или партицию с одного раздела на другой. При этом учитываются все ограничения, указанные для фоновых операций. Запрос самостоятельно инициирует процесс перемещения не дожидаясь фоновых операций. В случае недостатка места или неудовлетворения ограничениям пользователь получит сообщение об ошибке. Перемещения данных не взаимодействуют с репликацией данных, поэтому на разных репликах одной и той же таблицы могут быть указаны разные политики хранения. diff --git a/docs/ru/operations/table_engines/replacingmergetree.md b/docs/ru/engines/table_engines/mergetree_family/replacingmergetree.md similarity index 96% rename from docs/ru/operations/table_engines/replacingmergetree.md rename to docs/ru/engines/table_engines/mergetree_family/replacingmergetree.md index b403e485741..40a1eb1a9c6 100644 --- a/docs/ru/operations/table_engines/replacingmergetree.md +++ b/docs/ru/engines/table_engines/mergetree_family/replacingmergetree.md @@ -21,7 +21,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -Описание параметров запроса смотрите в [описании запроса](../../query_language/create.md). +Описание параметров запроса смотрите в [описании запроса](../../../engines/table_engines/mergetree_family/replacingmergetree.md). **Параметры ReplacingMergeTree** diff --git a/docs/ru/operations/table_engines/replication.md b/docs/ru/engines/table_engines/mergetree_family/replication.md similarity index 97% rename from docs/ru/operations/table_engines/replication.md rename to docs/ru/engines/table_engines/mergetree_family/replication.md index 67f8d5f0161..14a50a2b94b 100644 --- a/docs/ru/operations/table_engines/replication.md +++ b/docs/ru/engines/table_engines/mergetree_family/replication.md @@ -14,7 +14,7 @@ Репликация не зависит от шардирования. На каждом шарде репликация работает независимо. -Реплицируются сжатые данные запросов `INSERT`, `ALTER` (см. подробности в описании запроса [ALTER](../../query_language/alter.md#query_language_queries_alter)). +Реплицируются сжатые данные запросов `INSERT`, `ALTER` (см. подробности в описании запроса [ALTER](../../../engines/table_engines/mergetree_family/replication.md#query_language_queries_alter)). Запросы `CREATE`, `DROP`, `ATTACH`, `DETACH` и `RENAME` выполняются на одном сервере и не реплицируются: @@ -24,7 +24,7 @@ ClickHouse хранит метаинформацию о репликах в [Apache ZooKeeper](https://zookeeper.apache.org). Используйте ZooKeeper 3.4.5 или новее. -Для использовании репликации, установите параметры в секции [zookeeper](../server_settings/settings.md#server-settings_zookeeper) конфигурации сервера. +Для использовании репликации, установите параметры в секции [zookeeper](../../../operations/server_configuration_parameters/settings.md#server-settings_zookeeper) конфигурации сервера. !!! attention "Внимание" Не пренебрегайте настройками безопасности. ClickHouse поддерживает [ACL схему](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) `digest` подсистемы безопасности ZooKeeper. @@ -52,7 +52,7 @@ ClickHouse хранит метаинформацию о репликах в [Apa Если в конфигурационном файле не настроен ZooKeeper, то вы не сможете создать реплицируемые таблицы, а уже имеющиеся реплицируемые таблицы будут доступны в режиме только на чтение. -При запросах `SELECT`, ZooKeeper не используется, т.е. репликация не влияет на производительность `SELECT` и запросы работают так же быстро, как и для нереплицируемых таблиц. При запросах к распределенным реплицированным таблицам поведение ClickHouse регулируется настройками [max\_replica\_delay\_for\_distributed\_queries](../settings/settings.md#settings-max_replica_delay_for_distributed_queries) and [fallback\_to\_stale\_replicas\_for\_distributed\_queries](../settings/settings.md). +При запросах `SELECT`, ZooKeeper не используется, т.е. репликация не влияет на производительность `SELECT` и запросы работают так же быстро, как и для нереплицируемых таблиц. При запросах к распределенным реплицированным таблицам поведение ClickHouse регулируется настройками [max\_replica\_delay\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) and [fallback\_to\_stale\_replicas\_for\_distributed\_queries](../../../operations/settings/settings.md). При каждом запросе `INSERT`, делается около десятка записей в ZooKeeper в рамках нескольких транзакций. (Чтобы быть более точным, это для каждого вставленного блока данных; запрос INSERT содержит один блок или один блок на `max_insert_block_size = 1048576` строк.) Это приводит к некоторому увеличению задержек при `INSERT`, по сравнению с нереплицируемыми таблицами. Но если придерживаться обычных рекомендаций - вставлять данные пачками не более одного `INSERT` в секунду, то это не составляет проблем. На всём кластере ClickHouse, использующим для координации один кластер ZooKeeper, может быть в совокупности несколько сотен `INSERT` в секунду. Пропускная способность при вставке данных (количество строчек в секунду) такая же высокая, как для нереплицируемых таблиц. @@ -64,7 +64,7 @@ ClickHouse хранит метаинформацию о репликах в [Apa Каждый блок данных записывается атомарно. Запрос INSERT разбивается на блоки данных размером до `max_insert_block_size = 1048576` строк. То есть, если в запросе `INSERT` менее 1048576 строк, то он делается атомарно. -Блоки данных дедуплицируются. При многократной записи одного и того же блока данных (блоков данных одинакового размера, содержащих одни и те же строчки в одном и том же порядке), блок будет записан только один раз. Это сделано для того, чтобы в случае сбоя в сети, когда клиентское приложение не может понять, были ли данные записаны в БД, можно было просто повторить запрос `INSERT`. При этом не имеет значения, на какую реплику будут отправлены INSERT-ы с одинаковыми данными. Запрос `INSERT` идемпотентный. Параметры дедуплицирования регулируются настройками сервера [merge\_tree](../server_settings/settings.md#server_settings-merge_tree) +Блоки данных дедуплицируются. При многократной записи одного и того же блока данных (блоков данных одинакового размера, содержащих одни и те же строчки в одном и том же порядке), блок будет записан только один раз. Это сделано для того, чтобы в случае сбоя в сети, когда клиентское приложение не может понять, были ли данные записаны в БД, можно было просто повторить запрос `INSERT`. При этом не имеет значения, на какую реплику будут отправлены INSERT-ы с одинаковыми данными. Запрос `INSERT` идемпотентный. Параметры дедуплицирования регулируются настройками сервера [merge\_tree](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-merge_tree) При репликации, по сети передаются только исходные вставляемые данные. Дальнейшие преобразования данных (слияния) координируются и делаются на всех репликах одинаковым образом. За счёт этого минимизируется использование сети, и благодаря этому, репликация хорошо работает при расположении реплик в разных дата-центрах. (Стоит заметить, что дублирование данных в разных дата-центрах, по сути, является основной задачей репликации). diff --git a/docs/ru/operations/table_engines/summingmergetree.md b/docs/ru/engines/table_engines/mergetree_family/summingmergetree.md similarity index 91% rename from docs/ru/operations/table_engines/summingmergetree.md rename to docs/ru/engines/table_engines/mergetree_family/summingmergetree.md index 6b1a41384c0..b69f58c3dac 100644 --- a/docs/ru/operations/table_engines/summingmergetree.md +++ b/docs/ru/engines/table_engines/mergetree_family/summingmergetree.md @@ -19,7 +19,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -Описание параметров запроса смотрите в [описании запроса](../../query_language/create.md). +Описание параметров запроса смотрите в [описании запроса](../../../engines/table_engines/mergetree_family/summingmergetree.md). **Параметры SummingMergeTree** @@ -91,7 +91,7 @@ SELECT key, sum(value) FROM summtt GROUP BY key При вставке данных в таблицу они сохраняются как есть. Периодически ClickHouse выполняет слияние вставленных кусков данных и именно в этот момент производится суммирование и замена многих строк с одинаковым первичным ключом на одну для каждого результирующего куска данных. -ClickHouse может слить куски данных таким образом, что не все строки с одинаковым первичным ключом окажутся в одном финальном куске, т.е. суммирование будет не полным. Поэтому, при выборке данных (`SELECT`) необходимо использовать агрегатную функцию [sum()](../../query_language/agg_functions/reference.md#agg_function-sum) и секцию `GROUP BY` как описано в примере выше. +ClickHouse может слить куски данных таким образом, что не все строки с одинаковым первичным ключом окажутся в одном финальном куске, т.е. суммирование будет не полным. Поэтому, при выборке данных (`SELECT`) необходимо использовать агрегатную функцию [sum()](../../../engines/table_engines/mergetree_family/summingmergetree.md#agg_function-sum) и секцию `GROUP BY` как описано в примере выше. ### Общие правила суммирования {#obshchie-pravila-summirovaniia} @@ -105,7 +105,7 @@ ClickHouse может слить куски данных таким образо ### Суммирование в столбцах AggregateFunction {#summirovanie-v-stolbtsakh-aggregatefunction} -Для столбцов типа [AggregateFunction](../../data_types/nested_data_structures/aggregatefunction.md#data_type-aggregatefunction) ClickHouse выполняет агрегацию согласно заданной функции, повторяя поведение движка [AggregatingMergeTree](aggregatingmergetree.md). +Для столбцов типа [AggregateFunction](../../../engines/table_engines/mergetree_family/summingmergetree.md#data_type-aggregatefunction) ClickHouse выполняет агрегацию согласно заданной функции, повторяя поведение движка [AggregatingMergeTree](aggregatingmergetree.md). ### Вложенные структуры {#vlozhennye-struktury} @@ -127,7 +127,7 @@ ClickHouse может слить куски данных таким образо [(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] ``` -При запросе данных используйте функцию [sumMap(key, value)](../../query_language/agg_functions/reference.md) для агрегации `Map`. +При запросе данных используйте функцию [sumMap(key, value)](../../../engines/table_engines/mergetree_family/summingmergetree.md) для агрегации `Map`. Для вложенной структуры данных не нужно указывать её столбцы в кортеже столбцов для суммирования. diff --git a/docs/ru/operations/table_engines/versionedcollapsingmergetree.md b/docs/ru/engines/table_engines/mergetree_family/versionedcollapsingmergetree.md similarity index 99% rename from docs/ru/operations/table_engines/versionedcollapsingmergetree.md rename to docs/ru/engines/table_engines/mergetree_family/versionedcollapsingmergetree.md index a93b56f3c52..90647edd0eb 100644 --- a/docs/ru/operations/table_engines/versionedcollapsingmergetree.md +++ b/docs/ru/engines/table_engines/mergetree_family/versionedcollapsingmergetree.md @@ -24,7 +24,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -Подробности про `CREATE TABLE` смотрите в [описании запроса](../../query_language/create.md). +Подробности про `CREATE TABLE` смотрите в [описании запроса](../../../engines/table_engines/mergetree_family/versionedcollapsingmergetree.md). **Параметры движка** diff --git a/docs/ru/operations/table_engines/buffer.md b/docs/ru/engines/table_engines/special/buffer.md similarity index 100% rename from docs/ru/operations/table_engines/buffer.md rename to docs/ru/engines/table_engines/special/buffer.md diff --git a/docs/ru/operations/table_engines/dictionary.md b/docs/ru/engines/table_engines/special/dictionary.md similarity index 94% rename from docs/ru/operations/table_engines/dictionary.md rename to docs/ru/engines/table_engines/special/dictionary.md index 9f764a65ffa..fac22b5d2f2 100644 --- a/docs/ru/operations/table_engines/dictionary.md +++ b/docs/ru/engines/table_engines/special/dictionary.md @@ -1,6 +1,6 @@ # Dictionary {#dictionary} -Движок `Dictionary` отображает данные [словаря](../../query_language/dicts/external_dicts.md) как таблицу ClickHouse. +Движок `Dictionary` отображает данные [словаря](../../../engines/table_engines/special/dictionary.md) как таблицу ClickHouse. Рассмотрим для примера словарь `products` со следующей конфигурацией: @@ -57,7 +57,7 @@ WHERE name = 'products' └──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ ``` -В таком виде данные из словаря можно получить при помощи функций [dictGet\*](../../query_language/functions/ext_dict_functions.md#ext_dict_functions). +В таком виде данные из словаря можно получить при помощи функций [dictGet\*](../../../engines/table_engines/special/dictionary.md#ext_dict_functions). Такое представление неудобно, когда нам необходимо получить данные в чистом виде, а также при выполнении операции `JOIN`. Для этих случаев можно использовать движок `Dictionary`, который отобразит данные словаря в таблицу. diff --git a/docs/ru/operations/table_engines/distributed.md b/docs/ru/engines/table_engines/special/distributed.md similarity index 92% rename from docs/ru/operations/table_engines/distributed.md rename to docs/ru/engines/table_engines/special/distributed.md index 73547e76692..9fc7b470f1d 100644 --- a/docs/ru/operations/table_engines/distributed.md +++ b/docs/ru/engines/table_engines/special/distributed.md @@ -61,12 +61,12 @@ logs - имя кластера в конфигурационном файле с В качестве параметров для каждого сервера указываются `host`, `port` и, не обязательно, `user`, `password`, `secure`, `compression`: - `host` - адрес удалённого сервера. Может быть указан домен, или IPv4 или IPv6 адрес. В случае указания домена, при старте сервера делается DNS запрос, и результат запоминается на всё время работы сервера. Если DNS запрос неуспешен, то сервер не запускается. Если вы изменяете DNS-запись, перезапустите сервер. - `port` - TCP-порт для межсерверного взаимодействия (в конфиге - tcp\_port, обычно 9000). Не перепутайте с http\_port. -- `user` - имя пользователя для соединения с удалённым сервером. по умолчанию - default. Этот пользователь должен иметь доступ для соединения с указанным сервером. Доступы настраиваются в файле users.xml, подробнее смотрите в разделе [Права доступа](../../operations/access_rights.md). +- `user` - имя пользователя для соединения с удалённым сервером. по умолчанию - default. Этот пользователь должен иметь доступ для соединения с указанным сервером. Доступы настраиваются в файле users.xml, подробнее смотрите в разделе [Права доступа](../../../operations/access_rights.md). - `password` - пароль для соединения с удалённым сервером, в открытом виде. по умолчанию - пустая строка. - `secure` - Использовать шифрованное соединение ssl, Обычно используется с портом `port` = 9440. Сервер должен слушать порт 9440 с корректными настройками сертификатов. - `compression` - Использовать сжатие данных. По умолчанию: true. -При указании реплик, для каждого из шардов, при чтении, будет выбрана одна из доступных реплик. Можно настроить алгоритм балансировки нагрузки (то есть, предпочтения, на какую из реплик идти) - см. настройку [load\_balancing](../settings/settings.md#settings-load_balancing). +При указании реплик, для каждого из шардов, при чтении, будет выбрана одна из доступных реплик. Можно настроить алгоритм балансировки нагрузки (то есть, предпочтения, на какую из реплик идти) - см. настройку [load\_balancing](../../../operations/settings/settings.md#settings-load_balancing). Если соединение с сервером не установлено, то будет произведена попытка соединения с небольшим таймаутом. Если соединиться не удалось, то будет выбрана следующая реплика, и так для всех реплик. Если попытка соединения для всех реплик не удалась, то будут снова произведены попытки соединения по кругу, и так несколько раз. Это работает в пользу отказоустойчивости, хотя и не обеспечивает полную отказоустойчивость: удалённый сервер может принять соединение, но не работать, или плохо работать. @@ -78,7 +78,7 @@ logs - имя кластера в конфигурационном файле с Движок Distributed позволяет работать с кластером, как с локальным сервером. При этом, кластер является неэластичным: вы должны прописать его конфигурацию в конфигурационный файл сервера (лучше всех серверов кластера). -Как видно, движок Distributed требует прописывания кластера в конфигурационный файл; кластера из конфигурационного файла обновляются налету, без перезапуска сервера. Если вам необходимо каждый раз отправлять запрос на неизвестный набор шардов и реплик, вы можете не создавать Distributed таблицу, а воспользоваться табличной функцией remote. Смотрите раздел [Табличные функции](../../query_language/table_functions/index.md). +Как видно, движок Distributed требует прописывания кластера в конфигурационный файл; кластера из конфигурационного файла обновляются налету, без перезапуска сервера. Если вам необходимо каждый раз отправлять запрос на неизвестный набор шардов и реплик, вы можете не создавать Distributed таблицу, а воспользоваться табличной функцией remote. Смотрите раздел [Табличные функции](../../../engines/table_engines/special/distributed.md). Есть два способа записывать данные на кластер: @@ -107,10 +107,10 @@ logs - имя кластера в конфигурационном файле с - используются запросы, требующие соединение данных (IN, JOIN) по определённому ключу - тогда если данные шардированы по этому ключу, то можно использовать локальные IN, JOIN вместо GLOBAL IN, GLOBAL JOIN, что кардинально более эффективно. - используется большое количество серверов (сотни и больше) и большое количество маленьких запросов (запросы отдельных клиентов - сайтов, рекламодателей, партнёров) - тогда, для того, чтобы маленькие запросы не затрагивали весь кластер, имеет смысл располагать данные одного клиента на одном шарде, или (вариант, который используется в Яндекс.Метрике) сделать двухуровневое шардирование: разбить весь кластер на «слои», где слой может состоять из нескольких шардов; данные для одного клиента располагаются на одном слое, но в один слой можно по мере необходимости добавлять шарды, в рамках которых данные распределены произвольным образом; создаются распределённые таблицы на каждый слой и одна общая распределённая таблица для глобальных запросов. -Запись данных осуществляется полностью асинхронно. При вставке в таблицу, блок данных сначала записывается в файловую систему. Затем, в фоновом режиме отправляются на удалённые серверы при первой возможности. Период отправки регулируется настройками [distributed\_directory\_monitor\_sleep\_time\_ms](../settings/settings.md#distributed_directory_monitor_sleep_time_ms) и [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../settings/settings.md#distributed_directory_monitor_max_sleep_time_ms). Движок таблиц `Distributed` отправляет каждый файл со вставленными данными отдельно, но можно включить пакетную отправку данных настройкой [distributed\_directory\_monitor\_batch\_inserts](../settings/settings.md#distributed_directory_monitor_batch_inserts). Эта настройка улучшает производительность кластера за счет более оптимального использования ресурсов сервера-отправителя и сети. Необходимо проверять, что данные отправлены успешно, для этого проверьте список файлов (данных, ожидающих отправки) в каталоге таблицы `/var/lib/clickhouse/data/database/table/`. +Запись данных осуществляется полностью асинхронно. При вставке в таблицу, блок данных сначала записывается в файловую систему. Затем, в фоновом режиме отправляются на удалённые серверы при первой возможности. Период отправки регулируется настройками [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) и [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms). Движок таблиц `Distributed` отправляет каждый файл со вставленными данными отдельно, но можно включить пакетную отправку данных настройкой [distributed\_directory\_monitor\_batch\_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts). Эта настройка улучшает производительность кластера за счет более оптимального использования ресурсов сервера-отправителя и сети. Необходимо проверять, что данные отправлены успешно, для этого проверьте список файлов (данных, ожидающих отправки) в каталоге таблицы `/var/lib/clickhouse/data/database/table/`. Если после INSERT-а в Distributed таблицу, сервер перестал существовать или был грубо перезапущен (например, в следствие аппаратного сбоя), то записанные данные могут быть потеряны. Если в директории таблицы обнаружен повреждённый кусок данных, то он переносится в поддиректорию broken и больше не используется. -При выставлении опции max\_parallel\_replicas выполнение запроса распараллеливается по всем репликам внутри одного шарда. Подробнее смотрите раздел [max\_parallel\_replicas](../settings/settings.md#settings-max_parallel_replicas). +При выставлении опции max\_parallel\_replicas выполнение запроса распараллеливается по всем репликам внутри одного шарда. Подробнее смотрите раздел [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/distributed/) diff --git a/docs/ru/operations/table_engines/external_data.md b/docs/ru/engines/table_engines/special/external_data.md similarity index 100% rename from docs/ru/operations/table_engines/external_data.md rename to docs/ru/engines/table_engines/special/external_data.md diff --git a/docs/ru/operations/table_engines/file.md b/docs/ru/engines/table_engines/special/file.md similarity index 81% rename from docs/ru/operations/table_engines/file.md rename to docs/ru/engines/table_engines/special/file.md index 2d248c22081..138c2e47b89 100644 --- a/docs/ru/operations/table_engines/file.md +++ b/docs/ru/engines/table_engines/special/file.md @@ -14,13 +14,13 @@ File(Format) ``` -`Format` должен быть таким, который ClickHouse может использовать и в запросах `INSERT` и в запросах `SELECT`. Полный список поддерживаемых форматов смотрите в разделе [Форматы](../../interfaces/formats.md#formats). +`Format` должен быть таким, который ClickHouse может использовать и в запросах `INSERT` и в запросах `SELECT`. Полный список поддерживаемых форматов смотрите в разделе [Форматы](../../../interfaces/formats.md#formats). -Сервер ClickHouse не позволяет указать путь к файлу, с которым будет работать `File`. Используется путь к хранилищу, определенный параметром [path](../server_settings/settings.md) в конфигурации сервера. +Сервер ClickHouse не позволяет указать путь к файлу, с которым будет работать `File`. Используется путь к хранилищу, определенный параметром [path](../../../operations/server_configuration_parameters/settings.md) в конфигурации сервера. При создании таблицы с помощью `File(Format)` сервер ClickHouse создает в хранилище каталог с именем таблицы, а после добавления в таблицу данных помещает туда файл `data.Format`. -Можно вручную создать в хранилище каталог таблицы, поместить туда файл, затем на сервере ClickHouse добавить ([ATTACH](../../query_language/misc.md)) информацию о таблице, соответствующей имени каталога и прочитать из файла данные. +Можно вручную создать в хранилище каталог таблицы, поместить туда файл, затем на сервере ClickHouse добавить ([ATTACH](../../../engines/table_engines/special/file.md)) информацию о таблице, соответствующей имени каталога и прочитать из файла данные. !!! warning "Warning" Будьте аккуратны с этой функциональностью, поскольку сервер ClickHouse не отслеживает внешние изменения данных. Если в файл будет производиться запись одновременно со стороны сервера ClickHouse и с внешней стороны, то результат непредсказуем. @@ -58,7 +58,7 @@ SELECT * FROM file_engine_table ## Использование движка в clickhouse-local {#ispolzovanie-dvizhka-v-clickhouse-local} -В [clickhouse-local](../utils/clickhouse-local.md) движок в качестве параметра принимает не только формат, но и путь к файлу. В том числе можно указать стандартные потоки ввода/вывода цифровым или буквенным обозначением `0` или `stdin`, `1` или `stdout`. +В [clickhouse-local](../../../engines/table_engines/special/file.md) движок в качестве параметра принимает не только формат, но и путь к файлу. В том числе можно указать стандартные потоки ввода/вывода цифровым или буквенным обозначением `0` или `stdin`, `1` или `stdout`. **Пример:** diff --git a/docs/ru/engines/table_engines/special/generate.md b/docs/ru/engines/table_engines/special/generate.md new file mode 120000 index 00000000000..566dc4e5382 --- /dev/null +++ b/docs/ru/engines/table_engines/special/generate.md @@ -0,0 +1 @@ +../../../../en/engines/table_engines/special/generate.md \ No newline at end of file diff --git a/docs/ru/engines/table_engines/special/index.md b/docs/ru/engines/table_engines/special/index.md new file mode 100644 index 00000000000..22cebf295c1 --- /dev/null +++ b/docs/ru/engines/table_engines/special/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: Special +toc_priority: 31 +--- + diff --git a/docs/ru/operations/table_engines/join.md b/docs/ru/engines/table_engines/special/join.md similarity index 70% rename from docs/ru/operations/table_engines/join.md rename to docs/ru/engines/table_engines/special/join.md index a9c06d05ebf..0ca53f34acb 100644 --- a/docs/ru/operations/table_engines/join.md +++ b/docs/ru/engines/table_engines/special/join.md @@ -1,6 +1,6 @@ # Join {#join} -Подготовленная структура данных для использования в операциях [JOIN](../../query_language/select.md#select-join). +Подготовленная структура данных для использования в операциях [JOIN](../../../engines/table_engines/special/join.md#select-join). ## Создание таблицы {#creating-a-table} @@ -12,12 +12,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = Join(join_strictness, join_type, k1[, k2, ...]) ``` -Смотрите подробное описание запроса [CREATE TABLE](../../query_language/create.md#create-table-query). +Смотрите подробное описание запроса [CREATE TABLE](../../../engines/table_engines/special/join.md#create-table-query). **Параметры движка** -- `join_strictness` – [строгость JOIN](../../query_language/select.md#select-join-strictness). -- `join_type` – [тип JOIN](../../query_language/select.md#select-join-types). +- `join_strictness` – [строгость JOIN](../../../engines/table_engines/special/join.md#select-join-strictness). +- `join_type` – [тип JOIN](../../../engines/table_engines/special/join.md#select-join-types). - `k1[, k2, ...]` – ключевые столбцы секции `USING` с которыми выполняется операция `JOIN`. Вводите параметры `join_strictness` и `join_type` без кавычек, например, `Join(ANY, LEFT, col1)`. Они должны быть такими же как и в той операции `JOIN`, в которой таблица будет использоваться. Если параметры не совпадают, ClickHouse не генерирует исключение и может возвращать неверные данные. @@ -79,21 +79,21 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) Из таблиц нельзя выбрать данные с помощью запроса `SELECT`. Вместо этого, используйте один из следующих методов: - Используйте таблицу как правую в секции `JOIN`. -- Используйте функцию [joinGet](../../query_language/functions/other_functions.md#joinget), которая позволяет извлекать данные из таблицы таким же образом как из словаря. +- Используйте функцию [joinGet](../../../engines/table_engines/special/join.md#joinget), которая позволяет извлекать данные из таблицы таким же образом как из словаря. ### Ограничения и настройки {#join-limitations-and-settings} При создании таблицы, применяются следующие параметры : -- [join\_use\_nulls](../settings/settings.md#join_use_nulls) -- [max\_rows\_in\_join](../settings/query_complexity.md#settings-max_rows_in_join) -- [max\_bytes\_in\_join](../settings/query_complexity.md#settings-max_bytes_in_join) -- [join\_overflow\_mode](../settings/query_complexity.md#settings-join_overflow_mode) -- [join\_any\_take\_last\_row](../settings/settings.md#settings-join_any_take_last_row) +- [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) +- [max\_rows\_in\_join](../../../operations/settings/query_complexity.md#settings-max_rows_in_join) +- [max\_bytes\_in\_join](../../../operations/settings/query_complexity.md#settings-max_bytes_in_join) +- [join\_overflow\_mode](../../../operations/settings/query_complexity.md#settings-join_overflow_mode) +- [join\_any\_take\_last\_row](../../../operations/settings/settings.md#settings-join_any_take_last_row) Таблицы с движком `Join` нельзя использовать в операциях `GLOBAL JOIN`. -Движок `Join` позволяет использовать параметр [join\_use\_nulls](../settings/settings.md#join_use_nulls) в запросе `CREATE TABLE`, который также можно использовать в запросе [SELECT](../../query_language/select.md). Если у вас разные настройки `join_use_nulls`, вы можете получить сообщение об ошибке при объединении таблиц. Это зависит от типа соединения. Когда вы используете функцию [joinGet](../../query_language/functions/other_functions.md#joinget), вам необходимо использовать один и тот же параметр `join_use_nulls` в запросах `CRATE TABLE` и `SELECT`. +Движок `Join` позволяет использовать параметр [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) в запросе `CREATE TABLE`, который также можно использовать в запросе [SELECT](../../../engines/table_engines/special/join.md). Если у вас разные настройки `join_use_nulls`, вы можете получить сообщение об ошибке при объединении таблиц. Это зависит от типа соединения. Когда вы используете функцию [joinGet](../../../engines/table_engines/special/join.md#joinget), вам необходимо использовать один и тот же параметр `join_use_nulls` в запросах `CRATE TABLE` и `SELECT`. ## Хранение данных {#khranenie-dannykh} diff --git a/docs/ru/engines/table_engines/special/materializedview.md b/docs/ru/engines/table_engines/special/materializedview.md new file mode 100644 index 00000000000..2adcdb8df70 --- /dev/null +++ b/docs/ru/engines/table_engines/special/materializedview.md @@ -0,0 +1,5 @@ +# MaterializedView {#materializedview} + +Используется для реализации материализованных представлений (подробнее см. запрос [CREATE TABLE](../../../engines/table_engines/special/materializedview.md)). Для хранения данных, использует другой движок, который был указан при создании представления. При чтении из таблицы, просто использует этот движок. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/materializedview/) diff --git a/docs/ru/operations/table_engines/memory.md b/docs/ru/engines/table_engines/special/memory.md similarity index 100% rename from docs/ru/operations/table_engines/memory.md rename to docs/ru/engines/table_engines/special/memory.md diff --git a/docs/ru/operations/table_engines/merge.md b/docs/ru/engines/table_engines/special/merge.md similarity index 98% rename from docs/ru/operations/table_engines/merge.md rename to docs/ru/engines/table_engines/special/merge.md index 4f11cf77352..65dd8dc7a2c 100644 --- a/docs/ru/operations/table_engines/merge.md +++ b/docs/ru/engines/table_engines/special/merge.md @@ -52,7 +52,7 @@ FROM WatchLog ## Виртуальные столбцы {#virtualnye-stolbtsy} -- `_table` — содержит имя таблицы, из которой данные были прочитаны. Тип — [String](../../data_types/string.md). +- `_table` — содержит имя таблицы, из которой данные были прочитаны. Тип — [String](../../../engines/table_engines/special/merge.md). В секции `WHERE/PREWHERE` можно установить константное условие на столбец `_table` (например, `WHERE _table='xyz'`). В этом случае операции чтения выполняются только для тех таблиц, для которых выполняется условие на значение `_table`, таким образом, столбец `_table` работает как индекс. diff --git a/docs/ru/operations/table_engines/null.md b/docs/ru/engines/table_engines/special/null.md similarity index 100% rename from docs/ru/operations/table_engines/null.md rename to docs/ru/engines/table_engines/special/null.md diff --git a/docs/ru/operations/table_engines/set.md b/docs/ru/engines/table_engines/special/set.md similarity index 100% rename from docs/ru/operations/table_engines/set.md rename to docs/ru/engines/table_engines/special/set.md diff --git a/docs/ru/operations/table_engines/url.md b/docs/ru/engines/table_engines/special/url.md similarity index 95% rename from docs/ru/operations/table_engines/url.md rename to docs/ru/engines/table_engines/special/url.md index 6f9d22e887d..5f4696286d7 100644 --- a/docs/ru/operations/table_engines/url.md +++ b/docs/ru/engines/table_engines/special/url.md @@ -7,7 +7,7 @@ `Format` должен быть таким, который ClickHouse может использовать в запросах `SELECT` и, если есть необходимость, `INSERT`. Полный список поддерживаемых форматов смотрите в -разделе [Форматы](../../interfaces/formats.md#formats). +разделе [Форматы](../../../interfaces/formats.md#formats). `URL` должен соответствовать структуре Uniform Resource Locator. По указанному URL должен находится сервер работающий по протоколу HTTP или HTTPS. При этом не должно требоваться никаких @@ -17,7 +17,7 @@ соответственно. Для обработки `POST`-запросов удаленный сервер должен поддерживать [Chunked transfer encoding](https://ru.wikipedia.org/wiki/Chunked_transfer_encoding). -Максимальное количество переходов по редиректам при выполнении HTTP-запроса методом GET можно ограничить с помощью настройки [max\_http\_get\_redirects](../settings/settings.md#setting-max_http_get_redirects). +Максимальное количество переходов по редиректам при выполнении HTTP-запроса методом GET можно ограничить с помощью настройки [max\_http\_get\_redirects](../../../operations/settings/settings.md#setting-max_http_get_redirects). **Пример:** diff --git a/docs/ru/operations/table_engines/view.md b/docs/ru/engines/table_engines/special/view.md similarity index 100% rename from docs/ru/operations/table_engines/view.md rename to docs/ru/engines/table_engines/special/view.md diff --git a/docs/ru/faq/general.md b/docs/ru/faq/general.md index e5a5e0c00fa..5bfe8ea8f2d 100644 --- a/docs/ru/faq/general.md +++ b/docs/ru/faq/general.md @@ -25,7 +25,7 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 ### Секция INTO OUTFILE {#sektsiia-into-outfile} -Добавьте секцию [INTO OUTFILE](../query_language/select/#into-outfile-clause) к своему запросу. +Добавьте секцию [INTO OUTFILE](../sql_reference/statements/select.md#into-outfile-clause) к своему запросу. Например: @@ -33,7 +33,7 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 SELECT * FROM table INTO OUTFILE 'file' ``` -По умолчанию, для выдачи данных ClickHouse использует формат [TabSeparated](../interfaces/formats.md#tabseparated). Чтобы выбрать [формат данных](../interfaces/formats.md), используйте [секцию FORMAT](../query_language/select/#format-clause). +По умолчанию, для выдачи данных ClickHouse использует формат [TabSeparated](../interfaces/formats.md#tabseparated). Чтобы выбрать [формат данных](../interfaces/formats.md), используйте [секцию FORMAT](../sql_reference/statements/select.md#format-clause). Например: @@ -43,7 +43,7 @@ SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV ### Таблица с движком File {#tablitsa-s-dvizhkom-file} -Смотрите [File](../operations/table_engines/file.md). +Смотрите [File](../engines/table_engines/special/file.md). ### Перенаправление в командой строке {#perenapravlenie-v-komandoi-stroke} diff --git a/docs/ru/faq/index.md b/docs/ru/faq/index.md new file mode 100644 index 00000000000..2ee9d51e83b --- /dev/null +++ b/docs/ru/faq/index.md @@ -0,0 +1,6 @@ +--- +toc_folder_title: F.A.Q. +toc_priority: 76 +--- + + diff --git a/docs/ru/getting_started/example_datasets/index.md b/docs/ru/getting_started/example_datasets/index.md index c01f73ddc4d..5fbe0843f4b 100644 --- a/docs/ru/getting_started/example_datasets/index.md +++ b/docs/ru/getting_started/example_datasets/index.md @@ -1,3 +1,8 @@ +--- +toc_folder_title: Тестовые массивы данных +toc_priority: 12 +toc_title: Обзор +--- # Тестовые массивы данных Этот раздел описывает как получить тестовые массивы данных и загрузить их в ClickHouse. diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md index aa126f7b697..7711d6b94ee 100644 --- a/docs/ru/getting_started/index.md +++ b/docs/ru/getting_started/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: Начало работы +toc_hidden: true +toc_priority: 8 +toc_title: hidden +--- + # Начало работы {#nachalo-raboty} Если вы новичок в ClickHouse и хотите вживую оценить его производительность, прежде всего нужно пройти через [процесс установки](install.md). diff --git a/docs/ru/getting_started/install.md b/docs/ru/getting_started/install.md index 7caffb498e9..cd571156d03 100644 --- a/docs/ru/getting_started/install.md +++ b/docs/ru/getting_started/install.md @@ -57,7 +57,7 @@ sudo yum install clickhouse-server clickhouse-client Также есть возможность установить пакеты вручную, скачав отсюда: https://repo.yandex.ru/clickhouse/rpm/stable/x86\_64. -### Из tgz архивов {#from-tgz-archives} +### Из Tgz архивов {#from-tgz-archives} Команда ClickHouse в Яндексе рекомендует использовать предкомпилированные бинарники из `tgz` архивов для всех дистрибутивов, где невозможна установка `deb` и `rpm` пакетов. diff --git a/docs/ru/getting_started/tutorial.md b/docs/ru/getting_started/tutorial.md deleted file mode 100644 index 4f23dbe756d..00000000000 --- a/docs/ru/getting_started/tutorial.md +++ /dev/null @@ -1,668 +0,0 @@ ---- -en_copy: true ---- - -# ClickHouse Tutorial {#clickhouse-tutorial} - -## What to Expect from This Tutorial? {#what-to-expect-from-this-tutorial} - -By going through this tutorial, you’ll learn how to set up a simple ClickHouse cluster. It’ll be small, but fault-tolerant and scalable. Then we will use one of the example datasets to fill it with data and execute some demo queries. - -## Single Node Setup {#single-node-setup} - -To postpone the complexities of a distributed environment, we’ll start with deploying ClickHouse on a single server or virtual machine. ClickHouse is usually installed from [deb](index.md#install-from-deb-packages) or [rpm](index.md#from-rpm-packages) packages, but there are [alternatives](index.md#from-docker-image) for the operating systems that do no support them. - -For example, you have chosen `deb` packages and executed: - -``` bash -sudo apt-get install dirmngr -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 - -echo "deb http://repo.clickhouse.tech/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list -sudo apt-get update - -sudo apt-get install -y clickhouse-server clickhouse-client -``` - -What do we have in the packages that got installed: - -- `clickhouse-client` package contains [clickhouse-client](../interfaces/cli.md) application, interactive ClickHouse console client. -- `clickhouse-common` package contains a ClickHouse executable file. -- `clickhouse-server` package contains configuration files to run ClickHouse as a server. - -Server config files are located in `/etc/clickhouse-server/`. Before going further, please notice the `` element in `config.xml`. Path determines the location for data storage, so it should be located on volume with large disk capacity; the default value is `/var/lib/clickhouse/`. If you want to adjust the configuration, it’s not handy to directly edit `config.xml` file, considering it might get rewritten on future package updates. The recommended way to override the config elements is to create [files in config.d directory](../operations/configuration_files.md) which serve as “patches” to config.xml. - -As you might have noticed, `clickhouse-server` is not launched automatically after package installation. It won’t be automatically restarted after updates, either. The way you start the server depends on your init system, usually, it is: - -``` bash -sudo service clickhouse-server start -``` - -or - -``` bash -sudo /etc/init.d/clickhouse-server start -``` - -The default location for server logs is `/var/log/clickhouse-server/`. The server is ready to handle client connections once it logs the `Ready for connections` message. - -Once the `clickhouse-server` is up and running, we can use `clickhouse-client` to connect to the server and run some test queries like `SELECT "Hello, world!";`. - -
- -Quick tips for clickhouse-client -Interactive mode: - -``` bash -clickhouse-client -clickhouse-client --host=... --port=... --user=... --password=... -``` - -Enable multiline queries: - -``` bash -clickhouse-client -m -clickhouse-client --multiline -``` - -Run queries in batch-mode: - -``` bash -clickhouse-client --query='SELECT 1' -echo 'SELECT 1' | clickhouse-client -clickhouse-client <<< 'SELECT 1' -``` - -Insert data from a file in specified format: - -``` bash -clickhouse-client --query='INSERT INTO table VALUES' < data.txt -clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv -``` - -
- -## Import Sample Dataset {#import-sample-dataset} - -Now it’s time to fill our ClickHouse server with some sample data. In this tutorial, we’ll use the anonymized data of Yandex.Metrica, the first service that runs ClickHouse in production way before it became open-source (more on that in [history section](../introduction/history.md)). There are [multiple ways to import Yandex.Metrica dataset](example_datasets/metrica.md), and for the sake of the tutorial, we’ll go with the most realistic one. - -### Download and Extract Table Data {#download-and-extract-table-data} - -``` bash -curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv -``` - -The extracted files are about 10GB in size. - -### Create Tables {#create-tables} - -As in most databases management systems, ClickHouse logically groups tables into “databases”. There’s a `default` database, but we’ll create a new one named `tutorial`: - -``` bash -clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" -``` - -Syntax for creating tables is way more complicated compared to databases (see [reference](../query_language/create.md). In general `CREATE TABLE` statement has to specify three key things: - -1. Name of table to create. -2. Table schema, i.e. list of columns and their [data types](../data_types/index.md). -3. [Table engine](../operations/table_engines/index.md) and it’s settings, which determines all the details on how queries to this table will be physically executed. - -Yandex.Metrica is a web analytics service, and sample dataset doesn’t cover its full functionality, so there are only two tables to create: - -- `hits` is a table with each action done by all users on all websites covered by the service. -- `visits` is a table that contains pre-built sessions instead of individual actions. - -Let’s see and execute the real create table queries for these tables: - -``` sql -CREATE TABLE tutorial.hits_v1 -( - `WatchID` UInt64, - `JavaEnable` UInt8, - `Title` String, - `GoodEvent` Int16, - `EventTime` DateTime, - `EventDate` Date, - `CounterID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RegionID` UInt32, - `UserID` UInt64, - `CounterClass` Int8, - `OS` UInt8, - `UserAgent` UInt8, - `URL` String, - `Referer` String, - `URLDomain` String, - `RefererDomain` String, - `Refresh` UInt8, - `IsRobot` UInt8, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `FlashMinor2` String, - `NetMajor` UInt8, - `NetMinor` UInt8, - `UserAgentMajor` UInt16, - `UserAgentMinor` FixedString(2), - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `MobilePhone` UInt8, - `MobilePhoneModel` String, - `Params` String, - `IPNetworkID` UInt32, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `IsArtifical` UInt8, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `ClientTimeZone` Int16, - `ClientEventTime` DateTime, - `SilverlightVersion1` UInt8, - `SilverlightVersion2` UInt8, - `SilverlightVersion3` UInt32, - `SilverlightVersion4` UInt16, - `PageCharset` String, - `CodeVersion` UInt32, - `IsLink` UInt8, - `IsDownload` UInt8, - `IsNotBounce` UInt8, - `FUniqID` UInt64, - `HID` UInt32, - `IsOldCounter` UInt8, - `IsEvent` UInt8, - `IsParameter` UInt8, - `DontCountHits` UInt8, - `WithHash` UInt8, - `HitColor` FixedString(1), - `UTCEventTime` DateTime, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `WindowName` Int32, - `OpenerName` Int32, - `HistoryLength` Int16, - `BrowserLanguage` FixedString(2), - `BrowserCountry` FixedString(2), - `SocialNetwork` String, - `SocialAction` String, - `HTTPError` UInt16, - `SendTiming` Int32, - `DNSTiming` Int32, - `ConnectTiming` Int32, - `ResponseStartTiming` Int32, - `ResponseEndTiming` Int32, - `FetchTiming` Int32, - `RedirectTiming` Int32, - `DOMInteractiveTiming` Int32, - `DOMContentLoadedTiming` Int32, - `DOMCompleteTiming` Int32, - `LoadEventStartTiming` Int32, - `LoadEventEndTiming` Int32, - `NSToDOMContentLoadedTiming` Int32, - `FirstPaintTiming` Int32, - `RedirectCount` Int8, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `ParamPrice` Int64, - `ParamOrderID` String, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `GoalsReached` Array(UInt32), - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `RefererHash` UInt64, - `URLHash` UInt64, - `CLID` UInt32, - `YCLID` UInt64, - `ShareService` String, - `ShareURL` String, - `ShareTitle` String, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `IslandID` FixedString(16), - `RequestNum` UInt32, - `RequestTry` UInt8 -) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(EventDate) -ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -``` sql -CREATE TABLE tutorial.visits_v1 -( - `CounterID` UInt32, - `StartDate` Date, - `Sign` Int8, - `IsNew` UInt8, - `VisitID` UInt64, - `UserID` UInt64, - `StartTime` DateTime, - `Duration` UInt32, - `UTCStartTime` DateTime, - `PageViews` Int32, - `Hits` Int32, - `IsBounce` UInt8, - `Referer` String, - `StartURL` String, - `RefererDomain` String, - `StartURLDomain` String, - `EndURL` String, - `LinkURL` String, - `IsDownload` UInt8, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `PlaceID` Int32, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `IsYandex` UInt8, - `GoalReachesDepth` Int32, - `GoalReachesURL` Int32, - `GoalReachesAny` Int32, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `MobilePhoneModel` String, - `ClientEventTime` DateTime, - `RegionID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `IPNetworkID` UInt32, - `SilverlightVersion3` UInt32, - `CodeVersion` UInt32, - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `UserAgentMajor` UInt16, - `UserAgentMinor` UInt16, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `SilverlightVersion2` UInt8, - `SilverlightVersion4` UInt16, - `FlashVersion3` UInt16, - `FlashVersion4` UInt16, - `ClientTimeZone` Int16, - `OS` UInt8, - `UserAgent` UInt8, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `NetMajor` UInt8, - `NetMinor` UInt8, - `MobilePhone` UInt8, - `SilverlightVersion1` UInt8, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `JavaEnable` UInt8, - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `BrowserLanguage` UInt16, - `BrowserCountry` UInt16, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `Params` Array(String), - `Goals` Nested( - ID UInt32, - Serial UInt32, - EventTime DateTime, - Price Int64, - OrderID String, - CurrencyID UInt32), - `WatchIDs` Array(UInt64), - `ParamSumPrice` Int64, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `ClickLogID` UInt64, - `ClickEventID` Int32, - `ClickGoodEvent` Int32, - `ClickEventTime` DateTime, - `ClickPriorityID` Int32, - `ClickPhraseID` Int32, - `ClickPageID` Int32, - `ClickPlaceID` Int32, - `ClickTypeID` Int32, - `ClickResourceID` Int32, - `ClickCost` UInt32, - `ClickClientIP` UInt32, - `ClickDomainID` UInt32, - `ClickURL` String, - `ClickAttempt` UInt8, - `ClickOrderID` UInt32, - `ClickBannerID` UInt32, - `ClickMarketCategoryID` UInt32, - `ClickMarketPP` UInt32, - `ClickMarketCategoryName` String, - `ClickMarketPPName` String, - `ClickAWAPSCampaignName` String, - `ClickPageName` String, - `ClickTargetType` UInt16, - `ClickTargetPhraseID` UInt64, - `ClickContextType` UInt8, - `ClickSelectType` Int8, - `ClickOptions` String, - `ClickGroupBannerID` Int32, - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `FirstVisit` DateTime, - `PredLastVisit` Date, - `LastVisit` Date, - `TotalVisits` UInt32, - `TraficSource` Nested( - ID Int8, - SearchEngineID UInt16, - AdvEngineID UInt8, - PlaceID UInt16, - SocialSourceNetworkID UInt8, - Domain String, - SearchPhrase String, - SocialSourcePage String), - `Attendance` FixedString(16), - `CLID` UInt32, - `YCLID` UInt64, - `NormalizedRefererHash` UInt64, - `SearchPhraseHash` UInt64, - `RefererDomainHash` UInt64, - `NormalizedStartURLHash` UInt64, - `StartURLDomainHash` UInt64, - `NormalizedEndURLHash` UInt64, - `TopLevelDomain` UInt64, - `URLScheme` UInt64, - `OpenstatServiceNameHash` UInt64, - `OpenstatCampaignIDHash` UInt64, - `OpenstatAdIDHash` UInt64, - `OpenstatSourceIDHash` UInt64, - `UTMSourceHash` UInt64, - `UTMMediumHash` UInt64, - `UTMCampaignHash` UInt64, - `UTMContentHash` UInt64, - `UTMTermHash` UInt64, - `FromHash` UInt64, - `WebVisorEnabled` UInt8, - `WebVisorActivity` UInt32, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `Market` Nested( - Type UInt8, - GoalID UInt32, - OrderID String, - OrderPrice Int64, - PP UInt32, - DirectPlaceID UInt32, - DirectOrderID UInt32, - DirectBannerID UInt32, - GoodID String, - GoodName String, - GoodQuantity Int32, - GoodPrice Int64), - `IslandID` FixedString(16) -) -ENGINE = CollapsingMergeTree(Sign) -PARTITION BY toYYYYMM(StartDate) -ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -You can execute those queries using the interactive mode of `clickhouse-client` (just launch it in a terminal without specifying a query in advance) or try some [alternative interface](../interfaces/index.md) if you want. - -As we can see, `hits_v1` uses the [basic MergeTree engine](../operations/table_engines/mergetree.md), while the `visits_v1` uses the [Collapsing](../operations/table_engines/collapsingmergetree.md) variant. - -### Import Data {#import-data} - -Data import to ClickHouse is done via [INSERT INTO](../query_language/insert_into.md) query like in many other SQL databases. However, data is usually provided in one of the [supported serialization formats](../interfaces/formats.md) instead of `VALUES` clause (which is also supported). - -The files we downloaded earlier are in tab-separated format, so here’s how to import them via console client: - -``` bash -clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv -clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv -``` - -ClickHouse has a lot of [settings to tune](../operations/settings/index.md) and one way to specify them in console client is via arguments, as we can see with `--max_insert_block_size`. The easiest way to figure out what settings are available, what do they mean and what the defaults are is to query the `system.settings` table: - -``` sql -SELECT name, value, changed, description -FROM system.settings -WHERE name LIKE '%max_insert_b%' -FORMAT TSV - -max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." -``` - -Optionally you can [OPTIMIZE](../query_language/misc/#misc_operations-optimize) the tables after import. Tables that are configured with an engine from MergeTree-family always do merges of data parts in the background to optimize data storage (or at least check if it makes sense). These queries force the table engine to do storage optimization right now instead of some time later: - -``` bash -clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" -clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" -``` - -These queries start an I/O and CPU intensive operation, so if the table consistently receives new data, it’s better to leave it alone and let merges run in the background. - -Now we can check if the table import was successful: - -``` bash -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" -``` - -## Example Queries {#example-queries} - -``` sql -SELECT - StartURL AS URL, - AVG(Duration) AS AvgDuration -FROM tutorial.visits_v1 -WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' -GROUP BY URL -ORDER BY AvgDuration DESC -LIMIT 10 -``` - -``` sql -SELECT - sum(Sign) AS visits, - sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, - (100. * goal_visits) / visits AS goal_percent -FROM tutorial.visits_v1 -WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') -``` - -## Cluster Deployment {#cluster-deployment} - -ClickHouse cluster is a homogenous cluster. Steps to set up: - -1. Install ClickHouse server on all machines of the cluster -2. Set up cluster configs in configuration files -3. Create local tables on each instance -4. Create a [Distributed table](../operations/table_engines/distributed.md) - -[Distributed table](../operations/table_engines/distributed.md) is actually a kind of “view” to local tables of ClickHouse cluster. SELECT query from a distributed table executes using resources of all cluster’s shards. You may specify configs for multiple clusters and create multiple distributed tables providing views to different clusters. - -Example config for a cluster with three shards, one replica each: - -``` xml - - - - - example-perftest01j.yandex.ru - 9000 - - - - - example-perftest02j.yandex.ru - 9000 - - - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -For further demonstration, let’s create a new local table with the same `CREATE TABLE` query that we used for `hits_v1`, but different table name: - -``` sql -CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... -``` - -Creating a distributed table providing a view into local tables of the cluster: - -``` sql -CREATE TABLE tutorial.hits_all AS tutorial.hits_local -ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); -``` - -A common practice is to create similar Distributed tables on all machines of the cluster. It allows running distributed queries on any machine of the cluster. Also there’s an alternative option to create temporary distributed table for a given SELECT query using [remote](../query_language/table_functions/remote.md) table function. - -Let’s run [INSERT SELECT](../query_language/insert_into.md) into the Distributed table to spread the table to multiple servers. - -``` sql -INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; -``` - -!!! warning "Notice" - This approach is not suitable for the sharding of large tables. There’s a separate tool [clickhouse-copier](../operations/utils/clickhouse-copier.md) that can re-shard arbitrary large tables. - -As you could expect, computationally heavy queries run N times faster if they utilize 3 servers instead of one. - -In this case, we have used a cluster with 3 shards, and each contains a single replica. - -To provide resilience in a production environment, we recommend that each shard should contain 2-3 replicas spread between multiple availability zones or datacenters (or at least racks). Note that ClickHouse supports an unlimited number of replicas. - -Example config for a cluster of one shard containing three replicas: - -``` xml - - ... - - - - example-perftest01j.yandex.ru - 9000 - - - example-perftest02j.yandex.ru - 9000 - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -To enable native replication [ZooKeeper](http://zookeeper.apache.org/) is required. ClickHouse takes care of data consistency on all replicas and runs restore procedure after failure automatically. It’s recommended to deploy the ZooKeeper cluster on separate servers (where no other processes including ClickHouse are running). - -!!! note "Note" - ZooKeeper is not a strict requirement: in some simple cases, you can duplicate the data by writing it into all the replicas from your application code. This approach is **not** recommended, in this case, ClickHouse won’t be able to guarantee data consistency on all replicas. Thus it becomes the responsibility of your application. - -ZooKeeper locations are specified in the configuration file: - -``` xml - - - zoo01.yandex.ru - 2181 - - - zoo02.yandex.ru - 2181 - - - zoo03.yandex.ru - 2181 - - -``` - -Also, we need to set macros for identifying each shard and replica which are used on table creation: - -``` xml - - 01 - 01 - -``` - -If there are no replicas at the moment on replicated table creation, a new first replica is instantiated. If there are already live replicas, the new replica clones data from existing ones. You have an option to create all replicated tables first, and then insert data to it. Another option is to create some replicas and add the others after or during data insertion. - -``` sql -CREATE TABLE tutorial.hits_replica (...) -ENGINE = ReplcatedMergeTree( - '/clickhouse_perftest/tables/{shard}/hits', - '{replica}' -) -... -``` - -Here we use [ReplicatedMergeTree](../operations/table_engines/replication.md) table engine. In parameters we specify ZooKeeper path containing shard and replica identifiers. - -``` sql -INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; -``` - -Replication operates in multi-master mode. Data can be loaded into any replica, and the system then syncs it with other instances automatically. Replication is asynchronous so at a given moment, not all replicas may contain recently inserted data. At least one replica should be up to allow data ingestion. Others will sync up data and repair consistency once they will become active again. Note that this approach allows for the low possibility of a loss of recently inserted data. - -[Original article](https://clickhouse.tech/docs/en/getting_started/tutorial/) diff --git a/docs/ru/getting_started/tutorial.md b/docs/ru/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/ru/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/ru/guides/apply_catboost_model.md b/docs/ru/guides/apply_catboost_model.md index f6c9799a171..2f94753b7f2 100644 --- a/docs/ru/guides/apply_catboost_model.md +++ b/docs/ru/guides/apply_catboost_model.md @@ -178,7 +178,7 @@ LIMIT 10 ``` !!! note "Примечание" - Функция [modelEvaluate](../query_language/functions/other_functions.md#function-modelevaluate) возвращает кортежи (tuple) с исходными прогнозами по классам для моделей с несколькими классами. + Функция [modelEvaluate](../sql_reference/functions/other_functions.md#function-modelevaluate) возвращает кортежи (tuple) с исходными прогнозами по классам для моделей с несколькими классами. Спрогнозируйте вероятность: @@ -201,7 +201,7 @@ LIMIT 10 ``` !!! note "Примечание" - Подробнее про функцию [exp()](../query_language/functions/math_functions.md). + Подробнее про функцию [exp()](../sql_reference/functions/math_functions.md). Посчитайте логистическую функцию потерь (LogLoss) на всей выборке: @@ -227,4 +227,4 @@ FROM ``` !!! note "Примечание" - Подробнее про функции [avg()](../query_language/agg_functions/reference.md#agg_function-avg), [log()](../query_language/functions/math_functions.md). + Подробнее про функции [avg()](../sql_reference/aggregate_functions/reference.md#agg_function-avg), [log()](../sql_reference/functions/math_functions.md). diff --git a/docs/ru/guides/index.md b/docs/ru/guides/index.md index 1ece4b22b85..2ba84e9afc2 100644 --- a/docs/ru/guides/index.md +++ b/docs/ru/guides/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: Руководства +toc_priority: 38 +toc_title: Обзор +--- + # Руководства {#rukovodstva} Подробные пошаговые инструкции, которые помогут вам решать различные задачи с помощью ClickHouse. diff --git a/docs/ru/index.md b/docs/ru/index.md index c49c33adf6f..e537c240cb6 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -1,3 +1,8 @@ +--- +toc_priority: 0 +toc_title: Обзор +--- + # Что такое ClickHouse {#chto-takoe-clickhouse} ClickHouse - столбцовая система управления базами данных (СУБД) для онлайн обработки аналитических запросов (OLAP). @@ -77,48 +82,6 @@ ClickHouse - столбцовая система управления базам Например, для запроса «посчитать количество записей для каждой рекламной системы», требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия, возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть, такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике, такая скорость действительно достигается. -
- -Пример - -``` bash -$ clickhouse-client -ClickHouse client version 0.0.52053. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.52053. -``` - -``` sql -SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 -``` - -``` text -┌─CounterID─┬──count()─┐ -│ 114208 │ 56057344 │ -│ 115080 │ 51619590 │ -│ 3228 │ 44658301 │ -│ 38230 │ 42045932 │ -│ 145263 │ 42042158 │ -│ 91244 │ 38297270 │ -│ 154139 │ 26647572 │ -│ 150748 │ 24112755 │ -│ 242232 │ 21302571 │ -│ 338158 │ 13507087 │ -│ 62180 │ 12229491 │ -│ 82264 │ 12187441 │ -│ 232261 │ 12148031 │ -│ 146272 │ 11438516 │ -│ 168777 │ 11403636 │ -│ 4120072 │ 11227824 │ -│ 10938808 │ 10519739 │ -│ 74088 │ 9047015 │ -│ 115079 │ 8837972 │ -│ 337234 │ 8205961 │ -└───────────┴──────────┘ -``` - -
- ### По вычислениям {#po-vychisleniiam} Так как для выполнения запроса надо обработать достаточно большое количество строк, становится актуальным диспетчеризовывать все операции не для отдельных строк, а для целых векторов, или реализовать движок выполнения запроса так, чтобы издержки на диспетчеризацию были примерно нулевыми. Если этого не делать, то при любой не слишком плохой дисковой подсистеме, интерпретатор запроса неизбежно упрётся в CPU. diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index 749e93c34ad..b76e96cc1dc 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -88,7 +88,7 @@ clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {p ``` - `name` — идентификатор подстановки. В консольном клиенте его следует использовать как часть имени параметра `--param_ = value`. -- `data type` — [тип данных](../data_types/index.md) значения. Например, структура данных `(integer, ('string', integer))` может иметь тип данных `Tuple(UInt8, Tuple(String, UInt8))` ([целочисленный](../data_types/int_uint.md) тип может быть и другим). +- `data type` — [тип данных](../sql_reference/data_types/index.md) значения. Например, структура данных `(integer, ('string', integer))` может иметь тип данных `Tuple(UInt8, Tuple(String, UInt8))` ([целочисленный](../sql_reference/data_types/int_uint.md) тип может быть и другим). #### Пример {#primer} diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 8d5a38a3a63..b1707a55193 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -99,9 +99,9 @@ world Массивы форматируются в виде списка значений через запятую в квадратных скобках. Элементы массива - числа форматируются как обычно, а даты, даты-с-временем и строки - в одинарных кавычках с такими же правилами экранирования, как указано выше. -[NULL](../query_language/syntax.md) форматируется как `\N`. +[NULL](../sql_reference/syntax.md) форматируется как `\N`. -Каждый элемент структуры типа [Nested](../data_types/nested_data_structures/nested.md) представляется как отдельный массив. +Каждый элемент структуры типа [Nested](../sql_reference/data_types/nested_data_structures/nested.md) представляется как отдельный массив. Например: @@ -302,7 +302,7 @@ SearchPhrase=дизайн штор count()=1064 SearchPhrase=баку count()=1000 ``` -[NULL](../query_language/syntax.md) форматируется как `\N`. +[NULL](../sql_reference/syntax.md) форматируется как `\N`. ``` sql SELECT * FROM t_null FORMAT TSKV @@ -432,7 +432,7 @@ JSON совместим с JavaScript. Для этого, дополнитель Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). -ClickHouse поддерживает [NULL](../query_language/syntax.md), который при выводе JSON будет отображен как `null`. +ClickHouse поддерживает [NULL](../sql_reference/syntax.md), который при выводе JSON будет отображен как `null`. Смотрите также формат [JSONEachRow](#jsoneachrow) . @@ -507,7 +507,7 @@ ClickHouse игнорирует пробелы между элементами **Обработка пропущенных значений** -ClickHouse заменяет опущенные значения значениями по умолчанию для соответствующих [data types](../data_types/index.md). +ClickHouse заменяет опущенные значения значениями по умолчанию для соответствующих [data types](../sql_reference/data_types/index.md). Если указано `DEFAULT expr`, то ClickHouse использует различные правила подстановки в зависимости от настройки [input\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields). @@ -552,7 +552,7 @@ CREATE TABLE IF NOT EXISTS example_table ### Использование вложенных структур {#jsoneachrow-nested} -Если у вас есть таблица со столбцами типа [Nested](../data_types/nested_data_structures/nested.md), то в неё можно вставить данные из JSON-документа с такой же структурой. Функциональность включается настройкой [input\_format\_import\_nested\_json](../operations/settings/settings.md#settings-input_format_import_nested_json). +Если у вас есть таблица со столбцами типа [Nested](../sql_reference/data_types/nested_data_structures/nested.md), то в неё можно вставить данные из JSON-документа с такой же структурой. Функциональность включается настройкой [input\_format\_import\_nested\_json](../operations/settings/settings.md#settings-input_format_import_nested_json). Например, рассмотрим следующую таблицу: @@ -626,7 +626,7 @@ SELECT * FROM json_each_row_nested Рисуется полная сетка таблицы и, таким образом, каждая строчка занимает две строки в терминале. Каждый блок результата выводится в виде отдельной таблицы. Это нужно, чтобы можно было выводить блоки без буферизации результата (буферизация потребовалась бы, чтобы заранее вычислить видимую ширину всех значений.) -[NULL](../query_language/syntax.md) выводится как `ᴺᵁᴸᴸ`. +[NULL](../sql_reference/syntax.md) выводится как `ᴺᵁᴸᴸ`. ``` sql SELECT * FROM t_null @@ -728,7 +728,7 @@ FixedString представлены просто как последовате Array представлены как длина в формате varint (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), а затем элементы массива, подряд. -Для поддержки [NULL](../query_language/syntax.md#null-literal) перед каждым значением типа [Nullable](../data_types/nullable.md) следует байт содержащий 1 или 0. Если байт 1, то значение равно NULL, и этот байт интерпретируется как отдельное значение (т.е. после него следует значение следующего поля). Если байт 0, то после байта следует значение поля (не равно NULL). +Для поддержки [NULL](../sql_reference/syntax.md#null-literal) перед каждым значением типа [Nullable](../sql_reference/data_types/nullable.md) следует байт содержащий 1 или 0. Если байт 1, то значение равно NULL, и этот байт интерпретируется как отдельное значение (т.е. после него следует значение следующего поля). Если байт 0, то после байта следует значение поля (не равно NULL). ## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes} @@ -740,7 +740,7 @@ Array представлены как длина в формате varint (unsig ## Values {#data-format-values} -Выводит каждую строку в скобках. Строки разделены запятыми. После последней строки запятой нет. Значения внутри скобок также разделены запятыми. Числа выводятся в десятичном виде без кавычек. Массивы выводятся в квадратных скобках. Строки, даты, даты-с-временем выводятся в кавычках. Правила экранирования и особенности парсинга аналогичны формату [TabSeparated](#tabseparated). При форматировании, лишние пробелы не ставятся, а при парсинге - допустимы и пропускаются (за исключением пробелов внутри значений типа массив, которые недопустимы). [NULL](../query_language/syntax.md) представляется как `NULL`. +Выводит каждую строку в скобках. Строки разделены запятыми. После последней строки запятой нет. Значения внутри скобок также разделены запятыми. Числа выводятся в десятичном виде без кавычек. Массивы выводятся в квадратных скобках. Строки, даты, даты-с-временем выводятся в кавычках. Правила экранирования и особенности парсинга аналогичны формату [TabSeparated](#tabseparated). При форматировании, лишние пробелы не ставятся, а при парсинге - допустимы и пропускаются (за исключением пробелов внутри значений типа массив, которые недопустимы). [NULL](../sql_reference/syntax.md) представляется как `NULL`. Минимальный набор символов, которых вам необходимо экранировать при передаче в Values формате: одинарная кавычка и обратный слеш. @@ -750,7 +750,7 @@ Array представлены как длина в формате varint (unsig Выводит каждое значение на отдельной строке, с указанием имени столбца. Формат удобно использовать для вывода одной-нескольких строк, если каждая строка состоит из большого количества столбцов. -[NULL](../query_language/syntax.md) выводится как `ᴺᵁᴸᴸ`. +[NULL](../sql_reference/syntax.md) выводится как `ᴺᵁᴸᴸ`. Пример: @@ -928,7 +928,7 @@ message MessageType { ``` ClickHouse попытается найти столбец с именем `x.y.z` (или `x_y_z`, или `X.y_Z` и т.п.). -Вложенные сообщения удобно использовать в качестве соответствия для [вложенной структуры данных](../data_types/nested_data_structures/nested.md). +Вложенные сообщения удобно использовать в качестве соответствия для [вложенной структуры данных](../sql_reference/data_types/nested_data_structures/nested.md). Значения по умолчанию, определённые в схеме `proto2`, например, @@ -940,7 +940,7 @@ message MessageType { } ``` -не применяются; вместо них используются определенные в таблице [значения по умолчанию](../query_language/create.md#create-default-values). +не применяются; вместо них используются определенные в таблице [значения по умолчанию](../sql_reference/statements/create.md#create-default-values). ClickHouse пишет и читает сообщения `Protocol Buffers` в формате `length-delimited`. Это означает, что перед каждым сообщением пишется его длина в формате [varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). См. также [как читать и записывать сообщения Protocol Buffers в формате length-delimited в различных языках программирования](https://cwiki.apache.org/confluence/display/GEODE/Delimiting+Protobuf+Messages). @@ -951,25 +951,25 @@ ClickHouse пишет и читает сообщения `Protocol Buffers` в ### Соответствие типов данных {#sootvetstvie-tipov-dannykh} -Таблица ниже содержит поддерживаемые типы данных и их соответствие [типам данных](../data_types/index.md) ClickHouse для запросов `INSERT` и `SELECT`. +Таблица ниже содержит поддерживаемые типы данных и их соответствие [типам данных](../sql_reference/data_types/index.md) ClickHouse для запросов `INSERT` и `SELECT`. -| Тип данных Parquet (`INSERT`) | Тип данных ClickHouse | Тип данных Parquet (`SELECT`) | -|-------------------------------|---------------------------------------------|-------------------------------| -| `UINT8`, `BOOL` | [UInt8](../data_types/int_uint.md) | `UINT8` | -| `INT8` | [Int8](../data_types/int_uint.md) | `INT8` | -| `UINT16` | [UInt16](../data_types/int_uint.md) | `UINT16` | -| `INT16` | [Int16](../data_types/int_uint.md) | `INT16` | -| `UINT32` | [UInt32](../data_types/int_uint.md) | `UINT32` | -| `INT32` | [Int32](../data_types/int_uint.md) | `INT32` | -| `UINT64` | [UInt64](../data_types/int_uint.md) | `UINT64` | -| `INT64` | [Int64](../data_types/int_uint.md) | `INT64` | -| `FLOAT`, `HALF_FLOAT` | [Float32](../data_types/float.md) | `FLOAT` | -| `DOUBLE` | [Float64](../data_types/float.md) | `DOUBLE` | -| `DATE32` | [Date](../data_types/date.md) | `UINT16` | -| `DATE64`, `TIMESTAMP` | [DateTime](../data_types/datetime.md) | `UINT32` | -| `STRING`, `BINARY` | [String](../data_types/string.md) | `STRING` | -| — | [FixedString](../data_types/fixedstring.md) | `STRING` | -| `DECIMAL` | [Decimal](../data_types/decimal.md) | `DECIMAL` | +| Тип данных Parquet (`INSERT`) | Тип данных ClickHouse | Тип данных Parquet (`SELECT`) | +|-------------------------------|-----------------------------------------------------------|-------------------------------| +| `UINT8`, `BOOL` | [UInt8](../sql_reference/data_types/int_uint.md) | `UINT8` | +| `INT8` | [Int8](../sql_reference/data_types/int_uint.md) | `INT8` | +| `UINT16` | [UInt16](../sql_reference/data_types/int_uint.md) | `UINT16` | +| `INT16` | [Int16](../sql_reference/data_types/int_uint.md) | `INT16` | +| `UINT32` | [UInt32](../sql_reference/data_types/int_uint.md) | `UINT32` | +| `INT32` | [Int32](../sql_reference/data_types/int_uint.md) | `INT32` | +| `UINT64` | [UInt64](../sql_reference/data_types/int_uint.md) | `UINT64` | +| `INT64` | [Int64](../sql_reference/data_types/int_uint.md) | `INT64` | +| `FLOAT`, `HALF_FLOAT` | [Float32](../sql_reference/data_types/float.md) | `FLOAT` | +| `DOUBLE` | [Float64](../sql_reference/data_types/float.md) | `DOUBLE` | +| `DATE32` | [Date](../sql_reference/data_types/date.md) | `UINT16` | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql_reference/data_types/datetime.md) | `UINT32` | +| `STRING`, `BINARY` | [String](../sql_reference/data_types/string.md) | `STRING` | +| — | [FixedString](../sql_reference/data_types/fixedstring.md) | `STRING` | +| `DECIMAL` | [Decimal](../sql_reference/data_types/decimal.md) | `DECIMAL` | ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. @@ -991,7 +991,7 @@ $ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Pa $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} ``` -Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../operations/table_engines/hdfs.md). +Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../engines/table_engines/integrations/hdfs.md). ## ORC {#data-format-orc} @@ -999,24 +999,24 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ ### Соответствие типов данных {#sootvetstvie-tipov-dannykh-1} -Таблица показывает поддержанные типы данных и их соответствие [типам данных](../data_types/index.md) ClickHouse для запросов `INSERT`. +Таблица показывает поддержанные типы данных и их соответствие [типам данных](../sql_reference/data_types/index.md) ClickHouse для запросов `INSERT`. -| Тип данных ORC (`INSERT`) | Тип данных ClickHouse | -|---------------------------|---------------------------------------| -| `UINT8`, `BOOL` | [UInt8](../data_types/int_uint.md) | -| `INT8` | [Int8](../data_types/int_uint.md) | -| `UINT16` | [UInt16](../data_types/int_uint.md) | -| `INT16` | [Int16](../data_types/int_uint.md) | -| `UINT32` | [UInt32](../data_types/int_uint.md) | -| `INT32` | [Int32](../data_types/int_uint.md) | -| `UINT64` | [UInt64](../data_types/int_uint.md) | -| `INT64` | [Int64](../data_types/int_uint.md) | -| `FLOAT`, `HALF_FLOAT` | [Float32](../data_types/float.md) | -| `DOUBLE` | [Float64](../data_types/float.md) | -| `DATE32` | [Date](../data_types/date.md) | -| `DATE64`, `TIMESTAMP` | [DateTime](../data_types/datetime.md) | -| `STRING`, `BINARY` | [String](../data_types/string.md) | -| `DECIMAL` | [Decimal](../data_types/decimal.md) | +| Тип данных ORC (`INSERT`) | Тип данных ClickHouse | +|---------------------------|-----------------------------------------------------| +| `UINT8`, `BOOL` | [UInt8](../sql_reference/data_types/int_uint.md) | +| `INT8` | [Int8](../sql_reference/data_types/int_uint.md) | +| `UINT16` | [UInt16](../sql_reference/data_types/int_uint.md) | +| `INT16` | [Int16](../sql_reference/data_types/int_uint.md) | +| `UINT32` | [UInt32](../sql_reference/data_types/int_uint.md) | +| `INT32` | [Int32](../sql_reference/data_types/int_uint.md) | +| `UINT64` | [UInt64](../sql_reference/data_types/int_uint.md) | +| `INT64` | [Int64](../sql_reference/data_types/int_uint.md) | +| `FLOAT`, `HALF_FLOAT` | [Float32](../sql_reference/data_types/float.md) | +| `DOUBLE` | [Float64](../sql_reference/data_types/float.md) | +| `DATE32` | [Date](../sql_reference/data_types/date.md) | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql_reference/data_types/datetime.md) | +| `STRING`, `BINARY` | [String](../sql_reference/data_types/string.md) | +| `DECIMAL` | [Decimal](../sql_reference/data_types/decimal.md) | ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. @@ -1032,7 +1032,7 @@ ClickHouse поддерживает настраиваемую точность $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" ``` -Для обмена данных с Hadoop можно использовать [движок таблиц HDFS](../operations/table_engines/hdfs.md). +Для обмена данных с Hadoop можно использовать [движок таблиц HDFS](../engines/table_engines/integrations/hdfs.md). ## Схема формата {#formatschema} @@ -1045,6 +1045,6 @@ $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT OR относительно текущей директории на клиенте. Если клиент используется в [batch режиме](../interfaces/cli.md#cli_usage), то в записи схемы допускается только относительный путь, из соображений безопасности. Если для ввода/вывода данных используется [HTTP-интерфейс](../interfaces/http.md), то файл со схемой должен располагаться на сервере в каталоге, -указанном в параметре [format\_schema\_path](../operations/server_settings/settings.md#server_settings-format_schema_path) конфигурации сервера. +указанном в параметре [format\_schema\_path](../operations/server_configuration_parameters/settings.md#server_configuration_parameters-format_schema_path) конфигурации сервера. [Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/formats/) diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 9a89a25bec7..add57183824 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -3,7 +3,7 @@ HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах, HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. По умолчанию, clickhouse-server слушает HTTP на порту 8123 (это можно изменить в конфиге). -Если запросить GET / без параметров, то вернётся строка заданная с помощью настройки [http\_server\_default\_response](../operations/server_settings/settings.md#server_settings-http_server_default_response). Значение по умолчанию «Ok.» (с переводом строки на конце). +Если запросить GET / без параметров, то вернётся строка заданная с помощью настройки [http\_server\_default\_response](../operations/server_configuration_parameters/settings.md#server_configuration_parameters-http_server_default_response). Значение по умолчанию «Ok.» (с переводом строки на конце). ``` bash $ curl 'http://localhost:8123/' diff --git a/docs/ru/interfaces/index.md b/docs/ru/interfaces/index.md index 8f5a2e5f073..be49e640132 100644 --- a/docs/ru/interfaces/index.md +++ b/docs/ru/interfaces/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: Интерфейсы +toc_priority: 14 +toc_title: Введение +--- + # Интерфейсы {#interfaces} ClickHouse предоставляет два сетевых интерфейса (оба могут быть дополнительно обернуты в TLS для дополнительной безопасности): diff --git a/docs/ru/interfaces/mysql.md b/docs/ru/interfaces/mysql.md index 146947342cd..d550b430c69 100644 --- a/docs/ru/interfaces/mysql.md +++ b/docs/ru/interfaces/mysql.md @@ -1,6 +1,6 @@ # MySQL-интерфейс {#mysql-interface} -ClickHouse поддерживает взаимодействие по протоколу MySQL. Данная функция включается настройкой [mysql\_port](../operations/server_settings/settings.md#server_settings-mysql_port) в конфигурационном файле: +ClickHouse поддерживает взаимодействие по протоколу MySQL. Данная функция включается настройкой [mysql\_port](../operations/server_configuration_parameters/settings.md#server_configuration_parameters-mysql_port) в конфигурационном файле: ``` xml 9004 diff --git a/docs/ru/interfaces/third-party/index.md b/docs/ru/interfaces/third-party/index.md new file mode 100644 index 00000000000..8e768ad9d8e --- /dev/null +++ b/docs/ru/interfaces/third-party/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: От сторонних разработчиков +toc_priority: 24 +--- + diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index b835dc949bf..d16404e2f27 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -7,72 +7,78 @@ - Реляционные системы управления базами данных - [MySQL](https://www.mysql.com) - - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) + - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) + - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) + - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) - [PostgreSQL](https://www.postgresql.org) - - [clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) - - [infi.clickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (использует [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - - [pg2ch](https://github.com/mkabilov/pg2ch) - - [clickhouse\_fdw](https://github.com/adjust/clickhouse_fdw) + - [clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) + - [infi.clickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (использует [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [pg2ch](https://github.com/mkabilov/pg2ch) + - [clickhouse\_fdw](https://github.com/adjust/clickhouse_fdw) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - - [ClickHouseMightrator](https://github.com/zlzforever/ClickHouseMigrator) + - [ClickHouseMightrator](https://github.com/zlzforever/ClickHouseMigrator) - Очереди сообщений - [Kafka](https://kafka.apache.org) - - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (использует [Go client](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (использует [Go client](https://github.com/ClickHouse/clickhouse-go/)) +- Потоковая обработка + - [Flink](https://flink.apache.org) + - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) - Хранилища объектов - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - - [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup) + - [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup) - Оркестрация контейнеров - [Kubernetes](https://kubernetes.io) - - [clickhouse-operator](https://github.com/Altinity/clickhouse-operator) + - [clickhouse-operator](https://github.com/Altinity/clickhouse-operator) - Системы управления конфигурацией - [puppet](https://puppet.com) - - [innogames/clickhouse](https://forge.puppet.com/innogames/clickhouse) - - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) + - [innogames/clickhouse](https://forge.puppet.com/innogames/clickhouse) + - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) - Мониторинг - [Graphite](https://graphiteapp.org) - - [graphouse](https://github.com/yandex/graphouse) - - [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) + - - [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse) - - [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - оптимизирует партиции таблиц [\*GraphiteMergeTree](../../operations/table_engines/graphitemergetree.md#graphitemergetree) согласно правилам в [конфигурации rollup](../../operations/table_engines/graphitemergetree.md#rollup-configuration) + - [graphouse](https://github.com/yandex/graphouse) + - [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) + + - [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse) + - [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - оптимизирует партиции таблиц [\*GraphiteMergeTree](../../engines/table_engines/mergetree_family/graphitemergetree.md#graphitemergetree) согласно правилам в [конфигурации rollup](../../engines/table_engines/mergetree_family/graphitemergetree.md#rollup-configuration) - [Grafana](https://grafana.com/) - - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) + - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) - [Prometheus](https://prometheus.io/) - - [clickhouse\_exporter](https://github.com/f1yegor/clickhouse_exporter) - - [PromHouse](https://github.com/Percona-Lab/PromHouse) - - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) (использует [Go client](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_exporter](https://github.com/f1yegor/clickhouse_exporter) + - [PromHouse](https://github.com/Percona-Lab/PromHouse) + - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) (использует [Go client](https://github.com/kshvakov/clickhouse/)) - [Nagios](https://www.nagios.org/) - - [check\_clickhouse](https://github.com/exogroup/check_clickhouse/) - - [check\_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) + - [check\_clickhouse](https://github.com/exogroup/check_clickhouse/) + - [check\_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) - [Zabbix](https://www.zabbix.com) - - [clickhouse-zabbix-template](https://github.com/Altinity/clickhouse-zabbix-template) + - [clickhouse-zabbix-template](https://github.com/Altinity/clickhouse-zabbix-template) - [Sematext](https://sematext.com/) - - [clickhouse интеграция](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) + - [clickhouse интеграция](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) - Логирование - [rsyslog](https://www.rsyslog.com/) - - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - - [loghouse](https://github.com/flant/loghouse) (для [Kubernetes](https://kubernetes.io)) - - [logagent](https://www.sematext.com/logagent) - - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) + - [loghouse](https://github.com/flant/loghouse) (для [Kubernetes](https://kubernetes.io)) + - [Sematext](https://www.sematext.com/logagent) + - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) - Гео - [MaxMind](https://dev.maxmind.com/geoip/) - - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) + - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) ## Экосистемы вокруг языков программирования {#ekosistemy-vokrug-iazykov-programmirovaniia} - Python - [SQLAlchemy](https://www.sqlalchemy.org) - - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (использует [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (использует [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [pandas](https://pandas.pydata.org) - - [pandahouse](https://github.com/kszucs/pandahouse) + - [pandahouse](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) (использует [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) - Java - [Hadoop](http://hadoop.apache.org) - - [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) (использует [JDBC](../../query_language/table_functions/jdbc.md)) + - [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) (использует [JDBC](../../sql_reference/table_functions/jdbc.md)) - Scala - [Akka](https://akka.io) - [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client) diff --git a/docs/ru/introduction/adopters.md b/docs/ru/introduction/adopters.md deleted file mode 100644 index ef841b2fa05..00000000000 --- a/docs/ru/introduction/adopters.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -en_copy: true ---- - -# ClickHouse Adopters {#clickhouse-adopters} - -!!! warning "Disclaimer" - The following list of companies using ClickHouse and their success stories is assembled from public sources, thus might differ from current reality. We’d appreciate it if you share the story of adopting ClickHouse in your company and [add it to the list](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md), but please make sure you won’t have any NDA issues by doing so. Providing updates with publications from other companies is also useful. - -| Company | Industry | Usecase | Cluster Size | (Un)Compressed Data Size\* | Reference | -|-----------------------------------------------------------------------------|---------------------------------|-----------------------|------------------------------------------------------------|------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [2gis](https://2gis.ru) | Maps | Monitoring | — | — | [Talk in Russian, July 2019](https://youtu.be/58sPkXfq6nw) | -| [Aloha Browser](https://alohabrowser.com/) | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | -| [Amadeus](https://amadeus.com/) | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | -| [Appsflyer](https://www.appsflyer.com) | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | -| [ArenaData](https://arenadata.tech/) | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | -| [Badoo](https://badoo.com) | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | -| [Benocs](https://www.benocs.com/) | Network Telemetry and Analytics | Main Product | — | — | [Slides in English, October 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | -| [Bloomberg](https://www.bloomberg.com/) | Finance, Media | Monitoring | 102 servers | — | [Slides, May 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | -| [Bloxy](https://bloxy.info) | Blockchain | Analytics | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | -| `Dataliance/UltraPower` | Telecom | Analytics | — | — | [Slides in Chinese, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | -| [CARTO](https://carto.com/) | Business Intelligence | Geo analytics | — | — | [Geospatial processing with Clickhouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | -| [CERN](http://public.web.cern.ch/public/) | Research | Experiment | — | — | [Press release, April 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://cisco.com/) | Networking | Traffic analysis | — | — | [Lightning talk, October 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | -| [Citadel Securities](https://www.citadelsecurities.com/) | Finance | — | — | — | [Contribution, March 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | -| [Citymobil](https://city-mobil.ru) | Taxi | Analytics | — | — | [Blog Post in Russian, March 2020](https://habr.com/en/company/citymobil/blog/490660/) | -| [ContentSquare](https://contentsquare.com) | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | -| [Cloudflare](https://cloudflare.com) | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | -| [Corunet](https://coru.net/) | Analytics | Main product | — | — | [Slides in English, April 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [CraiditX 氪信](https://creditx.com) | Finance AI | Analysis | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | -| [Criteo/Storetail](https://www.criteo.com/) | Retail | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| [Deutsche Bank](https://db.com) | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | -| [Diva-e](https://www.diva-e.com) | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | -| [Exness](https://www.exness.com) | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | -| [Geniee](https://geniee.co.jp) | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| [HUYA](https://www.huya.com/) | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com) | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| [Infovista](https://www.infovista.com/) | Networks | Analytics | — | — | [Slides in English, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | -| [InnoGames](https://www.innogames.com) | Games | Metrics, Logging | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | -| [Integros](https://integros.com) | Platform for video services | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Kodiak Data](https://www.kodiakdata.com/) | Clouds | Main product | — | — | [Slides in Engish, April 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | -| [Kontur](https://kontur.ru) | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [LifeStreet](https://lifestreet.com/) | Ad network | Main product | 75 servers (3 replicas) | 5.27 PiB | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | -| [Mail.ru Cloud Solutions](https://mcs.mail.ru/) | Cloud services | Main product | — | — | [Running ClickHouse Instance, in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) | -| [MessageBird](https://www.messagebird.com) | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | -| [MGID](https://www.mgid.com/) | Ad network | Web-analytics | — | — | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | -| [OneAPM](https://www.oneapm.com/) | Monitorings and Data Analysis | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | -| [Pragma Innovation](http://www.pragma-innovation.fr/) | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | -| [QINGCLOUD](https://www.qingcloud.com/) | Cloud services | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | -| [Qrator](https://qrator.net) | DDoS protection | Main product | — | — | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [Beijing PERCENT Information Technology Co., Ltd.](https://www.percent.cn/) | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | -| [Rambler](https://rambler.ru) | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | -| [Tencent](https://www.tencent.com) | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | -| [Traffic Stars](https://trafficstars.com/) | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | -| [S7 Airlines](https://www.s7.ru) | Airlines | Metrics, Logging | — | — | [Talk in Russian, March 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| [SEMrush](https://www.semrush.com/) | Marketing | Main product | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | -| [scireum GmbH](https://www.scireum.de/) | e-Commerce | Main product | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | -| [Sentry](https://sentry.io/) | Software developer | Backend for product | — | — | [Blog Post in English, May 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | -| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | Goverment Social Security | Analytics | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | -| [seo.do](https://seo.do/) | Analytics | Main product | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | -| [Sina](http://english.sina.com/index.html) | News | — | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | -| [SMI2](https://smi2.ru/) | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) | -| [Splunk](https://www.splunk.com/) | Business Analytics | Main product | — | — | [Slides in English, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | -| [Spotify](https://www.spotify.com) | Music | Experimentation | — | — | [Slides, July 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | -| [Tencent](https://www.tencent.com) | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | -| [Uber](https://www.uber.com) | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | -| [VKontakte](https://vk.com) | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | -| [Wisebits](https://wisebits.com/) | IT Solutions | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [Xiaoxin Tech.](https://www.xiaoheiban.cn/) | Education | Common purpose | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | -| [Ximalaya](https://www.ximalaya.com/) | Audio sharing | OLAP | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | -| [Yandex Cloud](https://cloud.yandex.ru/services/managed-clickhouse) | Public Cloud | Main product | — | — | [Talk in Russian, December 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | -| [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Business Intelligence | Main product | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | -| [Yandex Market](https://market.yandex.ru/) | e-Commerce | Metrics, Logging | — | — | [Talk in Russian, January 2019](https://youtu.be/_l1qP0DyBcA?t=478) | -| [Yandex Metrica](https://metrica.yandex.com) | Web analytics | Main product | 360 servers in one cluster, 1862 servers in one department | 66.41 PiB / 5.68 PiB | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | -| [ЦВТ](https://htc-cs.ru/) | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| [МКБ](https://mkb.ru/) | Bank | Web-system monitoring | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | -| [金数据](https://jinshuju.net) | BI Analytics | Main product | — | — | [Slides in Chinese, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | - -[Original article](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/ru/introduction/adopters.md b/docs/ru/introduction/adopters.md new file mode 120000 index 00000000000..659153d5f6c --- /dev/null +++ b/docs/ru/introduction/adopters.md @@ -0,0 +1 @@ +../../en/introduction/adopters.md \ No newline at end of file diff --git a/docs/ru/introduction/distinctive_features.md b/docs/ru/introduction/distinctive_features.md index 093053a3b89..079a0667070 100644 --- a/docs/ru/introduction/distinctive_features.md +++ b/docs/ru/introduction/distinctive_features.md @@ -59,6 +59,13 @@ ClickHouse предоставляет различные способы разм Используется асинхронная multimaster репликация. После записи на любую доступную реплику, данные распространяются на все остальные реплики в фоне. Система поддерживает полную идентичность данных на разных репликах. Восстановление после большинства сбоев осуществляется автоматически, а в сложных случаях — полуавтоматически. При необходимости, можно [включить кворумную запись](../operations/settings/settings.md) данных. -Подробнее смотрите раздел [Репликация данных](../operations/table_engines/replication.md). +Подробнее смотрите раздел [Репликация данных](../engines/table_engines/mergetree_family/replication.md). + +## Особенности, которые могут считаться недостатками {#osobennosti-clickhouse-kotorye-mogut-schitatsia-nedostatkami} + +1. Отсутствие полноценных транзакций. +2. Возможность изменять или удалять ранее записанные данные с низкими задержками и высокой частотой запросов не предоставляется. Есть массовое удаление и изменение данных для очистки более не нужного или соответствия [GDPR](https://gdpr-info.eu). +3. Разреженный индекс делает ClickHouse плохо пригодным для точечных чтений одиночных строк по своим + ключам. [Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/distinctive_features/) diff --git a/docs/ru/introduction/features_considered_disadvantages.md b/docs/ru/introduction/features_considered_disadvantages.md deleted file mode 100644 index 0cd4838d908..00000000000 --- a/docs/ru/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1,8 +0,0 @@ -# Особенности ClickHouse, которые могут считаться недостатками {#osobennosti-clickhouse-kotorye-mogut-schitatsia-nedostatkami} - -1. Отсутствие полноценных транзакций. -2. Возможность изменять или удалять ранее записанные данные с низкими задержками и высокой частотой запросов не предоставляется. Есть массовое удаление и изменение данных для очистки более не нужного или соответствия [GDPR](https://gdpr-info.eu). -3. Разреженный индекс делает ClickHouse плохо пригодным для точечных чтений одиночных строк по своим - ключам. - -[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/features_considered_disadvantages/) diff --git a/docs/ru/introduction/history.md b/docs/ru/introduction/history.md index 856263704e5..65254b0f4f0 100644 --- a/docs/ru/introduction/history.md +++ b/docs/ru/introduction/history.md @@ -13,11 +13,11 @@ ClickHouse изначально разрабатывался для обеспе Также ClickHouse используется: -- для хранения данных Вебвизора; -- для обработки промежуточных данных; -- для построения глобальных отчётов Аналитиками; -- для выполнения запросов в целях отладки движка Метрики; -- для анализа логов работы API и пользовательского интерфейса. +- для хранения данных Вебвизора; +- для обработки промежуточных данных; +- для построения глобальных отчётов Аналитиками; +- для выполнения запросов в целях отладки движка Метрики; +- для анализа логов работы API и пользовательского интерфейса. ClickHouse имеет более десятка инсталляций в других отделах Яндекса: в Вертикальных сервисах, Маркете, Директе, БК, Бизнес аналитике, Мобильной разработке, AdFox, Персональных сервисах и т п. @@ -27,14 +27,14 @@ ClickHouse имеет более десятка инсталляций в дру Но агрегированные данные являются очень ограниченным решением, по следующим причинам: -- вы должны заранее знать перечень отчётов, необходимых пользователю; -- то есть, пользователь не может построить произвольный отчёт; -- при агрегации по большому количеству ключей, объём данных не уменьшается и агрегация бесполезна; -- при большом количестве отчётов, получается слишком много вариантов агрегации (комбинаторный взрыв); -- при агрегации по ключам высокой кардинальности (например, URL) объём данных уменьшается не сильно (менее чем в 2 раза); -- из-за этого, объём данных при агрегации может не уменьшиться, а вырасти; -- пользователи будут смотреть не все отчёты, которые мы для них посчитаем - то есть, большая часть вычислений бесполезна; -- возможно нарушение логической целостности данных для разных агрегаций; +- вы должны заранее знать перечень отчётов, необходимых пользователю; +- то есть, пользователь не может построить произвольный отчёт; +- при агрегации по большому количеству ключей, объём данных не уменьшается и агрегация бесполезна; +- при большом количестве отчётов, получается слишком много вариантов агрегации (комбинаторный взрыв); +- при агрегации по ключам высокой кардинальности (например, URL) объём данных уменьшается не сильно (менее чем в 2 раза); +- из-за этого, объём данных при агрегации может не уменьшиться, а вырасти; +- пользователи будут смотреть не все отчёты, которые мы для них посчитаем - то есть, большая часть вычислений бесполезна; +- возможно нарушение логической целостности данных для разных агрегаций; Как видно, если ничего не агрегировать, и работать с неагрегированными данными, то это даже может уменьшить объём вычислений. diff --git a/docs/ru/introduction/index.md b/docs/ru/introduction/index.md new file mode 100644 index 00000000000..5479b37428f --- /dev/null +++ b/docs/ru/introduction/index.md @@ -0,0 +1,6 @@ +--- +toc_folder_title: Введение +toc_priority: 1 +--- + + diff --git a/docs/ru/introduction/performance.md b/docs/ru/introduction/performance.md index 568fd835261..cbefa10cf7c 100644 --- a/docs/ru/introduction/performance.md +++ b/docs/ru/introduction/performance.md @@ -1,6 +1,6 @@ # Производительность {#proizvoditelnost} -По результатам внутреннего тестирования в Яндексе, ClickHouse обладает наиболее высокой производительностью (как наиболее высокой пропускной способностью на длинных запросах, так и наиболее низкой задержкой на коротких запросах), при соответствующем сценарии работы, среди доступных для тестирования систем подобного класса. Результаты тестирования можно посмотреть на [отдельной странице](https://clickhouse.tech/benchmark.html). +По результатам внутреннего тестирования в Яндексе, ClickHouse обладает наиболее высокой производительностью (как наиболее высокой пропускной способностью на длинных запросах, так и наиболее низкой задержкой на коротких запросах), при соответствующем сценарии работы, среди доступных для тестирования систем подобного класса. Результаты тестирования можно посмотреть на [отдельной странице](https://clickhouse.tech/benchmark/dbms/). Также это подтверждают многочисленные независимые бенчмарки. Их не сложно найти в Интернете самостоятельно, либо можно воспользоваться [небольшой коллекцией ссылок по теме](https://clickhouse.tech/#independent-benchmarks). diff --git a/docs/ru/operations/access_rights.md b/docs/ru/operations/access_rights.md index d4cd7793bf1..18c2a25377a 100644 --- a/docs/ru/operations/access_rights.md +++ b/docs/ru/operations/access_rights.md @@ -61,7 +61,7 @@ Здесь видно объявление двух пользователей - `default` и `web`. Пользователя `web` мы добавили самостоятельно. -Пользователь `default` выбирается в случаях, когда имя пользователя не передаётся. Также пользователь `default` может использоваться при распределённой обработке запроса - если в конфигурации кластера для сервера не указаны `user` и `password`. (см. раздел о движке [Distributed](../operations/table_engines/distributed.md)). +Пользователь `default` выбирается в случаях, когда имя пользователя не передаётся. Также пользователь `default` может использоваться при распределённой обработке запроса - если в конфигурации кластера для сервера не указаны `user` и `password`. (см. раздел о движке [Distributed](../engines/table_engines/special/distributed.md)). Пользователь, который используется для обмена информацией между серверами, объединенными в кластер, не должен иметь существенных ограничений или квот - иначе распределённые запросы сломаются. diff --git a/docs/ru/operations/backup.md b/docs/ru/operations/backup.md index 4888f2b418e..601eaa4d6d9 100644 --- a/docs/ru/operations/backup.md +++ b/docs/ru/operations/backup.md @@ -1,6 +1,6 @@ # Резервное копирование данных {#rezervnoe-kopirovanie-dannykh} -[Репликация](table_engines/replication.md) обеспечивает защиту от аппаратных сбоев, но не защищает от человеческих ошибок: случайного удаления данных, удаления не той таблицы, которую надо было, или таблицы на не том кластере, а также программных ошибок, которые приводят к неправильной обработке данных или их повреждению. Во многих случаях подобные ошибки влияют на все реплики. ClickHouse имеет встроенные средства защиты для предотвращения некоторых типов ошибок — например, по умолчанию [не получится удалить таблицы \*MergeTree, содержащие более 50 Гб данных, одной командой](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Однако эти средства защиты не охватывают все возможные случаи и могут быть обойдены. +[Репликация](../engines/table_engines/mergetree_family/replication.md) обеспечивает защиту от аппаратных сбоев, но не защищает от человеческих ошибок: случайного удаления данных, удаления не той таблицы, которую надо было, или таблицы на не том кластере, а также программных ошибок, которые приводят к неправильной обработке данных или их повреждению. Во многих случаях подобные ошибки влияют на все реплики. ClickHouse имеет встроенные средства защиты для предотвращения некоторых типов ошибок — например, по умолчанию [не получится удалить таблицы \*MergeTree, содержащие более 50 Гб данных, одной командой](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Однако эти средства защиты не охватывают все возможные случаи и могут быть обойдены. Для того чтобы эффективно уменьшить возможные человеческие ошибки, следует тщательно подготовить стратегию резервного копирования и восстановления данных **заранее**. @@ -15,11 +15,11 @@ ## Снимки файловой системы {#snimki-failovoi-sistemy} -Некоторые локальные файловые системы позволяют делать снимки (например, [ZFS](https://en.wikipedia.org/wiki/ZFS)), но они могут быть не лучшим выбором для обслуживания живых запросов. Возможным решением является создание дополнительных реплик с такой файловой системой и исключение их из [Distributed](table_engines/distributed.md) таблиц, используемых для запросов `SELECT`. Снимки на таких репликах будут недоступны для запросов, изменяющих данные. В качестве бонуса, эти реплики могут иметь особые конфигурации оборудования с большим количеством дисков, подключенных к серверу, что будет экономически эффективным. +Некоторые локальные файловые системы позволяют делать снимки (например, [ZFS](https://en.wikipedia.org/wiki/ZFS)), но они могут быть не лучшим выбором для обслуживания живых запросов. Возможным решением является создание дополнительных реплик с такой файловой системой и исключение их из [Distributed](../engines/table_engines/special/distributed.md) таблиц, используемых для запросов `SELECT`. Снимки на таких репликах будут недоступны для запросов, изменяющих данные. В качестве бонуса, эти реплики могут иметь особые конфигурации оборудования с большим количеством дисков, подключенных к серверу, что будет экономически эффективным. ## clickhouse-copier {#clickhouse-copier} -[clickhouse-copier](utils/clickhouse-copier.md) — это универсальный инструмент, который изначально был создан для перешардирования таблиц с петабайтами данных. Его также можно использовать для резервного копирования и восстановления, поскольку он надёжно копирует данные между таблицами и кластерами ClickHouse. +[clickhouse-copier](utilities/clickhouse-copier.md) — это универсальный инструмент, который изначально был создан для перешардирования таблиц с петабайтами данных. Его также можно использовать для резервного копирования и восстановления, поскольку он надёжно копирует данные между таблицами и кластерами ClickHouse. Для небольших объёмов данных можно применять `INSERT INTO ... SELECT ...` в удалённые таблицы. @@ -27,7 +27,7 @@ ClickHouse позволяет использовать запрос `ALTER TABLE ... FREEZE PARTITION ...` для создания локальной копии партиций таблицы. Это реализуется с помощью жестких ссылок (hardlinks) на каталог `/var/lib/clickhouse/shadow/`, поэтому такая копия обычно не занимает дополнительное место на диске для старых данных. Созданные копии файлов не обрабатываются сервером ClickHouse, поэтому вы можете просто оставить их там: у вас будет простая резервная копия, которая не требует дополнительной внешней системы, однако при аппаратных проблемах вы можете утратить и актуальные данные и сохраненную копию. По этой причине, лучше удаленно скопировать их в другое место, а затем удалить локальную копию. Распределенные файловые системы и хранилища объектов по-прежнему являются хорошими вариантами для этого, однако можно использовать и обычные присоединенные файловые серверы с достаточно большой ёмкостью (в этом случае передача будет происходить через сетевую файловую систему или, возможно, [rsync](https://en.wikipedia.org/wiki/Rsync)). -Дополнительные сведения о запросах, связанных с манипуляциями партициями, см. в разделе [ALTER](../query_language/alter.md#alter_manipulations-with-partitions). +Дополнительные сведения о запросах, связанных с манипуляциями партициями, см. в разделе [ALTER](../sql_reference/statements/alter.md#alter_manipulations-with-partitions). Для автоматизации этого подхода доступен инструмент от сторонних разработчиков: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup). diff --git a/docs/ru/operations/configuration_files.md b/docs/ru/operations/configuration_files.md index 0bcae78a128..19f7ea9d5db 100644 --- a/docs/ru/operations/configuration_files.md +++ b/docs/ru/operations/configuration_files.md @@ -12,7 +12,7 @@ Если указано `remove` - удалить элемент. -Также в конфиге могут быть указаны «подстановки». Если у элемента присутствует атрибут `incl`, то в качестве значения будет использована соответствующая подстановка из файла. По умолчанию, путь к файлу с подстановками - `/etc/metrika.xml`. Он может быть изменён в конфигурации сервера в элементе [include\_from](server_settings/settings.md#server_settings-include_from). Значения подстановок указываются в элементах `/yandex/имя_подстановки` этого файла. Если подстановка, заданная в `incl` отсутствует, то в лог попадает соответствующая запись. Чтобы ClickHouse не писал в лог об отсутствии подстановки, необходимо указать атрибут `optional="true"` (например, настройка [macros](server_settings/settings.md)). +Также в конфиге могут быть указаны «подстановки». Если у элемента присутствует атрибут `incl`, то в качестве значения будет использована соответствующая подстановка из файла. По умолчанию, путь к файлу с подстановками - `/etc/metrika.xml`. Он может быть изменён в конфигурации сервера в элементе [include\_from](server_configuration_parameters/settings.md#server_configuration_parameters-include_from). Значения подстановок указываются в элементах `/yandex/имя_подстановки` этого файла. Если подстановка, заданная в `incl` отсутствует, то в лог попадает соответствующая запись. Чтобы ClickHouse не писал в лог об отсутствии подстановки, необходимо указать атрибут `optional="true"` (например, настройка [macros](server_configuration_parameters/settings.md)). Подстановки могут также выполняться из ZooKeeper. Для этого укажите у элемента атрибут `from_zk = "/path/to/node"`. Значение элемента заменится на содержимое узла `/path/to/node` в ZooKeeper. В ZooKeeper-узел также можно положить целое XML-поддерево, оно будет целиком вставлено в исходный элемент. diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 2e85f1bf816..babb22cd8b4 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: Эксплуатация +toc_priority: 41 +toc_title: Введение +--- + # Эксплуатация {#ekspluatatsiia} Руководство по эксплуатации ClickHouse состоит из следующих основных разделов: @@ -12,7 +18,7 @@ - [Конфигурационные файлы](configuration_files.md) - [Квоты](quotas.md) - [Системные таблицы](system_tables.md) - - [Конфигурационные параметры сервера](server_settings/index.md) + - [Конфигурационные параметры сервера](server_configuration_parameters/index.md) - [Тестирование севреров с помощью ClickHouse](performance_test.md) - [Настройки](settings/index.md) - [Утилиты](utils/index.md) diff --git a/docs/ru/operations/monitoring.md b/docs/ru/operations/monitoring.md index 8681261bf6f..2629a4da931 100644 --- a/docs/ru/operations/monitoring.md +++ b/docs/ru/operations/monitoring.md @@ -21,7 +21,7 @@ ClickHouse не отслеживает состояние аппаратных Сервер ClickHouse имеет встроенные инструменты мониторинга. -Для отслеживания событий на сервере используйте логи. Подробнее смотрите в разделе конфигурационного файла [logger](server_settings/settings.md#server_settings-logger). +Для отслеживания событий на сервере используйте логи. Подробнее смотрите в разделе конфигурационного файла [logger](server_configuration_parameters/settings.md#server_configuration_parameters-logger). ClickHouse собирает: @@ -30,7 +30,9 @@ ClickHouse собирает: Метрики находятся в таблицах [system.metrics](system_tables.md#system_tables-metrics), [system.events](system_tables.md#system_tables-events) и [system.asynchronous\_metrics](system_tables.md#system_tables-asynchronous_metrics). -Можно настроить экспорт метрик из ClickHouse в [Graphite](https://github.com/graphite-project). Смотрите секцию [graphite](server_settings/settings.md#server_settings-graphite) конфигурационного файла ClickHouse. Перед настройкой экспорта метрик необходимо настроить Graphite, как указано в [официальном руководстве](https://graphite.readthedocs.io/en/latest/install.html). +Можно настроить экспорт метрик из ClickHouse в [Graphite](https://github.com/graphite-project). Смотрите секцию [graphite](server_configuration_parameters/settings.md#server_configuration_parameters-graphite) конфигурационного файла ClickHouse. Перед настройкой экспорта метрик необходимо настроить Graphite, как указано в [официальном руководстве](https://graphite.readthedocs.io/en/latest/install.html). + +Можно настроить экспорт метрик из ClickHouse в [Prometheus](https://prometheus.io). Смотрите [prometheus](server_configuration_parameters/settings.md#server_configuration_parameters-prometheus) конфигурационного файла ClickHouse. Перед настройкой экспорта метрик необходимо настроить Prometheus, как указано в [официальном руководстве](https://prometheus.io/docs/prometheus/latest/installation/). Также, можно отслеживать доступность сервера через HTTP API. Отправьте `HTTP GET` к ресурсу `/ping`. Если сервер доступен, он отвечает `200 OK`. diff --git a/docs/ru/operations/optimizing_performance/index.md b/docs/ru/operations/optimizing_performance/index.md new file mode 100644 index 00000000000..4f4cbb66d24 --- /dev/null +++ b/docs/ru/operations/optimizing_performance/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: Optimizing Performance +toc_priority: 52 +--- + diff --git a/docs/ru/operations/optimizing_performance/sampling_query_profiler.md b/docs/ru/operations/optimizing_performance/sampling_query_profiler.md new file mode 120000 index 00000000000..9f3b57cd086 --- /dev/null +++ b/docs/ru/operations/optimizing_performance/sampling_query_profiler.md @@ -0,0 +1 @@ +../../../en/operations/optimizing_performance/sampling_query_profiler.md \ No newline at end of file diff --git a/docs/ru/operations/performance/sampling_query_profiler.md b/docs/ru/operations/performance/sampling_query_profiler.md deleted file mode 100644 index 25368fcd883..00000000000 --- a/docs/ru/operations/performance/sampling_query_profiler.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -en_copy: true ---- - -# Sampling Query Profiler {#sampling-query-profiler} - -ClickHouse runs sampling profiler that allows analyzing query execution. Using profiler you can find source code routines that used the most frequently during query execution. You can trace CPU time and wall-clock time spent including idle time. - -To use profiler: - -- Setup the [trace\_log](../server_settings/settings.md#server_settings-trace_log) section of the server configuration. - - This section configures the [trace\_log](../system_tables.md#system_tables-trace_log) system table containing the results of the profiler functioning. It is configured by default. Remember that data in this table is valid only for a running server. After the server restart, ClickHouse doesn’t clean up the table and all the stored virtual memory address may become invalid. - -- Setup the [query\_profiler\_cpu\_time\_period\_ns](../settings/settings.md#query_profiler_cpu_time_period_ns) or [query\_profiler\_real\_time\_period\_ns](../settings/settings.md#query_profiler_real_time_period_ns) settings. Both settings can be used simultaneously. - - These settings allow you to configure profiler timers. As these are the session settings, you can get different sampling frequency for the whole server, individual users or user profiles, for your interactive session, and for each individual query. - -The default sampling frequency is one sample per second and both CPU and real timers are enabled. This frequency allows collecting enough information about ClickHouse cluster. At the same time, working with this frequency, profiler doesn’t affect ClickHouse server’s performance. If you need to profile each individual query try to use higher sampling frequency. - -To analyze the `trace_log` system table: - -- Install the `clickhouse-common-static-dbg` package. See [Install from DEB Packages](../../getting_started/install.md#install-from-deb-packages). - -- Allow introspection functions by the [allow\_introspection\_functions](../settings/settings.md#settings-allow_introspection_functions) setting. - - For security reasons, introspection functions are disabled by default. - -- Use the `addressToLine`, `addressToSymbol` and `demangle` [introspection functions](../../query_language/functions/introspection.md) to get function names and their positions in ClickHouse code. To get a profile for some query, you need to aggregate data from the `trace_log` table. You can aggregate data by individual functions or by the whole stack traces. - -If you need to visualize `trace_log` info, try [flamegraph](../../interfaces/third-party/gui/#clickhouse-flamegraph) and [speedscope](https://github.com/laplab/clickhouse-speedscope). - -## Example {#example} - -In this example we: - -- Filtering `trace_log` data by a query identifier and the current date. - -- Aggregating by stack trace. - -- Using introspection functions, we will get a report of: - - - Names of symbols and corresponding source code functions. - - Source code locations of these functions. - - - -``` sql -SELECT - count(), - arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym -FROM system.trace_log -WHERE (query_id = 'ebca3574-ad0a-400a-9cbc-dca382f5998c') AND (event_date = today()) -GROUP BY trace -ORDER BY count() DESC -LIMIT 10 -``` - -``` text -{% include "operations/performance/sampling_query_profiler_example_result.txt" %} -``` diff --git a/docs/ru/operations/performance/sampling_query_profiler_example_result.txt b/docs/ru/operations/performance/sampling_query_profiler_example_result.txt index a5f6d71ca95..56c2fdf9c65 100644 --- a/docs/ru/operations/performance/sampling_query_profiler_example_result.txt +++ b/docs/ru/operations/performance/sampling_query_profiler_example_result.txt @@ -1,7 +1,3 @@ ---- -en_copy: true ---- - Row 1: ────── count(): 6344 diff --git a/docs/ru/operations/performance_test.md b/docs/ru/operations/performance_test.md deleted file mode 100644 index ae4c5752703..00000000000 --- a/docs/ru/operations/performance_test.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -en_copy: true ---- - -# How To Test Your Hardware With ClickHouse {#how-to-test-your-hardware-with-clickhouse} - -With this instruction you can run basic ClickHouse performance test on any server without installation of ClickHouse packages. - -1. Go to “commits” page: https://github.com/ClickHouse/ClickHouse/commits/master - -2. Click on the first green check mark or red cross with green “ClickHouse Build Check” and click on the “Details” link near “ClickHouse Build Check”. - -3. Copy the link to “clickhouse” binary for amd64 or aarch64. - -4. ssh to the server and download it with wget: - - - - # For amd64: - wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578163263_binary/clickhouse - # For aarch64: - wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578161264_binary/clickhouse - # Then do: - chmod a+x clickhouse - -1. Download configs: - - - - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.xml - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/users.xml - mkdir config.d - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml - -1. Download benchmark files: - - - - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/benchmark-new.sh - chmod a+x benchmark-new.sh - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql - -1. Download test data according to the [Yandex.Metrica dataset](../getting_started/example_datasets/metrica.md) instruction (“hits” table containing 100 million rows). - - - - wget https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_100m_obfuscated_v1.tar.xz - tar xvf hits_100m_obfuscated_v1.tar.xz -C . - mv hits_100m_obfuscated_v1/* . - -1. Run the server: - - - - ./clickhouse server - -1. Check the data: ssh to the server in another terminal - - - - ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" - 100000000 - -1. Edit the benchmark-new.sh, change “clickhouse-client” to “./clickhouse client” and add “–max\_memory\_usage 100000000000” parameter. - - - - mcedit benchmark-new.sh - -1. Run the benchmark: - - - - ./benchmark-new.sh hits_100m_obfuscated - -1. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com - -All the results are published here: https://clickhouse.tech/benchmark\_hardware.html diff --git a/docs/ru/operations/performance_test.md b/docs/ru/operations/performance_test.md new file mode 120000 index 00000000000..a74c126c63f --- /dev/null +++ b/docs/ru/operations/performance_test.md @@ -0,0 +1 @@ +../../en/operations/performance_test.md \ No newline at end of file diff --git a/docs/ru/operations/quotas.md b/docs/ru/operations/quotas.md index f109b889350..399e80d2011 100644 --- a/docs/ru/operations/quotas.md +++ b/docs/ru/operations/quotas.md @@ -7,8 +7,8 @@ В отличие от них, квоты: -- ограничивают не один запрос, а множество запросов, которые могут быть выполнены за интервал времени; -- при распределённой обработке запроса, учитывают ресурсы, потраченные на всех удалённых серверах. +- ограничивают не один запрос, а множество запросов, которые могут быть выполнены за интервал времени; +- при распределённой обработке запроса, учитывают ресурсы, потраченные на всех удалённых серверах. Рассмотрим фрагмент файла users.xml, описывающего квоты. diff --git a/docs/ru/operations/requirements.md b/docs/ru/operations/requirements.md index 9fafe59343f..e6bc2f15e43 100644 --- a/docs/ru/operations/requirements.md +++ b/docs/ru/operations/requirements.md @@ -17,9 +17,9 @@ ClickHouse реализует параллельную обработку дан - Сложности запросов. - Объёма данных, обрабатываемых в запросах. -Для расчета объёма RAM необходимо оценить размер промежуточных данных для операций [GROUP BY](../query_language/select.md#select-group-by-clause), [DISTINCT](../query_language/select.md#select-distinct), [JOIN](../query_language/select.md#select-join) а также других операций, которыми вы пользуетесь. +Для расчета объёма RAM необходимо оценить размер промежуточных данных для операций [GROUP BY](../sql_reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql_reference/statements/select.md#select-distinct), [JOIN](../sql_reference/statements/select.md#select-join) а также других операций, которыми вы пользуетесь. -ClickHouse может использовать внешнюю память для промежуточных данных. Подробнее смотрите в разделе [GROUP BY во внешней памяти](../query_language/select.md#select-group-by-in-external-memory). +ClickHouse может использовать внешнюю память для промежуточных данных. Подробнее смотрите в разделе [GROUP BY во внешней памяти](../sql_reference/statements/select.md#select-group-by-in-external-memory). ## Файл подкачки {#fail-podkachki} diff --git a/docs/ru/operations/server_settings/index.md b/docs/ru/operations/server_configuration_parameters/index.md similarity index 93% rename from docs/ru/operations/server_settings/index.md rename to docs/ru/operations/server_configuration_parameters/index.md index dae21c2b475..91deb2973a7 100644 --- a/docs/ru/operations/server_settings/index.md +++ b/docs/ru/operations/server_configuration_parameters/index.md @@ -8,4 +8,4 @@ Перед изучением настроек ознакомьтесь с разделом [Конфигурационные файлы](../configuration_files.md#configuration_files), обратите внимание на использование подстановок (атрибуты `incl` и `optional`). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_settings/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_configuration_parameters/) diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_configuration_parameters/settings.md similarity index 81% rename from docs/ru/operations/server_settings/settings.md rename to docs/ru/operations/server_configuration_parameters/settings.md index a062f13c400..618bb9764ad 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_configuration_parameters/settings.md @@ -58,7 +58,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat База данных по умолчанию. -Перечень баз данных можно получить запросом [SHOW DATABASES](../../query_language/show.md#show-databases). +Перечень баз данных можно получить запросом [SHOW DATABASES](../../operations/server_configuration_parameters/settings.md#show-databases). **Пример** @@ -87,7 +87,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat - Указывается абсолютным или относительно конфигурационного файла сервера. - Может содержать wildcard-ы \* и ?. -Смотрите также «[Внешние словари](../../query_language/dicts/external_dicts.md)». +Смотрите также «[Внешние словари](../../operations/server_configuration_parameters/settings.md)». **Пример** @@ -111,7 +111,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat true ``` -## format\_schema\_path {#server_settings-format_schema_path} +## format\_schema\_path {#server_configuration_parameters-format_schema_path} Путь к каталогу со схемами для входных данных. Например со схемами для формата [CapnProto](../../interfaces/formats.md#capnproto). @@ -122,7 +122,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat format_schemas/ ``` -## graphite {#server_settings-graphite} +## graphite {#server_configuration_parameters-graphite} Отправка данных в [Graphite](https://github.com/graphite-project). @@ -133,10 +133,10 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat - interval – Период отправки в секундах. - timeout – Таймаут отправки данных в секундах. - root\_path – Префикс для ключей. -- metrics – Отправка данных из таблицы [system.metrics](../system_tables.md#system_tables-metrics). -- events – Отправка дельты данных, накопленной за промежуток времени из таблицы [system.events](../system_tables.md#system_tables-events). -- events\_cumulative – Отправка суммарных данных из таблицы [system.events](../system_tables.md#system_tables-events). -- asynchronous\_metrics – Отправка данных из таблицы [system.asynchronous\_metrics](../system_tables.md#system_tables-asynchronous_metrics). +- metrics – Отправка данных из таблицы [system.metrics](../../operations/server_configuration_parameters/settings.md#system_tables-metrics). +- events – Отправка дельты данных, накопленной за промежуток времени из таблицы [system.events](../../operations/server_configuration_parameters/settings.md#system_tables-events). +- events\_cumulative – Отправка суммарных данных из таблицы [system.events](../../operations/server_configuration_parameters/settings.md#system_tables-events). +- asynchronous\_metrics – Отправка данных из таблицы [system.asynchronous\_metrics](../../operations/server_configuration_parameters/settings.md#system_tables-asynchronous_metrics). Можно определить несколько секций ``, например, для передачи различных данных с различной частотой. @@ -156,11 +156,11 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## graphite\_rollup {#server_settings-graphite-rollup} +## graphite\_rollup {#server_configuration_parameters-graphite-rollup} Настройка прореживания данных для Graphite. -Подробнее читайте в разделе [GraphiteMergeTree](../table_engines/graphitemergetree.md). +Подробнее читайте в разделе [GraphiteMergeTree](../../operations/server_configuration_parameters/settings.md). **Пример** @@ -188,7 +188,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat Порт для обращений к серверу по протоколу HTTP(s). -Если указан `https_port`, то требуется конфигурирование [openSSL](#server_settings-openssl). +Если указан `https_port`, то требуется конфигурирование [openSSL](#server_configuration_parameters-openssl). Если указан `http_port`, то настройка openSSL игнорируется, даже если она задана. @@ -198,7 +198,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat 0000 ``` -## http\_server\_default\_response {#server_settings-http_server_default_response} +## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} Страница, показываемая по умолчанию, при обращении к HTTP(s) серверу ClickHouse. Значение по умолчанию «Ok.» (с переводом строки на конце). @@ -213,7 +213,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## include\_from {#server_settings-include_from} +## include\_from {#server_configuration_parameters-include_from} Путь к файлу с подстановками. @@ -251,7 +251,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ## interserver\_http\_credentials {#server-settings-interserver-http-credentials} -Имя пользователя и пароль, использующиеся для аутентификации при [репликации](../table_engines/replication.md) движками Replicated\*. Это имя пользователя и пароль используются только для взаимодействия между репликами кластера и никак не связаны с аутентификацией клиентов ClickHouse. Сервер проверяет совпадение имени и пароля для соединяющихся с ним реплик, а также использует это же имя и пароль для соединения с другими репликами. Соответственно, эти имя и пароль должны быть прописаны одинаковыми для всех реплик кластера. +Имя пользователя и пароль, использующиеся для аутентификации при [репликации](../../operations/server_configuration_parameters/settings.md) движками Replicated\*. Это имя пользователя и пароль используются только для взаимодействия между репликами кластера и никак не связаны с аутентификацией клиентов ClickHouse. Сервер проверяет совпадение имени и пароля для соединяющихся с ним реплик, а также использует это же имя и пароль для соединения с другими репликами. Соответственно, эти имя и пароль должны быть прописаны одинаковыми для всех реплик кластера. По умолчанию аутентификация не используется. Раздел содержит следующие параметры: @@ -278,7 +278,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat 3 ``` -## listen\_host {#server_settings-listen_host} +## listen\_host {#server_configuration_parameters-listen_host} Ограничение по хостам, с которых может прийти запрос. Если необходимо, чтобы сервер отвечал всем, то надо указать `::`. @@ -289,7 +289,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat 127.0.0.1 ``` -## logger {#server_settings-logger} +## logger {#server_configuration_parameters-logger} Настройки логирования. @@ -342,7 +342,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat Можно не указывать, если реплицируемых таблицы не используются. -Подробнее смотрите в разделе «[Создание реплицируемых таблиц](../../operations/table_engines/replication.md)». +Подробнее смотрите в разделе «[Создание реплицируемых таблиц](../../operations/server_configuration_parameters/settings.md)». **Пример** @@ -352,7 +352,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ## mark\_cache\_size {#server-mark-cache-size} -Приблизительный размер (в байтах) кэша засечек, используемых движками таблиц семейства [MergeTree](../table_engines/mergetree.md). +Приблизительный размер (в байтах) кэша засечек, используемых движками таблиц семейства [MergeTree](../../operations/server_configuration_parameters/settings.md). Кэш общий для сервера, память выделяется по мере необходимости. @@ -400,7 +400,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat Ограничение на удаление таблиц. -Если размер таблицы семейства [MergeTree](../table_engines/mergetree.md) превышает `max_table_size_to_drop` (в байтах), то ее нельзя удалить запросом DROP. +Если размер таблицы семейства [MergeTree](../../operations/server_configuration_parameters/settings.md) превышает `max_table_size_to_drop` (в байтах), то ее нельзя удалить запросом DROP. Если таблицу все же необходимо удалить, не перезапуская при этом сервер ClickHouse, то необходимо создать файл `/flags/force_drop_table` и выполнить запрос DROP. @@ -414,9 +414,9 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat 0 ``` -## merge\_tree {#server_settings-merge_tree} +## merge\_tree {#server_configuration_parameters-merge_tree} -Тонкая настройка таблиц семейства [MergeTree](../table_engines/mergetree.md). +Тонкая настройка таблиц семейства [MergeTree](../../operations/server_configuration_parameters/settings.md). Подробнее смотрите в заголовочном файле MergeTreeSettings.h. @@ -428,7 +428,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## openSSL {#server_settings-openssl} +## openSSL {#server_configuration_parameters-openssl} Настройки клиента/сервера SSL. @@ -487,17 +487,17 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## part\_log {#server_settings-part-log} +## part\_log {#server_configuration_parameters-part-log} -Логирование событий, связанных с данными типа [MergeTree](../table_engines/mergetree.md). Например, события добавления или мержа данных. Лог можно использовать для симуляции алгоритмов слияния, чтобы сравнивать их характеристики. Также, можно визуализировать процесс слияния. +Логирование событий, связанных с данными типа [MergeTree](../../operations/server_configuration_parameters/settings.md). Например, события добавления или мержа данных. Лог можно использовать для симуляции алгоритмов слияния, чтобы сравнивать их характеристики. Также, можно визуализировать процесс слияния. -Запросы логируются не в отдельный файл, а в таблицу [system.part\_log](../system_tables.md#system_tables-part-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). +Запросы логируются не в отдельный файл, а в таблицу [system.part\_log](../../operations/server_configuration_parameters/settings.md#system_tables-part-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). При настройке логирования используются следующие параметры: - `database` — имя базы данных; - `table` — имя таблицы; -- `partition_by` — устанавливает [произвольный ключ партиционирования](../../operations/table_engines/custom_partitioning_key.md); +- `partition_by` — устанавливает [произвольный ключ партиционирования](../../operations/server_configuration_parameters/settings.md); - `flush_interval_milliseconds` — период сброса данных из буфера в памяти в таблицу. **Пример** @@ -511,7 +511,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## path {#server_settings-path} +## path {#server_configuration_parameters-path} Путь к каталогу с данными. @@ -524,17 +524,41 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat /var/lib/clickhouse/ ``` -## query\_log {#server_settings-query-log} +## prometheus {#server_configuration_parameters-prometheus} + +Опубликовать данные о метриках, для сбора с помощью системы мониторинга [Prometheus](https://prometheus.io). + +Настройки: + +- `endpoint` – путь по которому будет осуществляться экспорт данных метрик по HTTP протоколу для сбора с помощью prometheus. Должен начинаться с '/'. +- `port` – порт по которому будет доступен endpoint для сбора метрик. +- `metrics` – флаг для экспорта текущих значений метрик из таблицы [system.metrics](../system_tables.md#system_tables-metrics). +- `events` – флаг для экспорта текущих значений метрик из таблицы [system.events](../system_tables.md#system_tables-events). +- `asynchronous_metrics` – флаг для экспорта текущих значений значения метрик из таблицы [system.asynchronous\_metrics](../system_tables.md#system_tables-asynchronous_metrics). + +**Пример** + +``` xml + + /metrics + 8001 + true + true + true + +``` + +## query\_log {#server_configuration_parameters-query-log} Настройка логирования запросов, принятых с настройкой [log\_queries=1](../settings/settings.md). -Запросы логируются не в отдельный файл, а в системную таблицу [system.query\_log](../system_tables.md#system_tables-query-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). +Запросы логируются не в отдельный файл, а в системную таблицу [system.query\_log](../../operations/server_configuration_parameters/settings.md#system_tables-query-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). При настройке логирования используются следующие параметры: - `database` — имя базы данных; - `table` — имя таблицы, куда будет записываться лог; -- `partition_by` — [произвольный ключ партиционирования](../../operations/table_engines/custom_partitioning_key.md) для таблицы с логами; +- `partition_by` — [произвольный ключ партиционирования](../../operations/server_configuration_parameters/settings.md) для таблицы с логами; - `flush_interval_milliseconds` — период сброса данных из буфера в памяти в таблицу. Если таблица не существует, то ClickHouse создаст её. Если структура журнала запросов изменилась при обновлении сервера ClickHouse, то таблица со старой структурой переименовывается, а новая таблица создается автоматически. @@ -550,17 +574,17 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## query\_thread\_log {#server_settings-query-thread-log} +## query\_thread\_log {#server_configuration_parameters-query-thread-log} Настройка логирования потоков выполнения запросов, принятых с настройкой [log\_query\_threads=1](../settings/settings.md#settings-log-query-threads). -Запросы логируются не в отдельный файл, а в системную таблицу [system.query\_thread\_log](../system_tables.md#system_tables-query-thread-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). +Запросы логируются не в отдельный файл, а в системную таблицу [system.query\_thread\_log](../../operations/server_configuration_parameters/settings.md#system_tables-query-thread-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). При настройке логирования используются следующие параметры: - `database` — имя базы данных; - `table` — имя таблицы, куда будет записываться лог; -- `partition_by` — [произвольный ключ партиционирования](../../operations/table_engines/custom_partitioning_key.md) для таблицы с логами; +- `partition_by` — [произвольный ключ партиционирования](../../operations/server_configuration_parameters/settings.md) для таблицы с логами; - `flush_interval_milliseconds` — период сброса данных из буфера в памяти в таблицу. Если таблица не существует, то ClickHouse создаст её. Если структура журнала запросов изменилась при обновлении сервера ClickHouse, то таблица со старой структурой переименовывается, а новая таблица создается автоматически. @@ -576,15 +600,15 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## trace\_log {#server_settings-trace_log} +## trace\_log {#server_configuration_parameters-trace_log} -Settings for the [trace\_log](../system_tables.md#system_tables-trace_log) system table operation. +Settings for the [trace\_log](../../operations/server_configuration_parameters/settings.md#system_tables-trace_log) system table operation. Parameters: - `database` — Database for storing a table. - `table` — Table name. -- `partition_by` — [Custom partitioning key](../../operations/table_engines/custom_partitioning_key.md) for a system table. +- `partition_by` — [Custom partitioning key](../../operations/server_configuration_parameters/settings.md) for a system table. - `flush_interval_milliseconds` — Interval for flushing data from the buffer in memory to the table. The default server configuration file `config.xml` contains the following settings section: @@ -600,7 +624,7 @@ The default server configuration file `config.xml` contains the following settin ## remote\_servers {#server-settings-remote-servers} -Конфигурация кластеров, которые использует движок таблиц [Distributed](../../operations/table_engines/distributed.md) и табличная функция `cluster`. +Конфигурация кластеров, которые использует движок таблиц [Distributed](../../operations/server_configuration_parameters/settings.md) и табличная функция `cluster`. **Пример** @@ -614,7 +638,7 @@ The default server configuration file `config.xml` contains the following settin - [skip\_unavailable\_shards](../settings/settings.md#settings-skip_unavailable_shards) -## timezone {#server_settings-timezone} +## timezone {#server_configuration_parameters-timezone} Временная зона сервера. @@ -628,7 +652,7 @@ The default server configuration file `config.xml` contains the following settin Europe/Moscow ``` -## tcp\_port {#server_settings-tcp_port} +## tcp\_port {#server_configuration_parameters-tcp_port} Порт для взаимодействия с клиентами по протоколу TCP. @@ -638,9 +662,9 @@ The default server configuration file `config.xml` contains the following settin 9000 ``` -## tcp\_port\_secure {#server_settings-tcp_port-secure} +## tcp\_port\_secure {#server_configuration_parameters-tcp_port-secure} -TCP порт для защищённого обмена данными с клиентами. Используйте с настройкой [OpenSSL](#server_settings-openssl). +TCP порт для защищённого обмена данными с клиентами. Используйте с настройкой [OpenSSL](#server_configuration_parameters-openssl). **Возможные значения** @@ -652,7 +676,7 @@ TCP порт для защищённого обмена данными с кли 9440 ``` -## mysql\_port {#server_settings-mysql_port} +## mysql\_port {#server_configuration_parameters-mysql_port} Порт для взаимодействия с клиентами по протоколу MySQL. @@ -677,7 +701,7 @@ TCP порт для защищённого обмена данными с кли ## uncompressed\_cache\_size {#server-settings-uncompressed_cache_size} -Размер кеша (в байтах) для несжатых данных, используемых движками таблиц семейства [MergeTree](../table_engines/mergetree.md). +Размер кеша (в байтах) для несжатых данных, используемых движками таблиц семейства [MergeTree](../../operations/server_configuration_parameters/settings.md). Кеш единый для сервера. Память выделяется по требованию. Кеш используется в том случае, если включена опция [use\_uncompressed\_cache](../settings/settings.md). @@ -689,9 +713,9 @@ TCP порт для защищённого обмена данными с кли 8589934592 ``` -## user\_files\_path {#server_settings-user_files_path} +## user\_files\_path {#server_configuration_parameters-user_files_path} -Каталог с пользовательскими файлами. Используется в табличной функции [file()](../../query_language/table_functions/file.md). +Каталог с пользовательскими файлами. Используется в табличной функции [file()](../../operations/server_configuration_parameters/settings.md). **Пример** @@ -763,7 +787,7 @@ ClickHouse использует ZooKeeper для хранения метадан **Смотрите также** -- [Репликация](../../operations/table_engines/replication.md) +- [Репликация](../../operations/server_configuration_parameters/settings.md) - [ZooKeeper Programmer’s Guide](http://zookeeper.apache.org/doc/current/zookeeperProgrammers.html) ## use\_minimalistic\_part\_header\_in\_zookeeper {#server-settings-use_minimalistic_part_header_in_zookeeper} @@ -772,20 +796,20 @@ ClickHouse использует ZooKeeper для хранения метадан Параметр применяется только к семейству таблиц `MergeTree`. Его можно установить: -- Глобально в разделе [merge\_tree](#server_settings-merge_tree) файла `config.xml`. +- Глобально в разделе [merge\_tree](#server_configuration_parameters-merge_tree) файла `config.xml`. ClickHouse использует этот параметр для всех таблиц на сервере. Вы можете изменить настройку в любое время. Существующие таблицы изменяют свое поведение при изменении параметра. - Для каждой отдельной таблицы. - При создании таблицы укажите соответствующую [настройку движка](../table_engines/mergetree.md#table_engine-mergetree-creating-a-table). Поведение существующей таблицы с установленным параметром не изменяется даже при изменении глобального параметра. + При создании таблицы укажите соответствующую [настройку движка](../../operations/server_configuration_parameters/settings.md#table_engine-mergetree-creating-a-table). Поведение существующей таблицы с установленным параметром не изменяется даже при изменении глобального параметра. **Возможные значения** - 0 — функциональность выключена. - 1 — функциональность включена. -Если `use_minimalistic_part_header_in_zookeeper = 1`, то [реплицированные](../table_engines/replication.md) таблицы хранят заголовки кусков данных в компактном виде, используя только одну `znode`. Если таблица содержит много столбцов, этот метод хранения значительно уменьшает объём данных, хранящихся в Zookeeper. +Если `use_minimalistic_part_header_in_zookeeper = 1`, то [реплицированные](../../operations/server_configuration_parameters/settings.md) таблицы хранят заголовки кусков данных в компактном виде, используя только одну `znode`. Если таблица содержит много столбцов, этот метод хранения значительно уменьшает объём данных, хранящихся в Zookeeper. !!! attention "Внимание" После того как вы установили `use_minimalistic_part_header_in_zookeeper = 1`, невозможно откатить ClickHouse до версии, которая не поддерживает этот параметр. Будьте осторожны при обновлении ClickHouse на серверах в кластере. Не обновляйте все серверы сразу. Безопаснее проверять новые версии ClickHouse в тестовой среде или только на некоторых серверах кластера. @@ -808,4 +832,4 @@ ClickHouse использует ZooKeeper для хранения метадан **Значение по умолчанию**: 15. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_settings/settings/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_configuration_parameters/settings/) diff --git a/docs/ru/operations/settings/merge_tree_settings.md b/docs/ru/operations/settings/merge_tree_settings.md new file mode 100644 index 00000000000..5297e359547 --- /dev/null +++ b/docs/ru/operations/settings/merge_tree_settings.md @@ -0,0 +1,96 @@ +# Настройки MergeTree таблиц {#merge-tree-settings} + +Значения настроек merge-tree (для всех MergeTree таблиц) можно посмотреть в таблице `system.merge_tree_settings`, их можно переопределить в `config.xml` в секции `merge_tree`, или задать в секции `SETTINGS` у каждой таблицы. + +Пример переопределения в `config.xml`: +```text + + 5 + +``` + +Пример для определения в `SETTINGS` у конкретной таблицы: +```sql +CREATE TABLE foo +( + `A` Int64 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS max_suspicious_broken_parts = 500; +``` + +Пример изменения настроек у конкретной таблицы командой `ALTER TABLE ... MODIFY SETTING`: +```sql +ALTER TABLE foo + MODIFY SETTING max_suspicious_broken_parts = 100; +``` + + +## parts_to_throw_insert {#parts-to-throw-insert} + +Eсли число кусков в партиции превышает значение `parts_to_throw_insert`, INSERT прерывается с исключением `Too many parts (N). Merges are processing significantly slower than inserts`. + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию: 300. + +Для достижения максимальной производительности запросов `SELECT` необходимо минимизировать количество обрабатываемых кусков, см. [Дизайн MergeTree](../../development/architecture.md#merge-tree). + +Можно установить большее значение 600 (1200), это уменьшит вероятность возникновения ошибки `Too many parts`, но в тоже время вы позже обнаружите возможную проблему со слияниями (например, из-за недостатка места на диске) и деградацию производительности `SELECT`. + + +## parts_to_delay_insert {#parts-to-delay-insert} + +Eсли число кусков в партиции превышает значение `parts_to_delay_insert`, `INSERT` искусственно замедляется. + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию: 150. + +ClickHouse искусственно выполняет `INSERT` дольше (добавляет 'sleep'), чтобы фоновый механизм слияния успевал слиять куски быстрее, чем они добавляются. + + +## max_delay_to_insert {#max-delay-to-insert} + +Величина в секундах, которая используется для расчета задержки `INSERT`, если число кусков в партиции превышает значение [parts_to_delay_insert](#parts-to-delay-insert). + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию: 1. + +Величина задержи (в миллисекундах) для `INSERT` вычисляется по формуле: + +```code +max_k = parts_to_throw_insert - parts_to_delay_insert +k = 1 + parts_count_in_partition - parts_to_delay_insert +delay_milliseconds = pow(max_delay_to_insert * 1000, k / max_k) +``` + +Т.е. если в партиции уже 299 кусков и parts_to_throw_insert = 300, parts_to_delay_insert = 150, max_delay_to_insert = 1, `INSERT` замедлится на `pow( 1 * 1000, (1 + 299 - 150) / (300 - 150) ) = 1000` миллисекунд. + +## old_parts_lifetime {#old-parts-lifetime} + +Время (в секундах) хранения неактивных кусков, для защиты от потери данных при спонтанной перезагрузке сервера или О.С. + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию: 480. + +После слияния нескольких кусков в новый кусок, ClickHouse помечает исходные куски как неактивные и удаляет их после `old_parts_lifetime` секунд. +Неактивные куски удаляются, если они не используются в текущих запросах, т.е. если счетчик ссылок куска -- `refcount` равен нулю. + +Неактивные куски удаляются не сразу, потому что при записи нового куска не вызывается `fsync`, т.е. некоторое время новый кусок находится только в оперативной памяти сервера (кеше О.С.). Т.о. при спонтанной перезагрузке сервера новый (смерженный) кусок может быть потерян или испорчен. В этом случае ClickHouse в процессе старта при проверке целостности кусков обнаружит проблему, вернет неактивные куски в список активных и позже заново их смержит. Сломанный кусок в этом случае переименовывается (добавляется префикс broken_) и перемещается в папку detached. Если проверка целостности не обнаруживает проблем в смерженном куске, то исходные неактивные куски переименовываются (добавляется префикс ignored_) и перемещаются в папку detached. + +Стандартное значение Linux dirty_expire_centisecs - 30 секунд (максимальное время, которое записанные данные хранятся только в оперативной памяти), но при больших нагрузках на дисковую систему, данные могут быть записаны намного позже. Экспериментально было найдено время - 480 секунд, за которое гарантированно новый кусок будет записан на диск. + + +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/merge_tree_settings/) diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md index 94791d79420..5ad28eed0a8 100644 --- a/docs/ru/operations/settings/query_complexity.md +++ b/docs/ru/operations/settings/query_complexity.md @@ -76,11 +76,11 @@ ## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} -Включает или отключает выполнение секций `GROUP BY` во внешней памяти. Смотрите [GROUP BY во внешней памяти](../../query_language/select.md#select-group-by-in-external-memory). +Включает или отключает выполнение секций `GROUP BY` во внешней памяти. Смотрите [GROUP BY во внешней памяти](../../sql_reference/statements/select.md#select-group-by-in-external-memory). Возможные значения: -- Максимальный объём RAM (в байтах), который может использовать отдельная операция [GROUP BY](../../query_language/select.md#select-group-by-clause). +- Максимальный объём RAM (в байтах), который может использовать отдельная операция [GROUP BY](../../sql_reference/statements/select.md#select-group-by-clause). - 0 — `GROUP BY` во внешней памяти отключен. Значение по умолчанию — 0. @@ -228,7 +228,7 @@ FORMAT Null; Ограничивает количество строк в хэш-таблице, используемой при соединении таблиц. -Параметр применяется к операциям [SELECT… JOIN](../../query_language/select.md#select-join) и к движку таблиц [Join](../table_engines/join.md). +Параметр применяется к операциям [SELECT… JOIN](../../sql_reference/statements/select.md#select-join) и к движку таблиц [Join](../../engines/table_engines/special/join.md). Если запрос содержит несколько `JOIN`, то ClickHouse проверяет значение настройки для каждого промежуточного результата. @@ -245,7 +245,7 @@ FORMAT Null; Ограничивает размер (в байтах) хэш-таблицы, используемой при объединении таблиц. -Параметр применяется к операциям [SELECT… JOIN](../../query_language/select.md#select-join) и к движку таблиц [Join](../table_engines/join.md). +Параметр применяется к операциям [SELECT… JOIN](../../sql_reference/statements/select.md#select-join) и к движку таблиц [Join](../../engines/table_engines/special/join.md). Если запрос содержит несколько `JOIN`, то ClickHouse проверяет значение настройки для каждого промежуточного результата. @@ -274,8 +274,8 @@ FORMAT Null; **Смотрите также** -- [Секция JOIN](../../query_language/select.md#select-join) -- [Движок таблиц Join](../table_engines/join.md) +- [Секция JOIN](../../sql_reference/statements/select.md#select-join) +- [Движоy таблиц Join](../../engines/table_engines/special/join.md) ## max\_partitions\_per\_insert\_block {#max-partitions-per-insert-block} diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 42b504086d0..732829494e1 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2,7 +2,7 @@ ## distributed\_product\_mode {#distributed-product-mode} -Изменяет поведение [распределенных подзапросов](../../query_language/select.md). +Изменяет поведение [распределенных подзапросов](../../sql_reference/statements/select.md). ClickHouse применяет настройку в тех случаях, когда запрос содержит произведение распределённых таблиц, т.е. когда запрос к распределенной таблице содержит не-GLOBAL подзапрос к также распределенной таблице. @@ -11,7 +11,7 @@ ClickHouse применяет настройку в тех случаях, ко - Только подзапросы для IN, JOIN. - Только если в секции FROM используется распределённая таблица, содержащая более одного шарда. - Если подзапрос касается распределенной таблицы, содержащей более одного шарда. -- Не используется в случае табличной функции [remote](../../query_language/table_functions/remote.md). +- Не используется в случае табличной функции [remote](../../sql_reference/table_functions/remote.md). Возможные значения: @@ -46,7 +46,7 @@ ClickHouse применяет настройку в тех случаях, ко ## fallback\_to\_stale\_replicas\_for\_distributed\_queries {#settings-fallback_to_stale_replicas_for_distributed_queries} -Форсирует запрос в устаревшую реплику в случае, если актуальные данные недоступны. См. [Репликация](../../operations/table_engines/replication.md). +Форсирует запрос в устаревшую реплику в случае, если актуальные данные недоступны. См. [Репликация](../../engines/table_engines/mergetree_family/replication.md). Из устаревших реплик таблицы ClickHouse выбирает наиболее актуальную. @@ -60,7 +60,7 @@ ClickHouse применяет настройку в тех случаях, ко Работает с таблицами семейства MergeTree. -При `force_index_by_date=1` ClickHouse проверяет, есть ли в запросе условие на ключ даты, которое может использоваться для отсечения диапазонов данных. Если подходящего условия нет - кидается исключение. При этом не проверяется, действительно ли условие уменьшает объём данных для чтения. Например, условие `Date != '2000-01-01'` подходит даже в том случае, когда соответствует всем данным в таблице (т.е. для выполнения запроса требуется full scan). Подробнее про диапазоны данных в таблицах MergeTree читайте в разделе [MergeTree](../table_engines/mergetree.md). +При `force_index_by_date=1` ClickHouse проверяет, есть ли в запросе условие на ключ даты, которое может использоваться для отсечения диапазонов данных. Если подходящего условия нет - кидается исключение. При этом не проверяется, действительно ли условие уменьшает объём данных для чтения. Например, условие `Date != '2000-01-01'` подходит даже в том случае, когда соответствует всем данным в таблице (т.е. для выполнения запроса требуется full scan). Подробнее про диапазоны данных в таблицах MergeTree читайте в разделе [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). ## force\_primary\_key {#settings-force-primary-key} @@ -68,7 +68,7 @@ ClickHouse применяет настройку в тех случаях, ко Работает с таблицами семейства MergeTree. -При `force_primary_key=1` ClickHouse проверяет, есть ли в запросе условие на первичный ключ, которое может использоваться для отсечения диапазонов данных. Если подходящего условия нет - кидается исключение. При этом не проверяется, действительно ли условие уменьшает объём данных для чтения. Подробнее про диапазоны данных в таблицах MergeTree читайте в разделе [MergeTree](../table_engines/mergetree.md). +При `force_primary_key=1` ClickHouse проверяет, есть ли в запросе условие на первичный ключ, которое может использоваться для отсечения диапазонов данных. Если подходящего условия нет - кидается исключение. При этом не проверяется, действительно ли условие уменьшает объём данных для чтения. Подробнее про диапазоны данных в таблицах MergeTree читайте в разделе [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). ## format\_schema {#format-schema} @@ -129,7 +129,7 @@ ClickHouse применяет настройку в тех случаях, ко ## max\_http\_get\_redirects {#setting-max_http_get_redirects} -Ограничивает максимальное количество переходов по редиректам в таблицах с движком [URL](../table_engines/url.md) при выполнении HTTP запросов методом GET. Настройка применяется для обоих типов таблиц: созданных запросом [CREATE TABLE](../../query_language/create/#create-table-query) и с помощью табличной функции [url](../../query_language/table_functions/url.md). +Ограничивает максимальное количество переходов по редиректам в таблицах с движком [URL](../../engines/table_engines/special/url.md) при выполнении HTTP запросов методом GET. Настройка применяется для обоих типов таблиц: созданных запросом [CREATE TABLE](../../sql_reference/create/#create-table-query) и с помощью табличной функции [url](../../sql_reference/table_functions/url.md). Возможные значения: @@ -165,7 +165,7 @@ ClickHouse применяет настройку в тех случаях, ко ## input\_format\_values\_interpret\_expressions {#settings-input_format_values_interpret_expressions} -Включает или отключает парсер SQL, если потоковый парсер не может проанализировать данные. Этот параметр используется только для формата [Values](../../interfaces/formats.md#data-format-values) при вставке данных. Дополнительные сведения о парсерах читайте в разделе [Синтаксис](../../query_language/syntax.md). +Включает или отключает парсер SQL, если потоковый парсер не может проанализировать данные. Этот параметр используется только для формата [Values](../../interfaces/formats.md#data-format-values) при вставке данных. Дополнительные сведения о парсерах читайте в разделе [Синтаксис](../../sql_reference/syntax.md). Возможные значения: @@ -181,7 +181,7 @@ ClickHouse применяет настройку в тех случаях, ко Пример использования: -Вставим значение типа [DateTime](../../data_types/datetime.md) при разных значения настройки. +Вставим значение типа [DateTime](../../sql_reference/data_types/datetime.md) при разных значения настройки. ``` sql SET input_format_values_interpret_expressions = 0; @@ -213,6 +213,49 @@ INSERT INTO datetime_t SELECT now() Ok. ``` +## input\_format\_values\_deduce\_templates\_of\_expressions {#settings-input_format_values_deduce_templates_of_expressions} + +Включает или отключает попытку вычисления шаблона для выражений SQL в формате [Values](../../interfaces/formats.md#data-format-values). Это позволяет гораздо быстрее парсить и интерпретировать выражения в `Values`, если выражения в последовательных строках имеют одинаковую структуру. ClickHouse пытается вычислить шаблон выражения, распарсить следующие строки с помощью этого шаблона и вычислить выражение в пачке успешно проанализированных строк. + +Возможные значения: + +- 0 — Выключена. +- 1 — Включена. + +Значение по умолчанию: 1. + +Для следующего запроса: + +``` sql +INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), (upper('Values')), ... +``` + +- Если `input_format_values_interpret_expressions=1` и `format_values_deduce_templates_of_expressions=0`, выражения интерпретируются отдельно для каждой строки (это очень медленно для большого количества строк). +- Если `input_format_values_interpret_expressions=0` и `format_values_deduce_templates_of_expressions=1`, выражения в первой, второй и третьей строках парсятся с помощью шаблона `lower(String)` и интерпретируется вместе, выражение в четвертой строке парсится с другим шаблоном (`upper(String)`). +- Если `input_format_values_interpret_expressions=1` и `format_values_deduce_templates_of_expressions=1`, то же самое, что и в предыдущем случае, но также позволяет выполнять резервную интерпретацию выражений отдельно, если невозможно вычислить шаблон. + +## input\_format\_values\_accurate\_types\_of\_literals {#settings-input-format-values-accurate-types-of-literals} + +Эта настройка используется, только когда `input_format_values_deduce_templates_of_expressions = 1`. Выражения для некоторых столбцов могут иметь одинаковую структуру, но содержат числовые литералы разных типов, например: + +``` sql +(..., abs(0), ...), -- UInt64 literal +(..., abs(3.141592654), ...), -- Float64 literal +(..., abs(-1), ...), -- Int64 literal +``` + +Возможные значения: + +- 0 — Выключена. + + В этом случае, ClickHouse может использовать более общий тип для некоторых литералов (например, `Float64` или `Int64` вместо `UInt64` для `42`), но это может привести к переполнению и проблемам с точностью. + +- 1 — Включена. + + В этом случае, ClickHouse проверяет фактический тип литерала и использует шаблон выражения соответствующего типа. В некоторых случаях это может значительно замедлить оценку выажения в `Values`. + +Значение по умолчанию: 1. + ## input\_format\_defaults\_for\_omitted\_fields {#session_settings-input_format_defaults_for_omitted_fields} При вставке данных запросом `INSERT`, заменяет пропущенные поля значениям по умолчанию для типа данных столбца. @@ -298,7 +341,7 @@ Ok. Выбор парсера для текстового представления дат и времени при обработке входного формата. -Настройка не применяется к [функциям для работы с датой и временем](../../query_language/functions/date_time_functions.md). +Настройка не применяется к [функциям для работы с датой и временем](../../sql_reference/functions/date_time_functions.md). Возможные значения: @@ -314,12 +357,12 @@ Ok. См. также: -- [Тип данных DateTime.](../../data_types/datetime.md) -- [Функции для работы с датой и временем.](../../query_language/functions/date_time_functions.md) +- [Тип данных DateTime.](../../sql_reference/data_types/datetime.md) +- [Функции для работы с датой и временем.](../../sql_reference/functions/date_time_functions.md) ## join\_default\_strictness {#settings-join_default_strictness} -Устанавливает строгость по умолчанию для [JOIN](../../query_language/select.md#select-join). +Устанавливает строгость по умолчанию для [JOIN](../../sql_reference/statements/select.md#select-join). Возможные значения @@ -334,7 +377,7 @@ Ok. Изменяет поведение операций, выполняемых со строгостью `ANY`. !!! warning "Внимание" - Настройка применяется только для операций `JOIN`, выполняемых над таблицами с движком [Join](../table_engines/join.md). + Настройка применяется только для операций `JOIN`, выполняемых над таблицами с движком [Join](../../engines/table_engines/special/join.md). Возможные значения: @@ -345,18 +388,18 @@ Ok. См. также: -- [Секция JOIN](../../query_language/select.md#select-join) -- [Движок таблиц Join](../table_engines/join.md) +- [Секция JOIN](../../sql_reference/statements/select.md#select-join) +- [Движок таблиц Join](../../engines/table_engines/special/join.md) - [join\_default\_strictness](#settings-join_default_strictness) ## join\_use\_nulls {#join_use_nulls} -Устанавливает тип поведения [JOIN](../../query_language/select.md). При объединении таблиц могут появиться пустые ячейки. ClickHouse заполняет их по-разному в зависимости от настроек. +Устанавливает тип поведения [JOIN](../../sql_reference/statements/select.md). При объединении таблиц могут появиться пустые ячейки. ClickHouse заполняет их по-разному в зависимости от настроек. Возможные значения - 0 — пустые ячейки заполняются значением по умолчанию соответствующего типа поля. -- 1 — `JOIN` ведёт себя как в стандартном SQL. Тип соответствующего поля преобразуется в [Nullable](../../data_types/nullable.md#data_type-nullable), а пустые ячейки заполняются значениями [NULL](../../query_language/syntax.md). +- 1 — `JOIN` ведёт себя как в стандартном SQL. Тип соответствующего поля преобразуется в [Nullable](../../sql_reference/data_types/nullable.md#data_type-nullable), а пустые ячейки заполняются значениями [NULL](../../sql_reference/syntax.md). Значение по умолчанию: 0. @@ -376,7 +419,7 @@ Ok. ## merge\_tree\_uniform\_read\_distribution {#setting-merge-tree-uniform-read-distribution} -При чтении из таблиц [MergeTree](../table_engines/mergetree.md) ClickHouse использует несколько потоков. Этот параметр включает/выключает равномерное распределение заданий по рабочим потокам. Алгоритм равномерного распределения стремится сделать время выполнения всех потоков примерно равным для одного запроса `SELECT`. +При чтении из таблиц [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) ClickHouse использует несколько потоков. Этот параметр включает/выключает равномерное распределение заданий по рабочим потокам. Алгоритм равномерного распределения стремится сделать время выполнения всех потоков примерно равным для одного запроса `SELECT`. Возможные значения: @@ -387,7 +430,7 @@ Ok. ## merge\_tree\_min\_rows\_for\_concurrent\_read {#setting-merge-tree-min-rows-for-concurrent-read} -Если количество строк, считываемых из файла таблицы [MergeTree](../table_engines/mergetree.md) превышает `merge_tree_min_rows_for_concurrent_read`, то ClickHouse пытается выполнить одновременное чтение из этого файла в несколько потоков. +Если количество строк, считываемых из файла таблицы [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) превышает `merge_tree_min_rows_for_concurrent_read`, то ClickHouse пытается выполнить одновременное чтение из этого файла в несколько потоков. Возможные значения: @@ -397,7 +440,7 @@ Ok. ## merge\_tree\_min\_bytes\_for\_concurrent\_read {#setting-merge-tree-min-bytes-for-concurrent-read} -Если число байтов, которое должно быть прочитано из одного файла таблицы с движком [MergeTree](../table_engines/mergetree.md), превышает значение `merge_tree_min_bytes_for_concurrent_read`, то ClickHouse выполняет одновременное чтение в несколько потоков из этого файла. +Если число байтов, которое должно быть прочитано из одного файла таблицы с движком [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md), превышает значение `merge_tree_min_bytes_for_concurrent_read`, то ClickHouse выполняет одновременное чтение в несколько потоков из этого файла. Возможное значение: @@ -439,7 +482,7 @@ Ok. Если требуется прочитать более, чем `merge_tree_max_rows_to_use_cache` строк в одном запросе, ClickHouse не используют кэш несжатых блоков. -Кэш несжатых блоков хранит данные, извлечённые при выполнении запросов. ClickHouse использует этот кэш для ускорения ответов на повторяющиеся небольшие запросы. Настройка защищает кэш от замусоривания запросами, для выполнения которых необходимо извлечь большое количество данных. Настройка сервера [uncompressed\_cache\_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) определяет размер кэша несжатых блоков. +Кэш несжатых блоков хранит данные, извлечённые при выполнении запросов. ClickHouse использует этот кэш для ускорения ответов на повторяющиеся небольшие запросы. Настройка защищает кэш от замусоривания запросами, для выполнения которых необходимо извлечь большое количество данных. Настройка сервера [uncompressed\_cache\_size](../server_configuration_parameters/settings.md#server-settings-uncompressed_cache_size) определяет размер кэша несжатых блоков. Возможные значения: @@ -451,7 +494,7 @@ Ok. Если требуется прочитать более, чем `merge_tree_max_bytes_to_use_cache` байтов в одном запросе, ClickHouse не используют кэш несжатых блоков. -Кэш несжатых блоков хранит данные, извлечённые при выполнении запросов. ClickHouse использует кэш для ускорения ответов на повторяющиеся небольшие запросы. Настройка защищает кэш от переполнения. Настройка сервера [uncompressed\_cache\_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) определяет размер кэша несжатых блоков. +Кэш несжатых блоков хранит данные, извлечённые при выполнении запросов. ClickHouse использует кэш для ускорения ответов на повторяющиеся небольшие запросы. Настройка защищает кэш от переполнения. Настройка сервера [uncompressed\_cache\_size](../server_configuration_parameters/settings.md#server-settings-uncompressed_cache_size) определяет размер кэша несжатых блоков. Возможное значение: @@ -476,7 +519,7 @@ ClickHouse использует этот параметр при чтении д Установка логирования запроса. -Запросы, переданные в ClickHouse с этой установкой, логируются согласно правилам конфигурационного параметра сервера [query\_log](../../operations/server_settings/settings.md#server_settings-query-log). +Запросы, переданные в ClickHouse с этой установкой, логируются согласно правилам конфигурационного параметра сервера [query\_log](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-query-log). Пример: @@ -488,7 +531,7 @@ log_queries=1 Установка логирования информации о потоках выполнения запроса. -Лог информации о потоках выполнения запросов, переданных в ClickHouse с этой установкой, записывается согласно правилам конфигурационного параметра сервера [query\_thread\_log](../server_settings/settings.md#server_settings-query-thread-log). +Лог информации о потоках выполнения запросов, переданных в ClickHouse с этой установкой, записывается согласно правилам конфигурационного параметра сервера [query\_thread\_log](../server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log). Пример: @@ -508,9 +551,31 @@ log_query_threads=1 Это значение намного больше, чем `max_block_size`. Это сделано, потому что некоторые движки таблиц (`*MergeTree`) будут на каждый вставляемый блок формировать кусок данных на диске, что является довольно большой сущностью. Также, в таблицах типа `*MergeTree`, данные сортируются при вставке, и достаточно большой размер блока позволяет отсортировать больше данных в оперативке. +## min_insert_block_size_rows {#min-insert-block-size-rows} + +Устанавливает минимальное количество строк в блоке, который может быть вставлен в таблицу запросом `INSERT`. Блоки меньшего размера склеиваются в блоки большего размера. + +Возможные значения: + +- Целое положительное число. +- 0 — Склейка блоков выключена. + +Значение по умолчанию: 1048576. + +## min_insert_block_size_bytes {#min-insert-block-size-bytes} + +Устанавливает минимальное количество байтов в блоке, который может быть вставлен в таблицу запросом `INSERT`. Блоки меньшего размера склеиваются в блоки большего размера. + +Возможные значения: + +- Целое положительное число. +- 0 — Склейка блоков выключена. + +Значение по умолчанию: 268435456. + ## max\_replica\_delay\_for\_distributed\_queries {#settings-max_replica_delay_for_distributed_queries} -Отключает отстающие реплики при распределенных запросах. См. [Репликация](../../operations/table_engines/replication.md). +Отключает отстающие реплики при распределенных запросах. См. [Репликация](../../engines/table_engines/mergetree_family/replication.md). Устанавливает время в секундах. Если отставание реплики больше установленного значения, то реплика не используется. @@ -555,7 +620,7 @@ log_query_threads=1 ## min\_compress\_block\_size {#min-compress-block-size} -Для таблиц типа [MergeTree](../table_engines/mergetree.md). В целях уменьшения задержек при обработке запросов, блок сжимается при записи следующей засечки, если его размер не меньше min\_compress\_block\_size. По умолчанию - 65 536. +Для таблиц типа [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). В целях уменьшения задержек при обработке запросов, блок сжимается при записи следующей засечки, если его размер не меньше min\_compress\_block\_size. По умолчанию - 65 536. Реальный размер блока, если несжатых данных меньше max\_compress\_block\_size, будет не меньше этого значения и не меньше объёма данных на одну засечку. @@ -634,7 +699,7 @@ log_query_threads=1 Использовать ли кэш разжатых блоков. Принимает 0 или 1. По умолчанию - 0 (выключено). -Использование кэша несжатых блоков (только для таблиц семейства MergeTree) может существенно сократить задержку и увеличить пропускную способность при работе с большим количеством коротких запросов. Включите эту настройку для пользователей, от которых идут частые короткие запросы. Также обратите внимание на конфигурационный параметр [uncompressed\_cache\_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) (настраивается только в конфигурационном файле) – размер кэша разжатых блоков. По умолчанию - 8 GiB. Кэш разжатых блоков заполняется по мере надобности, а наиболее невостребованные данные автоматически удаляются. +Использование кэша несжатых блоков (только для таблиц семейства MergeTree) может существенно сократить задержку и увеличить пропускную способность при работе с большим количеством коротких запросов. Включите эту настройку для пользователей, от которых идут частые короткие запросы. Также обратите внимание на конфигурационный параметр [uncompressed\_cache\_size](../server_configuration_parameters/settings.md#server-settings-uncompressed_cache_size) (настраивается только в конфигурационном файле) – размер кэша разжатых блоков. По умолчанию - 8 GiB. Кэш разжатых блоков заполняется по мере надобности, а наиболее невостребованные данные автоматически удаляются. Для запросов, читающих хоть немного приличный объём данных (миллион строк и больше), кэш разжатых блоков автоматически выключается, чтобы оставить место для действительно мелких запросов. Поэтому, можно держать настройку `use_uncompressed_cache` всегда выставленной в 1. @@ -850,7 +915,7 @@ ClickHouse генерирует исключение Значение по умолчанию: 1. -По умолчанию блоки, вставляемые в реплицируемые таблицы оператором `INSERT`, дедуплицируются (см. [Репликация данных](../table_engines/replication.md)). +По умолчанию блоки, вставляемые в реплицируемые таблицы оператором `INSERT`, дедуплицируются (см. [Репликация данных](../../engines/table_engines/mergetree_family/replication.md)). ## deduplicate\_blocks\_in\_dependent\_materialized\_views {#settings-deduplicate-blocks-in-dependent-materialized-views} @@ -869,15 +934,15 @@ ClickHouse генерирует исключение ## count\_distinct\_implementation {#settings-count_distinct_implementation} -Задаёт, какая из функций `uniq*` используется при выполнении конструкции [COUNT(DISTINCT …)](../../query_language/agg_functions/reference.md#agg_function-count). +Задаёт, какая из функций `uniq*` используется при выполнении конструкции [COUNT(DISTINCT …)](../../sql_reference/aggregate_functions/reference.md#agg_function-count). Возможные значения: -- [uniq](../../query_language/agg_functions/reference.md#agg_function-uniq) -- [uniqCombined](../../query_language/agg_functions/reference.md#agg_function-uniqcombined) -- [uniqCombined64](../../query_language/agg_functions/reference.md#agg_function-uniqcombined64) -- [uniqHLL12](../../query_language/agg_functions/reference.md#agg_function-uniqhll12) -- [uniqExact](../../query_language/agg_functions/reference.md#agg_function-uniqexact) +- [uniq](../../sql_reference/aggregate_functions/reference.md#agg_function-uniq) +- [uniqCombined](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqcombined) +- [uniqCombined64](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqcombined64) +- [uniqHLL12](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqhll12) +- [uniqExact](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqexact) Значение по умолчанию: `uniqExact`. @@ -957,7 +1022,7 @@ ClickHouse генерирует исключение ## optimize\_throw\_if\_noop {#setting-optimize_throw_if_noop} -Включает или отключает генерирование исключения в в случаях, когда запрос [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) не выполняет мёрж. +Включает или отключает генерирование исключения в в случаях, когда запрос [OPTIMIZE](../../sql_reference/statements/misc.md#misc_operations-optimize) не выполняет мёрж. По умолчанию, `OPTIMIZE` завершается успешно и в тех случаях, когда он ничего не сделал. Настройка позволяет отделить подобные случаи и включает генерирование исключения с поясняющим сообщением. @@ -970,7 +1035,7 @@ ClickHouse генерирует исключение ## distributed\_directory\_monitor\_sleep\_time\_ms {#distributed_directory_monitor_sleep_time_ms} -Основной интервал отправки данных движком таблиц [Distributed](../table_engines/distributed.md). Фактический интервал растёт экспоненциально при возникновении ошибок. +Основной интервал отправки данных движком таблиц [Distributed](../../engines/table_engines/special/distributed.md). Фактический интервал растёт экспоненциально при возникновении ошибок. Возможные значения: @@ -980,7 +1045,7 @@ ClickHouse генерирует исключение ## distributed\_directory\_monitor\_max\_sleep\_time\_ms {#distributed_directory_monitor_max_sleep_time_ms} -Максимальный интервал отправки данных движком таблиц [Distributed](../table_engines/distributed.md). Ограничивает экпоненциальный рост интервала, установленого настройкой [distributed\_directory\_monitor\_sleep\_time\_ms](#distributed_directory_monitor_sleep_time_ms). +Максимальный интервал отправки данных движком таблиц [Distributed](../../engines/table_engines/special/distributed.md). Ограничивает экпоненциальный рост интервала, установленого настройкой [distributed\_directory\_monitor\_sleep\_time\_ms](#distributed_directory_monitor_sleep_time_ms). Возможные значения: @@ -992,7 +1057,7 @@ ClickHouse генерирует исключение Включает/выключает пакетную отправку вставленных данных. -Если пакетная отправка включена, то движок таблиц [Distributed](../table_engines/distributed.md) вместо того, чтобы отправлять каждый файл со вставленными данными по отдельности, старается отправить их все за одну операцию. Пакетная отправка улучшает производительность кластера за счет более оптимального использования ресурсов сервера и сети. +Если пакетная отправка включена, то движок таблиц [Distributed](../../engines/table_engines/special/distributed.md) вместо того, чтобы отправлять каждый файл со вставленными данными по отдельности, старается отправить их все за одну операцию. Пакетная отправка улучшает производительность кластера за счет более оптимального использования ресурсов сервера и сети. Возможные значения: @@ -1018,7 +1083,7 @@ ClickHouse генерирует исключение ## query\_profiler\_real\_time\_period\_ns {#query_profiler_real_time_period_ns} -Sets the period for a real clock timer of the [query profiler](../../operations/performance/sampling_query_profiler.md). Real clock timer counts wall-clock time. +Sets the period for a real clock timer of the [query profiler](../../operations/optimizing_performance/sampling_query_profiler.md). Real clock timer counts wall-clock time. Possible values: @@ -1031,7 +1096,7 @@ Possible values: - 0 for turning off the timer. -Type: [UInt64](../../data_types/int_uint.md). +Type: [UInt64](../../sql_reference/data_types/int_uint.md). Default value: 1000000000 nanoseconds (once a second). @@ -1041,7 +1106,7 @@ See also: ## query\_profiler\_cpu\_time\_period\_ns {#query_profiler_cpu_time_period_ns} -Sets the period for a CPU clock timer of the [query profiler](../../operations/performance/sampling_query_profiler.md). This timer counts only CPU time. +Sets the period for a CPU clock timer of the [query profiler](../../operations/optimizing_performance/sampling_query_profiler.md). This timer counts only CPU time. Possible values: @@ -1054,7 +1119,7 @@ Possible values: - 0 for turning off the timer. -Type: [UInt64](../../data_types/int_uint.md). +Type: [UInt64](../../sql_reference/data_types/int_uint.md). Default value: 1000000000 nanoseconds. @@ -1064,7 +1129,7 @@ See also: ## allow\_introspection\_functions {#settings-allow_introspection_functions} -Enables of disables [introspections functions](../../query_language/functions/introspection.md) for query profiling. +Enables of disables [introspections functions](../../sql_reference/functions/introspection.md) for query profiling. Possible values: @@ -1075,7 +1140,7 @@ Default value: 0. **See Also** -- [Sampling Query Profiler](../performance/sampling_query_profiler.md) +- [Sampling Query Profiler](../optimizing_performance/sampling_query_profiler.md) - System table [trace\_log](../../operations/system_tables.md#system_tables-trace_log) [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) diff --git a/docs/ru/operations/settings/settings_users.md b/docs/ru/operations/settings/settings_users.md index b010358e547..1719f21a031 100644 --- a/docs/ru/operations/settings/settings_users.md +++ b/docs/ru/operations/settings/settings_users.md @@ -139,6 +139,6 @@ ``` -Элемент `filter` содержать любое выражение, возвращающее значение типа [UInt8](../../data_types/int_uint.md). Обычно он содержит сравнения и логические операторы. Строки `database_name.table1`, для которых фильтр возвращает 0 не выдаются пользователю. Фильтрация несовместима с операциями `PREWHERE` и отключает оптимизацию `WHERE→PREWHERE`. +Элемент `filter` содержать любое выражение, возвращающее значение типа [UInt8](../../sql_reference/data_types/int_uint.md). Обычно он содержит сравнения и логические операторы. Строки `database_name.table1`, для которых фильтр возвращает 0 не выдаются пользователю. Фильтрация несовместима с операциями `PREWHERE` и отключает оптимизацию `WHERE→PREWHERE`. [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings_users/) diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index a8ccd369562..b68aa570f52 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -12,8 +12,8 @@ Столбцы: -- `metric` ([String](../data_types/string.md)) — название метрики. -- `value` ([Float64](../data_types/float.md)) — значение метрики. +- `metric` ([String](../sql_reference/data_types/string.md)) — название метрики. +- `value` ([Float64](../sql_reference/data_types/float.md)) — значение метрики. **Пример** @@ -63,7 +63,7 @@ user String — имя пользователя, которого использ Содержит информацию о столбцах всех таблиц. -С помощью этой таблицы можно получить информацию аналогично запросу [DESCRIBE TABLE](../query_language/misc.md#misc-describe-table), но для многих таблиц сразу. +С помощью этой таблицы можно получить информацию аналогично запросу [DESCRIBE TABLE](../sql_reference/statements/misc.md#misc-describe-table), но для многих таблиц сразу. Таблица `system.columns` содержит столбцы (тип столбца указан в скобках): @@ -131,31 +131,70 @@ SELECT * FROM system.contributors WHERE name='Olga Khvostikova' ## system.detached\_parts {#system_tables-detached_parts} -Содержит информацию об отсоединённых кусках таблиц семейства [MergeTree](table_engines/mergetree.md). Столбец `reason` содержит причину, по которой кусок был отсоединён. Для кусов, отсоединённых пользователем, `reason` содержит пустую строку. -Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION\|PART](../query_language/query_language/alter/#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). -Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). +Содержит информацию об отсоединённых кусках таблиц семейства [MergeTree](../engines/table_engines/mergetree_family/mergetree.md). Столбец `reason` содержит причину, по которой кусок был отсоединён. Для кусов, отсоединённых пользователем, `reason` содержит пустую строку. +Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION\|PART](../sql_reference/alter/#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). +Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../sql_reference/alter/#alter_drop-detached). -## system.dictionaries {#system-dictionaries} +## system.dictionaries {#system_tables-dictionaries} -Содержит информацию о внешних словарях. +Содержит информацию о [внешних словарях](../sql_reference/dictionaries/external_dictionaries/external_dicts.md). Столбцы: -- `name String` — Имя словаря. -- `type String` — Тип словаря: Flat, Hashed, Cache. -- `origin String` — Путь к конфигурационному файлу, в котором описан словарь. -- `attribute.names Array(String)` — Массив имён атрибутов, предоставляемых словарём. -- `attribute.types Array(String)` — Соответствующий массив типов атрибутов, предоставляемых словарём. -- `has_hierarchy UInt8` — Является ли словарь иерархическим. -- `bytes_allocated UInt64` — Количество оперативной памяти, которое использует словарь. -- `hit_rate Float64` — Для cache-словарей - доля использований, для которых значение было в кэше. -- `element_count UInt64` — Количество хранящихся в словаре элементов. -- `load_factor Float64` — Доля заполненности словаря (для hashed словаря - доля заполнения хэш-таблицы). -- `creation_time DateTime` — Время создания или последней успешной перезагрузки словаря. -- `last_exception String` — Текст ошибки, возникшей при создании или перезагрузке словаря, если словарь не удалось создать. -- `source String` - Текст, описывающий источник данных для словаря. +- `database` ([String](../sql_reference/data_types/string.md)) — Имя базы данных, в которой находится словарь, созданный с помощью DDL-запроса. Пустая строка для других словарей. +- `name` ([String](../sql_reference/data_types/string.md)) — [Имя словаря](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md). +- `status` ([Enum8](../sql_reference/data_types/enum.md)) — Статус словаря. Возможные значения: + - `NOT_LOADED` — Словарь не загружен, потому что не использовался. + - `LOADED` — Словарь загружен успешно. + - `FAILED` — Словарь не загружен в результате ошибки. + - `LOADING` — Словарь в процессе загрузки. + - `LOADED_AND_RELOADING` — Словарь загружен успешно, сейчас перезагружается (частые причины: запрос [SYSTEM RELOAD DICTIONARY](../sql_reference/statements/system.md#query_language-system-reload-dictionary), таймаут, изменение настроек словаря). + - `FAILED_AND_RELOADING` — Словарь не загружен в результате ошибки, сейчас перезагружается. +- `origin` ([String](../sql_reference/data_types/string.md)) — Путь к конфигурационному файлу, описывающему словарь. +- `type` ([String](../sql_reference/data_types/string.md)) — Тип размещения словаря. [Хранение словарей в памяти](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md). +- `key` — [Тип ключа](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-key): Числовой ключ ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) или Составной ключ ([String](../sql_reference/data_types/string.md)) — строка вида “(тип 1, тип 2, …, тип n)”. +- `attribute.names` ([Array](../sql_reference/data_types/array.md)([String](../sql_reference/data_types/string.md))) — Массив [имен атрибутов](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes), предоставляемых справочником. +- `attribute.types` ([Array](../sql_reference/data_types/array.md)([String](../sql_reference/data_types/string.md))) — Соответствующий массив [типов атрибутов](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes), предоставляемых справочником. +- `bytes_allocated` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Объем оперативной памяти, используемый словарем. +- `query_count` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Количество запросов с момента загрузки словаря или с момента последней успешной перезагрузки. +- `hit_rate` ([Float64](../sql_reference/data_types/float.md)) — Для cache-словарей — процент закэшированных значений. +- `element_count` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Количество элементов, хранящихся в словаре. +- `load_factor` ([Float64](../sql_reference/data_types/float.md)) — Процент заполнения словаря (для хэшированного словаря — процент заполнения хэш-таблицы). +- `source` ([String](../sql_reference/data_types/string.md)) — Текст, описывающий [источник данных](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md) для словаря. +- `lifetime_min` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Минимальное [время обновления](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md) словаря в памяти, по истечении которого Clickhouse попытается перезагрузить словарь (если задано `invalidate_query`, то только если он изменился). Задается в секундах. +- `lifetime_max` ([UInt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Максимальное [время обновления](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md) словаря в памяти, по истечении которого Clickhouse попытается перезагрузить словарь (если задано `invalidate_query`, то только если он изменился). Задается в секундах. +- `loading_start_time` ([DateTime](../sql_reference/data_types/datetime.md)) — Время начала загрузки словаря. +- `loading_duration` ([Float32](../sql_reference/data_types/float.md)) — Время, затраченное на загрузку словаря. +- `last_exception` ([String](../sql_reference/data_types/string.md)) — Текст ошибки, возникающей при создании или перезагрузке словаря, если словарь не удалось создать. -Заметим, что количество оперативной памяти, которое использует словарь, не является пропорциональным количеству элементов, хранящихся в словаре. Так, для flat и cached словарей, все ячейки памяти выделяются заранее, независимо от реальной заполненности словаря. +**Пример** + +Настройте словарь. + +``` sql +CREATE DICTIONARY dictdb.dict +( + `key` Int64 DEFAULT -1, + `value_default` String DEFAULT 'world', + `value_expression` String DEFAULT 'xxx' EXPRESSION 'toString(127 * 172)' +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dicttbl' DB 'dictdb')) +LIFETIME(MIN 0 MAX 1) +LAYOUT(FLAT()) +``` + +Убедитесь, что словарь загружен. + +``` sql +SELECT * FROM system.dictionaries +``` + +``` text +┌─database─┬─name─┬─status─┬─origin──────┬─type─┬─key────┬─attribute.names──────────────────────┬─attribute.types─────┬─bytes_allocated─┬─query_count─┬─hit_rate─┬─element_count─┬───────────load_factor─┬─source─────────────────────┬─lifetime_min─┬─lifetime_max─┬──loading_start_time─┌──last_successful_update_time─┬──────loading_duration─┬─last_exception─┐ +│ dictdb │ dict │ LOADED │ dictdb.dict │ Flat │ UInt64 │ ['value_default','value_expression'] │ ['String','String'] │ 74032 │ 0 │ 1 │ 1 │ 0.0004887585532746823 │ ClickHouse: dictdb.dicttbl │ 0 │ 1 │ 2020-03-04 04:17:34 │ 2020-03-04 04:30:34 │ 0.002 │ │ +└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ +``` ## system.events {#system_tables-events} @@ -163,9 +202,9 @@ SELECT * FROM system.contributors WHERE name='Olga Khvostikova' Столбцы: -- `event` ([String](../data_types/string.md)) — имя события. -- `value` ([UInt64](../data_types/int_uint.md)) — количество произошедших событий. -- `description` ([String](../data_types/string.md)) — описание события. +- `event` ([String](../sql_reference/data_types/string.md)) — имя события. +- `value` ([UInt64](../sql_reference/data_types/int_uint.md)) — количество произошедших событий. +- `description` ([String](../sql_reference/data_types/string.md)) — описание события. **Пример** @@ -201,7 +240,7 @@ SELECT * FROM system.events LIMIT 5 ## system.graphite\_retentions {#system-graphite-retentions} -Содержит информацию о том, какие параметры [graphite\_rollup](server_settings/settings.md#server_settings-graphite_rollup) используются в таблицах с движками [\*GraphiteMergeTree](table_engines/graphitemergetree.md). +Содержит информацию о том, какие параметры [graphite\_rollup](server_configuration_parameters/settings.md#server_configuration_parameters-graphite_rollup) используются в таблицах с движками [\*GraphiteMergeTree](../engines/table_engines/mergetree_family/graphitemergetree.md). Столбцы: @@ -241,9 +280,9 @@ SELECT * FROM system.events LIMIT 5 Столбцы: -- `metric` ([String](../data_types/string.md)) — название метрики. -- `value` ([Int64](../data_types/int_uint.md)) — значение метрики. -- `description` ([String](../data_types/string.md)) — описание метрики. +- `metric` ([String](../sql_reference/data_types/string.md)) — название метрики. +- `value` ([Int64](../sql_reference/data_types/int_uint.md)) — значение метрики. +- `description` ([String](../sql_reference/data_types/string.md)) — описание метрики. Список поддержанных метрик смотрите в файле [src/Common/CurrentMetrics.cpp](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/CurrentMetrics.cpp). @@ -350,13 +389,13 @@ CurrentMetric_ReplicatedChecks: 0 ## system.parts {#system_tables-parts} -Содержит информацию о кусках данных таблиц семейства [MergeTree](table_engines/mergetree.md). +Содержит информацию о кусках данных таблиц семейства [MergeTree](../engines/table_engines/mergetree_family/mergetree.md). Каждая строка описывает один кусок данных. Столбцы: -- `partition` (`String`) – Имя партиции. Что такое партиция можно узнать из описания запроса [ALTER](../query_language/alter.md#query_language_queries_alter). +- `partition` (`String`) – Имя партиции. Что такое партиция можно узнать из описания запроса [ALTER](../sql_reference/statements/alter.md#sql_reference_queries_alter). Форматы: @@ -407,7 +446,7 @@ CurrentMetric_ReplicatedChecks: 0 - `primary_key_bytes_in_memory_allocated` (`UInt64`) – объём памяти (в байтах) выделенный для размещения первичных ключей. -- `is_frozen` (`UInt8`) – Признак, показывающий существование бэкапа партиции. 1, бэкап есть. 0, бэкапа нет. Смотрите раздел [FREEZE PARTITION](../query_language/alter.md#alter_freeze-partition). +- `is_frozen` (`UInt8`) – Признак, показывающий существование бэкапа партиции. 1, бэкап есть. 0, бэкапа нет. Смотрите раздел [FREEZE PARTITION](../sql_reference/statements/alter.md#alter_freeze-partition). - `database` (`String`) – имя базы данных. @@ -419,11 +458,11 @@ CurrentMetric_ReplicatedChecks: 0 - `disk` (`String`) – имя диска, на котором находится кусок данных. -- `hash_of_all_files` (`String`) – значение [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) для сжатых файлов. +- `hash_of_all_files` (`String`) – значение [sipHash128](../sql_reference/functions/hash_functions.md#hash_functions-siphash128) для сжатых файлов. -- `hash_of_uncompressed_files` (`String`) – значение [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) несжатых файлов (файлы с засечками, первичным ключом и пр.) +- `hash_of_uncompressed_files` (`String`) – значение [sipHash128](../sql_reference/functions/hash_functions.md#hash_functions-siphash128) несжатых файлов (файлы с засечками, первичным ключом и пр.) -- `uncompressed_hash_of_compressed_files` (`String`) – значение [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) данных в сжатых файлах как если бы они были разжатыми. +- `uncompressed_hash_of_compressed_files` (`String`) – значение [sipHash128](../sql_reference/functions/hash_functions.md#hash_functions-siphash128) данных в сжатых файлах как если бы они были разжатыми. - `bytes` (`UInt64`) – алиас для `bytes_on_disk`. @@ -431,9 +470,9 @@ CurrentMetric_ReplicatedChecks: 0 ## system.part\_log {#system_tables-part-log} -Системная таблица `system.part_log` создается только в том случае, если задана серверная настройка [part\_log](server_settings/settings.md#server_settings-part-log). +Системная таблица `system.part_log` создается только в том случае, если задана серверная настройка [part\_log](server_configuration_parameters/settings.md#server_configuration_parameters-part-log). -Содержит информацию о всех событиях, произошедших с [кусками данных](table_engines/custom_partitioning_key.md) таблиц семейства [MergeTree](table_engines/mergetree.md) (например, события добавления, удаления или слияния данных). +Содержит информацию о всех событиях, произошедших с [кусками данных](../engines/table_engines/mergetree_family/custom_partitioning_key.md) таблиц семейства [MergeTree](../engines/table_engines/mergetree_family/mergetree.md) (например, события добавления, удаления или слияния данных). Столбцы: @@ -441,7 +480,7 @@ CurrentMetric_ReplicatedChecks: 0 - `NEW_PART` — вставка нового куска. - `MERGE_PARTS` — слияние кусков. - `DOWNLOAD_PART` — загрузка с реплики. - - `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../query_language/alter.md#alter_detach-partition). + - `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../sql_reference/statements/alter.md#alter_detach-partition). - `MUTATE_PART` — изменение куска. - `MOVE_PART` — перемещение куска между дисками. - `event_date` (Date) — дата события. @@ -485,7 +524,7 @@ CurrentMetric_ReplicatedChecks: 0 !!! note "Внимание" Таблица не содержит входных данных для запросов `INSERT`. -ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_log](server_settings/settings.md#server_settings-query-log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. +ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. Чтобы включить логирование, задайте значение параметра [log\_queries](settings/settings.md#settings-log-queries) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md). @@ -555,14 +594,14 @@ ClickHouse создаёт таблицу только в том случае, к 2. Если во время обработки запроса произошла ошибка, создаются два события с типами 1 и 4. 3. Если ошибка произошла до запуска запроса, создается одно событие с типом 3. -По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_log](server_settings/settings.md#server_settings-query-log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. +По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. Если таблицу удалить вручную, она пересоздастся автоматически «на лету». При этом все логи на момент удаления таблицы будут удалены. !!! note "Примечание" Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов. -Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_log](server_settings/settings.md#server_settings-query-log) (параметр `partition_by`). +Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`). ## system.query\_log {#system_tables-query_log} @@ -571,7 +610,7 @@ Contains information about execution of queries. For each query, you can see pro !!! note "Note" The table doesn’t contain input data for `INSERT` queries. -ClickHouse creates this table only if the [query\_log](server_settings/settings.md#server_settings-query-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. +ClickHouse creates this table only if the [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. To enable query logging, set the [log\_queries](settings/settings.md#settings-log-queries) parameter to 1. For details, see the [Settings](settings/settings.md) section. @@ -641,19 +680,19 @@ Each query creates one or two rows in the `query_log` table, depending on the st 2. If an error occurred during query processing, two events with types 1 and 4 are created. 3. If an error occurred before launching the query, a single event with type 3 is created. -By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_log](server_settings/settings.md#server_settings-query-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. +By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted. !!! note "Note" The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself. -You can specify an arbitrary partitioning key for the `system.query_log` table in the [query\_log](server_settings/settings.md#server_settings-query-log) server setting (see the `partition_by` parameter). +You can specify an arbitrary partitioning key for the `system.query_log` table in the [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `partition_by` parameter). \#\# system.query\_thread\_log {\#system\_tables-query-thread-log} Содержит информацию о каждом потоке выполняемых запросов. -ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_thread\_log](server_settings/settings.md#server_settings-query-thread-log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. +ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_thread\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. Чтобы включить логирование, задайте значение параметра [log\_query\_threads](settings/settings.md#settings-log-query-threads) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md). @@ -704,43 +743,43 @@ ClickHouse создаёт таблицу только в том случае, к - `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events - `ProfileEvents.Values` (Array(UInt64)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`. -По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_thread\_log](server_settings/settings.md#server_settings-query-thread-log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. +По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_thread\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. Если таблицу удалить вручную, она пересоздастся автоматически «на лету». При этом все логи на момент удаления таблицы будут удалены. !!! note "Примечание" Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов. -Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_thread\_log](server_settings/settings.md#server_settings-query-thread-log) (параметр `partition_by`). +Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_thread\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log) (параметр `partition_by`). ## system.trace\_log {#system_tables-trace_log} Contains stack traces collected by the sampling query profiler. -ClickHouse creates this table when the [trace\_log](server_settings/settings.md#server_settings-trace_log) server configuration section is set. Also the [query\_profiler\_real\_time\_period\_ns](settings/settings.md#query_profiler_real_time_period_ns) and [query\_profiler\_cpu\_time\_period\_ns](settings/settings.md#query_profiler_cpu_time_period_ns) settings should be set. +ClickHouse creates this table when the [trace\_log](server_configuration_parameters/settings.md#server_configuration_parameters-trace_log) server configuration section is set. Also the [query\_profiler\_real\_time\_period\_ns](settings/settings.md#query_profiler_real_time_period_ns) and [query\_profiler\_cpu\_time\_period\_ns](settings/settings.md#query_profiler_cpu_time_period_ns) settings should be set. To analyze logs, use the `addressToLine`, `addressToSymbol` and `demangle` introspection functions. Columns: -- `event_date`([Date](../data_types/date.md)) — Date of sampling moment. +- `event_date`([Date](../sql_reference/data_types/date.md)) — Date of sampling moment. -- `event_time`([DateTime](../data_types/datetime.md)) — Timestamp of sampling moment. +- `event_time`([DateTime](../sql_reference/data_types/datetime.md)) — Timestamp of sampling moment. -- `revision`([UInt32](../data_types/int_uint.md)) — ClickHouse server build revision. +- `revision`([UInt32](../sql_reference/data_types/int_uint.md)) — ClickHouse server build revision. When connecting to server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server. -- `timer_type`([Enum8](../data_types/enum.md)) — Timer type: +- `timer_type`([Enum8](../sql_reference/data_types/enum.md)) — Timer type: - `Real` represents wall-clock time. - `CPU` represents CPU time. -- `thread_number`([UInt32](../data_types/int_uint.md)) — Thread identifier. +- `thread_number`([UInt32](../sql_reference/data_types/int_uint.md)) — Thread identifier. -- `query_id`([String](../data_types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) system table. +- `query_id`([String](../sql_reference/data_types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) system table. -- `trace`([Array(UInt64)](../data_types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. +- `trace`([Array(UInt64)](../sql_reference/data_types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. **Example** @@ -882,33 +921,33 @@ WHERE Если этот запрос ничего не возвращает - значит всё хорошо. -## system.settings {#system-tables-system-settings} +## system.settings {#system-tables-system-settings} Содержит информацию о сессионных настройках для текущего пользователя. Столбцы: -- `name` ([String](../data_types/string.md)) — имя настройки. -- `value` ([String](../data_types/string.md)) — значение настройки. -- `changed` ([UInt8](../data_types/int_uint.md#uint-ranges)) — показывает, изменена ли настройка по отношению к значению по умолчанию. -- `description` ([String](../data_types/string.md)) — краткое описание настройки. -- `min` ([Nullable](../data_types/nullable.md)([String](../data_types/string.md))) — минимальное значение настройки, если задано [ограничение](settings/constraints_on_settings.md#constraints-on-settings). Если нет, то поле содержит [NULL](../query_language/syntax.md#null-literal). -- `max` ([Nullable](../data_types/nullable.md)([String](../data_types/string.md))) — максимальное значение настройки, если задано [ограничение](settings/constraints_on_settings.md#constraints-on-settings). Если нет, то поле содержит [NULL](../query_language/syntax.md#null-literal). -- `readonly` ([UInt8](../data_types/int_uint.md#uint-ranges)) — Показывает, может ли пользователь изменять настройку: - - `0` — Текущий пользователь может изменять настройку. - - `1` — Текущий пользователь не может изменять настройку. +- `name` ([String](../sql_reference/data_types/string.md)) — имя настройки. +- `value` ([String](../sql_reference/data_types/string.md)) — значение настройки. +- `changed` ([UInt8](../sql_reference/data_types/int_uint.md#uint-ranges)) — показывает, изменена ли настройка по отношению к значению по умолчанию. +- `description` ([String](../sql_reference/data_types/string.md)) — краткое описание настройки. +- `min` ([Nullable](../sql_reference/data_types/nullable.md)([String](../sql_reference/data_types/string.md))) — минимальное значение настройки, если задано [ограничение](settings/constraints_on_settings.md#constraints-on-settings). Если нет, то поле содержит [NULL](../sql_reference/syntax.md#null-literal). +- `max` ([Nullable](../sql_reference/data_types/nullable.md)([String](../sql_reference/data_types/string.md))) — максимальное значение настройки, если задано [ограничение](settings/constraints_on_settings.md#constraints-on-settings). Если нет, то поле содержит [NULL](../sql_reference/syntax.md#null-literal). +- `readonly` ([UInt8](../sql_reference/data_types/int_uint.md#uint-ranges)) — Показывает, может ли пользователь изменять настройку: + - `0` — Текущий пользователь может изменять настройку. + - `1` — Текущий пользователь не может изменять настройку. **Пример** Пример показывает как получить информацию о настройках, имена которых содержат `min_i`. -```sql -SELECT * -FROM system.settings +``` sql +SELECT * +FROM system.settings WHERE name LIKE '%min_i%' ``` -```text +``` text ┌─name────────────────────────────────────────┬─value─────┬─changed─┬─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─min──┬─max──┬─readonly─┐ │ min_insert_block_size_rows │ 1048576 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ │ min_insert_block_size_bytes │ 268435456 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ @@ -918,21 +957,23 @@ WHERE name LIKE '%min_i%' Использование `WHERE changed` может быть полезно, например, если необходимо проверить: -- Что настройки корректно загрузились из конфигурационного файла и используются. -- Настройки, изменённые в текущей сессии. +- Что настройки корректно загрузились из конфигурационного файла и используются. +- Настройки, изменённые в текущей сессии. -```sql + + +``` sql SELECT * FROM system.settings WHERE changed AND name='load_balancing' ``` - **Cм. также** -- [Настройки](settings/index.md#settings) -- [Разрешения для запросов](settings/permissions_for_queries.md#settings_readonly) -- [Ограничения для значений настроек](settings/constraints_on_settings.md) +- [Настройки](settings/index.md#settings) +- [Разрешения для запросов](settings/permissions_for_queries.md#settings_readonly) +- [Ограничения для значений настроек](settings/constraints_on_settings.md) + +## system.table\_engines {#system.table_engines} -## system.table_engines ``` text ┌─name───────────────────┬─value───────┬─changed─┐ │ max_threads │ 8 │ 1 │ @@ -974,9 +1015,9 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') **Смотрите также** -- [Секции движка](table_engines/mergetree/#mergetree-query-clauses) семейства MergeTree -- [Настройки](table_engines/kafka.md#table_engine-kafka-creating-a-table) Kafka -- [Настройки](table_engines/join/#join-limitations-and-settings) Join +- [Секции движка](../engines/table_engines/mergetree_family/mergetree.md#mergetree-query-clauses) семейства MergeTree +- [Настройки](../engines/table_engines/integrations/kafka.md#table_engine-kafka-creating-a-table) Kafka +- [Настройки](../engines/table_engines/special/join.md#join-limitations-and-settings) Join ## system.tables {#system-tables} @@ -992,7 +1033,7 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') - `metadata_path` (String) — путь к табличным метаданным в файловой системе. - `metadata_modification_time` (DateTime) — время последней модификации табличных метаданных. - `dependencies_database` (Array(String)) — зависимости базы данных. -- `dependencies_table` (Array(String)) — табличные зависимости (таблицы [MaterializedView](table_engines/materializedview.md), созданные на базе текущей таблицы). +- `dependencies_table` (Array(String)) — табличные зависимости (таблицы [MaterializedView](../engines/table_engines/special/materializedview.md), созданные на базе текущей таблицы). - `create_table_query` (String) — запрос, которым создавалась таблица. - `engine_full` (String) — параметры табличного движка. - `partition_key` (String) — ключ партиционирования таблицы. @@ -1075,7 +1116,7 @@ path: /clickhouse/tables/01-08/visits/replicas ## system.mutations {#system_tables-mutations} -Таблица содержит информацию о ходе выполнения [мутаций](../query_language/alter.md#alter-mutations) MergeTree-таблиц. Каждой команде мутации соответствует одна строка. В таблице есть следующие столбцы: +Таблица содержит информацию о ходе выполнения [мутаций](../sql_reference/statements/alter.md#alter-mutations) MergeTree-таблиц. Каждой команде мутации соответствует одна строка. В таблице есть следующие столбцы: **database**, **table** - имя БД и таблицы, к которой была применена мутация. @@ -1101,28 +1142,28 @@ path: /clickhouse/tables/01-08/visits/replicas ## system.disks {#system_tables-disks} -Cодержит информацию о дисках, заданных в [конфигурации сервера](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). +Cодержит информацию о дисках, заданных в [конфигурации сервера](../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). Столбцы: -- `name` ([String](../data_types/string.md)) — имя диска в конфигурации сервера. -- `path` ([String](../data_types/string.md)) — путь к точке монтирования в файловой системе. -- `free_space` ([UInt64](../data_types/int_uint.md)) — свободное место на диске в байтах. -- `total_space` ([UInt64](../data_types/int_uint.md)) — объём диска в байтах. -- `keep_free_space` ([UInt64](../data_types/int_uint.md)) — место, которое должно остаться свободным на диске в байтах. Задаётся значением параметра `keep_free_space_bytes` конфигурации дисков. +- `name` ([String](../sql_reference/data_types/string.md)) — имя диска в конфигурации сервера. +- `path` ([String](../sql_reference/data_types/string.md)) — путь к точке монтирования в файловой системе. +- `free_space` ([UInt64](../sql_reference/data_types/int_uint.md)) — свободное место на диске в байтах. +- `total_space` ([UInt64](../sql_reference/data_types/int_uint.md)) — объём диска в байтах. +- `keep_free_space` ([UInt64](../sql_reference/data_types/int_uint.md)) — место, которое должно остаться свободным на диске в байтах. Задаётся значением параметра `keep_free_space_bytes` конфигурации дисков. ## system.storage\_policies {#system_tables-storage_policies} -Содержит информацию о политиках хранения и томах, заданных в [конфигурации сервера](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). +Содержит информацию о политиках хранения и томах, заданных в [конфигурации сервера](../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). Столбцы: -- `policy_name` ([String](../data_types/string.md)) — имя политики хранения. -- `volume_name` ([String](../data_types/string.md)) — имя тома, который содержится в политике хранения. -- `volume_priority` ([UInt64](../data_types/int_uint.md)) — порядковый номер тома согласно конфигурации. -- `disks` ([Array(String)](../data_types/array.md)) — имена дисков, содержащихся в политике хранения. -- `max_data_part_size` ([UInt64](../data_types/int_uint.md)) — максимальный размер куска данных, который может храниться на дисках тома (0 — без ограничений). -- `move_factor` ([Float64](../data_types/float.md))\` — доля свободного места, при превышении которой данные начинают перемещаться на следующий том. +- `policy_name` ([String](../sql_reference/data_types/string.md)) — имя политики хранения. +- `volume_name` ([String](../sql_reference/data_types/string.md)) — имя тома, который содержится в политике хранения. +- `volume_priority` ([UInt64](../sql_reference/data_types/int_uint.md)) — порядковый номер тома согласно конфигурации. +- `disks` ([Array(String)](../sql_reference/data_types/array.md)) — имена дисков, содержащихся в политике хранения. +- `max_data_part_size` ([UInt64](../sql_reference/data_types/int_uint.md)) — максимальный размер куска данных, который может храниться на дисках тома (0 — без ограничений). +- `move_factor` ([Float64](../sql_reference/data_types/float.md))\` — доля свободного места, при превышении которой данные начинают перемещаться на следующий том. Если политика хранения содержит несколько томов, то каждому тому соответствует отдельная запись в таблице. diff --git a/docs/ru/operations/table_engines/generate.md b/docs/ru/operations/table_engines/generate.md deleted file mode 100644 index 051369d2e1c..00000000000 --- a/docs/ru/operations/table_engines/generate.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -en_copy: true ---- - -# GenerateRandom {#table_engines-generate} - -The GenerateRandom table engine produces random data for given table schema. - -Usage examples: - -- Use in test to populate reproducible large table. -- Generate random input for fuzzing tests. - -## Usage in ClickHouse Server {#usage-in-clickhouse-server} - -``` sql -ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) -``` - -The `max_array_length` and `max_string_length` parameters specify maximum length of all -array columns and strings correspondingly in generated data. - -Generate table engine supports only `SELECT` queries. - -It supports all [DataTypes](../../data_types/index.md) that can be stored in a table except `LowCardinality` and `AggregateFunction`. - -**Example:** - -**1.** Set up the `generate_engine_table` table: - -``` sql -CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE = GenerateRandom(1, 5, 3) -``` - -**2.** Query the data: - -``` sql -SELECT * FROM generate_engine_table LIMIT 3 -``` - -``` text -┌─name─┬──────value─┐ -│ c4xJ │ 1412771199 │ -│ r │ 1791099446 │ -│ 7#$ │ 124312908 │ -└──────┴────────────┘ -``` - -## Details of Implementation {#details-of-implementation} - -- Not supported: - - `ALTER` - - `SELECT ... SAMPLE` - - `INSERT` - - Indices - - Replication - -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/generate/) diff --git a/docs/ru/operations/table_engines/materializedview.md b/docs/ru/operations/table_engines/materializedview.md deleted file mode 100644 index 6bcc528782a..00000000000 --- a/docs/ru/operations/table_engines/materializedview.md +++ /dev/null @@ -1,5 +0,0 @@ -# MaterializedView {#materializedview} - -Используется для реализации материализованных представлений (подробнее см. запрос [CREATE TABLE](../../query_language/create.md)). Для хранения данных, использует другой движок, который был указан при создании представления. При чтении из таблицы, просто использует этот движок. - -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/materializedview/) diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md index 4aa4605defb..271a6a35e25 100644 --- a/docs/ru/operations/tips.md +++ b/docs/ru/operations/tips.md @@ -1,6 +1,6 @@ # Советы по эксплуатации {#sovety-po-ekspluatatsii} -## CPU scaling governor {#cpu-scaling-governor} +## CPU Scaling Governor {#cpu-scaling-governor} Всегда используйте `performance` scaling governor. `ondemand` scaling governor работает намного хуже при постоянно высоком спросе. @@ -25,7 +25,7 @@ $ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_gov $ echo 0 | sudo tee /proc/sys/vm/overcommit_memory ``` -## Huge pages {#huge-pages} +## Huge Pages {#huge-pages} Механизм прозрачных huge pages нужно отключить. Он мешает работе аллокаторов памяти, что приводит к значительной деградации производительности. diff --git a/docs/ru/operations/troubleshooting.md b/docs/ru/operations/troubleshooting.md index 4c9fbf49bd6..a045de41ccc 100644 --- a/docs/ru/operations/troubleshooting.md +++ b/docs/ru/operations/troubleshooting.md @@ -7,7 +7,7 @@ ## Установка дистрибутива {#troubleshooting-installation-errors} -### Не получается скачать deb-пакеты из репозитория ClickHouse с помощью apt-get {#ne-poluchaetsia-skachat-deb-pakety-iz-repozitoriia-clickhouse-s-pomoshchiu-apt-get} +### Не получается скачать deb-пакеты из репозитория ClickHouse с помощью Apt-get {#ne-poluchaetsia-skachat-deb-pakety-iz-repozitoriia-clickhouse-s-pomoshchiu-apt-get} - Проверьте настройки брандмауэра. - Если по какой-либо причине вы не можете получить доступ к репозиторию, скачайте пакеты как описано в разделе [Начало работы](../getting_started/index.md) и установите их вручную командой `sudo dpkg -i `. Также, необходим пакет `tzdata`. @@ -98,7 +98,7 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se - Параметры endpoint. - Проверьте настройки [listen_host](server_settings/settings.md#server_settings-listen_host) и [tcp_port](server_settings/settings.md#server_settings-tcp_port). + Проверьте настройки [listen_host](server_configuration_parameters/settings.md#server_configuration_parameters-listen_host) и [tcp_port](server_configuration_parameters/settings.md#server_configuration_parameters-tcp_port). По умолчанию, сервер ClickHouse принимает только локальные подключения. diff --git a/docs/ru/operations/utilities/clickhouse-benchmark.md b/docs/ru/operations/utilities/clickhouse-benchmark.md new file mode 120000 index 00000000000..3695c9fbdd3 --- /dev/null +++ b/docs/ru/operations/utilities/clickhouse-benchmark.md @@ -0,0 +1 @@ +../../../en/operations/utilities/clickhouse-benchmark.md \ No newline at end of file diff --git a/docs/ru/operations/utils/clickhouse-copier.md b/docs/ru/operations/utilities/clickhouse-copier.md similarity index 100% rename from docs/ru/operations/utils/clickhouse-copier.md rename to docs/ru/operations/utilities/clickhouse-copier.md diff --git a/docs/ru/operations/utils/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md similarity index 97% rename from docs/ru/operations/utils/clickhouse-local.md rename to docs/ru/operations/utilities/clickhouse-local.md index bcc34deea4e..7dfa9587686 100644 --- a/docs/ru/operations/utils/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -1,6 +1,6 @@ # clickhouse-local {#clickhouse-local} -Принимает на вход данные, которые можно представить в табличном виде и выполняет над ними операции, заданные на [языке запросов](../../query_language/index.md) ClickHouse. +Принимает на вход данные, которые можно представить в табличном виде и выполняет над ними операции, заданные на [языке запросов](../../operations/utilities/clickhouse-local.md) ClickHouse. `clickhouse-local` использует движок сервера ClickHouse, т.е. поддерживает все форматы данных и движки таблиц, с которыми работает ClickHouse, при этом для выполнения операций не требуется запущенный сервер. diff --git a/docs/ru/operations/utils/index.md b/docs/ru/operations/utilities/index.md similarity index 100% rename from docs/ru/operations/utils/index.md rename to docs/ru/operations/utilities/index.md diff --git a/docs/ru/operations/utils/clickhouse-benchmark.md b/docs/ru/operations/utils/clickhouse-benchmark.md deleted file mode 100644 index 1d8ac3dec46..00000000000 --- a/docs/ru/operations/utils/clickhouse-benchmark.md +++ /dev/null @@ -1,153 +0,0 @@ ---- -en_copy: true ---- - -# clickhouse-benchmark {#clickhouse-benchmark} - -Connects to a ClickHouse server and repeatedly sends specified queries. - -Syntax: - -``` bash -$ echo "single query" | clickhouse-benchmark [keys] -``` - -or - -``` bash -$ clickhouse-benchmark [keys] <<< "single query" -``` - -If you want to send a set of queries, create a text file and place each query on the individual string in this file. For example: - -``` sql -SELECT * FROM system.numbers LIMIT 10000000 -SELECT 1 -``` - -Then pass this file to a standard input of `clickhouse-benchmark`. - -``` bash -clickhouse-benchmark [keys] < queries_file -``` - -## Keys {#clickhouse-benchmark-keys} - -- `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` sends simultaneously. Default value: 1. -- `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1. -- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. For the [comparison mode](#clickhouse-benchmark-comparison-mode) you can use multiple `-h` keys. -- `-p N`, `--port=N` — Server port. Default value: 9000. For the [comparison mode](#clickhouse-benchmark-comparison-mode) you can use multiple `-p` keys. -- `-i N`, `--iterations=N` — Total number of queries. Default value: 0. -- `-r`, `--randomize` — Random order of queries execution if there is more then one input query. -- `-s`, `--secure` — Using TLS connection. -- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` stops sending queries when the specified time limit is reached. Default value: 0 (time limit disabled). -- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [comparison mode](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` performs the [Independent two-sample Student’s t-test](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) test to determine whether the two distributions aren’t different with the selected level of confidence. -- `--cumulative` — Printing cumulative data instead of data per interval. -- `--database=DATABASE_NAME` — ClickHouse database name. Default value: `default`. -- `--json=FILEPATH` — JSON output. When the key is set, `clickhouse-benchmark` outputs a report to the specified JSON-file. -- `--user=USERNAME` — ClickHouse user name. Default value: `default`. -- `--password=PSWD` — ClickHouse user password. Default value: empty string. -- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` outputs stack traces of exceptions. -- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` at the specified stage. Possible values: `complete`, `fetch_columns`, `with_mergeable_state`. Default value: `complete`. -- `--help` — Shows the help message. - -If you want to apply some [settings](../../operations/settings/index.md) for queries, pass them as a key `--= SETTING_VALUE`. For example, `--max_memory_usage=1048576`. - -## Output {#clickhouse-benchmark-output} - -By default, `clickhouse-benchmark` reports for each `--delay` interval. - -Example of the report: - -``` text -Queries executed: 10. - -localhost:9000, queries 10, QPS: 6.772, RPS: 67904487.440, MiB/s: 518.070, result RPS: 67721584.984, result MiB/s: 516.675. - -0.000% 0.145 sec. -10.000% 0.146 sec. -20.000% 0.146 sec. -30.000% 0.146 sec. -40.000% 0.147 sec. -50.000% 0.148 sec. -60.000% 0.148 sec. -70.000% 0.148 sec. -80.000% 0.149 sec. -90.000% 0.150 sec. -95.000% 0.150 sec. -99.000% 0.150 sec. -99.900% 0.150 sec. -99.990% 0.150 sec. -``` - -In the report you can find: - -- Number of queries in the `Queries executed:` field. - -- Status string containing (in order): - - - Endpoint of ClickHouse server. - - Number of processed queries. - - QPS: QPS: How many queries server performed per second during a period specified in the `--delay` argument. - - RPS: How many rows server read per second during a period specified in the `--delay` argument. - - MiB/s: How many mebibytes server read per second during a period specified in the `--delay` argument. - - result RPS: How many rows placed by server to the result of a query per second during a period specified in the `--delay` argument. - - result MiB/s. How many mebibytes placed by server to the result of a query per second during a period specified in the `--delay` argument. - -- Percentiles of queries execution time. - -## Comparison mode {#clickhouse-benchmark-comparison-mode} - -`clickhouse-benchmark` can compare performances for two running ClickHouse servers. - -To use the comparison mode, specify endpoints of both servers by two pairs of `--host`, `--port` keys. Keys matched together by position in arguments list, the first `--host` is matched with the first `--port` and so on. `clickhouse-benchmark` establishes connections to both servers, then sends queries. Each query addressed to a randomly selected server. The results are shown for each server separately. - -## Example {#clickhouse-benchmark-example} - -``` bash -$ echo "SELECT * FROM system.numbers LIMIT 10000000 OFFSET 10000000" | clickhouse-benchmark -i 10 -``` - -``` text -Loaded 1 queries. - -Queries executed: 6. - -localhost:9000, queries 6, QPS: 6.153, RPS: 123398340.957, MiB/s: 941.455, result RPS: 61532982.200, result MiB/s: 469.459. - -0.000% 0.159 sec. -10.000% 0.159 sec. -20.000% 0.159 sec. -30.000% 0.160 sec. -40.000% 0.160 sec. -50.000% 0.162 sec. -60.000% 0.164 sec. -70.000% 0.165 sec. -80.000% 0.166 sec. -90.000% 0.166 sec. -95.000% 0.167 sec. -99.000% 0.167 sec. -99.900% 0.167 sec. -99.990% 0.167 sec. - - - -Queries executed: 10. - -localhost:9000, queries 10, QPS: 6.082, RPS: 121959604.568, MiB/s: 930.478, result RPS: 60815551.642, result MiB/s: 463.986. - -0.000% 0.159 sec. -10.000% 0.159 sec. -20.000% 0.160 sec. -30.000% 0.163 sec. -40.000% 0.164 sec. -50.000% 0.165 sec. -60.000% 0.166 sec. -70.000% 0.166 sec. -80.000% 0.167 sec. -90.000% 0.167 sec. -95.000% 0.170 sec. -99.000% 0.172 sec. -99.900% 0.172 sec. -99.990% 0.172 sec. -``` diff --git a/docs/ru/query_language/index.md b/docs/ru/query_language/index.md deleted file mode 100644 index d57d8971a7d..00000000000 --- a/docs/ru/query_language/index.md +++ /dev/null @@ -1,9 +0,0 @@ -# Справка по SQL {#spravka-po-sql} - -- [SELECT](select.md) -- [INSERT INTO](insert_into.md) -- [CREATE](create.md) -- [ALTER](alter.md#query_language_queries_alter) -- [Прочие виды запросов](misc.md) - -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) diff --git a/docs/ru/roadmap.md b/docs/ru/roadmap.md deleted file mode 100644 index 0d17b67a3a9..00000000000 --- a/docs/ru/roadmap.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -en_copy: true ---- - -# Roadmap {#roadmap} - -## Q1 2020 {#q1-2020} - -- Role-based access control - -## Q2 2020 {#q2-2020} - -- Integration with external authentication services -- Resource pools for more precise distribution of cluster capacity between users - -{## [Original article](https://clickhouse.tech/docs/en/roadmap/) ##} diff --git a/docs/ru/query_language/agg_functions/combinators.md b/docs/ru/sql_reference/aggregate_functions/combinators.md similarity index 96% rename from docs/ru/query_language/agg_functions/combinators.md rename to docs/ru/sql_reference/aggregate_functions/combinators.md index bc0616ef5de..e4295f0d76e 100644 --- a/docs/ru/query_language/agg_functions/combinators.md +++ b/docs/ru/sql_reference/aggregate_functions/combinators.md @@ -27,9 +27,9 @@ Для работы с промежуточными состояниями предназначены: -- Движок таблиц [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md). -- Функция [finalizeAggregation](../functions/other_functions.md#function-finalizeaggregation). -- Функция [runningAccumulate](../functions/other_functions.md#function-runningaccumulate). +- Движок таблиц [AggregatingMergeTree](../../engines/table_engines/mergetree_family/aggregatingmergetree.md). +- Функция [finalizeAggregation](../../sql_reference/aggregate_functions/combinators.md#function-finalizeaggregation). +- Функция [runningAccumulate](../../sql_reference/aggregate_functions/combinators.md#function-runningaccumulate). - Комбинатор [-Merge](#aggregate_functions_combinators_merge). - Комбинатор [-MergeState](#aggregate_functions_combinators_mergestate). diff --git a/docs/ru/query_language/agg_functions/index.md b/docs/ru/sql_reference/aggregate_functions/index.md similarity index 95% rename from docs/ru/query_language/agg_functions/index.md rename to docs/ru/sql_reference/aggregate_functions/index.md index e0c07a1c95e..d28e105f6fc 100644 --- a/docs/ru/query_language/agg_functions/index.md +++ b/docs/ru/sql_reference/aggregate_functions/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: Агрегатные функции +toc_priority: 33 +toc_title: Введение +--- + # Агрегатные функции {#aggregate-functions} Агрегатные функции работают в [привычном](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) для специалистов по базам данных смысле. diff --git a/docs/ru/query_language/agg_functions/parametric_functions.md b/docs/ru/sql_reference/aggregate_functions/parametric_functions.md similarity index 94% rename from docs/ru/query_language/agg_functions/parametric_functions.md rename to docs/ru/sql_reference/aggregate_functions/parametric_functions.md index 22fc7eb2934..70430d21eae 100644 --- a/docs/ru/query_language/agg_functions/parametric_functions.md +++ b/docs/ru/sql_reference/aggregate_functions/parametric_functions.md @@ -17,7 +17,7 @@ **Возвращаемые значения** -- [Массив](../../data_types/array.md) [кортежей](../../data_types/tuple.md) следующего вида: +- [Массив](../../sql_reference/data_types/array.md) [кортежей](../../sql_reference/data_types/tuple.md) следующего вида: ``` [(lower_1, upper_1, height_1), ... (lower_N, upper_N, height_N)] @@ -44,7 +44,7 @@ FROM ( └─────────────────────────────────────────────────────────────────────────┘ ``` -С помощью функции [bar](../functions/other_functions.md#function-bar) можно визуализировать гистограмму, например: +С помощью функции [bar](../../sql_reference/aggregate_functions/parametric_functions.md#function-bar) можно визуализировать гистограмму, например: ``` sql WITH histogram(5)(rand() % 100) AS hist @@ -86,7 +86,7 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) - `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). -- `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../data_types/int_uint.md). +- `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../sql_reference/aggregate_functions/parametric_functions.md). - `cond1`, `cond2` — условия, описывающие цепочку событий. Тип данных — `UInt8`. Можно использовать до 32 условий. Функция учитывает только те события, которые указаны в условиях. Функция пропускает данные из последовательности, если они не описаны ни в одном из условий. @@ -173,7 +173,7 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...) - `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). -- `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../data_types/int_uint.md). +- `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../sql_reference/aggregate_functions/parametric_functions.md). - `cond1`, `cond2` — условия, описывающие цепочку событий. Тип данных — `UInt8`. Можно использовать до 32 условий. Функция учитывает только те события, которые указаны в условиях. Функция пропускает данные из последовательности, если они не описаны ни в одном из условий. @@ -234,10 +234,10 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **Параметры** -- `window` — ширина скользящего окна по времени в секундах. [UInt](../../data_types/int_uint.md). +- `window` — ширина скользящего окна по времени в секундах. [UInt](../../sql_reference/aggregate_functions/parametric_functions.md). - `mode` - необязательный параметр. Если установлено значение `'strict'`, то функция `windowFunnel()` применяет условия только для уникальных значений. -- `timestamp` — имя столбца, содержащего временные отметки. [Date](../../data_types/date.md), [DateTime](../../data_types/datetime.md#data_type-datetime) и другие параметры с типом `Integer`. В случае хранения меток времени в столбцах с типом `UInt64`, максимально допустимое значение соответствует ограничению для типа `Int64`, т.е. равно `2^63-1`. -- `cond` — условия или данные, описывающие цепочку событий. [UInt8](../../data_types/int_uint.md). +- `timestamp` — имя столбца, содержащего временные отметки. [Date](../../sql_reference/aggregate_functions/parametric_functions.md), [DateTime](../../sql_reference/aggregate_functions/parametric_functions.md#data_type-datetime) и другие параметры с типом `Integer`. В случае хранения меток времени в столбцах с типом `UInt64`, максимально допустимое значение соответствует ограничению для типа `Int64`, т.е. равно `2^63-1`. +- `cond` — условия или данные, описывающие цепочку событий. [UInt8](../../sql_reference/aggregate_functions/parametric_functions.md). **Возвращаемое значение** @@ -299,7 +299,7 @@ ORDER BY level ASC Аналитическая функция, которая показывает, насколько выдерживаются те или иные условия, например, удержание динамики/уровня [посещаемости сайта](https://yandex.ru/support/partner2/statistics/metrika-visitors-statistics.html?lang=ru). -Функция принимает набор (от 1 до 32) логических условий, как в [WHERE](../select.md#select-where), и применяет их к заданному набору данных. +Функция принимает набор (от 1 до 32) логических условий, как в [WHERE](../../sql_reference/statements/select.md#select-where), и применяет их к заданному набору данных. Условия, кроме первого, применяются попарно: результат второго будет истинным, если истинно первое и второе, третьего - если истинно первое и третье и т. д. diff --git a/docs/ru/query_language/agg_functions/reference.md b/docs/ru/sql_reference/aggregate_functions/reference.md similarity index 91% rename from docs/ru/query_language/agg_functions/reference.md rename to docs/ru/sql_reference/aggregate_functions/reference.md index 3a6c3679c54..b84e92ef7a8 100644 --- a/docs/ru/query_language/agg_functions/reference.md +++ b/docs/ru/sql_reference/aggregate_functions/reference.md @@ -19,9 +19,9 @@ ClickHouse поддерживает следующие виды синтакси **Возвращаемое значение** - Если функция вызывается без параметров, она вычисляет количество строк. -- Если передаётся [выражение](../syntax.md#syntax-expressions) , то функция вычисляет количество раз, когда выражение возвращает не NULL. Если выражение возвращает значение типа [Nullable](../../data_types/nullable.md), то результат `count` не становится `Nullable`. Функция возвращает 0, если выражение возвращает `NULL` для всех строк. +- Если передаётся [выражение](../syntax.md#syntax-expressions) , то функция вычисляет количество раз, когда выражение возвращает не NULL. Если выражение возвращает значение типа [Nullable](../../sql_reference/data_types/nullable.md), то результат `count` не становится `Nullable`. Функция возвращает 0, если выражение возвращает `NULL` для всех строк. -В обоих случаях тип возвращаемого значения [UInt64](../../data_types/int_uint.md). +В обоих случаях тип возвращаемого значения [UInt64](../../sql_reference/data_types/int_uint.md). **Подробности** @@ -240,7 +240,7 @@ binary decimal ## groupBitmap {#groupbitmap} -Bitmap или агрегатные вычисления для столбца с типом данных `UInt*`, возвращают кардинальность в виде значения типа UInt64, если добавить суффикс -State, то возвращают [объект bitmap](../functions/bitmap_functions.md). +Bitmap или агрегатные вычисления для столбца с типом данных `UInt*`, возвращают кардинальность в виде значения типа UInt64, если добавить суффикс -State, то возвращают [объект bitmap](../../sql_reference/aggregate_functions/reference.md). ``` sql groupBitmap(expr) @@ -376,7 +376,7 @@ skewPop(expr) **Возвращаемое значение** -Коэффициент асимметрии заданного распределения. Тип — [Float64](../../data_types/float.md) +Коэффициент асимметрии заданного распределения. Тип — [Float64](../../sql_reference/aggregate_functions/reference.md) **Пример** @@ -400,7 +400,7 @@ skewSamp(expr) **Возвращаемое значение** -Коэффициент асимметрии заданного распределения. Тип — [Float64](../../data_types/float.md). Если `n <= 1` (`n` — размер выборки), тогда функция возвращает `nan`. +Коэффициент асимметрии заданного распределения. Тип — [Float64](../../sql_reference/aggregate_functions/reference.md). Если `n <= 1` (`n` — размер выборки), тогда функция возвращает `nan`. **Пример** @@ -422,7 +422,7 @@ kurtPop(expr) **Возвращаемое значение** -Коэффициент эксцесса заданного распределения. Тип — [Float64](../../data_types/float.md) +Коэффициент эксцесса заданного распределения. Тип — [Float64](../../sql_reference/aggregate_functions/reference.md) **Пример** @@ -446,7 +446,7 @@ kurtSamp(expr) **Возвращаемое значение** -Коэффициент эксцесса заданного распределения. Тип — [Float64](../../data_types/float.md). Если `n <= 1` (`n` — размер выборки), тогда функция возвращает `nan`. +Коэффициент эксцесса заданного распределения. Тип — [Float64](../../sql_reference/aggregate_functions/reference.md). Если `n <= 1` (`n` — размер выборки), тогда функция возвращает `nan`. **Пример** @@ -536,8 +536,8 @@ avgWeighted(x, weight) **Параметры** -- `x` — Значения. [Целые числа](../../data_types/int_uint.md) или [числа с плавающей запятой](../../data_types/float.md). -- `weight` — Веса отдельных значений. [Целые числа](../../data_types/int_uint.md) или [числа с плавающей запятой](../../data_types/float.md). +- `x` — Значения. [Целые числа](../../sql_reference/aggregate_functions/reference.md) или [числа с плавающей запятой](../../sql_reference/aggregate_functions/reference.md). +- `weight` — Веса отдельных значений. [Целые числа](../../sql_reference/aggregate_functions/reference.md) или [числа с плавающей запятой](../../sql_reference/aggregate_functions/reference.md). Типы параметров должны совпадать. @@ -546,7 +546,7 @@ avgWeighted(x, weight) - Среднее арифметическое взвешенное. - `NaN`, если все веса равны 0. -Тип: [Float64](../../data_types/float.md) +Тип: [Float64](../../sql_reference/aggregate_functions/reference.md) **Пример** @@ -580,7 +580,7 @@ uniq(x[, ...]) **Возвращаемое значение** -- Значение с типом данных [UInt64](../../data_types/int_uint.md). +- Значение с типом данных [UInt64](../../sql_reference/aggregate_functions/reference.md). **Детали реализации** @@ -621,7 +621,7 @@ uniqCombined(HLL_precision)(x[, ...]) **Возвращаемое значение** -- Число типа [UInt64](../../data_types/int_uint.md). +- Число типа [UInt64](../../sql_reference/aggregate_functions/reference.md). **Детали реализации** @@ -669,7 +669,7 @@ uniqHLL12(x[, ...]) **Возвращаемое значение** -- Значение хэша с типом данных [UInt64](../../data_types/int_uint.md). +- Значение хэша с типом данных [UInt64](../../sql_reference/aggregate_functions/reference.md). **Детали реализации** @@ -723,10 +723,13 @@ uniqExact(x[, ...]) В некоторых случаях, вы всё же можете рассчитывать на порядок выполнения запроса. Это — случаи, когда `SELECT` идёт из подзапроса, в котором используется `ORDER BY`. -## groupArrayInsertAt(x) {#grouparrayinsertatx} +## groupArrayInsertAt(value, position) {#grouparrayinsertatvalue-position} Вставляет в массив значение в заданную позицию. +!!! note "Примечание" + Эта функция использует нумерацию массивов с нуля, в отличие от принятой в SQL нумерации с единицы. + Принимает на вход значение и позицию. Если на одну и ту же позицию вставляется несколько значений, в результирующем массиве может оказаться любое (первое в случае однопоточного выполнения). Если в позицию не вставляется ни одного значения, то позиции присваивается значение по умолчанию. Опциональные параметры: @@ -905,7 +908,7 @@ quantile(level)(expr) **Параметры** - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../data_types/index.md#data_types) или типов [Date](../../data_types/date.md), [DateTime](../../data_types/datetime.md). +- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../sql_reference/aggregate_functions/reference.md#data_types) или типов [Date](../../sql_reference/aggregate_functions/reference.md), [DateTime](../../sql_reference/aggregate_functions/reference.md). **Возвращаемое значение** @@ -913,9 +916,9 @@ quantile(level)(expr) Тип: -- [Float64](../../data_types/float.md) для входных данных числового типа. -- [Date](../../data_types/date.md), если входные значения имеют тип `Date`. -- [DateTime](../../data_types/datetime.md), если входные значения имеют тип `DateTime`. +- [Float64](../../sql_reference/aggregate_functions/reference.md) для входных данных числового типа. +- [Date](../../sql_reference/aggregate_functions/reference.md), если входные значения имеют тип `Date`. +- [DateTime](../../sql_reference/aggregate_functions/reference.md), если входные значения имеют тип `DateTime`. **Пример** @@ -968,7 +971,7 @@ quantileDeterministic(level)(expr, determinator) **Параметры** - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../data_types/index.md#data_types) или типов [Date](../../data_types/date.md), [DateTime](../../data_types/datetime.md). +- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../sql_reference/aggregate_functions/reference.md#data_types) или типов [Date](../../sql_reference/aggregate_functions/reference.md), [DateTime](../../sql_reference/aggregate_functions/reference.md). - `determinator` — Число, хэш которого используется при сэмплировании в алгоритме reservoir sampling, чтобы сделать результат детерминированным. В качестве детерминатора можно использовать любое определённое положительное число, например, идентификатор пользователя или события. Если одно и то же значение детерминатора попадается в выборке слишком часто, то функция выдаёт некорректный результат. **Возвращаемое значение** @@ -977,9 +980,9 @@ quantileDeterministic(level)(expr, determinator) Тип: -- [Float64](../../data_types/float.md) для входных данных числового типа. -- [Date](../../data_types/date.md) если входные значения имеют тип `Date`. -- [DateTime](../../data_types/datetime.md) если входные значения имеют тип `DateTime`. +- [Float64](../../sql_reference/aggregate_functions/reference.md) для входных данных числового типа. +- [Date](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `Date`. +- [DateTime](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `DateTime`. **Пример** @@ -1032,7 +1035,7 @@ quantileExact(level)(expr) **Параметры** - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../data_types/index.md#data_types) или типов [Date](../../data_types/date.md), [DateTime](../../data_types/datetime.md). +- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../sql_reference/aggregate_functions/reference.md#data_types) или типов [Date](../../sql_reference/aggregate_functions/reference.md), [DateTime](../../sql_reference/aggregate_functions/reference.md). **Возвращаемое значение** @@ -1040,9 +1043,9 @@ quantileExact(level)(expr) Тип: -- [Float64](../../data_types/float.md) для входных данных числового типа. -- [Date](../../data_types/date.md) если входные значения имеют тип `Date`. -- [DateTime](../../data_types/datetime.md) если входные значения имеют тип `DateTime`. +- [Float64](../../sql_reference/aggregate_functions/reference.md) для входных данных числового типа. +- [Date](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `Date`. +- [DateTime](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `DateTime`. **Пример** @@ -1084,7 +1087,7 @@ quantileExactWeighted(level)(expr, weight) **Параметры** - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../data_types/index.md#data_types) или типов [Date](../../data_types/date.md), [DateTime](../../data_types/datetime.md). +- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../sql_reference/aggregate_functions/reference.md#data_types) или типов [Date](../../sql_reference/aggregate_functions/reference.md), [DateTime](../../sql_reference/aggregate_functions/reference.md). - `weight` — Столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. **Возвращаемое значение** @@ -1093,9 +1096,9 @@ quantileExactWeighted(level)(expr, weight) Тип: -- [Float64](../../data_types/float.md) для входных данных числового типа. -- [Date](../../data_types/date.md) если входные значения имеют тип `Date`. -- [DateTime](../../data_types/datetime.md) если входные значения имеют тип `DateTime`. +- [Float64](../../sql_reference/aggregate_functions/reference.md) для входных данных числового типа. +- [Date](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `Date`. +- [DateTime](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `DateTime`. **Пример** @@ -1149,7 +1152,7 @@ quantileTiming(level)(expr) - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — [Выражение](../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../data_types/float.md). +- `expr` — [Выражение](../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../sql_reference/aggregate_functions/reference.md). - Если в функцию передать отрицательные значения, то её поведение не определено. - Если значение больше, чем 30 000 (например, время загрузки страницы превышает 30 секунд), то оно приравнивается к 30 000. @@ -1173,7 +1176,7 @@ quantileTiming(level)(expr) Тип: `Float32`. !!! note "Примечания" - Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../data_types/float.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../select.md#select-order-by). + Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../sql_reference/aggregate_functions/reference.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../../sql_reference/statements/select.md#select-order-by). **Пример** @@ -1232,7 +1235,7 @@ quantileTimingWeighted(level)(expr, weight) - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — [Выражение](../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../data_types/float.md). +- `expr` — [Выражение](../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../sql_reference/aggregate_functions/reference.md). - Если в функцию передать отрицательные значения, то её поведение не определено. - Если значение больше, чем 30 000 (например, время загрузки страницы превышает 30 секунд), то оно приравнивается к 30 000. @@ -1258,7 +1261,7 @@ quantileTimingWeighted(level)(expr, weight) Тип: `Float32`. !!! note "Примечания" - Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../data_types/float.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../select.md#select-order-by). + Если в функцию `quantileTimingIf` не передать значений, то вернётся [NaN](../../sql_reference/aggregate_functions/reference.md#data_type-float-nan-inf). Это необходимо для отделения подобных случаев от случаев, когда результат 0. Подробности про сортировку `NaN` cмотрите в разделе [Секция ORDER BY](../../sql_reference/statements/select.md#select-order-by). **Пример** @@ -1315,7 +1318,7 @@ quantileTDigest(level)(expr) **Параметры** - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../data_types/index.md#data_types) или типов [Date](../../data_types/date.md), [DateTime](../../data_types/datetime.md). +- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../sql_reference/aggregate_functions/reference.md#data_types) или типов [Date](../../sql_reference/aggregate_functions/reference.md), [DateTime](../../sql_reference/aggregate_functions/reference.md). **Возвращаемое значение** @@ -1323,9 +1326,9 @@ quantileTDigest(level)(expr) Тип: -- [Float64](../../data_types/float.md) для входных данных числового типа. -- [Date](../../data_types/date.md) если входные значения имеют тип `Date`. -- [DateTime](../../data_types/datetime.md) если входные значения имеют тип `DateTime`. +- [Float64](../../sql_reference/aggregate_functions/reference.md) для входных данных числового типа. +- [Date](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `Date`. +- [DateTime](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `DateTime`. **Пример** @@ -1369,7 +1372,7 @@ quantileTDigestWeighted(level)(expr, weight) **Параметры** - `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../data_types/index.md#data_types) или типов [Date](../../data_types/date.md), [DateTime](../../data_types/datetime.md). +- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../sql_reference/aggregate_functions/reference.md#data_types) или типов [Date](../../sql_reference/aggregate_functions/reference.md), [DateTime](../../sql_reference/aggregate_functions/reference.md). - `weight` — Столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. **Возвращаемое значение** @@ -1378,9 +1381,9 @@ quantileTDigestWeighted(level)(expr, weight) Тип: -- [Float64](../../data_types/float.md) для входных данных числового типа. -- [Date](../../data_types/date.md) если входные значения имеют тип `Date`. -- [DateTime](../../data_types/datetime.md) если входные значения имеют тип `DateTime`. +- [Float64](../../sql_reference/aggregate_functions/reference.md) для входных данных числового типа. +- [Date](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `Date`. +- [DateTime](../../sql_reference/aggregate_functions/reference.md) если входные значения имеют тип `DateTime`. **Пример** @@ -1522,7 +1525,7 @@ topKWeighted(N)(x, weight) **Аргументы** - `x` – значение. -- `weight` — вес. [UInt8](../../data_types/int_uint.md). +- `weight` — вес. [UInt8](../../sql_reference/aggregate_functions/reference.md). **Возвращаемое значение** diff --git a/docs/ru/data_types/nested_data_structures/aggregatefunction.md b/docs/ru/sql_reference/data_types/aggregatefunction.md similarity index 87% rename from docs/ru/data_types/nested_data_structures/aggregatefunction.md rename to docs/ru/sql_reference/data_types/aggregatefunction.md index 641d8aa8386..d7ee1211845 100644 --- a/docs/ru/data_types/nested_data_structures/aggregatefunction.md +++ b/docs/ru/sql_reference/data_types/aggregatefunction.md @@ -23,7 +23,7 @@ CREATE TABLE t ) ENGINE = ... ``` -[uniq](../../query_language/agg_functions/reference.md#agg_function-uniq), anyIf ([any](../../query_language/agg_functions/reference.md#agg_function-any)+[If](../../query_language/agg_functions/combinators.md#agg-functions-combinator-if)) и [quantiles](../../query_language/agg_functions/reference.md) — агрегатные функции, поддержанные в ClickHouse. +[uniq](../../sql_reference/data_types/aggregatefunction.md#agg_function-uniq), anyIf ([any](../../sql_reference/data_types/aggregatefunction.md#agg_function-any)+[If](../../sql_reference/data_types/aggregatefunction.md#agg-functions-combinator-if)) и [quantiles](../../sql_reference/data_types/aggregatefunction.md) — агрегатные функции, поддержанные в ClickHouse. ## Особенности использования {#osobennosti-ispolzovaniia} @@ -58,6 +58,6 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP ## Пример использования {#primer-ispolzovaniia} -Смотрите в описании движка [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md). +Смотрите в описании движка [AggregatingMergeTree](../../sql_reference/data_types/aggregatefunction.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/aggregatefunction/) diff --git a/docs/ru/data_types/array.md b/docs/ru/sql_reference/data_types/array.md similarity index 92% rename from docs/ru/data_types/array.md rename to docs/ru/sql_reference/data_types/array.md index 0fa13d54cae..acc2e4940d0 100644 --- a/docs/ru/data_types/array.md +++ b/docs/ru/sql_reference/data_types/array.md @@ -42,7 +42,7 @@ SELECT [1, 2] AS x, toTypeName(x) ## Особенности работы с типами данных {#osobennosti-raboty-s-tipami-dannykh} -При создании массива «на лету» ClickHouse автоматически определяет тип аргументов как наиболее узкий тип данных, в котором можно хранить все перечисленные аргументы. Если среди аргументов есть [NULL](../query_language/syntax.md#null-literal) или аргумент типа [Nullable](nullable.md#data_type-nullable), то тип элементов массива — [Nullable](nullable.md). +При создании массива «на лету» ClickHouse автоматически определяет тип аргументов как наиболее узкий тип данных, в котором можно хранить все перечисленные аргументы. Если среди аргументов есть [NULL](../../sql_reference/data_types/array.md#null-literal) или аргумент типа [Nullable](nullable.md#data_type-nullable), то тип элементов массива — [Nullable](nullable.md). Если ClickHouse не смог подобрать тип данных, то он сгенерирует исключение. Это произойдёт, например, при попытке создать массив одновременно со строками и числами `SELECT array(1, 'a')`. diff --git a/docs/ru/data_types/boolean.md b/docs/ru/sql_reference/data_types/boolean.md similarity index 100% rename from docs/ru/data_types/boolean.md rename to docs/ru/sql_reference/data_types/boolean.md diff --git a/docs/ru/data_types/date.md b/docs/ru/sql_reference/data_types/date.md similarity index 100% rename from docs/ru/data_types/date.md rename to docs/ru/sql_reference/data_types/date.md diff --git a/docs/ru/data_types/datetime.md b/docs/ru/sql_reference/data_types/datetime.md similarity index 86% rename from docs/ru/data_types/datetime.md rename to docs/ru/sql_reference/data_types/datetime.md index 957ffe717a3..e52ca549907 100644 --- a/docs/ru/data_types/datetime.md +++ b/docs/ru/sql_reference/data_types/datetime.md @@ -18,13 +18,13 @@ DateTime([timezone]) Список поддерживаемых временных зон можно найти в [IANA Time Zone Database](https://www.iana.org/time-zones). Пакет `tzdata`, содержащий [базу данных часовых поясов IANA](https://www.iana.org/time-zones), должен быть установлен в системе. Используйте команду `timedatectl list-timezones` для получения списка часовых поясов, известных локальной системе. -Часовой пояс для столбца типа `DateTime` можно в явном виде установить при создании таблицы. Если часовой пояс не установлен, то ClickHouse использует значение параметра [timezone](../operations/server_settings/settings.md#server_settings-timezone), установленное в конфигурации сервера или в настройках операционной системы на момент запуска сервера. +Часовой пояс для столбца типа `DateTime` можно в явном виде установить при создании таблицы. Если часовой пояс не установлен, то ClickHouse использует значение параметра [timezone](../../sql_reference/data_types/datetime.md#server_configuration_parameters-timezone), установленное в конфигурации сервера или в настройках операционной системы на момент запуска сервера. -Консольный клиент ClickHouse по умолчанию использует часовой пояс сервера, если для значения `DateTime` часовой пояс не был задан в явном виде при инициализации типа данных. Чтобы использовать часовой пояс клиента, запустите [clickhouse-client](../interfaces/cli.md) с параметром `--use_client_time_zone`. +Консольный клиент ClickHouse по умолчанию использует часовой пояс сервера, если для значения `DateTime` часовой пояс не был задан в явном виде при инициализации типа данных. Чтобы использовать часовой пояс клиента, запустите [clickhouse-client](../../interfaces/cli.md) с параметром `--use_client_time_zone`. -ClickHouse отображает значения типа `DateTime` в формате `YYYY-MM-DD hh:mm:ss`. Отображение можно поменять с помощью функции [formatDateTime](../query_language/functions/date_time_functions.md#formatdatetime). +ClickHouse отображает значения типа `DateTime` в формате `YYYY-MM-DD hh:mm:ss`. Отображение можно поменять с помощью функции [formatDateTime](../../sql_reference/data_types/datetime.md#formatdatetime). -При вставке данных в ClickHouse, можно использовать различные форматы даты и времени в зависимости от значения настройки [date\_time\_input\_format](../operations/settings/settings.md#settings-date_time_input_format). +При вставке данных в ClickHouse, можно использовать различные форматы даты и времени в зависимости от значения настройки [date\_time\_input\_format](../../operations/settings/settings.md#settings-date_time_input_format). ## Примеры {#primery} @@ -111,12 +111,12 @@ FROM dt ## See Also {#see-also} -- [Функции преобразования типов](../query_language/functions/type_conversion_functions.md) -- [Функции для работы с датой и временем](../query_language/functions/date_time_functions.md) -- [Функции для работы с массивами](../query_language/functions/array_functions.md) -- [Настройка `date_time_input_format`](../operations/settings/settings.md#settings-date_time_input_format) -- [Конфигурационный параметр сервера `timezone`](../operations/server_settings/settings.md#server_settings-timezone) -- [Операторы для работы с датой и временем](../query_language/operators.md#operators-datetime) +- [Функции преобразования типов](../../sql_reference/data_types/datetime.md) +- [Функции для работы с датой и временем](../../sql_reference/data_types/datetime.md) +- [Функции для работы с массивами](../../sql_reference/data_types/datetime.md) +- [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format) +- [Конфигурационный параметр сервера `timezone`](../../sql_reference/data_types/datetime.md#server_configuration_parameters-timezone) +- [Операторы для работы с датой и временем](../../sql_reference/data_types/datetime.md#operators-datetime) - [Тип данных `Date`](date.md) [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/datetime/) diff --git a/docs/ru/data_types/datetime64.md b/docs/ru/sql_reference/data_types/datetime64.md similarity index 92% rename from docs/ru/data_types/datetime64.md rename to docs/ru/sql_reference/data_types/datetime64.md index 8e3277dd52f..9e126885058 100644 --- a/docs/ru/data_types/datetime64.md +++ b/docs/ru/sql_reference/data_types/datetime64.md @@ -87,11 +87,11 @@ FROM dt ## See Also {#see-also} -- [Функции преобразования типов](../query_language/functions/type_conversion_functions.md) -- [Функции для работы с датой и временем](../query_language/functions/date_time_functions.md) -- [Функции для работы с массивами](../query_language/functions/array_functions.md) -- [Настройка `date_time_input_format`](../operations/settings/settings.md#settings-date_time_input_format) -- [Конфигурационный параметр сервера `timezone`](../operations/server_settings/settings.md#server_settings-timezone) -- [Операторы для работы с датой и временем](../query_language/operators.md#operators-datetime) +- [Функции преобразования типов](../../sql_reference/data_types/datetime64.md) +- [Функции для работы с датой и временем](../../sql_reference/data_types/datetime64.md) +- [Функции для работы с массивами](../../sql_reference/data_types/datetime64.md) +- [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format) +- [Конфигурационный параметр сервера `timezone`](../../sql_reference/data_types/datetime64.md#server_configuration_parameters-timezone) +- [Операторы для работы с датой и временем](../../sql_reference/data_types/datetime64.md#operators-datetime) - [Тип данных `Date`](date.md) - [Тип данных `DateTime`](datetime.md) diff --git a/docs/ru/data_types/decimal.md b/docs/ru/sql_reference/data_types/decimal.md similarity index 100% rename from docs/ru/data_types/decimal.md rename to docs/ru/sql_reference/data_types/decimal.md diff --git a/docs/ru/sql_reference/data_types/domains/index.md b/docs/ru/sql_reference/data_types/domains/index.md new file mode 100644 index 00000000000..d4496cf8d5b --- /dev/null +++ b/docs/ru/sql_reference/data_types/domains/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: Domains +toc_priority: 56 +--- + diff --git a/docs/ru/data_types/domains/ipv4.md b/docs/ru/sql_reference/data_types/domains/ipv4.md similarity index 100% rename from docs/ru/data_types/domains/ipv4.md rename to docs/ru/sql_reference/data_types/domains/ipv4.md diff --git a/docs/ru/data_types/domains/ipv6.md b/docs/ru/sql_reference/data_types/domains/ipv6.md similarity index 100% rename from docs/ru/data_types/domains/ipv6.md rename to docs/ru/sql_reference/data_types/domains/ipv6.md diff --git a/docs/ru/data_types/domains/overview.md b/docs/ru/sql_reference/data_types/domains/overview.md similarity index 100% rename from docs/ru/data_types/domains/overview.md rename to docs/ru/sql_reference/data_types/domains/overview.md diff --git a/docs/ru/data_types/enum.md b/docs/ru/sql_reference/data_types/enum.md similarity index 99% rename from docs/ru/data_types/enum.md rename to docs/ru/sql_reference/data_types/enum.md index 2ee7c77028b..58f2a4b188e 100644 --- a/docs/ru/data_types/enum.md +++ b/docs/ru/sql_reference/data_types/enum.md @@ -86,7 +86,7 @@ SELECT toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)')) Для каждого из значений прописывается число в диапазоне `-128 .. 127` для `Enum8` или в диапазоне `-32768 .. 32767` для `Enum16`. Все строки должны быть разными, числа - тоже. Разрешена пустая строка. При указании такого типа (в определении таблицы), числа могут идти не подряд и в произвольном порядке. При этом, порядок не имеет значения. -Ни строка, ни цифровое значение в `Enum` не могут быть [NULL](../query_language/syntax.md). +Ни строка, ни цифровое значение в `Enum` не могут быть [NULL](../../sql_reference/syntax.md#null-literal). `Enum` может быть передан в тип [Nullable](nullable.md). Таким образом, если создать таблицу запросом diff --git a/docs/ru/data_types/fixedstring.md b/docs/ru/sql_reference/data_types/fixedstring.md similarity index 89% rename from docs/ru/data_types/fixedstring.md rename to docs/ru/sql_reference/data_types/fixedstring.md index ba91fcde9b3..7e2fdd5d525 100644 --- a/docs/ru/data_types/fixedstring.md +++ b/docs/ru/sql_reference/data_types/fixedstring.md @@ -51,6 +51,6 @@ WHERE a = 'b\0' Это поведение отличается от поведения MySQL для типа `CHAR`, где строки дополняются пробелами, а пробелы перед выводом вырезаются. -Обратите внимание, что длина значения `FixedString(N)` постоянна. Функция [length](../query_language/functions/array_functions.md#array_functions-length) возвращает `N` даже если значение `FixedString(N)` заполнено только нулевыми байтами, однако функция [empty](../query_language/functions/string_functions.md#empty) в этом же случае возвращает `1`. +Обратите внимание, что длина значения `FixedString(N)` постоянна. Функция [length](../../sql_reference/data_types/fixedstring.md#array_functions-length) возвращает `N` даже если значение `FixedString(N)` заполнено только нулевыми байтами, однако функция [empty](../../sql_reference/data_types/fixedstring.md#empty) в этом же случае возвращает `1`. [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/fixedstring/) diff --git a/docs/ru/data_types/float.md b/docs/ru/sql_reference/data_types/float.md similarity index 97% rename from docs/ru/data_types/float.md rename to docs/ru/sql_reference/data_types/float.md index 20eab345573..91d4b655e2a 100644 --- a/docs/ru/data_types/float.md +++ b/docs/ru/sql_reference/data_types/float.md @@ -75,6 +75,6 @@ SELECT 0 / 0 └──────────────┘ ``` - Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY](../query_language/select.md). + Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY](../sql_reference/data_types/float.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/float/) diff --git a/docs/ru/data_types/index.md b/docs/ru/sql_reference/data_types/index.md similarity index 83% rename from docs/ru/data_types/index.md rename to docs/ru/sql_reference/data_types/index.md index a362b4708e6..1fd52b9bf9e 100644 --- a/docs/ru/data_types/index.md +++ b/docs/ru/sql_reference/data_types/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: Типы данных +toc_priority: 37 +toc_title: Введение +--- + # Типы данных {#data_types} ClickHouse может сохранять в ячейках таблиц данные различных типов. diff --git a/docs/ru/data_types/int_uint.md b/docs/ru/sql_reference/data_types/int_uint.md similarity index 100% rename from docs/ru/data_types/int_uint.md rename to docs/ru/sql_reference/data_types/int_uint.md diff --git a/docs/ru/data_types/nested_data_structures/index.md b/docs/ru/sql_reference/data_types/nested_data_structures/index.md similarity index 100% rename from docs/ru/data_types/nested_data_structures/index.md rename to docs/ru/sql_reference/data_types/nested_data_structures/index.md diff --git a/docs/ru/data_types/nested_data_structures/nested.md b/docs/ru/sql_reference/data_types/nested_data_structures/nested.md similarity index 100% rename from docs/ru/data_types/nested_data_structures/nested.md rename to docs/ru/sql_reference/data_types/nested_data_structures/nested.md diff --git a/docs/ru/data_types/nullable.md b/docs/ru/sql_reference/data_types/nullable.md similarity index 83% rename from docs/ru/data_types/nullable.md rename to docs/ru/sql_reference/data_types/nullable.md index 1ee6777254b..dfbd0fb0512 100644 --- a/docs/ru/data_types/nullable.md +++ b/docs/ru/sql_reference/data_types/nullable.md @@ -1,6 +1,6 @@ # Nullable(TypeName) {#data_type-nullable} -Позволяет работать как со значением типа `TypeName` так и с отсутствием этого значения ([NULL](../query_language/syntax.md)) в одной и той же переменной, в том числе хранить `NULL` в таблицах вместе со значения типа `TypeName`. Например, в столбце типа `Nullable(Int8)` можно хранить значения типа `Int8`, а в тех строках, где значения нет, будет храниться `NULL`. +Позволяет работать как со значением типа `TypeName` так и с отсутствием этого значения ([NULL](../../sql_reference/data_types/nullable.md)) в одной и той же переменной, в том числе хранить `NULL` в таблицах вместе со значения типа `TypeName`. Например, в столбце типа `Nullable(Int8)` можно хранить значения типа `Int8`, а в тех строках, где значения нет, будет храниться `NULL`. В качестве `TypeName` нельзя использовать составные типы данных [Array](array.md#data_type-array) и [Tuple](tuple.md). Составные типы данных могут содержать значения типа `Nullable`, например `Array(Nullable(Int8))`. diff --git a/docs/ru/data_types/special_data_types/expression.md b/docs/ru/sql_reference/data_types/special_data_types/expression.md similarity index 100% rename from docs/ru/data_types/special_data_types/expression.md rename to docs/ru/sql_reference/data_types/special_data_types/expression.md diff --git a/docs/ru/data_types/special_data_types/index.md b/docs/ru/sql_reference/data_types/special_data_types/index.md similarity index 100% rename from docs/ru/data_types/special_data_types/index.md rename to docs/ru/sql_reference/data_types/special_data_types/index.md diff --git a/docs/ru/data_types/special_data_types/interval.md b/docs/ru/sql_reference/data_types/special_data_types/interval.md similarity index 84% rename from docs/ru/data_types/special_data_types/interval.md rename to docs/ru/sql_reference/data_types/special_data_types/interval.md index 22912bdbca1..1721b8631ad 100644 --- a/docs/ru/data_types/special_data_types/interval.md +++ b/docs/ru/sql_reference/data_types/special_data_types/interval.md @@ -1,6 +1,6 @@ # Interval {#data-type-interval} -Семейство типов данных, представляющих интервалы дат и времени. Оператор [INTERVAL](../../query_language/operators.md#operator-interval) возвращает значения этих типов. +Семейство типов данных, представляющих интервалы дат и времени. Оператор [INTERVAL](../../../sql_reference/data_types/special_data_types/interval.md#operator-interval) возвращает значения этих типов. !!! warning "Внимание" Нельзя использовать типы данных `Interval` для хранения данных в таблице. @@ -35,7 +35,7 @@ SELECT toTypeName(INTERVAL 4 DAY) ## Использование {#data-type-interval-usage-remarks} -Значения типов `Interval` можно использовать в арифметических операциях со значениями типов [Date](../../data_types/date.md) и [DateTime](../../data_types/datetime.md). Например, можно добавить 4 дня к текущей дате: +Значения типов `Interval` можно использовать в арифметических операциях со значениями типов [Date](../../../sql_reference/data_types/special_data_types/interval.md) и [DateTime](../../../sql_reference/data_types/special_data_types/interval.md). Например, можно добавить 4 дня к текущей дате: ``` sql SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY @@ -74,5 +74,5 @@ Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argu ## Смотрите также {#smotrite-takzhe} -- Оператор[INTERVAL](../../query_language/operators.md#operator-interval) -- Функция приведения типа [toInterval](../../query_language/functions/type_conversion_functions.md#function-tointerval) +- Оператор[INTERVAL](../../../sql_reference/data_types/special_data_types/interval.md#operator-interval) +- Функция приведения типа [toInterval](../../../sql_reference/data_types/special_data_types/interval.md#function-tointerval) diff --git a/docs/ru/data_types/special_data_types/nothing.md b/docs/ru/sql_reference/data_types/special_data_types/nothing.md similarity index 63% rename from docs/ru/data_types/special_data_types/nothing.md rename to docs/ru/sql_reference/data_types/special_data_types/nothing.md index ab4b96cc519..c23de847077 100644 --- a/docs/ru/data_types/special_data_types/nothing.md +++ b/docs/ru/sql_reference/data_types/special_data_types/nothing.md @@ -1,8 +1,8 @@ # Nothing {#nothing} -Этот тип данных предназначен только для того, чтобы представлять [NULL](../../query_language/syntax.md), т.е. отсутствие значения. +Этот тип данных предназначен только для того, чтобы представлять [NULL](../../../sql_reference/data_types/special_data_types/nothing.md), т.е. отсутствие значения. -Невозможно создать значение типа `Nothing`, поэтому он используется там, где значение не подразумевается. Например, `NULL` записывается как `Nullable(Nothing)` ([Nullable](../../data_types/nullable.md) — это тип данных, позволяющий хранить `NULL` в таблицах). Также тип `Nothing` используется для обозначения пустых массивов: +Невозможно создать значение типа `Nothing`, поэтому он используется там, где значение не подразумевается. Например, `NULL` записывается как `Nullable(Nothing)` ([Nullable](../../../sql_reference/data_types/special_data_types/nothing.md) — это тип данных, позволяющий хранить `NULL` в таблицах). Также тип `Nothing` используется для обозначения пустых массивов: ``` sql SELECT toTypeName(Array()) diff --git a/docs/ru/data_types/special_data_types/set.md b/docs/ru/sql_reference/data_types/special_data_types/set.md similarity index 100% rename from docs/ru/data_types/special_data_types/set.md rename to docs/ru/sql_reference/data_types/special_data_types/set.md diff --git a/docs/ru/data_types/string.md b/docs/ru/sql_reference/data_types/string.md similarity index 100% rename from docs/ru/data_types/string.md rename to docs/ru/sql_reference/data_types/string.md diff --git a/docs/ru/data_types/tuple.md b/docs/ru/sql_reference/data_types/tuple.md similarity index 87% rename from docs/ru/data_types/tuple.md rename to docs/ru/sql_reference/data_types/tuple.md index 17732d04953..e8f5f335278 100644 --- a/docs/ru/data_types/tuple.md +++ b/docs/ru/sql_reference/data_types/tuple.md @@ -2,7 +2,7 @@ Кортеж из элементов любого [типа](index.md#data_types). Элементы кортежа могут быть одного или разных типов. -Кортежи используются для временной группировки столбцов. Столбцы могут группироваться при использовании выражения IN в запросе, а также для указания нескольких формальных параметров лямбда-функций. Подробнее смотрите разделы [Операторы IN](../query_language/select.md), [Функции высшего порядка](../query_language/functions/higher_order_functions.md#higher_order_functions). +Кортежи используются для временной группировки столбцов. Столбцы могут группироваться при использовании выражения IN в запросе, а также для указания нескольких формальных параметров лямбда-функций. Подробнее смотрите разделы [Операторы IN](../../sql_reference/data_types/tuple.md), [Функции высшего порядка](../../sql_reference/data_types/tuple.md#higher_order_functions). Кортежи могут быть результатом запроса. В этом случае, в текстовых форматах кроме JSON, значения выводятся в круглых скобках через запятую. В форматах JSON, кортежи выводятся в виде массивов (в квадратных скобках). @@ -28,7 +28,7 @@ SELECT tuple(1,'a') AS x, toTypeName(x) ## Особенности работы с типами данных {#osobennosti-raboty-s-tipami-dannykh} -При создании кортежа «на лету» ClickHouse автоматически определяет тип каждого аргументов как минимальный из типов, который может сохранить значение аргумента. Если аргумент — [NULL](../query_language/syntax.md#null-literal), то тип элемента кортежа — [Nullable](nullable.md). +При создании кортежа «на лету» ClickHouse автоматически определяет тип каждого аргументов как минимальный из типов, который может сохранить значение аргумента. Если аргумент — [NULL](../../sql_reference/data_types/tuple.md#null-literal), то тип элемента кортежа — [Nullable](nullable.md). Пример автоматического определения типа данных: diff --git a/docs/ru/data_types/uuid.md b/docs/ru/sql_reference/data_types/uuid.md similarity index 82% rename from docs/ru/data_types/uuid.md rename to docs/ru/sql_reference/data_types/uuid.md index 24f43cc5d24..d62ec22eecb 100644 --- a/docs/ru/data_types/uuid.md +++ b/docs/ru/sql_reference/data_types/uuid.md @@ -16,7 +16,7 @@ ## Как сгенерировать UUID {#kak-sgenerirovat-uuid} -Для генерации UUID-значений предназначена функция [generateUUIDv4](../query_language/functions/uuid_functions.md). +Для генерации UUID-значений предназначена функция [generateUUIDv4](../../sql_reference/data_types/uuid.md). ## Примеры использования {#primery-ispolzovaniia} @@ -65,8 +65,8 @@ SELECT * FROM t_uuid ## Ограничения {#ogranicheniia} -Тип данных UUID можно использовать только с функциями, которые поддерживаются типом данных [String](string.md) (например, [min](../query_language/agg_functions/reference.md#agg_function-min), [max](../query_language/agg_functions/reference.md#agg_function-max), и [count](../query_language/agg_functions/reference.md#agg_function-count)). +Тип данных UUID можно использовать только с функциями, которые поддерживаются типом данных [String](string.md) (например, [min](../../sql_reference/data_types/uuid.md#agg_function-min), [max](../../sql_reference/data_types/uuid.md#agg_function-max), и [count](../../sql_reference/data_types/uuid.md#agg_function-count)). -Тип данных UUID не поддерживается арифметическими операциями (например, [abs](../query_language/functions/arithmetic_functions.md#arithm_func-abs)) или агрегатными функциями, такими как [sum](../query_language/agg_functions/reference.md#agg_function-sum) и [avg](../query_language/agg_functions/reference.md#agg_function-avg). +Тип данных UUID не поддерживается арифметическими операциями (например, [abs](../../sql_reference/data_types/uuid.md#arithm_func-abs)) или агрегатными функциями, такими как [sum](../../sql_reference/data_types/uuid.md#agg_function-sum) и [avg](../../sql_reference/data_types/uuid.md#agg_function-avg). [Original article](https://clickhouse.tech/docs/en/data_types/uuid/) diff --git a/docs/ru/query_language/dicts/external_dicts.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts.md similarity index 79% rename from docs/ru/query_language/dicts/external_dicts.md rename to docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts.md index 586e443e3a9..4929e571758 100644 --- a/docs/ru/query_language/dicts/external_dicts.md +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts.md @@ -5,11 +5,11 @@ ClickHouse: - Полностью или частично хранит словари в оперативной памяти. - Периодически обновляет их и динамически подгружает отсутствующие значения. -- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../create.md#create-dictionary-query). +- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../../sql_reference/statements/create.md#create-dictionary-query). -Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../operations/server_settings/settings.md). +Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../../sql_reference/dictionaries/external_dictionaries/external_dicts.md). -Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../operations/server_settings/settings.md). +Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../../sql_reference/dictionaries/external_dictionaries/external_dicts.md). Конфигурационный файл словарей имеет вид: @@ -35,10 +35,10 @@ ClickHouse: В одном файле можно [сконфигурировать](external_dicts_dict.md) произвольное количество словарей. -Если вы создаёте внешние словари [DDL-запросами](../create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера. +Если вы создаёте внешние словари [DDL-запросами](../../../sql_reference/statements/create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера. !!! attention "Внимание" - Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../functions/other_functions.md)). Эта функциональность не связана с внешними словарями. + Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../../../sql_reference/dictionaries/external_dictionaries/external_dicts.md)). Эта функциональность не связана с внешними словарями. ## Смотрите также {#ext-dicts-see-also} @@ -47,6 +47,6 @@ ClickHouse: - [Обновление словарей](external_dicts_dict_lifetime.md) - [Источники внешних словарей](external_dicts_dict_sources.md) - [Ключ и поля словаря](external_dicts_dict_structure.md) -- [Функции для работы с внешними словарями](../functions/ext_dict_functions.md#ext_dict_functions) +- [Функции для работы с внешними словарями](../../../sql_reference/dictionaries/external_dictionaries/external_dicts.md#ext_dict_functions) [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts/) diff --git a/docs/ru/query_language/dicts/external_dicts_dict.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md similarity index 91% rename from docs/ru/query_language/dicts/external_dicts_dict.md rename to docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md index dcb8c6652b0..2e3068882bf 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict.md +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md @@ -24,7 +24,7 @@ XML-конфигурация словаря имеет следующую стр ``` -Соответствующий [DDL-запрос](../create.md#create-dictionary-query) имеет следующий вид: +Соответствующий [DDL-запрос](../../../sql_reference/statements/create.md#create-dictionary-query) имеет следующий вид: ``` sql CREATE DICTIONARY dict_name diff --git a/docs/ru/query_language/dicts/external_dicts_dict_hierarchical.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md similarity index 90% rename from docs/ru/query_language/dicts/external_dicts_dict_hierarchical.md rename to docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md index 335b882a97c..ef9b79c0444 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_hierarchical.md +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md @@ -32,7 +32,7 @@ ClickHouse поддерживает иерархические словари с ClickHouse поддерживает свойство [hierarchical](external_dicts_dict_structure.md#hierarchical-dict-attr) для атрибутов [внешнего словаря](index.md). Это свойство позволяет конфигурировать словари, подобные описанному выше. -С помощью функции [dictGetHierarchy](../functions/ext_dict_functions.md#dictgethierarchy) можно получить цепочку предков элемента. +С помощью функции [dictGetHierarchy](../../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md#dictgethierarchy) можно получить цепочку предков элемента. Структура словаря для нашего примера может выглядеть следующим образом: diff --git a/docs/ru/query_language/dicts/external_dicts_dict_layout.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md similarity index 99% rename from docs/ru/query_language/dicts/external_dicts_dict_layout.md rename to docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md index 458593e82aa..898fe45b15a 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md @@ -34,7 +34,7 @@ ``` -Соответствущий [DDL-запрос](../create.md#create-dictionary-query): +Соответствущий [DDL-запрос](../../../sql_reference/statements/create.md#create-dictionary-query): ``` sql CREATE DICTIONARY (...) diff --git a/docs/ru/query_language/dicts/external_dicts_dict_lifetime.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md similarity index 100% rename from docs/ru/query_language/dicts/external_dicts_dict_lifetime.md rename to docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md diff --git a/docs/ru/query_language/dicts/external_dicts_dict_sources.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md similarity index 93% rename from docs/ru/query_language/dicts/external_dicts_dict_sources.md rename to docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md index 8b9961ee7fc..102a7cc0e7f 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_sources.md +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md @@ -19,7 +19,7 @@ ``` -Аналогичный [DDL-запрос](../create.md#create-dictionary-query): +Аналогичный [DDL-запрос](../../../sql_reference/statements/create.md#create-dictionary-query): ``` sql CREATE DICTIONARY dict_name (...) @@ -30,6 +30,25 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration Источник настраивается в разделе `source`. +Для типов источников +[Локальный файл](#dicts-external_dicts_dict_sources-local_file), +[Исполняемый файл](#dicts-external_dicts_dict_sources-executable), +[HTTP(s)](#dicts-external_dicts_dict_sources-http), +[ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) +доступны дополнительные настройки: + +``` xml + + + /opt/dictionaries/os.tsv + TabSeparated + + + 0 + + +``` + Типы источников (`source_type`): - [Локальный файл](#dicts-external_dicts_dict_sources-local_file) @@ -64,7 +83,7 @@ SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) Поля настройки: - `path` — Абсолютный путь к файлу. -- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../interfaces/formats.md#formats)». +- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)». ## Исполняемый файл {#dicts-external_dicts_dict_sources-executable} @@ -90,7 +109,7 @@ SOURCE(EXECUTABLE(command 'cat /opt/dictionaries/os.tsv' format 'TabSeparated')) Поля настройки: - `command` — Абсолютный путь к исполняемому файлу или имя файла (если каталог программы прописан в `PATH`). -- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../interfaces/formats.md#formats)». +- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)». ## HTTP(s) {#dicts-external_dicts_dict_sources-http} @@ -128,12 +147,12 @@ SOURCE(HTTP( )) ``` -Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../operations/server_settings/settings.md) в конфигурации сервера. +Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md) в конфигурации сервера. Поля настройки: - `url` — URL источника. -- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../interfaces/formats.md#formats)». +- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)». ## ODBC {#dicts-external_dicts_dict_sources-odbc} @@ -172,7 +191,7 @@ SOURCE(ODBC( ClickHouse получает от ODBC-драйвера информацию о квотировании и квотирует настройки в запросах к драйверу, поэтому имя таблицы нужно указывать в соответствии с регистром имени таблицы в базе данных. -Если у вас есть проблемы с кодировками при использовании Oracle, ознакомьтесь с соответствующим разделом [FAQ](../../faq/general.md#oracle-odbc-encodings). +Если у вас есть проблемы с кодировками при использовании Oracle, ознакомьтесь с соответствующим разделом [FAQ](../../../faq/general.md#oracle-odbc-encodings). ### Выявленная уязвимость в функционировании ODBC словарей {#vyiavlennaia-uiazvimost-v-funktsionirovanii-odbc-slovarei} @@ -509,7 +528,7 @@ SOURCE(CLICKHOUSE( Поля настройки: -- `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../operations/table_engines/distributed.md) и прописать её в дальнейших настройках. +- `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md) и прописать её в дальнейших настройках. - `port` — порт сервера ClickHouse. - `user` — имя пользователя ClickHouse. - `password` — пароль пользователя ClickHouse. diff --git a/docs/ru/query_language/dicts/external_dicts_dict_structure.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md similarity index 90% rename from docs/ru/query_language/dicts/external_dicts_dict_structure.md rename to docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md index 7fa762c063a..f83b7657b54 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_structure.md +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md @@ -154,14 +154,14 @@ CREATE DICTIONARY somename ( | Тег | Описание | Обязательный | |------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| | `name` | Имя столбца. | Да | -| `type` | Тип данных ClickHouse.
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../data_types/nullable.md) не поддерживается. | Да | +| `type` | Тип данных ClickHouse.
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md) не поддерживается. | Да | | `null_value` | Значение по умолчанию для несуществующего элемента.
В примере это пустая строка. Нельзя указать значение `NULL`. | Да | -| `expression` | [Выражение](../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.
Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.

Значение по умолчанию: нет выражения. | Нет | +| `expression` | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.
Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.

Значение по умолчанию: нет выражения. | Нет | | `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external_dicts_dict_hierarchical.md).

Default value: `false`. | No | | `is_object_id` | Признак того, что запрос выполняется к документу MongoDB по `ObjectID`.

Значение по умолчанию: `false`. | Нет | ## Смотрите также {#smotrite-takzhe} -- [Функции для работы с внешними словарями](../functions/ext_dict_functions.md). +- [Функции для работы с внешними словарями](../../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_structure/) diff --git a/docs/ru/sql_reference/dictionaries/external_dictionaries/index.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/index.md new file mode 100644 index 00000000000..9af8b4f2f12 --- /dev/null +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: External Dictionaries +toc_priority: 37 +--- + diff --git a/docs/ru/query_language/dicts/index.md b/docs/ru/sql_reference/dictionaries/index.md similarity index 82% rename from docs/ru/query_language/dicts/index.md rename to docs/ru/sql_reference/dictionaries/index.md index e53b7f394d8..e876b92c9bf 100644 --- a/docs/ru/query_language/dicts/index.md +++ b/docs/ru/sql_reference/dictionaries/index.md @@ -8,7 +8,7 @@ ClickHouse поддерживает специальные функции для ClickHouse поддерживает: -- [Встроенные словари](internal_dicts.md#internal_dicts) со специфическим [набором функций](../functions/ym_dict_functions.md). -- [Подключаемые (внешние) словари](external_dicts.md) с [набором функций](../functions/ext_dict_functions.md). +- [Встроенные словари](internal_dicts.md#internal_dicts) со специфическим [набором функций](../../sql_reference/dictionaries/external_dictionaries/index.md). +- [Подключаемые (внешние) словари](external_dictionaries/external_dicts.md) с [набором функций](../../sql_reference/dictionaries/external_dictionaries/index.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/) diff --git a/docs/ru/query_language/dicts/internal_dicts.md b/docs/ru/sql_reference/dictionaries/internal_dicts.md similarity index 100% rename from docs/ru/query_language/dicts/internal_dicts.md rename to docs/ru/sql_reference/dictionaries/internal_dicts.md diff --git a/docs/ru/query_language/functions/arithmetic_functions.md b/docs/ru/sql_reference/functions/arithmetic_functions.md similarity index 100% rename from docs/ru/query_language/functions/arithmetic_functions.md rename to docs/ru/sql_reference/functions/arithmetic_functions.md diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/sql_reference/functions/array_functions.md similarity index 93% rename from docs/ru/query_language/functions/array_functions.md rename to docs/ru/sql_reference/functions/array_functions.md index ce757921bf5..82a6ceea001 100644 --- a/docs/ru/query_language/functions/array_functions.md +++ b/docs/ru/sql_reference/functions/array_functions.md @@ -55,7 +55,7 @@ arrayConcat(arrays) **Параметры** -- `arrays` – произвольное количество элементов типа [Array](../../data_types/array.md) +- `arrays` – произвольное количество элементов типа [Array](../../sql_reference/functions/array_functions.md) **Пример** @@ -359,7 +359,7 @@ arrayPushBack(array, single_value) **Параметры** - `array` - Массив. -- `single_value` - Одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../data_types/index.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. +- `single_value` - Одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql_reference/functions/array_functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. **Пример** @@ -384,7 +384,7 @@ arrayPushFront(array, single_value) **Параметры** - `array` - Массив. -- `single_value` - Одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../data_types/index.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. +- `single_value` - Одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql_reference/functions/array_functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. **Пример** @@ -881,7 +881,7 @@ flatten(array_of_arrays) **Параметры** -- `array_of_arrays` — [Массивов](../../data_types/array.md) массивов. Например, `[[1,2,3], [4,5]]`. +- `array_of_arrays` — [Массивов](../../sql_reference/functions/array_functions.md) массивов. Например, `[[1,2,3], [4,5]]`. **Примеры** @@ -907,7 +907,7 @@ arrayCompact(arr) **Параметры** -`arr` — [Массив](../../data_types/array.md) для обхода. +`arr` — [Массив](../../sql_reference/functions/array_functions.md) для обхода. **Возвращаемое значение** @@ -931,4 +931,42 @@ SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) └────────────────────────────────────────────┘ ``` +## arrayZip {#arrayzip} + +Объединяет несколько массивов в один. Результирующий массив содержит соответственные элементы исходных массивов, сгруппированные в кортежи в указанном порядке аргументов. + +**Синтаксис** + +``` sql +arrayZip(arr1, arr2, ..., arrN) +``` + +**Параметры** + +- `arrN` — [Массив](../data_types/array.md). + +Функция принимает любое количество массивов, которые могут быть различных типов. Все массивы должны иметь одинаковую длину. + +**Возвращаемое значение** + +- Массив с элементами исходных массивов, сгруппированными в [кортежи](../data_types/tuple.md). Типы данных в кортежах соответствуют типам данных входных массивов и следуют в том же порядке, в котором переданы массивы. + +Тип: [Массив](../data_types/array.md). + +**Пример** + +Запрос: + +``` sql +SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]) +``` + +Ответ: + +``` text +┌─arrayZip(['a', 'b', 'c'], [5, 2, 1])─┐ +│ [('a',5),('b',2),('c',1)] │ +└──────────────────────────────────────┘ +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/array_functions/) diff --git a/docs/ru/query_language/functions/array_join.md b/docs/ru/sql_reference/functions/array_join.md similarity index 100% rename from docs/ru/query_language/functions/array_join.md rename to docs/ru/sql_reference/functions/array_join.md diff --git a/docs/ru/query_language/functions/bit_functions.md b/docs/ru/sql_reference/functions/bit_functions.md similarity index 94% rename from docs/ru/query_language/functions/bit_functions.md rename to docs/ru/sql_reference/functions/bit_functions.md index 53efa9db06b..09eb9d17bc2 100644 --- a/docs/ru/query_language/functions/bit_functions.md +++ b/docs/ru/sql_reference/functions/bit_functions.md @@ -207,7 +207,7 @@ bitCount(x) **Параметры** -- `x` — [Целое число](../../data_types/int_uint.md) или [число с плавающей запятой](../../data_types/float.md). Функция использует представление числа в памяти, что позволяет поддержать числа с плавающей запятой. +- `x` — [Целое число](../../sql_reference/functions/bit_functions.md) или [число с плавающей запятой](../../sql_reference/functions/bit_functions.md). Функция использует представление числа в памяти, что позволяет поддержать числа с плавающей запятой. **Возвращаемое значение** diff --git a/docs/ru/query_language/functions/bitmap_functions.md b/docs/ru/sql_reference/functions/bitmap_functions.md similarity index 97% rename from docs/ru/query_language/functions/bitmap_functions.md rename to docs/ru/sql_reference/functions/bitmap_functions.md index 90adee46b15..0a6288498be 100644 --- a/docs/ru/query_language/functions/bitmap_functions.md +++ b/docs/ru/sql_reference/functions/bitmap_functions.md @@ -61,8 +61,8 @@ bitmapSubsetLimit(bitmap, range_start, cardinality_limit) **Параметры** - `bitmap` – Битмап. [Bitmap object](#bitmap_functions-bitmapbuild). -- `range_start` – Начальная точка подмножества. [UInt32](../../data_types/int_uint.md). -- `cardinality_limit` – Верхний предел подмножества. [UInt32](../../data_types/int_uint.md). +- `range_start` – Начальная точка подмножества. [UInt32](../../sql_reference/functions/bitmap_functions.md). +- `cardinality_limit` – Верхний предел подмножества. [UInt32](../../sql_reference/functions/bitmap_functions.md). **Возвращаемое значение** @@ -97,7 +97,7 @@ bitmapContains(haystack, needle) **Параметры** - `haystack` – [объект Bitmap](#bitmap_functions-bitmapbuild), в котором функция ищет значение. -- `needle` – значение, которое функция ищет. Тип — [UInt32](../../data_types/int_uint.md). +- `needle` – значение, которое функция ищет. Тип — [UInt32](../../sql_reference/functions/bitmap_functions.md). **Возвращаемые значения** diff --git a/docs/ru/query_language/functions/comparison_functions.md b/docs/ru/sql_reference/functions/comparison_functions.md similarity index 100% rename from docs/ru/query_language/functions/comparison_functions.md rename to docs/ru/sql_reference/functions/comparison_functions.md diff --git a/docs/ru/query_language/functions/conditional_functions.md b/docs/ru/sql_reference/functions/conditional_functions.md similarity index 98% rename from docs/ru/query_language/functions/conditional_functions.md rename to docs/ru/sql_reference/functions/conditional_functions.md index f2c97330d20..f3c57aac38b 100644 --- a/docs/ru/query_language/functions/conditional_functions.md +++ b/docs/ru/sql_reference/functions/conditional_functions.md @@ -14,7 +14,7 @@ SELECT if(cond, then, else) **Параметры** -- `cond` – Условие, которое может быть равно 0 или нет. Может быть [UInt8](../../data_types/int_uint.md) или `NULL`. +- `cond` – Условие, которое может быть равно 0 или нет. Может быть [UInt8](../../sql_reference/functions/conditional_functions.md) или `NULL`. - `then` - Возвращается результат выражения, если условие `cond` истинно. - `else` - Возвращается результат выражения, если условие `cond` ложно. diff --git a/docs/ru/query_language/functions/date_time_functions.md b/docs/ru/sql_reference/functions/date_time_functions.md similarity index 97% rename from docs/ru/query_language/functions/date_time_functions.md rename to docs/ru/sql_reference/functions/date_time_functions.md index 51a097fb31b..9f2f5b8afd2 100644 --- a/docs/ru/query_language/functions/date_time_functions.md +++ b/docs/ru/sql_reference/functions/date_time_functions.md @@ -215,9 +215,9 @@ dateDiff('unit', startdate, enddate, [timezone]) |quarter | |year | -- `startdate` — Первая дата. [Date](../../data_types/date.md) или [DateTime](../../data_types/datetime.md). +- `startdate` — Первая дата. [Date](../../sql_reference/functions/date_time_functions.md) или [DateTime](../../sql_reference/functions/date_time_functions.md). -- `enddate` — Вторая дата. [Date](../../data_types/date.md) или [DateTime](../../data_types/datetime.md). +- `enddate` — Вторая дата. [Date](../../sql_reference/functions/date_time_functions.md) или [DateTime](../../sql_reference/functions/date_time_functions.md). - `timezone` — Опциональный параметр. Если определен, применяется к обоим значениям: `startdate` и `enddate`. Если не определен, используются часовые пояса `startdate` и `enddate`. Если часовые пояса не совпадают, вернется неожидаемый результат. diff --git a/docs/ru/query_language/functions/encoding_functions.md b/docs/ru/sql_reference/functions/encoding_functions.md similarity index 91% rename from docs/ru/query_language/functions/encoding_functions.md rename to docs/ru/sql_reference/functions/encoding_functions.md index e6fbeb133c5..b4c4716d9c7 100644 --- a/docs/ru/query_language/functions/encoding_functions.md +++ b/docs/ru/sql_reference/functions/encoding_functions.md @@ -12,7 +12,7 @@ char(number_1, [number_2, ..., number_n]); **Параметры** -- `number_1, number_2, ..., number_n` — Числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../data_types/int_uint.md), [Float](../../data_types/float.md). +- `number_1, number_2, ..., number_n` — Числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../sql_reference/functions/encoding_functions.md), [Float](../../sql_reference/functions/encoding_functions.md). **Возвращаемое значение** @@ -104,7 +104,7 @@ Values of floating point and Decimal types are encoded as their representation i **Parameters** -- `arg` — A value to convert to hexadecimal. Types: [String](../../data_types/string.md), [UInt](../../data_types/int_uint.md), [Float](../../data_types/float.md), [Decimal](../../data_types/decimal.md), [Date](../../data_types/date.md) or [DateTime](../../data_types/datetime.md). +- `arg` — A value to convert to hexadecimal. Types: [String](../../sql_reference/functions/encoding_functions.md), [UInt](../../sql_reference/functions/encoding_functions.md), [Float](../../sql_reference/functions/encoding_functions.md), [Decimal](../../sql_reference/functions/encoding_functions.md), [Date](../../sql_reference/functions/encoding_functions.md) or [DateTime](../../sql_reference/functions/encoding_functions.md). **Returned value** diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/sql_reference/functions/ext_dict_functions.md similarity index 85% rename from docs/ru/query_language/functions/ext_dict_functions.md rename to docs/ru/sql_reference/functions/ext_dict_functions.md index dd7aff463f5..1a1d383e4bb 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/sql_reference/functions/ext_dict_functions.md @@ -1,6 +1,6 @@ # Функции для работы с внешними словарями {#ext_dict_functions} -Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../dicts/external_dicts.md). +Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../../sql_reference/functions/ext_dict_functions.md). ## dictGet {#dictget} @@ -15,12 +15,12 @@ dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `attr_name` — имя столбца словаря. [Строковый литерал](../syntax.md#syntax-string-literal). -- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../data_types/int_uint.md) или [Tuple](../../data_types/tuple.md) в зависимости от конфигурации словаря. +- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql_reference/functions/ext_dict_functions.md) или [Tuple](../../sql_reference/functions/ext_dict_functions.md) в зависимости от конфигурации словаря. - `default_value_expr` — значение, возвращаемое в том случае, когда словарь не содержит строки с заданным ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращающее значение с типом данных, сконфигурированным для атрибута `attr_name`. **Возвращаемое значение** -- Значение атрибута, соответствующее ключу `id_expr`, если ClickHouse смог привести это значение к [заданному типу данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes). +- Значение атрибута, соответствующее ключу `id_expr`, если ClickHouse смог привести это значение к [заданному типу данных](../../sql_reference/functions/ext_dict_functions.md#ext_dict_structure-attributes). - Если ключа, соответствующего `id_expr` в словаре нет, то: @@ -90,7 +90,7 @@ LIMIT 3 **Смотрите также** -- [Внешние словари](../dicts/external_dicts.md) +- [Внешние словари](../../sql_reference/functions/ext_dict_functions.md) ## dictHas {#dicthas} @@ -103,7 +103,7 @@ dictHas('dict_name', id) **Параметры** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). -- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../data_types/int_uint.md). +- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql_reference/functions/ext_dict_functions.md). **Возвращаемое значение** @@ -114,7 +114,7 @@ dictHas('dict_name', id) ## dictGetHierarchy {#dictgethierarchy} -Создаёт массив, содержащий цепочку предков для заданного ключа в [иерархическом словаре](../dicts/external_dicts_dict_hierarchical.md). +Создаёт массив, содержащий цепочку предков для заданного ключа в [иерархическом словаре](../dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md). **Синтаксис** @@ -125,13 +125,13 @@ dictGetHierarchy('dict_name', key) **Параметры** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). -- `key` — значение ключа. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../data_types/int_uint.md). +- `key` — значение ключа. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql_reference/functions/ext_dict_functions.md). **Возвращаемое значение** - Цепочка предков заданного ключа. -Type: [Array(UInt64)](../../data_types/array.md). +Type: [Array(UInt64)](../../sql_reference/functions/ext_dict_functions.md). ## dictIsIn {#dictisin} @@ -142,8 +142,8 @@ Type: [Array(UInt64)](../../data_types/array.md). **Параметры** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). -- `child_id_expr` — ключ для проверки. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../data_types/int_uint.md). -- `ancestor_id_expr` — предполагаемый предок ключа `child_id_expr`. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../data_types/int_uint.md). +- `child_id_expr` — ключ для проверки. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql_reference/functions/ext_dict_functions.md). +- `ancestor_id_expr` — предполагаемый предок ключа `child_id_expr`. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql_reference/functions/ext_dict_functions.md). **Возвращаемое значение** @@ -179,12 +179,12 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `attr_name` — имя столбца словаря. [Строковый литерал](../syntax.md#syntax-string-literal). -- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../data_types/int_uint.md). +- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql_reference/functions/ext_dict_functions.md). - `default_value_expr` — значение, возвращаемое в том случае, когда словарь не содержит строки с заданным ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращающее значение с типом данных, сконфигурированным для атрибута `attr_name`. **Возвращаемое значение** -- Если ClickHouse успешно обработал атрибут в соответствии с [заданным типом данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), то функции возвращают значение атрибута, соответствующее ключу `id_expr`. +- Если ClickHouse успешно обработал атрибут в соответствии с [заданным типом данных](../../sql_reference/functions/ext_dict_functions.md#ext_dict_structure-attributes), то функции возвращают значение атрибута, соответствующее ключу `id_expr`. - Если запрошенного `id_expr` нет в словаре, то: diff --git a/docs/ru/query_language/functions/functions_for_nulls.md b/docs/ru/sql_reference/functions/functions_for_nulls.md similarity index 97% rename from docs/ru/query_language/functions/functions_for_nulls.md rename to docs/ru/sql_reference/functions/functions_for_nulls.md index 1782a5a0cf2..0d07cbeb62d 100644 --- a/docs/ru/query_language/functions/functions_for_nulls.md +++ b/docs/ru/sql_reference/functions/functions_for_nulls.md @@ -204,7 +204,7 @@ SELECT nullIf(1, 2) ## assumeNotNull {#assumenotnull} -Приводит значение типа [Nullable](../../data_types/nullable.md) к не `Nullable`, если значение не `NULL`. +Приводит значение типа [Nullable](../../sql_reference/functions/functions_for_nulls.md) к не `Nullable`, если значение не `NULL`. ``` sql assumeNotNull(x) diff --git a/docs/ru/query_language/functions/geo.md b/docs/ru/sql_reference/functions/geo.md similarity index 86% rename from docs/ru/query_language/functions/geo.md rename to docs/ru/sql_reference/functions/geo.md index 07e2e4c1da6..db51ac05166 100644 --- a/docs/ru/query_language/functions/geo.md +++ b/docs/ru/sql_reference/functions/geo.md @@ -107,8 +107,8 @@ pointInPolygon((x, y), [(a, b), (c, d) ...], ...) **Входные значения** -- `(x, y)` — координаты точки на плоскости. Тип данных — [Tuple](../../data_types/tuple.md) — кортеж из двух чисел. -- `[(a, b), (c, d) ...]` — вершины многоугольника. Тип данных — [Array](../../data_types/array.md). Каждая вершина представлена парой координат `(a, b)`. Вершины следует указывать в порядке обхода по или против часовой стрелки. Минимальное количество вершин — 3. Многоугольник должен быть константным. +- `(x, y)` — координаты точки на плоскости. Тип данных — [Tuple](../../sql_reference/functions/geo.md) — кортеж из двух чисел. +- `[(a, b), (c, d) ...]` — вершины многоугольника. Тип данных — [Array](../../sql_reference/functions/geo.md). Каждая вершина представлена парой координат `(a, b)`. Вершины следует указывать в порядке обхода по или против часовой стрелки. Минимальное количество вершин — 3. Многоугольник должен быть константным. - функция поддерживает также многоугольники с дырками (вырезанными кусками). Для этого случая, добавьте многоугольники, описывающие вырезанные куски, дополнительными аргументами функции. Функция не поддерживает не односвязные многоугольники. **Возвращаемые значения** @@ -196,14 +196,14 @@ h3IsValid(h3index) **Входные значения** -- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../data_types/int_uint.md). +- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../sql_reference/functions/geo.md). **Возвращаемые значения** - 0 — число не является H3-индексом - 1 — число является H3-индексом -Тип — [UInt8](../../data_types/int_uint.md). +Тип — [UInt8](../../sql_reference/functions/geo.md). **Пример** @@ -227,14 +227,14 @@ h3GetResolution(h3index) **Входные значения** -- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../data_types/int_uint.md). +- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../sql_reference/functions/geo.md). **Возвращаемые значения** - Разрешение сетки, от 0 до 15. - Для несуществующего идентификатора может быть возвращено произвольное значение, используйте [h3IsValid](#h3isvalid) для проверки идентификаторов -Тип — [UInt8](../../data_types/int_uint.md). +Тип — [UInt8](../../sql_reference/functions/geo.md). **Пример** @@ -258,11 +258,11 @@ h3EdgeAngle(resolution) **Входные значения** -- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значений — `[0, 15]`. +- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../sql_reference/functions/geo.md). Диапазон возможных значений — `[0, 15]`. **Возвращаемые значения** -Средняя длина стороны многоугольника H3 в градусах, тип — [Float64](../../data_types/float.md). +Средняя длина стороны многоугольника H3 в градусах, тип — [Float64](../../sql_reference/functions/geo.md). **Пример** @@ -286,11 +286,11 @@ h3EdgeLengthM(resolution) **Входные значения** -- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значений — `[0, 15]`. +- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../sql_reference/functions/geo.md). Диапазон возможных значений — `[0, 15]`. **Возвращаемые значения** -Средняя длина стороны многоугольника H3 в метрах, тип — [Float64](../../data_types/float.md). +Средняя длина стороны многоугольника H3 в метрах, тип — [Float64](../../sql_reference/functions/geo.md). **Пример** @@ -320,16 +320,16 @@ geoToH3(lon, lat, resolution) **Параметры** -- `lon` — географическая долгота. Тип данных — [Float64](../../data_types/float.md). -- `lat` — географическая широта. Тип данных — [Float64](../../data_types/float.md). -- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значений — `[0, 15]`. +- `lon` — географическая долгота. Тип данных — [Float64](../../sql_reference/functions/geo.md). +- `lat` — географическая широта. Тип данных — [Float64](../../sql_reference/functions/geo.md). +- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../sql_reference/functions/geo.md). Диапазон возможных значений — `[0, 15]`. **Возвращаемые значения** - Порядковый номер шестиугольника. - 0 в случае ошибки. -Тип — [UInt64](../../data_types/int_uint.md). +Тип — [UInt64](../../sql_reference/functions/geo.md). **Пример** @@ -357,12 +357,12 @@ h3kRing(h3index, k) **Входные значения** -- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../data_types/int_uint.md). -- `k` — радиус. Тип данных — [целое число](../../data_types/int_uint.md) +- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../sql_reference/functions/geo.md). +- `k` — радиус. Тип данных — [целое число](../../sql_reference/functions/geo.md) **Возвращаемые значения** -[Массив](../../data_types/array.md) из H3-индексов типа [UInt64](../../data_types/int_uint.md). +[Массив](../../sql_reference/functions/geo.md) из H3-индексов типа [UInt64](../../sql_reference/functions/geo.md). **Пример** diff --git a/docs/ru/query_language/functions/hash_functions.md b/docs/ru/sql_reference/functions/hash_functions.md similarity index 83% rename from docs/ru/query_language/functions/hash_functions.md rename to docs/ru/sql_reference/functions/hash_functions.md index 32e701cbd23..9dcd67ba35f 100644 --- a/docs/ru/query_language/functions/hash_functions.md +++ b/docs/ru/sql_reference/functions/hash_functions.md @@ -4,7 +4,7 @@ ## halfMD5 {#hash-functions-halfmd5} -[Интерпретирует](../../query_language/functions/type_conversion_functions.md#type_conversion_functions-reinterpretAsString) все входные параметры как строки и вычисляет хэш [MD5](https://ru.wikipedia.org/wiki/MD5) для каждой из них. Затем объединяет хэши, берет первые 8 байт хэша результирующей строки и интерпретирует их как значение типа `UInt64` с big-endian порядком байтов. +[Интерпретирует](../../sql_reference/functions/hash_functions.md#type_conversion_functions-reinterpretAsString) все входные параметры как строки и вычисляет хэш [MD5](https://ru.wikipedia.org/wiki/MD5) для каждой из них. Затем объединяет хэши, берет первые 8 байт хэша результирующей строки и интерпретирует их как значение типа `UInt64` с big-endian порядком байтов. ``` sql halfMD5(par1, ...) @@ -15,11 +15,11 @@ halfMD5(par1, ...) **Параметры** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../data_types/index.md). +Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -Значение хэша с типом данных [UInt64](../../data_types/int_uint.md). +Значение хэша с типом данных [UInt64](../../sql_reference/functions/hash_functions.md). **Пример** @@ -49,7 +49,7 @@ sipHash64(par1,...) Это криптографическая хэш-функция. Она работает по крайней мере в три раза быстрее, чем функция [MD5](#hash_functions-md5). -Функция [интерпретирует](../../query_language/functions/type_conversion_functions.md#type_conversion_functions-reinterpretAsString) все входные параметры как строки и вычисляет хэш MD5 для каждой из них. Затем комбинирует хэши по следующему алгоритму. +Функция [интерпретирует](../../sql_reference/functions/hash_functions.md#type_conversion_functions-reinterpretAsString) все входные параметры как строки и вычисляет хэш MD5 для каждой из них. Затем комбинирует хэши по следующему алгоритму. 1. После хэширования всех входных параметров функция получает массив хэшей. 2. Функция принимает первый и второй элементы и вычисляет хэш для массива из них. @@ -58,11 +58,11 @@ sipHash64(par1,...) **Параметры** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../data_types/index.md). +Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -Значение хэша с типом данных [UInt64](../../data_types/int_uint.md). +Значение хэша с типом данных [UInt64](../../sql_reference/functions/hash_functions.md). **Пример** @@ -94,11 +94,11 @@ cityHash64(par1,...) **Параметры** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../data_types/index.md). +Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -Значение хэша с типом данных [UInt64](../../data_types/int_uint.md). +Значение хэша с типом данных [UInt64](../../sql_reference/functions/hash_functions.md). **Примеры** @@ -160,11 +160,11 @@ farmHash64(par1, ...) **Параметры** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../data_types/index.md). +Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -Значение хэша с типом данных [UInt64](../../data_types/int_uint.md). +Значение хэша с типом данных [UInt64](../../sql_reference/functions/hash_functions.md). **Пример** @@ -288,11 +288,11 @@ metroHash64(par1, ...) **Параметры** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../data_types/index.md). +Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -Значение хэша с типом данных [UInt64](../../data_types/int_uint.md). +Значение хэша с типом данных [UInt64](../../sql_reference/functions/hash_functions.md). **Пример** @@ -323,12 +323,12 @@ murmurHash2_64(par1, ...) **Параметры** -Обе функции принимают переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../data_types/index.md). +Обе функции принимают переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -- Функция `murmurHash2_32` возвращает значение типа [UInt32](../../data_types/int_uint.md). -- Функция `murmurHash2_64` возвращает значение типа [UInt64](../../data_types/int_uint.md). +- Функция `murmurHash2_32` возвращает значение типа [UInt32](../../sql_reference/functions/hash_functions.md). +- Функция `murmurHash2_64` возвращает значение типа [UInt64](../../sql_reference/functions/hash_functions.md). **Пример** @@ -342,6 +342,44 @@ SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23: └──────────────────────┴────────┘ ``` +## gccMurmurHash {#gccmurmurhash} + +Вычисляет 64-битное значение [MurmurHash2](https://github.com/aappleby/smhasher), используя те же hash seed, что и [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191). + +**Синтаксис** + +```sql +gccMurmurHash(par1, ...); +``` + +**Параметры** + +- `par1, ...` — Переменное число параметров. Каждый параметр может быть любого из [поддерживаемых типов данных](../../sql_reference/data_types/index.md). + +**Возвращаемое значение** + +- Вычисленный хэш-код. + +Тип: [UInt64](../../sql_reference/data_types/int_uint.md). + +**Примеры** + +Запрос: + +```sql +SELECT + gccMurmurHash(1, 2, 3) AS res1, + gccMurmurHash(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))) AS res2 +``` + +Результат: + +``` text +┌─────────────────res1─┬────────────────res2─┐ +│ 12384823029245979431 │ 1188926775431157506 │ +└──────────────────────┴─────────────────────┘ +``` + ## murmurHash3\_32, murmurHash3\_64 {#murmurhash3-32-murmurhash3-64} Генерирует значение [MurmurHash3](https://github.com/aappleby/smhasher). @@ -353,12 +391,12 @@ murmurHash3_64(par1, ...) **Параметры** -Обе функции принимают переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../data_types/index.md). +Обе функции принимают переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -- Функция `murmurHash3_32` возвращает значение типа [UInt32](../../data_types/int_uint.md). -- Функция `murmurHash3_64` возвращает значение типа [UInt64](../../data_types/int_uint.md). +- Функция `murmurHash3_32` возвращает значение типа [UInt32](../../sql_reference/functions/hash_functions.md). +- Функция `murmurHash3_64` возвращает значение типа [UInt64](../../sql_reference/functions/hash_functions.md). **Пример** @@ -382,11 +420,11 @@ murmurHash3_128( expr ) **Параметры** -- `expr` — [выражение](../syntax.md#syntax-expressions) возвращающее значение типа[String](../../data_types/string.md). +- `expr` — [выражение](../syntax.md#syntax-expressions) возвращающее значение типа[String](../../sql_reference/functions/hash_functions.md). **Возвращаемое значение** -Хэш-значение типа [FixedString(16)](../../data_types/fixedstring.md). +Хэш-значение типа [FixedString(16)](../../sql_reference/functions/hash_functions.md). **Пример** diff --git a/docs/ru/query_language/functions/higher_order_functions.md b/docs/ru/sql_reference/functions/higher_order_functions.md similarity index 100% rename from docs/ru/query_language/functions/higher_order_functions.md rename to docs/ru/sql_reference/functions/higher_order_functions.md diff --git a/docs/ru/query_language/functions/in_functions.md b/docs/ru/sql_reference/functions/in_functions.md similarity index 93% rename from docs/ru/query_language/functions/in_functions.md rename to docs/ru/sql_reference/functions/in_functions.md index 70c8c1bb6c3..a5cdb1dc217 100644 --- a/docs/ru/query_language/functions/in_functions.md +++ b/docs/ru/sql_reference/functions/in_functions.md @@ -2,7 +2,7 @@ ## in, notIn, globalIn, globalNotIn {#in-functions} -Смотрите раздел [Операторы IN](../select.md#select-in-operators). +Смотрите раздел [Операторы IN](../statements/select.md#select-in-operators). ## tuple(x, y, …), оператор (x, y, …) {#tuplex-y-operator-x-y} diff --git a/docs/ru/query_language/functions/index.md b/docs/ru/sql_reference/functions/index.md similarity index 100% rename from docs/ru/query_language/functions/index.md rename to docs/ru/sql_reference/functions/index.md diff --git a/docs/ru/query_language/functions/introspection.md b/docs/ru/sql_reference/functions/introspection.md similarity index 94% rename from docs/ru/query_language/functions/introspection.md rename to docs/ru/sql_reference/functions/introspection.md index 50b4cbb44bf..41ffb114e09 100644 --- a/docs/ru/query_language/functions/introspection.md +++ b/docs/ru/sql_reference/functions/introspection.md @@ -29,7 +29,7 @@ addressToLine(address_of_binary_instruction) **Параметры** -- `address_of_binary_instruction` ([Тип UInt64](../../data_types/int_uint.md))- Адрес инструкции в запущенном процессе. +- `address_of_binary_instruction` ([Тип UInt64](../../sql_reference/functions/introspection.md))- Адрес инструкции в запущенном процессе. **Возвращаемое значение** @@ -41,7 +41,7 @@ addressToLine(address_of_binary_instruction) - Пустая строка, если адрес не является допустимым. -Тип: [String](../../data_types/string.md). +Тип: [String](../../sql_reference/functions/introspection.md). **Пример** @@ -120,14 +120,14 @@ addressToSymbol(address_of_binary_instruction) **Параметры** -- `address_of_binary_instruction` ([Тип uint64](../../data_types/int_uint.md)) — Адрес инструкции в запущенном процессе. +- `address_of_binary_instruction` ([Тип uint64](../../sql_reference/functions/introspection.md)) — Адрес инструкции в запущенном процессе. **Возвращаемое значение** - Символ из объектных файлов ClickHouse. - Пустая строка, если адрес не является допустимым. -Тип: [String](../../data_types/string.md). +Тип: [String](../../sql_reference/functions/introspection.md). **Пример** @@ -217,14 +217,14 @@ demangle(symbol) **Параметры** -- `symbol` ([Строка](../../data_types/string.md)) - Символ из объектного файла. +- `symbol` ([Строка](../../sql_reference/functions/introspection.md)) - Символ из объектного файла. **Возвращаемое значение** - Имя функции C++. - Пустая строка, если символ не является допустимым. -Тип: [Строка](../../data_types/string.md). +Тип: [Строка](../../sql_reference/functions/introspection.md). **Пример** diff --git a/docs/ru/query_language/functions/ip_address_functions.md b/docs/ru/sql_reference/functions/ip_address_functions.md similarity index 95% rename from docs/ru/query_language/functions/ip_address_functions.md rename to docs/ru/sql_reference/functions/ip_address_functions.md index 87c1da4114b..57485f6a812 100644 --- a/docs/ru/query_language/functions/ip_address_functions.md +++ b/docs/ru/sql_reference/functions/ip_address_functions.md @@ -176,7 +176,7 @@ SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32) ## toIPv4(string) {#toipv4string} -Псевдоним функции `IPv4StringToNum()` которая принимает строку с адресом IPv4 и возвращает значение типа [IPv4](../../data_types/domains/ipv4.md), которое равно значению, возвращаемому функцией `IPv4StringToNum()`. +Псевдоним функции `IPv4StringToNum()` которая принимает строку с адресом IPv4 и возвращает значение типа [IPv4](../../sql_reference/functions/ip_address_functions.md), которое равно значению, возвращаемому функцией `IPv4StringToNum()`. ``` sql WITH @@ -208,7 +208,7 @@ SELECT ## toIPv6(string) {#toipv6string} -Псевдоним функции `IPv6StringToNum()` которая принимает строку с адресом IPv6 и возвращает значение типа [IPv6](../../data_types/domains/ipv6.md), которое равно значению, возвращаемому функцией `IPv6StringToNum()`. +Псевдоним функции `IPv6StringToNum()` которая принимает строку с адресом IPv6 и возвращает значение типа [IPv6](../../sql_reference/functions/ip_address_functions.md), которое равно значению, возвращаемому функцией `IPv6StringToNum()`. ``` sql WITH diff --git a/docs/ru/query_language/functions/json_functions.md b/docs/ru/sql_reference/functions/json_functions.md similarity index 100% rename from docs/ru/query_language/functions/json_functions.md rename to docs/ru/sql_reference/functions/json_functions.md diff --git a/docs/ru/query_language/functions/logical_functions.md b/docs/ru/sql_reference/functions/logical_functions.md similarity index 100% rename from docs/ru/query_language/functions/logical_functions.md rename to docs/ru/sql_reference/functions/logical_functions.md diff --git a/docs/ru/query_language/functions/machine_learning_functions.md b/docs/ru/sql_reference/functions/machine_learning_functions.md similarity index 51% rename from docs/ru/query_language/functions/machine_learning_functions.md rename to docs/ru/sql_reference/functions/machine_learning_functions.md index 8a51d1dd4c6..c5dd27d96af 100644 --- a/docs/ru/query_language/functions/machine_learning_functions.md +++ b/docs/ru/sql_reference/functions/machine_learning_functions.md @@ -6,8 +6,8 @@ ### Stochastic Linear Regression {#stochastic-linear-regression} -Агрегатная функция [stochasticLinearRegression](../agg_functions/reference.md#agg_functions-stochasticlinearregression) реализует стохастический градиентный спуск, использую линейную модель и функцию потерь MSE. +Агрегатная функция [stochasticLinearRegression](../../sql_reference/functions/machine_learning_functions.md#agg_functions-stochasticlinearregression) реализует стохастический градиентный спуск, использую линейную модель и функцию потерь MSE. ### Stochastic Logistic Regression {#stochastic-logistic-regression} -Агрегатная функция [stochasticLogisticRegression](../agg_functions/reference.md#agg_functions-stochasticlogisticregression) реализует стохастический градиентный спуск для задачи бинарной классификации. +Агрегатная функция [stochasticLogisticRegression](../../sql_reference/functions/machine_learning_functions.md#agg_functions-stochasticlogisticregression) реализует стохастический градиентный спуск для задачи бинарной классификации. diff --git a/docs/ru/query_language/functions/math_functions.md b/docs/ru/sql_reference/functions/math_functions.md similarity index 100% rename from docs/ru/query_language/functions/math_functions.md rename to docs/ru/sql_reference/functions/math_functions.md diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/sql_reference/functions/other_functions.md similarity index 97% rename from docs/ru/query_language/functions/other_functions.md rename to docs/ru/sql_reference/functions/other_functions.md index e4f7440b7cd..a0568cb262e 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/sql_reference/functions/other_functions.md @@ -48,7 +48,7 @@ basename( expr ) **Параметры** -- `expr` — Выражение, возвращающее значение типа [String](../../data_types/string.md). В результирующем значении все бэкслэши должны быть экранированы. +- `expr` — Выражение, возвращающее значение типа [String](../../sql_reference/functions/other_functions.md). В результирующем значении все бэкслэши должны быть экранированы. **Возвращаемое значение** @@ -186,8 +186,8 @@ SELECT currentUser(); **Параметры** -- `x` — Значение, которое нужно проверить на бесконечность. Тип: [Float\*](../../data_types/float.md). -- `y` — Запасное значение. Тип: [Float\*](../../data_types/float.md). +- `x` — Значение, которое нужно проверить на бесконечность. Тип: [Float\*](../../sql_reference/functions/other_functions.md). +- `y` — Запасное значение. Тип: [Float\*](../../sql_reference/functions/other_functions.md). **Возвращаемые значения** @@ -420,7 +420,7 @@ neighbor(column, offset[, default_value]) **Параметры** - `column` — Имя столбца или скалярное выражение. -- `offset` - Смещение от текущей строки `column`. [Int64](../../data_types/int_uint.md). +- `offset` - Смещение от текущей строки `column`. [Int64](../../sql_reference/functions/other_functions.md). - `default_value` - Опциональный параметр. Значение, которое будет возвращено, если смещение выходит за пределы блока данных. **Возвращаемое значение** @@ -603,7 +603,7 @@ WHERE diff != 1 ## getSizeOfEnumType {#getsizeofenumtype} -Возвращает количество полей в [Enum](../../data_types/enum.md). +Возвращает количество полей в [Enum](../../sql_reference/functions/other_functions.md). ``` sql getSizeOfEnumType(value) @@ -716,7 +716,7 @@ defaultValueOfArgumentType(expression) - `0` для чисел; - Пустая строка для строк; -- `ᴺᵁᴸᴸ` для [Nullable](../../data_types/nullable.md). +- `ᴺᵁᴸᴸ` для [Nullable](../../sql_reference/functions/other_functions.md). **Пример** @@ -791,7 +791,7 @@ filesystemAvailable() - Объём доступного для записи данных места в байтах. -Тип: [UInt64](../../data_types/int_uint.md). +Тип: [UInt64](../../sql_reference/functions/other_functions.md). **Пример** @@ -823,7 +823,7 @@ filesystemFree() - Объем свободного места в байтах. -Тип: [UInt64](../../data_types/int_uint.md). +Тип: [UInt64](../../sql_reference/functions/other_functions.md). **Пример** @@ -843,7 +843,7 @@ SELECT formatReadableSize(filesystemFree()) AS "Free space", toTypeName(filesyst ## filesystemCapacity {#filesystemcapacity} -Возвращает информацию о ёмкости файловой системы в байтах. Для оценки должен быть настроен [путь](../../operations/server_settings/settings.md#server_settings-path) к каталогу с данными. +Возвращает информацию о ёмкости файловой системы в байтах. Для оценки должен быть настроен [путь](../../sql_reference/functions/other_functions.md#server_configuration_parameters-path) к каталогу с данными. **Синтаксис** @@ -855,7 +855,7 @@ filesystemCapacity() - Информация о ёмкости файловой системы в байтах. -Тип: [UInt64](../../data_types/int_uint.md). +Тип: [UInt64](../../sql_reference/functions/other_functions.md). **Пример** @@ -883,9 +883,9 @@ SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesy ## joinGet {#joinget} -Функция позволяет извлекать данные из таблицы таким же образом как из [словаря](../../query_language/dicts/index.md). +Функция позволяет извлекать данные из таблицы таким же образом как из [словаря](../../sql_reference/functions/other_functions.md). -Получает данные из таблиц [Join](../../operations/table_engines/join.md#creating-a-table) по ключу. +Получает данные из таблиц [Join](../../sql_reference/functions/other_functions.md#creating-a-table) по ключу. Поддерживаются только таблицы, созданные с `ENGINE = Join(ANY, LEFT, )`. @@ -907,7 +907,7 @@ joinGet(join_storage_table_name, `value_column`, join_keys) Если значения не существует в исходной таблице, вернется `0` или `null` в соответствии с настройками [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls). -Подробнее о настройке `join_use_nulls` в [операциях Join](../../operations/table_engines/join.md). +Подробнее о настройке `join_use_nulls` в [операциях Join](../../sql_reference/functions/other_functions.md). **Пример** @@ -1010,7 +1010,7 @@ randomPrintableASCII(length) - Строка со случайным набором печатных символов [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters). -Тип: [String](../../data_types/string.md) +Тип: [String](../../sql_reference/functions/other_functions.md) **Пример** diff --git a/docs/ru/query_language/functions/random_functions.md b/docs/ru/sql_reference/functions/random_functions.md similarity index 100% rename from docs/ru/query_language/functions/random_functions.md rename to docs/ru/sql_reference/functions/random_functions.md diff --git a/docs/ru/query_language/functions/rounding_functions.md b/docs/ru/sql_reference/functions/rounding_functions.md similarity index 98% rename from docs/ru/query_language/functions/rounding_functions.md rename to docs/ru/sql_reference/functions/rounding_functions.md index 9c5f73815b5..9e7947c109d 100644 --- a/docs/ru/query_language/functions/rounding_functions.md +++ b/docs/ru/sql_reference/functions/rounding_functions.md @@ -30,7 +30,7 @@ round(expression [, decimal_places]) **Параметры:** -- `expression` — Число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../data_types/index.md#data_types). +- `expression` — Число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../sql_reference/functions/rounding_functions.md#data_types). - `decimal-places` — Целое значение. - Если `decimal-places > 0`, то функция округляет значение справа от запятой. - Если `decimal-places < 0` то функция округляет значение слева от запятой. @@ -109,7 +109,7 @@ roundBankers(expression [, decimal_places]) **Параметры** -- `expression` — Число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../data_types/index.md#data_types). +- `expression` — Число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../sql_reference/functions/rounding_functions.md#data_types). - `decimal-places` — Десятичный разряд. Целое число. - `decimal-places > 0` — Функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции справа от запятой. Например, `roundBankers(3.55, 1) = 3.6`. - `decimal-places < 0` — Функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции слева от запятой. Например, `roundBankers(24.55, -1) = 20`. diff --git a/docs/ru/query_language/functions/splitting_merging_functions.md b/docs/ru/sql_reference/functions/splitting_merging_functions.md similarity index 100% rename from docs/ru/query_language/functions/splitting_merging_functions.md rename to docs/ru/sql_reference/functions/splitting_merging_functions.md diff --git a/docs/ru/query_language/functions/string_functions.md b/docs/ru/sql_reference/functions/string_functions.md similarity index 97% rename from docs/ru/query_language/functions/string_functions.md rename to docs/ru/sql_reference/functions/string_functions.md index ef2793d2c18..5c51ad7c73c 100644 --- a/docs/ru/query_language/functions/string_functions.md +++ b/docs/ru/sql_reference/functions/string_functions.md @@ -70,7 +70,7 @@ toValidUTF8( input_string ) Параметры: -- input\_string — произвольный набор байтов, представленный как объект типа [String](../../data_types/string.md). +- input\_string — произвольный набор байтов, представленный как объект типа [String](../../sql_reference/functions/string_functions.md). Возвращаемое значение: Корректная строка UTF-8. @@ -98,8 +98,8 @@ repeat(s, n) **Параметры** -- `s` — Строка для повторения. [String](../../data_types/string.md). -- `n` — Количество повторов. [UInt](../../data_types/int_uint.md). +- `s` — Строка для повторения. [String](../../sql_reference/functions/string_functions.md). +- `n` — Количество повторов. [UInt](../../sql_reference/functions/string_functions.md). **Возвращаемое значение** @@ -322,8 +322,8 @@ trim([[LEADING|TRAILING|BOTH] trim_character FROM] input_string) **Параметры** -- `trim_character` — один или несколько символов, подлежащие удалению. [String](../../data_types/string.md). -- `input_string` — строка для обрезки. [String](../../data_types/string.md). +- `trim_character` — один или несколько символов, подлежащие удалению. [String](../../sql_reference/functions/string_functions.md). +- `input_string` — строка для обрезки. [String](../../sql_reference/functions/string_functions.md). **Возвращаемое значение** @@ -361,7 +361,7 @@ trimLeft(input_string) **Параметры** -- `input_string` — строка для обрезки. [String](../../data_types/string.md). +- `input_string` — строка для обрезки. [String](../../sql_reference/functions/string_functions.md). **Возвращаемое значение** @@ -399,7 +399,7 @@ trimRight(input_string) **Параметры** -- `input_string` — строка для обрезки. [String](../../data_types/string.md). +- `input_string` — строка для обрезки. [String](../../sql_reference/functions/string_functions.md). **Возвращаемое значение** @@ -437,7 +437,7 @@ trimBoth(input_string) **Параметры** -- `input_string` — строка для обрезки. [String](../../data_types/string.md). +- `input_string` — строка для обрезки. [String](../../sql_reference/functions/string_functions.md). **Возвращаемое значение** diff --git a/docs/ru/query_language/functions/string_replace_functions.md b/docs/ru/sql_reference/functions/string_replace_functions.md similarity index 100% rename from docs/ru/query_language/functions/string_replace_functions.md rename to docs/ru/sql_reference/functions/string_replace_functions.md diff --git a/docs/ru/query_language/functions/string_search_functions.md b/docs/ru/sql_reference/functions/string_search_functions.md similarity index 100% rename from docs/ru/query_language/functions/string_search_functions.md rename to docs/ru/sql_reference/functions/string_search_functions.md diff --git a/docs/ru/query_language/functions/type_conversion_functions.md b/docs/ru/sql_reference/functions/type_conversion_functions.md similarity index 90% rename from docs/ru/query_language/functions/type_conversion_functions.md rename to docs/ru/sql_reference/functions/type_conversion_functions.md index 00582cd61cb..be4de7e7c89 100644 --- a/docs/ru/query_language/functions/type_conversion_functions.md +++ b/docs/ru/sql_reference/functions/type_conversion_functions.md @@ -8,7 +8,7 @@ ## toInt(8\|16\|32\|64) {#toint8163264} -Преобразует входное значение к типу [Int](../../data_types/int_uint.md). Семейство функций включает: +Преобразует входное значение к типу [Int](../../sql_reference/functions/type_conversion_functions.md). Семейство функций включает: - `toInt8(expr)` — возвращает значение типа `Int8`. - `toInt16(expr)` — возвращает значение типа `Int16`. @@ -25,7 +25,7 @@ Функции используют [округление к нулю](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), т.е. обрезают дробную часть числа. -Поведение функций для аргументов [NaN и Inf](../../data_types/float.md#data_type-float-nan-inf) не определено. При использовании функций помните о возможных проблемах при [преобразовании чисел](#numeric-conversion-issues). +Поведение функций для аргументов [NaN и Inf](../../sql_reference/functions/type_conversion_functions.md#data_type-float-nan-inf) не определено. При использовании функций помните о возможных проблемах при [преобразовании чисел](#numeric-conversion-issues). **Пример** @@ -73,7 +73,7 @@ select toInt64OrNull('123123'), toInt8OrNull('123qwe123') ## toUInt(8\|16\|32\|64) {#touint8163264} -Преобраует входное значение к типу [UInt](../../data_types/int_uint.md). Семейство функций включает: +Преобраует входное значение к типу [UInt](../../sql_reference/functions/type_conversion_functions.md). Семейство функций включает: - `toUInt8(expr)` — возвращает значение типа `UInt8`. - `toUInt16(expr)` — возвращает значение типа `UInt16`. @@ -90,7 +90,7 @@ select toInt64OrNull('123123'), toInt8OrNull('123qwe123') Функции используют [округление к нулю](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), т.е. обрезают дробную часть числа. -Поведение функций для аргументов [NaN и Inf](../../data_types/float.md#data_type-float-nan-inf) не определено. Если передать строку, содержащую отрицательное число, например `'-32'`, ClickHouse генерирует исключение. При использовании функций помните о возможных проблемах при [преобразовании чисел](#numeric-conversion-issues). +Поведение функций для аргументов [NaN и Inf](../../sql_reference/functions/type_conversion_functions.md#data_type-float-nan-inf) не определено. Если передать строку, содержащую отрицательное число, например `'-32'`, ClickHouse генерирует исключение. При использовании функций помните о возможных проблемах при [преобразовании чисел](#numeric-conversion-issues). **Пример** @@ -128,7 +128,7 @@ SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8) ## toDecimal(32\|64\|128) {#todecimal3264128} -Преобразует `value` к типу данных [Decimal](../../data_types/decimal.md) с точностью `S`. `value` может быть числом или строкой. Параметр `S` (scale) задаёт число десятичных знаков. +Преобразует `value` к типу данных [Decimal](../../sql_reference/functions/type_conversion_functions.md) с точностью `S`. `value` может быть числом или строкой. Параметр `S` (scale) задаёт число десятичных знаков. - `toDecimal32(value, S)` - `toDecimal64(value, S)` @@ -136,7 +136,7 @@ SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8) ## toDecimal(32\|64\|128)OrNull {#todecimal3264128ornull} -Преобразует входную строку в значение с типом данных [Nullable (Decimal (P, S))](../../data_types/decimal.md). Семейство функций включает в себя: +Преобразует входную строку в значение с типом данных [Nullable (Decimal (P, S))](../../sql_reference/functions/type_conversion_functions.md). Семейство функций включает в себя: - `toDecimal32OrNull(expr, S)` — Возвращает значение типа `Nullable(Decimal32(S))`. - `toDecimal64OrNull(expr, S)` — Возвращает значение типа `Nullable(Decimal64(S))`. @@ -146,7 +146,7 @@ SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8) **Параметры** -- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../data_types/string.md). ClickHouse ожидает текстовое представление десятичного числа. Например, `'1.111'`. +- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql_reference/functions/type_conversion_functions.md). ClickHouse ожидает текстовое представление десятичного числа. Например, `'1.111'`. - `S` — количество десятичных знаков в результирующем значении. **Возвращаемое значение** @@ -180,7 +180,7 @@ SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val) ## toDecimal(32\|64\|128)OrZero {#todecimal3264128orzero} -Преобразует тип входного значения в [Decimal (P, S)](../../data_types/decimal.md). Семейство функций включает в себя: +Преобразует тип входного значения в [Decimal (P, S)](../../sql_reference/functions/type_conversion_functions.md). Семейство функций включает в себя: - `toDecimal32OrZero( expr, S)` — возвращает значение типа `Decimal32(S)`. - `toDecimal64OrZero( expr, S)` — возвращает значение типа `Decimal64(S)`. @@ -190,7 +190,7 @@ SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val) **Параметры** -- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../data_types/string.md). ClickHouse ожидает текстовое представление десятичного числа. Например, `'1.111'`. +- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql_reference/functions/type_conversion_functions.md). ClickHouse ожидает текстовое представление десятичного числа. Например, `'1.111'`. - `S` — количество десятичных знаков в результирующем значении. **Возвращаемое значение** @@ -332,7 +332,7 @@ SELECT Преобразование в FixedString(N) работает только для аргументов типа String или FixedString(N). -Поддержано преобразование к типу [Nullable](../../data_types/nullable.md) и обратно. Пример: +Поддержано преобразование к типу [Nullable](../../sql_reference/functions/type_conversion_functions.md) и обратно. Пример: ``` sql SELECT toTypeName(x) FROM t_null @@ -358,7 +358,7 @@ SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null ## toInterval(Year\|Quarter\|Month\|Week\|Day\|Hour\|Minute\|Second) {#function-tointerval} -Приводит аргумент из числового типа данных к типу данных [IntervalType](../../data_types/special_data_types/interval.md). +Приводит аргумент из числового типа данных к типу данных [IntervalType](../../sql_reference/functions/type_conversion_functions.md). **Синтаксис** @@ -401,7 +401,7 @@ SELECT ## parseDateTimeBestEffort {#parsedatetimebesteffort} -Преобразует дату и время в [строковом](../../data_types/string.md) представлении к типу данных [DateTime](../../data_types/datetime.md#data_type-datetime). +Преобразует дату и время в [строковом](../../sql_reference/functions/type_conversion_functions.md) представлении к типу данных [DateTime](../../sql_reference/functions/type_conversion_functions.md#data_type-datetime). Функция распознаёт форматы [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), [RFC 1123 - 5.2.14 RFC-822 Date and Time Specification](https://tools.ietf.org/html/rfc1123#page-55), формат даты времени ClickHouse's а также некоторые другие форматы. @@ -413,8 +413,8 @@ parseDateTimeBestEffort(time_string[, time_zone]); **Параметры** -- `time_string` — строка, содержащая дату и время для преобразования. [String](../../data_types/string.md). -- `time_zone` — часовой пояс. Функция анализирует `time_string` в соответствии с заданным часовым поясом. [String](../../data_types/string.md). +- `time_string` — строка, содержащая дату и время для преобразования. [String](../../sql_reference/functions/type_conversion_functions.md). +- `time_zone` — часовой пояс. Функция анализирует `time_string` в соответствии с заданным часовым поясом. [String](../../sql_reference/functions/type_conversion_functions.md). **Поддерживаемые нестандартные форматы** diff --git a/docs/ru/query_language/functions/url_functions.md b/docs/ru/sql_reference/functions/url_functions.md similarity index 98% rename from docs/ru/query_language/functions/url_functions.md rename to docs/ru/sql_reference/functions/url_functions.md index 6dc62b9c193..f024cfdcfd9 100644 --- a/docs/ru/query_language/functions/url_functions.md +++ b/docs/ru/sql_reference/functions/url_functions.md @@ -20,7 +20,7 @@ domain(url) **Параметры** -- `url` — URL. Тип — [String](../../data_types/string.md). +- `url` — URL. Тип — [String](../../sql_reference/functions/url_functions.md). URL может быть указан со схемой или без неё. Примеры: @@ -71,7 +71,7 @@ topLevelDomain(url) **Параметры** -- `url` — URL. Тип — [String](../../data_types/string.md). +- `url` — URL. Тип — [String](../../sql_reference/functions/url_functions.md). URL может быть указан со схемой или без неё. Примеры: diff --git a/docs/ru/query_language/functions/uuid_functions.md b/docs/ru/sql_reference/functions/uuid_functions.md similarity index 92% rename from docs/ru/query_language/functions/uuid_functions.md rename to docs/ru/sql_reference/functions/uuid_functions.md index 1f52ba27e31..960e2b10e80 100644 --- a/docs/ru/query_language/functions/uuid_functions.md +++ b/docs/ru/sql_reference/functions/uuid_functions.md @@ -10,7 +10,7 @@ generateUUIDv4() **Возвращаемое значение** -Значение типа [UUID](../../data_types/uuid.md). +Значение типа [UUID](../../sql_reference/functions/uuid_functions.md). **Пример использования** @@ -56,7 +56,7 @@ SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') AS uuid ## UUIDStringToNum {#uuidstringtonum} -Принимает строку, содержащую 36 символов в формате `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`, и возвращает в виде набора байт в [FixedString(16)](../../data_types/fixedstring.md). +Принимает строку, содержащую 36 символов в формате `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`, и возвращает в виде набора байт в [FixedString(16)](../../sql_reference/functions/uuid_functions.md). ``` sql UUIDStringToNum(String) @@ -82,7 +82,7 @@ SELECT ## UUIDNumToString {#uuidnumtostring} -Принимает значение типа [FixedString(16)](../../data_types/fixedstring.md). Возвращает строку из 36 символов в текстовом виде. +Принимает значение типа [FixedString(16)](../../sql_reference/functions/uuid_functions.md). Возвращает строку из 36 символов в текстовом виде. ``` sql UUIDNumToString(FixedString(16)) diff --git a/docs/ru/query_language/functions/ym_dict_functions.md b/docs/ru/sql_reference/functions/ym_dict_functions.md similarity index 98% rename from docs/ru/query_language/functions/ym_dict_functions.md rename to docs/ru/sql_reference/functions/ym_dict_functions.md index d5e11658a4f..5d7aece88b9 100644 --- a/docs/ru/query_language/functions/ym_dict_functions.md +++ b/docs/ru/sql_reference/functions/ym_dict_functions.md @@ -113,8 +113,8 @@ regionToTopContinent(id[, geobase]); **Параметры** -- `id` — Идентификатор региона из геобазы Яндекса. [UInt32](../../data_types/int_uint.md). -- `geobase` — Ключ словаря. Смотрите [Множественные геобазы](#multiple-geobases). [String](../../data_types/string.md). Опциональный параметр. +- `id` — Идентификатор региона из геобазы Яндекса. [UInt32](../../sql_reference/functions/ym_dict_functions.md). +- `geobase` — Ключ словаря. Смотрите [Множественные геобазы](#multiple-geobases). [String](../../sql_reference/functions/ym_dict_functions.md). Опциональный параметр. **Возвращаемое значение** diff --git a/docs/ru/sql_reference/index.md b/docs/ru/sql_reference/index.md new file mode 100644 index 00000000000..ca21b0df3ed --- /dev/null +++ b/docs/ru/sql_reference/index.md @@ -0,0 +1,16 @@ +--- +toc_folder_title: Справка по SQL +toc_hidden: true +toc_priority: 28 +toc_title: hidden +--- + +# Справка по SQL {#spravka-po-sql} + +- [SELECT](statements/select.md) +- [INSERT INTO](statements/insert_into.md) +- [CREATE](statements/create.md) +- [ALTER](statements/alter.md) +- [Прочие виды запросов](statements/misc.md) + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) diff --git a/docs/ru/query_language/operators.md b/docs/ru/sql_reference/operators.md similarity index 89% rename from docs/ru/query_language/operators.md rename to docs/ru/sql_reference/operators.md index 670990b0967..2cff1ad6aed 100644 --- a/docs/ru/query_language/operators.md +++ b/docs/ru/sql_reference/operators.md @@ -55,7 +55,7 @@ ## Операторы для работы с множествами {#operatory-dlia-raboty-s-mnozhestvami} -*Смотрите раздел [Операторы IN](select.md#select-in-operators).* +*Смотрите раздел [Операторы IN](../sql_reference/statements/select.md#select-in-operators).* `a IN ...` - функция `in(a, b)` @@ -86,7 +86,7 @@ EXTRACT(part FROM date); Эти значения могут быть указаны также в нижнем регистре (`day`, `month`). -В параметре `date` указывается исходная дата. Поддерживаются типы [Date](../data_types/date.md) и [DateTime](../data_types/datetime.md). +В параметре `date` указывается исходная дата. Поддерживаются типы [Date](../sql_reference/data_types/date.md) и [DateTime](../sql_reference/data_types/datetime.md). Примеры: @@ -133,7 +133,7 @@ FROM test.Orders; ### INTERVAL {#operator-interval} -Создаёт значение типа [Interval](../data_types/special_data_types/interval.md) которое должно использоваться в арифметических операциях со значениями типов [Date](../data_types/date.md) и [DateTime](../data_types/datetime.md). +Создаёт значение типа [Interval](../sql_reference/operators.md) которое должно использоваться в арифметических операциях со значениями типов [Date](../sql_reference/operators.md) и [DateTime](../sql_reference/operators.md). Типы интервалов: - `SECOND` @@ -162,8 +162,8 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL **Смотрите также** -- Тип данных [Interval](../data_types/special_data_types/interval.md) -- Функции преобразования типов [toInterval](functions/type_conversion_functions.md#function-tointerval) +- Тип данных [Interval](../sql_reference/operators.md) +- Функции преобразования типов [toInterval](../sql_reference/operators.md#function-tointerval) ## Оператор логического отрицания {#operator-logicheskogo-otritsaniia} @@ -183,7 +183,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL Примечание: -Условный оператор сначала вычисляет значения b и c, затем проверяет выполнение условия a, и только после этого возвращает соответствующее значение. Если в качестве b или с выступает функция [arrayJoin()](functions/array_join.md#functions_arrayjoin), то размножение каждой строки произойдет вне зависимости от условия а. +Условный оператор сначала вычисляет значения b и c, затем проверяет выполнение условия a, и только после этого возвращает соответствующее значение. Если в качестве b или с выступает функция [arrayJoin()](../sql_reference/operators.md#functions_arrayjoin), то размножение каждой строки произойдет вне зависимости от условия а. ## Условное выражение {#operator_case} @@ -232,7 +232,7 @@ ClickHouse поддерживает операторы `IS NULL` и `IS NOT NULL ### IS NULL {#operator-is-null} -- Для значений типа [Nullable](../data_types/nullable.md) оператор `IS NULL` возвращает: +- Для значений типа [Nullable](../sql_reference/operators.md) оператор `IS NULL` возвращает: - `1`, если значение — `NULL`. - `0` в обратном случае. - Для прочих значений оператор `IS NULL` всегда возвращает `0`. @@ -251,7 +251,7 @@ SELECT x+100 FROM t_null WHERE y IS NULL ### IS NOT NULL {#is-not-null} -- Для значений типа [Nullable](../data_types/nullable.md) оператор `IS NOT NULL` возвращает: +- Для значений типа [Nullable](../sql_reference/operators.md) оператор `IS NOT NULL` возвращает: - `0`, если значение — `NULL`. - `1`, в обратном случае. - Для прочих значений оператор `IS NOT NULL` всегда возвращает `1`. diff --git a/docs/ru/query_language/alter.md b/docs/ru/sql_reference/statements/alter.md similarity index 93% rename from docs/ru/query_language/alter.md rename to docs/ru/sql_reference/statements/alter.md index 401d7e3bcbc..5c08d68623e 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/sql_reference/statements/alter.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: ALTER +--- + ## ALTER {#query_language_queries_alter} Запрос `ALTER` поддерживается только для таблиц типа `*MergeTree`, а также `Merge` и `Distributed`. Запрос имеет несколько вариантов. @@ -33,7 +38,7 @@ ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] Если указано `IF NOT EXISTS`, запрос не будет возвращать ошибку, если столбец уже существует. Если указано `AFTER name_after` (имя другого столбца), то столбец добавляется (в список столбцов таблицы) после указанного. Иначе, столбец добавляется в конец таблицы. Обратите внимание, ClickHouse не позволяет добавлять столбцы в начало таблицы. Для цепочки действий, `name_after` может быть именем столбца, который добавляется в одном из предыдущих действий. -Добавление столбца всего лишь меняет структуру таблицы, и не производит никаких действий с данными - соответствующие данные не появляются на диске после ALTER-а. При чтении из таблицы, если для какого-либо столбца отсутствуют данные, то он заполняется значениями по умолчанию (выполняя выражение по умолчанию, если такое есть, или нулями, пустыми строками). Также, столбец появляется на диске при слиянии кусков данных (см. [MergeTree](../operations/table_engines/mergetree.md)). +Добавление столбца всего лишь меняет структуру таблицы, и не производит никаких действий с данными - соответствующие данные не появляются на диске после ALTER-а. При чтении из таблицы, если для какого-либо столбца отсутствуют данные, то он заполняется значениями по умолчанию (выполняя выражение по умолчанию, если такое есть, или нулями, пустыми строками). Также, столбец появляется на диске при слиянии кусков данных (см. [MergeTree](../../sql_reference/statements/alter.md)). Такая схема позволяет добиться мгновенной работы запроса `ALTER` и отсутствия необходимости увеличивать объём старых данных. @@ -107,11 +112,11 @@ MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] - TTL - Примеры изменения TTL столбца смотрите в разделе [TTL столбца](../operations/table_engines/mergetree.md#mergetree-column-ttl). + Примеры изменения TTL столбца смотрите в разделе [TTL столбца](../../sql_reference/statements/alter.md#mergetree-column-ttl). Если указано `IF EXISTS`, запрос не возвращает ошибку, если столбца не существует. -При изменении типа, значения преобразуются так, как если бы к ним была применена функция [toType](functions/type_conversion_functions.md). Если изменяется только выражение для умолчания, запрос не делает никакой сложной работы и выполняется мгновенно. +При изменении типа, значения преобразуются так, как если бы к ним была применена функция [toType](../../sql_reference/statements/alter.md). Если изменяется только выражение для умолчания, запрос не делает никакой сложной работы и выполняется мгновенно. Пример запроса: @@ -139,11 +144,11 @@ ALTER TABLE visits MODIFY COLUMN browser Array(String) Отсутствует возможность удалять столбцы, входящие в первичный ключ или ключ для сэмплирования (в общем, входящие в выражение `ENGINE`). Изменение типа у столбцов, входящих в первичный ключ возможно только в том случае, если это изменение не приводит к изменению данных (например, разрешено добавление значения в Enum или изменение типа с `DateTime` на `UInt32`). -Если возможностей запроса `ALTER` не хватает для нужного изменения таблицы, вы можете создать новую таблицу, скопировать туда данные с помощью запроса [INSERT SELECT](insert_into.md#insert_query_insert-select), затем поменять таблицы местами с помощью запроса [RENAME](misc.md#misc_operations-rename), и удалить старую таблицу. В качестве альтернативы для запроса `INSERT SELECT`, можно использовать инструмент [clickhouse-copier](../operations/utils/clickhouse-copier.md). +Если возможностей запроса `ALTER` не хватает для нужного изменения таблицы, вы можете создать новую таблицу, скопировать туда данные с помощью запроса [INSERT SELECT](insert_into.md#insert_query_insert-select), затем поменять таблицы местами с помощью запроса [RENAME](misc.md#misc_operations-rename), и удалить старую таблицу. В качестве альтернативы для запроса `INSERT SELECT`, можно использовать инструмент [clickhouse-copier](../../sql_reference/statements/alter.md). Запрос `ALTER` блокирует все чтения и записи для таблицы. То есть, если на момент запроса `ALTER`, выполнялся долгий `SELECT`, то запрос `ALTER` сначала дождётся его выполнения. И в это время, все новые запросы к той же таблице, будут ждать, пока завершится этот `ALTER`. -Для таблиц, которые не хранят данные самостоятельно (типа [Merge](../operations/table_engines/merge.md) и [Distributed](../operations/table_engines/distributed.md)), `ALTER` всего лишь меняет структуру таблицы, но не меняет структуру подчинённых таблиц. Для примера, при ALTER-е таблицы типа `Distributed`, вам также потребуется выполнить запрос `ALTER` для таблиц на всех удалённых серверах. +Для таблиц, которые не хранят данные самостоятельно (типа [Merge](../../sql_reference/statements/alter.md) и [Distributed](../../sql_reference/statements/alter.md)), `ALTER` всего лишь меняет структуру таблицы, но не меняет структуру подчинённых таблиц. Для примера, при ALTER-е таблицы типа `Distributed`, вам также потребуется выполнить запрос `ALTER` для таблиц на всех удалённых серверах. ### Манипуляции с ключевыми выражениями таблиц {#manipuliatsii-s-kliuchevymi-vyrazheniiami-tablits} @@ -153,8 +158,8 @@ ALTER TABLE visits MODIFY COLUMN browser Array(String) MODIFY ORDER BY new_expression ``` -Работает только для таблиц семейства [`MergeTree`](../operations/table_engines/mergetree.md) (в том числе [реплицированных](../operations/table_engines/replication.md)). После выполнения запроса -[ключ сортировки](../operations/table_engines/mergetree.md) таблицы +Работает только для таблиц семейства [`MergeTree`](../../sql_reference/statements/alter.md) (в том числе [реплицированных](../../sql_reference/statements/alter.md)). После выполнения запроса +[ключ сортировки](../../sql_reference/statements/alter.md) таблицы заменяется на `new_expression` (выражение или кортеж выражений). Первичный ключ при этом остаётся прежним. Операция затрагивает только метаданные. Чтобы сохранить свойство упорядоченности кусков данных по ключу @@ -197,7 +202,7 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; ### Манипуляции с партициями и кусками {#alter_manipulations-with-partitions} -Для работы с [партициями](../operations/table_engines/custom_partitioning_key.md) доступны следующие операции: +Для работы с [партициями](../../sql_reference/statements/alter.md) доступны следующие операции: - [DETACH PARTITION](#alter_detach-partition) – перенести партицию в директорию `detached`; - [DROP PARTITION](#alter_drop-partition) – удалить партицию; @@ -229,7 +234,7 @@ ALTER TABLE visits DETACH PARTITION 201901 После того как запрос будет выполнен, вы сможете производить любые операции с данными в директории `detached`. Например, можно удалить их из файловой системы. -Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../operations/system_tables.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплики-лидера, запрос вернет ошибку. +Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../../operations/system_tables.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплики-лидера, запрос вернет ошибку. #### DROP PARTITION {#alter_drop-partition} @@ -355,7 +360,7 @@ ALTER TABLE table_name FREEZE [PARTITION partition_expr] - `N` — инкрементальный номер резервной копии. !!! note "Примечание" - При использовании [нескольких дисков для хранения данных таблицы](../operations/table_engines/mergetree.md#table_engine-mergetree-multiple-volumes) директория `shadow/N` появляется на каждом из дисков, на которых были куски, попавшие под выражение `PARTITION`. + При использовании [нескольких дисков для хранения данных таблицы](../../sql_reference/statements/alter.md#table_engine-mergetree-multiple-volumes) директория `shadow/N` появляется на каждом из дисков, на которых были куски, попавшие под выражение `PARTITION`. Структура директорий внутри резервной копии такая же, как внутри `/var/lib/clickhouse/`. Запрос выполнит ‘chmod’ для всех файлов, запрещая запись в них. @@ -373,7 +378,7 @@ ALTER TABLE table_name FREEZE [PARTITION partition_expr] Восстановление данных из резервной копии не требует остановки сервера. -Подробнее о резервном копировании и восстановлении данных читайте в разделе [Резервное копирование данных](../operations/backup.md). +Подробнее о резервном копировании и восстановлении данных читайте в разделе [Резервное копирование данных](../../operations/backup.md). #### FETCH PARTITION {#alter_fetch-partition} @@ -406,7 +411,7 @@ ALTER TABLE users ATTACH PARTITION 201902; #### MOVE PARTITION\|PART {#alter_move-partition} -Перемещает партицию или кусок данных на другой том или диск для таблиц с движком `MergeTree`. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](../operations/table_engines/mergetree.md#table_engine-mergetree-multiple-volumes). +Перемещает партицию или кусок данных на другой том или диск для таблиц с движком `MergeTree`. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](../../sql_reference/statements/alter.md#table_engine-mergetree-multiple-volumes). ``` sql ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name' @@ -429,10 +434,10 @@ ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd' Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: -- Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../operations/system_tables.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. +- Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../../operations/system_tables.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. - Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. - Строковый идентификатор партиции. Идентификатор партиции используется для именования кусков партиции на файловой системе и в ZooKeeper. В запросах `ALTER` идентификатор партиции нужно указывать в секции `PARTITION ID`, в одинарных кавычках. Например, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached\_parts](../operations/system_tables.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. +- Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached\_parts](../../operations/system_tables.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. Использование кавычек в имени партиций зависит от типа данных столбца, по которому задано партиционирование. Например, для столбца с типом `String` имя партиции необходимо указывать в кавычках (одинарных). Для типов `Date` и `Int*` кавычки указывать не нужно. @@ -448,7 +453,7 @@ OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; ### Манипуляции с TTL таблицы {#manipuliatsii-s-ttl-tablitsy} -Вы можете изменить [TTL для таблицы](../operations/table_engines/mergetree.md#mergetree-table-ttl) запросом следующего вида: +Вы можете изменить [TTL для таблицы](../../sql_reference/statements/alter.md#mergetree-table-ttl) запросом следующего вида: ``` sql ALTER TABLE table-name MODIFY TTL ttl-expression @@ -493,7 +498,7 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name Мутации линейно упорядочены между собой и накладываются на каждый кусок в порядке добавления. Мутации также упорядочены со вставками - гарантируется, что данные, вставленные в таблицу до начала выполнения запроса мутации, будут изменены, а данные, вставленные после окончания запроса мутации, изменены не будут. При этом мутации никак не блокируют вставки. -Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../operations/system_tables.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](misc.md#kill-mutation). +Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../../operations/system_tables.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](misc.md#kill-mutation). Записи о последних выполненных мутациях удаляются не сразу (количество сохраняемых мутаций определяется параметром движка таблиц `finished_mutations_to_keep`). Более старые записи удаляются. diff --git a/docs/ru/query_language/create.md b/docs/ru/sql_reference/statements/create.md similarity index 94% rename from docs/ru/query_language/create.md rename to docs/ru/sql_reference/statements/create.md index dfaae11a359..e09877d3495 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/sql_reference/statements/create.md @@ -1,3 +1,8 @@ +--- +toc_priority: 35 +toc_title: CREATE +--- + ## CREATE DATABASE {#query-language-create-database} Создает базу данных. @@ -20,11 +25,11 @@ CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(.. - `ENGINE` - - [MySQL](../database_engines/mysql.md) + - [MySQL](../../sql_reference/statements/create.md) Позволяет получать данные с удаленного сервера MySQL. - По умолчанию ClickHouse использует собственный [движок баз данных](../database_engines/index.md). + По умолчанию ClickHouse использует собственный [движок баз данных](../../sql_reference/statements/create.md). ## CREATE TABLE {#create-table-query} @@ -65,7 +70,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... Во всех случаях, если указано `IF NOT EXISTS`, то запрос не будет возвращать ошибку, если таблица уже существует. В этом случае, запрос будет ничего не делать. -После секции `ENGINE` в запросе могут использоваться и другие секции в зависимости от движка. Подробную документацию по созданию таблиц смотрите в описаниях [движков таблиц](../operations/table_engines/index.md#table_engines). +После секции `ENGINE` в запросе могут использоваться и другие секции в зависимости от движка. Подробную документацию по созданию таблиц смотрите в описаниях [движков таблиц](../../sql_reference/statements/create.md#table_engines). ### Значения по умолчанию {#create-default-values} @@ -123,11 +128,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ### Выражение для TTL {#vyrazhenie-dlia-ttl} -Определяет время хранения значений. Может быть указано только для таблиц семейства MergeTree. Подробнее смотрите в [TTL для столбцов и таблиц](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). +Определяет время хранения значений. Может быть указано только для таблиц семейства MergeTree. Подробнее смотрите в [TTL для столбцов и таблиц](../../sql_reference/statements/create.md#table_engine-mergetree-ttl). ### Кодеки сжатия столбцов {#codecs} -По умолчанию, ClickHouse применяет к столбцу метод сжатия, определённый в [конфигурации сервера](../operations/server_settings/settings.md#compression). Кроме этого, можно задать метод сжатия для каждого отдельного столбца в запросе `CREATE TABLE`. +По умолчанию, ClickHouse применяет к столбцу метод сжатия, определённый в [конфигурации сервера](../../sql_reference/statements/create.md#compression). Кроме этого, можно задать метод сжатия для каждого отдельного столбца в запросе `CREATE TABLE`. ``` sql CREATE TABLE codec_example @@ -149,10 +154,10 @@ ENGINE = Сжатие поддерживается для следующих движков таблиц: -- [MergeTree family](../operations/table_engines/mergetree.md) -- [Log family](../operations/table_engines/log_family.md) -- [Set](../operations/table_engines/set.md) -- [Join](../operations/table_engines/join.md) +- [MergeTree family](../../sql_reference/statements/create.md) +- [Log family](../../sql_reference/statements/create.md) +- [Set](../../sql_reference/statements/create.md) +- [Join](../../sql_reference/statements/create.md) ClickHouse поддерживает кодеки общего назначения и специализированные кодеки. @@ -213,7 +218,7 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name В большинстве случаев, временные таблицы создаются не вручную, а при использовании внешних данных для запроса, или при распределённом `(GLOBAL) IN`. Подробнее см. соответствующие разделы -Вместо временных можно использовать обычные таблицы с [ENGINE = Memory](../operations/table_engines/memory.md). +Вместо временных можно использовать обычные таблицы с [ENGINE = Memory](../../sql_reference/statements/create.md). ## Распределенные DDL запросы (секция ON CLUSTER) {#raspredelennye-ddl-zaprosy-sektsiia-on-cluster} @@ -289,12 +294,12 @@ LAYOUT(LAYOUT_NAME([param_name param_value])) LIFETIME([MIN val1] MAX val2) ``` -Создаёт [внешний словарь](dicts/external_dicts.md) с заданной [структурой](dicts/external_dicts_dict_structure.md), [источником](dicts/external_dicts_dict_sources.md), [способом размещения в памяти](dicts/external_dicts_dict_layout.md) и [периодом обновления](dicts/external_dicts_dict_lifetime.md). +Создаёт [внешний словарь](../../sql_reference/statements/create.md) с заданной [структурой](../../sql_reference/statements/create.md), [источником](../../sql_reference/statements/create.md), [способом размещения в памяти](../../sql_reference/statements/create.md) и [периодом обновления](../../sql_reference/statements/create.md). Структура внешнего словаря состоит из атрибутов. Атрибуты словаря задаются как столбцы таблицы. Единственным обязательным свойством атрибута является его тип, все остальные свойства могут иметь значения по умолчанию. -В зависимости от [способа размещения словаря в памяти](dicts/external_dicts_dict_layout.md), ключами словаря могут быть один и более атрибутов. +В зависимости от [способа размещения словаря в памяти](../../sql_reference/statements/create.md), ключами словаря могут быть один и более атрибутов. -Смотрите [Внешние словари](dicts/external_dicts.md). +Смотрите [Внешние словари](../../sql_reference/statements/create.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/create/) diff --git a/docs/ru/sql_reference/statements/index.md b/docs/ru/sql_reference/statements/index.md new file mode 100644 index 00000000000..5b461b73c92 --- /dev/null +++ b/docs/ru/sql_reference/statements/index.md @@ -0,0 +1,5 @@ +--- +toc_folder_title: Выражения +toc_priority: 31 +--- + diff --git a/docs/ru/query_language/insert_into.md b/docs/ru/sql_reference/statements/insert_into.md similarity index 87% rename from docs/ru/query_language/insert_into.md rename to docs/ru/sql_reference/statements/insert_into.md index 7a9540e059c..e9c80466a58 100644 --- a/docs/ru/query_language/insert_into.md +++ b/docs/ru/sql_reference/statements/insert_into.md @@ -1,3 +1,8 @@ +--- +toc_priority: 34 +toc_title: INSERT INTO +--- + ## INSERT {#insert} Добавление данных. @@ -13,9 +18,9 @@ INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), . - Значения, вычисляемые из `DEFAULT` выражений, указанных в определении таблицы. - Нули и пустые строки, если `DEFAULT` не определены. -Если [strict\_insert\_defaults=1](../operations/settings/settings.md), то столбцы, для которых не определены `DEFAULT`, необходимо перечислить в запросе. +Если [strict\_insert\_defaults=1](../../operations/settings/settings.md), то столбцы, для которых не определены `DEFAULT`, необходимо перечислить в запросе. -В INSERT можно передавать данные любого [формата](../interfaces/formats.md#formats), который поддерживает ClickHouse. Для этого формат необходимо указать в запросе в явном виде: +В INSERT можно передавать данные любого [формата](../../interfaces/formats.md#formats), который поддерживает ClickHouse. Для этого формат необходимо указать в запросе в явном виде: ``` sql INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set @@ -37,7 +42,7 @@ INSERT INTO t FORMAT TabSeparated 22 Qwerty ``` -С помощью консольного клиента или HTTP интерфейса можно вставлять данные отдельно от запроса. Как это сделать, читайте в разделе «[Интерфейсы](../interfaces/index.md#interfaces)». +С помощью консольного клиента или HTTP интерфейса можно вставлять данные отдельно от запроса. Как это сделать, читайте в разделе «[Интерфейсы](../../interfaces/index.md#interfaces)». ### Ограничения (constraints) {#ogranicheniia-constraints} @@ -56,7 +61,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... Не поддерживаются другие запросы на модификацию части данных: `UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`. Вы можете удалять старые данные с помощью запроса `ALTER TABLE ... DROP PARTITION`. -Для табличной функции [input()](table_functions/input.md) после секции `SELECT` должна следовать +Для табличной функции [input()](../table_functions/input.md) после секции `SELECT` должна следовать секция `FORMAT`. ### Замечания о производительности {#zamechaniia-o-proizvoditelnosti} diff --git a/docs/ru/query_language/misc.md b/docs/ru/sql_reference/statements/misc.md similarity index 87% rename from docs/ru/query_language/misc.md rename to docs/ru/sql_reference/statements/misc.md index 476c57d34a5..d8278534d46 100644 --- a/docs/ru/query_language/misc.md +++ b/docs/ru/sql_reference/statements/misc.md @@ -25,17 +25,17 @@ CHECK TABLE [db.]name Запрос `CHECK TABLE` сравнивает текущие размеры файлов (в которых хранятся данные из колонок) с ожидаемыми значениями. Если значения не совпадают, данные в таблице считаются поврежденными. Искажение возможно, например, из-за сбоя при записи данных. -Ответ содержит колонку `result`, содержащую одну строку с типом [Boolean](../data_types/boolean.md). Допустимые значения: +Ответ содержит колонку `result`, содержащую одну строку с типом [Boolean](../../sql_reference/data_types/boolean.md). Допустимые значения: - 0 - данные в таблице повреждены; - 1 - данные не повреждены. Запрос `CHECK TABLE` поддерживает следующие движки таблиц: -- [Log](../operations/table_engines/log.md) -- [TinyLog](../operations/table_engines/tinylog.md) -- [StripeLog](../operations/table_engines/stripelog.md) -- [Семейство MergeTree](../operations/table_engines/mergetree.md) +- [Log](../../engines/table_engines/log_family/log.md) +- [TinyLog](../../engines/table_engines/log_family/tinylog.md) +- [StripeLog](../../engines/table_engines/log_family/stripelog.md) +- [Семейство MergeTree](../../engines/table_engines/mergetree_family/index.md) При попытке выполнить запрос с таблицами с другими табличными движками, ClickHouse генерирует исключение. @@ -48,7 +48,7 @@ CHECK TABLE [db.]name В этом случае можно скопировать оставшиеся неповрежденные данные в другую таблицу. Для этого: 1. Создайте новую таблицу с такой же структурой, как у поврежденной таблицы. Для этого выполните запрос `CREATE TABLE AS `. -2. Установите значение параметра [max\_threads](../operations/settings/settings.md#settings-max_threads) в 1. Это нужно для того, чтобы выполнить следующий запрос в одном потоке. Установить значение параметра можно через запрос: `SET max_threads = 1`. +2. Установите значение параметра [max\_threads](../../operations/settings/settings.md#settings-max_threads) в 1. Это нужно для того, чтобы выполнить следующий запрос в одном потоке. Установить значение параметра можно через запрос: `SET max_threads = 1`. 3. Выполните запрос `INSERT INTO SELECT * FROM `. В результате неповрежденные данные будут скопированы в другую таблицу. Обратите внимание, будут скопированы только те данные, которые следуют до поврежденного участка. 4. Перезапустите `clickhouse-client`, чтобы вернуть предыдущее значение параметра `max_threads`. @@ -153,7 +153,7 @@ KILL MUTATION [ON CLUSTER cluster] [FORMAT format] ``` -Пытается остановить выполняющиеся в данные момент [мутации](alter.md#alter-mutations). Мутации для остановки выбираются из таблицы [`system.mutations`](../operations/system_tables.md#system_tables-mutations) с помощью условия, указанного в секции `WHERE` запроса `KILL`. +Пытается остановить выполняющиеся в данные момент [мутации](alter.md#alter-mutations). Мутации для остановки выбираются из таблицы [`system.mutations`](../../operations/system_tables.md#system_tables-mutations) с помощью условия, указанного в секции `WHERE` запроса `KILL`. Тестовый вариант запроса (`TEST`) только проверяет права пользователя и выводит список запросов для остановки. @@ -177,11 +177,11 @@ KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = ' OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] ``` -Запрос пытается запустить внеплановый мёрж кусков данных для таблиц семейства [MergeTree](../operations/table_engines/mergetree.md). Другие движки таблиц не поддерживаются. +Запрос пытается запустить внеплановый мёрж кусков данных для таблиц семейства [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). Другие движки таблиц не поддерживаются. -Если `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../operations/table_engines/replication.md), ClickHouse создаёт задачу на мёрж и ожидает её исполнения на всех узлах (если активирована настройка `replication_alter_partitions_sync`). +Если `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table_engines/mergetree_family/replication.md), ClickHouse создаёт задачу на мёрж и ожидает её исполнения на всех узлах (если активирована настройка `replication_alter_partitions_sync`). -- Если `OPTIMIZE` не выполняет мёрж по любой причине, ClickHouse не оповещает об этом клиента. Чтобы включить оповещения, используйте настройку [optimize\_throw\_if\_noop](../operations/settings/settings.md#setting-optimize_throw_if_noop). +- Если `OPTIMIZE` не выполняет мёрж по любой причине, ClickHouse не оповещает об этом клиента. Чтобы включить оповещения, используйте настройку [optimize\_throw\_if\_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop). - Если указать `PARTITION`, то оптимизация выполняется только для указанной партиции. [Как задавать имя партиции в запросах](alter.md#alter-how-to-specify-part-expr). - Если указать `FINAL`, то оптимизация выполняется даже в том случае, если все данные уже лежат в одном куске. - Если указать `DEDUPLICATE`, то произойдет схлопывание полностью одинаковых строк (сравниваются значения во всех колонках), имеет смысл только для движка MergeTree. @@ -205,7 +205,7 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... SET param = value ``` -Устанавливает значение `value` для [настройки](../operations/settings/index.md) `param` в текущей сессии. [Конфигурационные параметры сервера](../operations/server_settings/index.md) нельзя изменить подобным образом. +Устанавливает значение `value` для [настройки](../../operations/settings/index.md) `param` в текущей сессии. [Конфигурационные параметры сервера](../../operations/server_configuration_parameters/settings.md) нельзя изменить подобным образом. Можно одним запросом установить все настройки из заданного профиля настроек. @@ -213,7 +213,7 @@ SET param = value SET profile = 'profile-name-from-the-settings-file' ``` -Подробности смотрите в разделе [Настройки](../operations/settings/settings.md). +Подробности смотрите в разделе [Настройки](../../operations/settings/settings.md). ## TRUNCATE {#truncate} @@ -223,7 +223,7 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Удаляет все данные из таблицы. Если условие `IF EXISTS` не указано, запрос вернет ошибку, если таблицы не существует. -Запрос `TRUNCATE` не поддерживается для следующих движков: [View](../operations/table_engines/view.md), [File](../operations/table_engines/file.md), [URL](../operations/table_engines/url.md) и [Null](../operations/table_engines/null.md). +Запрос `TRUNCATE` не поддерживается для следующих движков: [View](../../engines/table_engines/special/view.md), [File](../../engines/table_engines/special/file.md), [URL](../../engines/table_engines/special/url.md) и [Null](../../engines/table_engines/special/null.md). ## USE {#use} diff --git a/docs/ru/query_language/select.md b/docs/ru/sql_reference/statements/select.md similarity index 91% rename from docs/ru/query_language/select.md rename to docs/ru/sql_reference/statements/select.md index 759a65f155a..2ceeb3489c2 100644 --- a/docs/ru/query_language/select.md +++ b/docs/ru/sql_reference/statements/select.md @@ -1,3 +1,8 @@ +--- +toc_priority: 33 +toc_title: SELECT +--- + # Синтаксис запросов SELECT {#sintaksis-zaprosov-select} `SELECT` осуществляет выборку данных. @@ -107,7 +112,7 @@ Cекция `FROM` определяет источник данных: - Таблица - Подзапрос -- [Табличная функция](table_functions/index.md) +- [Табличная функция](../../sql_reference/statements/select.md) Также могут присутствовать `ARRAY JOIN` и обычный `JOIN` (смотрите ниже). @@ -117,17 +122,17 @@ Cекция `FROM` определяет источник данных: Для выполнения запроса, из соответствующей таблицы, вынимаются все столбцы, перечисленные в запросе. Из подзапросов выкидываются столбцы, не нужные для внешнего запроса. Если в запросе не перечислено ни одного столбца (например, `SELECT count() FROM t`), то из таблицы всё равно вынимается один какой-нибудь столбец (предпочитается самый маленький), для того, чтобы можно было посчитать количество строк. -Модификатор `FINAL` может быть использован в запросе `SELECT` из таблиц семейства [MergeTree](../operations/table_engines/mergetree.md). При указании `FINAL`, данные будут выбираться полностью «домерженными». Стоит учитывать, что использование `FINAL` приводит к чтению также столбцов, относящихся к первичному ключу. Также, запрос будет выполняться в один поток, и при выполнении запроса будет выполняться слияние данных. Это приводит к тому, что при использовании `FINAL`, запрос выполняется медленнее. В большинстве случаев, следует избегать использования `FINAL`. +Модификатор `FINAL` может быть использован в запросе `SELECT` из таблиц семейства [MergeTree](../../engines/table_engines/mergetree_family/index.md). При указании `FINAL`, данные будут выбираться полностью «домерженными». Стоит учитывать, что использование `FINAL` приводит к чтению также столбцов, относящихся к первичному ключу. Также, запрос будет выполняться в один поток, и при выполнении запроса будет выполняться слияние данных. Это приводит к тому, что при использовании `FINAL`, запрос выполняется медленнее. В большинстве случаев, следует избегать использования `FINAL`. Модификатор `FINAL` может быть использован для всех таблиц семейства `MergeTree`, которые производят преобразования данных в процессе фоновых слияний (кроме GraphiteMergeTree). #### FINAL Modifier {#select-from-final} -Применим при выборке данных из таблиц с движками таблиц семейства [MergeTree](../operations/table_engines/mergetree.md), кроме `GraphiteMergeTree`. Если в запросе используется `FINAL`, то ClickHouse полностью мёржит данные перед выдачей результата, таким образом выполняя все преобразования данных, которые производятся движком таблиц при мёржах. +Применим при выборке данных из таблиц с движками таблиц семейства [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)), кроме `GraphiteMergeTree`. Если в запросе используется `FINAL`, то ClickHouse полностью мёржит данные перед выдачей результата, таким образом выполняя все преобразования данных, которые производятся движком таблиц при мёржах. Также поддержан для движков: -- [Replicated](../operations/table_engines/replication.md)-версий `MergeTree`. -- [View](../operations/table_engines/view.md), [Buffer](../operations/table_engines/buffer.md), [Distributed](../operations/table_engines/distributed.md), и [MaterializedView](../operations/table_engines/materializedview.md), которые работают поверх других движков, если они созданы для таблиц с движками семейства `MergeTree`. +- [Replicated](../../engines/table_engines/mergetree_family/replication.md)-версий `MergeTree`. +- [View](../../engines/table_engines/special/view.md), [Buffer](../../engines/table_engines/special/buffer.md), [Distributed](../../engines/table_engines/special/distributed.md), и [MaterializedView](../../engines/table_engines/special/materializedview.md), которые работают поверх других движков, если они созданы для таблиц с движками семейства `MergeTree`. Запросы, использующие `FINAL` исполняются медленнее аналогичных запросов без `FINAL`, поскольку: @@ -153,9 +158,9 @@ Cекция `FROM` определяет источник данных: - Сэмплирование работает детерминированно. При многократном выполнении одного и того же запроса `SELECT .. SAMPLE`, результат всегда будет одинаковым. - Сэмплирование поддерживает консистентность для разных таблиц. Имеется в виду, что для таблиц с одним и тем же ключом сэмплирования, подмножество данных в выборках будет одинаковым (выборки при этом должны быть сформированы для одинаковой доли данных). Например, выборка по идентификаторам посетителей выберет из разных таблиц строки с одинаковым подмножеством всех возможных идентификаторов. Это свойство позволяет использовать выборки в подзапросах в секции [IN](#select-in-operators), а также объединять выборки с помощью [JOIN](#select-join). -- Сэмплирование позволяет читать меньше данных с диска. Обратите внимание, для этого необходимо корректно указать ключ сэмплирования. Подробнее см. в разделе [Создание таблицы MergeTree](../operations/table_engines/mergetree.md#table_engine-mergetree-creating-a-table). +- Сэмплирование позволяет читать меньше данных с диска. Обратите внимание, для этого необходимо корректно указать ключ сэмплирования. Подробнее см. в разделе [Создание таблицы MergeTree](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table). -Сэмплирование поддерживается только таблицами семейства [MergeTree](../operations/table_engines/mergetree.md) и только в том случае, если для таблиц был указан ключ сэмплирования (выражение, на основе которого должна производиться выборка). Подробнее см. в разделе [Создание таблиц MergeTree](../operations/table_engines/mergetree.md#table_engine-mergetree-creating-a-table). +Сэмплирование поддерживается только таблицами семейства [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table) и только в том случае, если для таблиц был указан ключ сэмплирования (выражение, на основе которого должна производиться выборка). Подробнее см. в разделе [Создание таблиц MergeTree](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table). Выражение `SAMPLE` в запросе можно задать следующими способами: @@ -198,7 +203,7 @@ ORDER BY PageViews DESC LIMIT 1000 При выполнении `SAMPLE n` коэффициент сэмплирования заранее неизвестен (то есть нет информации о том, относительно какого количества данных будет сформирована выборка). Чтобы узнать коэффициент сэмплирования, используйте столбец `_sample_factor`. -Виртуальный столбец `_sample_factor` автоматически создается в тех таблицах, для которых задано выражение `SAMPLE BY` (подробнее см. в разделе [Создание таблицы MergeTree](../operations/table_engines/mergetree.md#table_engine-mergetree-creating-a-table)). В столбце содержится коэффициент сэмплирования для таблицы – он рассчитывается динамически по мере добавления данных в таблицу. Ниже приведены примеры использования столбца `_sample_factor`. +Виртуальный столбец `_sample_factor` автоматически создается в тех таблицах, для которых задано выражение `SAMPLE BY` (подробнее см. в разделе [Создание таблицы MergeTree](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table)). В столбце содержится коэффициент сэмплирования для таблицы – он рассчитывается динамически по мере добавления данных в таблицу. Ниже приведены примеры использования столбца `_sample_factor`. Предположим, у нас есть таблица, в которой ведется статистика посещений сайта. Пример ниже показывает, как рассчитать суммарное число просмотров: @@ -252,7 +257,7 @@ SAMPLE 1/10 OFFSET 1/2 ### Секция ARRAY JOIN {#select-array-join-clause} -Позволяет выполнить `JOIN` с массивом или вложенной структурой данных. Смысл похож на функцию [arrayJoin](functions/array_join.md#functions_arrayjoin), но функциональность более широкая. +Позволяет выполнить `JOIN` с массивом или вложенной структурой данных. Смысл похож на функцию [arrayJoin](../../sql_reference/statements/select.md#functions_arrayjoin), но функциональность более широкая. ``` sql SELECT @@ -271,7 +276,7 @@ FROM - `ARRAY JOIN` — в этом случае результат `JOIN` не будет содержать пустые массивы; - `LEFT ARRAY JOIN` — пустые массивы попадут в результат выполнения `JOIN`. В качестве значения для пустых массивов устанавливается значение по умолчанию. Обычно это 0, пустая строка или NULL, в зависимости от типа элементов массива. -Рассмотрим примеры использования `ARRAY JOIN` и `LEFT ARRAY JOIN`. Для начала создадим таблицу, содержащую столбец с типом [Array](../data_types/array.md), и добавим в него значение: +Рассмотрим примеры использования `ARRAY JOIN` и `LEFT ARRAY JOIN`. Для начала создадим таблицу, содержащую столбец с типом [Array](../../sql_reference/statements/select.md), и добавим в него значение: ``` sql CREATE TABLE arrays_test @@ -389,7 +394,7 @@ ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS ma └───────┴─────────┴───┴─────┴────────┘ ``` -В примере ниже используется функция [arrayEnumerate](functions/array_functions.md#array_functions-arrayenumerate): +В примере ниже используется функция [arrayEnumerate](../../sql_reference/statements/select.md#array_functions-arrayenumerate): ``` sql SELECT s, arr, a, num, arrayEnumerate(arr) @@ -409,7 +414,7 @@ ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; #### ARRAY JOIN с вложенными структурами данных {#array-join-s-vlozhennymi-strukturami-dannykh} -`ARRAY JOIN` также работает с [вложенными структурами данных](../data_types/nested_data_structures/nested.md). Пример: +`ARRAY JOIN` также работает с [вложенными структурами данных](../../sql_reference/statements/select.md). Пример: ``` sql CREATE TABLE nested_test @@ -502,7 +507,7 @@ ARRAY JOIN nest AS n; └───────┴─────┴─────┴─────────┴────────────┘ ``` -Пример использования функции [arrayEnumerate](functions/array_functions.md#array_functions-arrayenumerate): +Пример использования функции [arrayEnumerate](../../sql_reference/statements/select.md#array_functions-arrayenumerate): ``` sql SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num @@ -534,7 +539,7 @@ FROM (ON )|(USING ) ... ``` -Вместо `` и `` можно указать имена таблиц. Это эквивалентно подзапросу `SELECT * FROM table`, за исключением особого случая таблицы с движком [Join](../operations/table_engines/join.md) – массива, подготовленного для присоединения. +Вместо `` и `` можно указать имена таблиц. Это эквивалентно подзапросу `SELECT * FROM table`, за исключением особого случая таблицы с движком [Join](../../sql_reference/statements/select.md) – массива, подготовленного для присоединения. #### Поддерживаемые типы `JOIN` {#select-join-types} @@ -618,9 +623,9 @@ USING (equi_column1, ... equi_columnN, asof_column) `ASOF JOIN` принимает метку времени пользовательского события из `table_1` и находит такое событие в `table_2` метка времени которого наиболее близка к метке времени события из `table_1` в соответствии с условием на ближайшее совпадение. При этом столбец `user_id` используется для объединения по равенству, а столбец `ev_time` для объединения по ближайшему совпадению. В нашем примере `event_1_1` может быть объединено с `event_2_1`, `event_1_2` может быть объединено с `event_2_3`, а `event_2_2` не объединяется. !!! note "Примечание" - `ASOF JOIN` не поддержан для движка таблиц [Join](../operations/table_engines/join.md). + `ASOF JOIN` не поддержан для движка таблиц [Join](../../sql_reference/statements/select.md). -Чтобы задать значение строгости по умолчанию, используйте сессионный параметр [join\_default\_strictness](../operations/settings/settings.md#settings-join_default_strictness). +Чтобы задать значение строгости по умолчанию, используйте сессионный параметр [join\_default\_strictness](../../operations/settings/settings.md#settings-join_default_strictness). #### GLOBAL JOIN {#global-join} @@ -684,27 +689,27 @@ LIMIT 10 «Правая» таблица (результат подзапроса) располагается в оперативной памяти. Если её не хватает, вы не сможете выполнить `JOIN`. -Каждый раз для выполнения запроса с одинаковым `JOIN`, подзапрос выполняется заново — результат не кэшируется. Это можно избежать, используя специальный движок таблиц [Join](../operations/table_engines/join.md), представляющий собой подготовленное множество для соединения, которое всегда находится в оперативке. +Каждый раз для выполнения запроса с одинаковым `JOIN`, подзапрос выполняется заново — результат не кэшируется. Это можно избежать, используя специальный движок таблиц [Join](../../engines/table_engines/special/join.md), представляющий собой подготовленное множество для соединения, которое всегда находится в оперативке. В некоторых случаях более эффективно использовать `IN` вместо `JOIN`. Среди разных типов `JOIN`, наиболее эффективен `ANY LEFT JOIN`, следующий по эффективности `ANY INNER JOIN`. Наименее эффективны `ALL LEFT JOIN` и `ALL INNER JOIN`. -Если `JOIN` необходим для соединения с таблицами измерений (dimension tables - сравнительно небольшие таблицы, которые содержат свойства измерений - например, имена для рекламных кампаний), то использование `JOIN` может быть не очень удобным из-за громоздкости синтаксиса, а также из-за того, что правая таблица читается заново при каждом запросе. Специально для таких случаев существует функциональность «Внешние словари», которую следует использовать вместо `JOIN`. Дополнительные сведения смотрите в разделе [Внешние словари](dicts/external_dicts.md). +Если `JOIN` необходим для соединения с таблицами измерений (dimension tables - сравнительно небольшие таблицы, которые содержат свойства измерений - например, имена для рекламных кампаний), то использование `JOIN` может быть не очень удобным из-за громоздкости синтаксиса, а также из-за того, что правая таблица читается заново при каждом запросе. Специально для таких случаев существует функциональность «Внешние словари», которую следует использовать вместо `JOIN`. Дополнительные сведения смотрите в разделе [Внешние словари](../../sql_reference/statements/select.md). **Ограничения по памяти** ClickHouse использует алгоритм [hash join](https://en.wikipedia.org/wiki/Hash_join). ClickHouse принимает `` и создает для него хэш-таблицу в RAM. Чтобы ограничить потребление памяти операцией `JOIN`, используйте следующие параметры: -- [max\_rows\_in\_join](../operations/settings/query_complexity.md#settings-max_rows_in_join) — ограничивает количество строк в хэш-таблице. -- [max\_bytes\_in\_join](../operations/settings/query_complexity.md#settings-max_bytes_in_join) — ограничивает размер хэш-таблицы. +- [max\_rows\_in\_join](../../operations/settings/query_complexity.md#settings-max_rows_in_join) — ограничивает количество строк в хэш-таблице. +- [max\_bytes\_in\_join](../../operations/settings/query_complexity.md#settings-max_bytes_in_join) — ограничивает размер хэш-таблицы. -По достижении любого из этих ограничений, ClickHouse действует в соответствии с настройкой [join\_overflow\_mode](../operations/settings/query_complexity.md#settings-join_overflow_mode). +По достижении любого из этих ограничений, ClickHouse действует в соответствии с настройкой [join\_overflow\_mode](../../operations/settings/query_complexity.md#settings-join_overflow_mode). #### Обработка пустых ячеек и NULL {#obrabotka-pustykh-iacheek-i-null} -При слиянии таблиц могут появляться пустые ячейки. То, каким образом ClickHouse заполняет эти ячейки, определяется настройкой [join\_use\_nulls](../operations/settings/settings.md#join_use_nulls). +При слиянии таблиц могут появляться пустые ячейки. То, каким образом ClickHouse заполняет эти ячейки, определяется настройкой [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls). -Если ключами `JOIN` выступают поля типа [Nullable](../data_types/nullable.md), то строки, где хотя бы один из ключей имеет значение [NULL](syntax.md#null-literal), не соединяются. +Если ключами `JOIN` выступают поля типа [Nullable](../../sql_reference/statements/select.md), то строки, где хотя бы один из ключей имеет значение [NULL](../syntax.md#null-literal), не соединяются. #### Ограничения синтаксиса {#ogranicheniia-sintaksisa} @@ -723,9 +728,9 @@ ClickHouse использует алгоритм [hash join](https://en.wikipedi Результат выражения должен иметь тип `UInt8`. -ClickHouse использует в выражении индексы, если это позволяет [движок таблицы](../operations/table_engines/index.md). +ClickHouse использует в выражении индексы, если это позволяет [движок таблицы](../../sql_reference/statements/select.md). -Если в секции необходимо проверить [NULL](syntax.md#null-literal), то используйте операторы [IS NULL](operators.md#operator-is-null) и [IS NOT NULL](operators.md#is-not-null), а также соответствующие функции `isNull` и `isNotNull`. В противном случае выражение будет считаться всегда не выполненным. +Если в секции необходимо проверить [NULL](../syntax.md#null-literal), то используйте операторы [IS NULL](../operators.md#operator-is-null) и [IS NOT NULL](../operators.md#is-not-null), а также соответствующие функции `isNull` и `isNotNull`. В противном случае выражение будет считаться всегда не выполненным. Пример проверки на `NULL`: @@ -796,7 +801,7 @@ GROUP BY вычисляет для каждого встретившегося #### Обработка NULL {#obrabotka-null} -При группировке, ClickHouse рассматривает [NULL](syntax.md) как значение, причём `NULL=NULL`. +При группировке, ClickHouse рассматривает [NULL](../syntax.md) как значение, причём `NULL=NULL`. Рассмотрим, что это значит на примере. @@ -854,11 +859,11 @@ GROUP BY вычисляет для каждого встретившегося #### GROUP BY во внешней памяти {#select-group-by-in-external-memory} Можно включить сброс временных данных на диск, чтобы ограничить потребление оперативной памяти при выполнении `GROUP BY`. -Настройка [max\_bytes\_before\_external\_group\_by](../operations/settings/settings.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. +Настройка [max\_bytes\_before\_external\_group\_by](../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. При использовании `max_bytes_before_external_group_by`, рекомендуем выставить `max_memory_usage` приблизительно в два раза больше. Это следует сделать, потому что агрегация выполняется в две стадии: чтение и формирование промежуточных данных (1) и слияние промежуточных данных (2). Сброс данных на файловую систему может производиться только на стадии 1. Если сброса временных данных не было, то на стадии 2 может потребляться до такого же объёма памяти, как на стадии 1. -Например, если [max\_memory\_usage](../operations/settings/settings.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а max\_memory\_usage в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. +Например, если [max\_memory\_usage](../../operations/settings/settings.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а max\_memory\_usage в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. При распределённой обработке запроса внешняя агрегация производится на удалённых серверах. Для того чтобы на сервере-инициаторе запроса использовалось немного оперативки, нужно выставить настройку `distributed_aggregation_memory_efficient` в 1. @@ -870,7 +875,7 @@ GROUP BY вычисляет для каждого встретившегося ### Секция LIMIT BY {#sektsiia-limit-by} -Запрос с секцией `LIMIT n BY expressions` выбирает первые `n` строк для каждого отличного значения `expressions`. Ключ `LIMIT BY` может содержать любое количество [выражений](syntax.md#syntax-expressions). +Запрос с секцией `LIMIT n BY expressions` выбирает первые `n` строк для каждого отличного значения `expressions`. Ключ `LIMIT BY` может содержать любое количество [выражений](../syntax.md#syntax-expressions). ClickHouse поддерживает следующий синтаксис: @@ -936,7 +941,7 @@ LIMIT 100 Запрос выберет топ 5 рефереров для каждой пары `domain, device_type`, но не более 100 строк (`LIMIT n BY + LIMIT`). -`LIMIT n BY` работает с [NULL](syntax.md) как если бы это было конкретное значение. Т.е. в результате запроса пользователь получит все комбинации полей, указанных в `BY`. +`LIMIT n BY` работает с [NULL](../syntax.md) как если бы это было конкретное значение. Т.е. в результате запроса пользователь получит все комбинации полей, указанных в `BY`. ### Секция HAVING {#sektsiia-having} @@ -1007,7 +1012,7 @@ WHERE и HAVING отличаются тем, что WHERE выполняется ### Секция SELECT {#select-select} -[Выражения](syntax.md#syntax-expressions) указанные в секции `SELECT` анализируются после завершения всех вычислений из секций, описанных выше. Вернее, анализируются выражения, стоящие над агрегатными функциями, если есть агрегатные функции. +[Выражения](../syntax.md#syntax-expressions) указанные в секции `SELECT` анализируются после завершения всех вычислений из секций, описанных выше. Вернее, анализируются выражения, стоящие над агрегатными функциями, если есть агрегатные функции. Сами агрегатные функции и то, что под ними, вычисляются при агрегации (`GROUP BY`). Эти выражения работают так, как будто применяются к отдельным строкам результата. Если в результат необходимо включить все столбцы, используйте символ звёздочка (`*`). Например, `SELECT * FROM ...`. @@ -1080,7 +1085,7 @@ Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of `DISTINCT` не поддерживается, если в `SELECT` присутствует хотя бы один столбец типа массив. -`DISTINCT` работает с [NULL](syntax.md) как если бы `NULL` был конкретным значением, причём `NULL=NULL`. Т.е. в результате `DISTINCT` разные комбинации с `NULL` встретятся только по одному разу. +`DISTINCT` работает с [NULL](../syntax.md) как если бы `NULL` был конкретным значением, причём `NULL=NULL`. Т.е. в результате `DISTINCT` разные комбинации с `NULL` встретятся только по одному разу. ClickHouse поддерживает использование в одном запросе секций `DISTINCT` и `ORDER BY` для разных столбцов. Секция `DISTINCT` исполняется перед секцией `ORDER BY`. @@ -1237,7 +1242,7 @@ ORDER BY EventDate ASC #### Обработка NULL {#obrabotka-null-1} -При обработке запроса оператор IN будет считать, что результат операции с [NULL](syntax.md) всегда равен `0`, независимо от того, находится `NULL` в правой или левой части оператора. Значения `NULL` не входят ни в какое множество, не соответствуют друг другу и не могут сравниваться. +При обработке запроса оператор IN будет считать, что результат операции с [NULL](../syntax.md) всегда равен `0`, независимо от того, находится `NULL` в правой или левой части оператора. Значения `NULL` не входят ни в какое множество, не соответствуют друг другу и не могут сравниваться. Рассмотрим для примера таблицу `t_null`: @@ -1275,7 +1280,7 @@ FROM t_null Существует два варианта IN-ов с подзапросами (аналогично для JOIN-ов): обычный `IN` / `JOIN` и `GLOBAL IN` / `GLOBAL JOIN`. Они отличаются способом выполнения при распределённой обработке запроса. !!! attention "Attention" - Помните, что алгоритмы, описанные ниже, могут работать иначе в зависимости от [настройки](../operations/settings/settings.md) `distributed_product_mode`. + Помните, что алгоритмы, описанные ниже, могут работать иначе в зависимости от [настройки](../../operations/settings/settings.md) `distributed_product_mode`. При использовании обычного IN-а, запрос отправляется на удалённые серверы, и на каждом из них выполняются подзапросы в секциях `IN` / `JOIN`. @@ -1375,7 +1380,7 @@ SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL Вы можете получить в дополнение к результату также минимальные и максимальные значения по столбцам результата. Для этого выставите настройку **extremes** в 1. Минимумы и максимумы считаются для числовых типов, дат, дат-с-временем. Для остальных столбцов будут выведены значения по умолчанию. -Вычисляются дополнительные две строчки - минимумы и максимумы, соответственно. Эти две дополнительные строки выводятся в [форматах](../interfaces/formats.md) `JSON*`, `TabSeparated*`, и `Pretty*` отдельно от остальных строчек. В остальных форматах они не выводится. +Вычисляются дополнительные две строчки - минимумы и максимумы, соответственно. Эти две дополнительные строки выводятся в [форматах](../../interfaces/formats.md) `JSON*`, `TabSeparated*`, и `Pretty*` отдельно от остальных строчек. В остальных форматах они не выводится. Во форматах `JSON*`, экстремальные значения выводятся отдельным полем ‘extremes’. В форматах `TabSeparated*`, строка выводится после основного результата и после ‘totals’ если есть. Перед ней (после остальных данных) вставляется пустая строка. В форматах `Pretty*`, строка выводится отдельной таблицей после основного результата и после `totals` если есть. diff --git a/docs/ru/query_language/show.md b/docs/ru/sql_reference/statements/show.md similarity index 90% rename from docs/ru/query_language/show.md rename to docs/ru/sql_reference/statements/show.md index 03f99378ccc..545301d6166 100644 --- a/docs/ru/query_language/show.md +++ b/docs/ru/sql_reference/statements/show.md @@ -23,7 +23,7 @@ SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] ``` -Выводит содержимое таблицы [system.processes](../operations/system_tables.md#system_tables-processes), которая содержит список запросов, выполняющихся в данный момент времени, кроме самих запросов `SHOW PROCESSLIST`. +Выводит содержимое таблицы [system.processes](../../operations/system_tables.md#system_tables-processes), которая содержит список запросов, выполняющихся в данный момент времени, кроме самих запросов `SHOW PROCESSLIST`. Запрос `SELECT * FROM system.processes` возвращает данные обо всех текущих запросах. @@ -66,7 +66,7 @@ SHOW TABLES FROM system LIKE '%co%' LIMIT 2 ## SHOW DICTIONARIES {#show-dictionaries} -Выводит список [внешних словарей](dicts/external_dicts.md). +Выводит список [внешних словарей](../../sql_reference/statements/show.md). ``` sql SHOW DICTIONARIES [FROM ] [LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] diff --git a/docs/ru/query_language/system.md b/docs/ru/sql_reference/statements/system.md similarity index 80% rename from docs/ru/query_language/system.md rename to docs/ru/sql_reference/statements/system.md index 12909c12ce2..2a4acd15e7f 100644 --- a/docs/ru/query_language/system.md +++ b/docs/ru/sql_reference/statements/system.md @@ -17,7 +17,7 @@ ## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} Перегружает все словари, которые были успешно загружены до этого. -По умолчанию включена ленивая загрузка [dictionaries\_lazy\_load](../operations/server_settings/settings.md#dictionaries-lazy-load), поэтому словари не загружаются автоматически при старте, а только при первом обращении через dictGet или SELECT к ENGINE=Dictionary. После этого такие словари (LOADED) будут перегружаться командой `system reload dictionaries`. +По умолчанию включена ленивая загрузка [dictionaries\_lazy\_load](../../sql_reference/statements/system.md#dictionaries-lazy-load), поэтому словари не загружаются автоматически при старте, а только при первом обращении через dictGet или SELECT к ENGINE=Dictionary. После этого такие словари (LOADED) будут перегружаться командой `system reload dictionaries`. Всегда возвращает `Ok.`, вне зависимости от результата обновления словарей. ## RELOAD DICTIONARY dictionary\_name {#query_language-system-reload-dictionary} @@ -58,7 +58,7 @@ SELECT name, status FROM system.dictionaries; ## Управление распределёнными таблицами {#query-language-system-distributed} -ClickHouse может оперировать [распределёнными](../operations/table_engines/distributed.md) таблицами. Когда пользователь вставляет данные в эти таблицы, ClickHouse сначала формирует очередь из данных, которые должны быть отправлены на узлы кластера, а затем асинхронно отправляет подготовленные данные. Вы можете управлять очередью с помощью запросов [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) и [FLUSH DISTRIBUTED](#query_language-system-flush-distributed). Также есть возможность синхронно вставлять распределенные данные с помощью настройки `insert_distributed_sync`. +ClickHouse может оперировать [распределёнными](../../sql_reference/statements/system.md) таблицами. Когда пользователь вставляет данные в эти таблицы, ClickHouse сначала формирует очередь из данных, которые должны быть отправлены на узлы кластера, а затем асинхронно отправляет подготовленные данные. Вы можете управлять очередью с помощью запросов [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) и [FLUSH DISTRIBUTED](#query_language-system-flush-distributed). Также есть возможность синхронно вставлять распределенные данные с помощью настройки `insert_distributed_sync`. ### STOP DISTRIBUTED SENDS {#query_language-system-stop-distributed-sends} diff --git a/docs/ru/query_language/syntax.md b/docs/ru/sql_reference/syntax.md similarity index 97% rename from docs/ru/query_language/syntax.md rename to docs/ru/sql_reference/syntax.md index 0219de90685..b9576e48a59 100644 --- a/docs/ru/query_language/syntax.md +++ b/docs/ru/sql_reference/syntax.md @@ -68,13 +68,13 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') - Иначе — ошибка. Соответствующее значение будет иметь тип минимального размера, который вмещает значение. -Например, 1 парсится как `UInt8`, а 256 как `UInt16`. Подробнее о типах данных читайте в разделе [Типы данных](../data_types/index.md). +Например, 1 парсится как `UInt8`, а 256 как `UInt16`. Подробнее о типах данных читайте в разделе [Типы данных](../sql_reference/syntax.md). Примеры: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. ### Строковые {#syntax-string-literal} -Поддерживаются только строковые литералы в одинарных кавычках. Символы внутри могут быть экранированы с помощью обратного слеша. Следующие escape-последовательности имеют соответствующее специальное значение: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. Во всех остальных случаях, последовательности вида `\c`, где `c` — любой символ, преобразуется в `c` . Таким образом, могут быть использованы последовательности `\'` и `\\`. Значение будет иметь тип [String](../data_types/string.md). +Поддерживаются только строковые литералы в одинарных кавычках. Символы внутри могут быть экранированы с помощью обратного слеша. Следующие escape-последовательности имеют соответствующее специальное значение: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. Во всех остальных случаях, последовательности вида `\c`, где `c` — любой символ, преобразуется в `c` . Таким образом, могут быть использованы последовательности `\'` и `\\`. Значение будет иметь тип [String](../sql_reference/syntax.md). Минимальный набор символов, которых вам необходимо экранировать в строковых литералах: `'` и `\`. Одинарная кавычка может быть экранирована одинарной кавычкой, литералы `'It\'s'` и `'It''s'` эквивалентны. @@ -83,13 +83,13 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') Поддерживаются конструкции для массивов: `[1, 2, 3]` и кортежей: `(1, 'Hello, world!', 2)`. На самом деле, это вовсе не литералы, а выражение с оператором создания массива и оператором создания кортежа, соответственно. Массив должен состоять хотя бы из одного элемента, а кортеж - хотя бы из двух. -Кортежи носят служебное значение для использования в секции `IN` запроса `SELECT`. Кортежи могут быть получены как результат запроса, но они не могут быть сохранены в базе данных (за исключением таблицы [Memory](../operations/table_engines/memory.md).) +Кортежи носят служебное значение для использования в секции `IN` запроса `SELECT`. Кортежи могут быть получены как результат запроса, но они не могут быть сохранены в базе данных (за исключением таблицы [Memory](../sql_reference/syntax.md).) ### NULL {#null-literal} Обозначает, что значение отсутствует. -Чтобы в поле таблицы можно было хранить `NULL`, оно должно быть типа [Nullable](../data_types/nullable.md). +Чтобы в поле таблицы можно было хранить `NULL`, оно должно быть типа [Nullable](../sql_reference/syntax.md). В зависимости от формата данных (входных или выходных) `NULL` может иметь различное представление. Подробнее смотрите в документации для [форматов данных](../interfaces/formats.md#formats). @@ -123,7 +123,7 @@ expr AS alias Например, `SELECT table_name_alias.column_name FROM table_name table_name_alias`. - В функции [CAST](functions/type_conversion_functions.md#type_conversion_function-cast), ключевое слово `AS` имеет другое значение. Смотрите описание функции. + В функции [CAST](sql_reference/syntax.md#type_conversion_function-cast), ключевое слово `AS` имеет другое значение. Смотрите описание функции. - `expr` — любое выражение, которое поддерживает ClickHouse. diff --git a/docs/ru/query_language/table_functions/file.md b/docs/ru/sql_reference/table_functions/file.md similarity index 92% rename from docs/ru/query_language/table_functions/file.md rename to docs/ru/sql_reference/table_functions/file.md index d415b20858b..e0da3ddc15f 100644 --- a/docs/ru/query_language/table_functions/file.md +++ b/docs/ru/sql_reference/table_functions/file.md @@ -8,7 +8,7 @@ file(path, format, structure) **Входные параметры** -- `path` — относительный путь до файла от [user\_files\_path](../../operations/server_settings/settings.md#server_settings-user_files_path). Путь к файлу поддерживает следующие шаблоны в режиме доступа только для чтения `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, \``'abc', 'def'` — строки. +- `path` — относительный путь до файла от [user\_files\_path](../../sql_reference/table_functions/file.md#server_configuration_parameters-user_files_path). Путь к файлу поддерживает следующие шаблоны в режиме доступа только для чтения `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, \``'abc', 'def'` — строки. - `format` — [формат](../../interfaces/formats.md#formats) файла. - `structure` — структура таблицы. Формат `'colunmn1_name column1_ype, column2_name column2_type, ...'`. diff --git a/docs/ru/query_language/table_functions/generate.md b/docs/ru/sql_reference/table_functions/generate.md similarity index 100% rename from docs/ru/query_language/table_functions/generate.md rename to docs/ru/sql_reference/table_functions/generate.md diff --git a/docs/ru/query_language/table_functions/hdfs.md b/docs/ru/sql_reference/table_functions/hdfs.md similarity index 100% rename from docs/ru/query_language/table_functions/hdfs.md rename to docs/ru/sql_reference/table_functions/hdfs.md diff --git a/docs/ru/query_language/table_functions/index.md b/docs/ru/sql_reference/table_functions/index.md similarity index 60% rename from docs/ru/query_language/table_functions/index.md rename to docs/ru/sql_reference/table_functions/index.md index 446aa554243..70c701d1713 100644 --- a/docs/ru/query_language/table_functions/index.md +++ b/docs/ru/sql_reference/table_functions/index.md @@ -1,14 +1,20 @@ +--- +toc_folder_title: Табличные функции +toc_priority: 34 +toc_title: Введение +--- + # Табличные функции {#tablichnye-funktsii} Табличные функции — это метод создания таблиц. Табличные функции можно использовать в: -- Секции [FROM](../select.md#select-from) запроса `SELECT`. +- Секции [FROM](../statements/select.md#select-from) запроса `SELECT`. Это способ создания временной таблицы, которая доступна только в текущем запросе. -- Запросе [CREATE TABLE AS \](../create.md#create-table-query). +- Запросе [CREATE TABLE AS \](../statements/create.md#create-table-query). Это один из методов создания таблицы. @@ -17,14 +23,14 @@ | Функция | Описание | |-----------------------|----------------------------------------------------------------------------------------------------------------------------------------| -| [file](file.md) | Создаёт таблицу с движком [File](../../operations/table_engines/file.md). | -| [merge](merge.md) | Создаёт таблицу с движком [Merge](../../operations/table_engines/merge.md). | +| [file](file.md) | Создаёт таблицу с движком [File](../../sql_reference/table_functions/index.md). | +| [merge](merge.md) | Создаёт таблицу с движком [Merge](../../sql_reference/table_functions/index.md). | | [numbers](numbers.md) | Создаёт таблицу с единственным столбцом, заполненным целыми числами. | -| [remote](remote.md) | Предоставляет доступ к удалённым серверам, не создавая таблицу с движком [Distributed](../../operations/table_engines/distributed.md). | -| [url](url.md) | Создаёт таблицу с движком [Url](../../operations/table_engines/url.md). | -| [mysql](mysql.md) | Создаёт таблицу с движком [MySQL](../../operations/table_engines/mysql.md). | -| [jdbc](jdbc.md) | Создаёт таблицу с дижком [JDBC](../../operations/table_engines/jdbc.md). | -| [odbc](odbc.md) | Создаёт таблицу с движком [ODBC](../../operations/table_engines/odbc.md). | -| [hdfs](hdfs.md) | Создаёт таблицу с движком [HDFS](../../operations/table_engines/hdfs.md). | +| [remote](remote.md) | Предоставляет доступ к удалённым серверам, не создавая таблицу с движком [Distributed](../../sql_reference/table_functions/index.md). | +| [url](url.md) | Создаёт таблицу с движком [Url](../../sql_reference/table_functions/index.md). | +| [mysql](mysql.md) | Создаёт таблицу с движком [MySQL](../../sql_reference/table_functions/index.md). | +| [jdbc](jdbc.md) | Создаёт таблицу с дижком [JDBC](../../sql_reference/table_functions/index.md). | +| [odbc](odbc.md) | Создаёт таблицу с движком [ODBC](../../sql_reference/table_functions/index.md). | +| [hdfs](hdfs.md) | Создаёт таблицу с движком [HDFS](../../sql_reference/table_functions/index.md). | [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/) diff --git a/docs/ru/query_language/table_functions/input.md b/docs/ru/sql_reference/table_functions/input.md similarity index 100% rename from docs/ru/query_language/table_functions/input.md rename to docs/ru/sql_reference/table_functions/input.md diff --git a/docs/ru/query_language/table_functions/jdbc.md b/docs/ru/sql_reference/table_functions/jdbc.md similarity index 100% rename from docs/ru/query_language/table_functions/jdbc.md rename to docs/ru/sql_reference/table_functions/jdbc.md diff --git a/docs/ru/query_language/table_functions/merge.md b/docs/ru/sql_reference/table_functions/merge.md similarity index 100% rename from docs/ru/query_language/table_functions/merge.md rename to docs/ru/sql_reference/table_functions/merge.md diff --git a/docs/ru/query_language/table_functions/mysql.md b/docs/ru/sql_reference/table_functions/mysql.md similarity index 93% rename from docs/ru/query_language/table_functions/mysql.md rename to docs/ru/sql_reference/table_functions/mysql.md index 228b0bbf38e..a26f89ced2f 100644 --- a/docs/ru/query_language/table_functions/mysql.md +++ b/docs/ru/sql_reference/table_functions/mysql.md @@ -73,7 +73,7 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123') ## Смотрите также {#smotrite-takzhe} -- [Движок таблиц ‘MySQL’](../../operations/table_engines/mysql.md) -- [Использование MySQL как источника данных для внешнего словаря](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql) +- [Движок таблиц ‘MySQL’](../../sql_reference/table_functions/mysql.md) +- [Использование MySQL как источника данных для внешнего словаря](../../sql_reference/table_functions/mysql.md#dicts-external_dicts_dict_sources-mysql) [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/mysql/) diff --git a/docs/ru/query_language/table_functions/numbers.md b/docs/ru/sql_reference/table_functions/numbers.md similarity index 100% rename from docs/ru/query_language/table_functions/numbers.md rename to docs/ru/sql_reference/table_functions/numbers.md diff --git a/docs/ru/query_language/table_functions/odbc.md b/docs/ru/sql_reference/table_functions/odbc.md similarity index 95% rename from docs/ru/query_language/table_functions/odbc.md rename to docs/ru/sql_reference/table_functions/odbc.md index bff2c23cf47..0d277b2b26d 100644 --- a/docs/ru/query_language/table_functions/odbc.md +++ b/docs/ru/sql_reference/table_functions/odbc.md @@ -95,7 +95,7 @@ SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test') ## Смотрите также {#smotrite-takzhe} -- [Внешние словари ODBC](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc) -- [Движок таблиц ODBC](../../operations/table_engines/odbc.md). +- [Внешние словари ODBC](../../sql_reference/table_functions/odbc.md#dicts-external_dicts_dict_sources-odbc) +- [Движок таблиц ODBC](../../sql_reference/table_functions/odbc.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/jdbc/) diff --git a/docs/ru/query_language/table_functions/remote.md b/docs/ru/sql_reference/table_functions/remote.md similarity index 100% rename from docs/ru/query_language/table_functions/remote.md rename to docs/ru/sql_reference/table_functions/remote.md diff --git a/docs/ru/query_language/table_functions/url.md b/docs/ru/sql_reference/table_functions/url.md similarity index 100% rename from docs/ru/query_language/table_functions/url.md rename to docs/ru/sql_reference/table_functions/url.md diff --git a/docs/ru/whats_new/changelog/2017.md b/docs/ru/whats_new/changelog/2017.md new file mode 120000 index 00000000000..a098eddf1d8 --- /dev/null +++ b/docs/ru/whats_new/changelog/2017.md @@ -0,0 +1 @@ +../../../en/whats_new/changelog/2017.md \ No newline at end of file diff --git a/docs/ru/whats_new/changelog/2018.md b/docs/ru/whats_new/changelog/2018.md new file mode 120000 index 00000000000..124fb19e175 --- /dev/null +++ b/docs/ru/whats_new/changelog/2018.md @@ -0,0 +1 @@ +../../../en/whats_new/changelog/2018.md \ No newline at end of file diff --git a/docs/ru/whats_new/changelog/2019.md b/docs/ru/whats_new/changelog/2019.md new file mode 120000 index 00000000000..740d1edd238 --- /dev/null +++ b/docs/ru/whats_new/changelog/2019.md @@ -0,0 +1 @@ +../../../en/whats_new/changelog/2019.md \ No newline at end of file diff --git a/docs/ru/whats_new/changelog/index.md b/docs/ru/whats_new/changelog/index.md new file mode 100644 index 00000000000..969e71fbf42 --- /dev/null +++ b/docs/ru/whats_new/changelog/index.md @@ -0,0 +1,650 @@ +--- +toc_folder_title: Changelog +toc_priority: 74 +toc_title: '2020' +--- + +## ClickHouse release v20.3 + +### ClickHouse release v20.3.4.10, 2020-03-20 + +#### Bug Fix +* This release also contains all bug fixes from 20.1.8.41 +* Fix missing `rows_before_limit_at_least` for queries over http (with processors pipeline). This fixes [#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + + +### ClickHouse release v20.3.3.6, 2020-03-17 + +#### Bug Fix +* This release also contains all bug fixes from 20.1.7.38 +* Fix bug in a replication that doesn't allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. +* Add setting `use_compact_format_in_distributed_parts_names` which allows to write files for `INSERT` queries into `Distributed` table with more compact format. This fixes [#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. + +### ClickHouse release v20.3.2.1, 2020-03-12 + +#### Backward Incompatible Change + +* Fixed the issue `file name too long` when sending data for `Distributed` tables for a large number of replicas. Fixed the issue that replica credentials were exposed in the server log. The format of directory name on disk was changed to `[shard{shard_index}[_replica{replica_index}]]`. [#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([Mikhail Korotov](https://github.com/millb)) After you upgrade to the new version, you will not be able to downgrade without manual intervention, because old server version does not recognize the new directory format. If you want to downgrade, you have to manually rename the corresponding directories to the old format. This change is relevant only if you have used asynchronous `INSERT`s to `Distributed` tables. In the version 20.3.3 we will introduce a setting that will allow you to enable the new format gradually. +* Changed the format of replication log entries for mutation commands. You have to wait for old mutations to process before installing the new version. +* Implement simple memory profiler that dumps stacktraces to `system.trace_log` every N bytes over soft allocation limit [#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([Ivan](https://github.com/abyss7)) [#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([alexey-milovidov](https://github.com/alexey-milovidov)) The column of `system.trace_log` was renamed from `timer_type` to `trace_type`. This will require changes in third-party performance analysis and flamegraph processing tools. +* Use OS thread id everywhere instead of internal thread number. This fixes [#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) Old `clickhouse-client` cannot receive logs that are send from the server when the setting `send_logs_level` is enabled, because the names and types of the structured log messages were changed. On the other hand, different server versions can send logs with different types to each other. When you don't use the `send_logs_level` setting, you should not care. [#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove `indexHint` function [#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove `findClusterIndex`, `findClusterValue` functions. This fixes [#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). If you were using these functions, send an email to `clickhouse-feedback@yandex-team.com` [#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now it's not allowed to create columns or add columns with `SELECT` subquery as default expression. [#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([alesapin](https://github.com/alesapin)) +* Require aliases for subqueries in JOIN. [#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) +* Improved `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn't change type of column and `MODIFY` type doesn't loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alesapin](https://github.com/alesapin)) +* Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* The setting `experimental_use_processors` is enabled by default. This setting enables usage of the new query pipeline. This is internal refactoring and we expect no visible changes. If you will see any issues, set it to back zero. [#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### New Feature +* Add `Avro` and `AvroConfluent` input/output formats [#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([Andrew Onyshchuk](https://github.com/oandrew)) [#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([Andrew Onyshchuk](https://github.com/oandrew)) [#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Multi-threaded and non-blocking updates of expired keys in `cache` dictionaries (with optional permission to read old ones). [#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Add query `ALTER ... MATERIALIZE TTL`. It runs mutation that forces to remove expired data by TTL and recalculates meta-information about TTL in all parts. [#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([Anton Popov](https://github.com/CurtizJ)) +* Switch from HashJoin to MergeJoin (on disk) if needed [#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([Artem Zuikov](https://github.com/4ertus2)) +* Added `MOVE PARTITION` command for `ALTER TABLE` [#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Reloading storage configuration from configuration file on the fly. [#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Allowed to change `storage_policy` to not less rich one. [#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Added support for globs/wildcards for S3 storage and table function. [#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Implement `bitAnd`, `bitOr`, `bitXor`, `bitNot` for `FixedString(N)` datatype. [#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Added function `bitCount`. This fixes [#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([alexey-milovidov](https://github.com/alexey-milovidov)) [#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([ikopylov](https://github.com/ikopylov)) +* Add `generateRandom` table function to generate random rows with given schema. Allows to populate arbitrary test table with data. [#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([Ilya Yatsishin](https://github.com/qoega)) +* `JSONEachRowFormat`: support special case when objects enclosed in top-level array. [#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([Kruglov Pavel](https://github.com/Avogar)) +* Now it's possible to create a column with `DEFAULT` expression which depends on a column with default `ALIAS` expression. [#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([alesapin](https://github.com/alesapin)) +* Allow to specify `--limit` more than the source data size in `clickhouse-obfuscator`. The data will repeat itself with different random seed. [#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added `groupArraySample` function (similar to `groupArray`) with reservior sampling algorithm. [#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([Amos Bird](https://github.com/amosbird)) +* Now you can monitor the size of update queue in `cache`/`complex_key_cache` dictionaries via system metrics. [#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Allow to use CRLF as a line separator in CSV output format with setting `output_format_csv_crlf_end_of_line` is set to 1 [#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([Mikhail Korotov](https://github.com/millb)) +* Implement more functions of the [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` and `stringToH3` [#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([Nico Mandery](https://github.com/nmandery)) +* New setting introduced: `max_parser_depth` to control maximum stack size and allow large complex queries. This fixes [#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) and [#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([Maxim Smirnov](https://github.com/qMBQx8GH)) +* Add a setting `force_optimize_skip_unused_shards` setting to throw if skipping of unused shards is not possible [#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([Azat Khuzhin](https://github.com/azat)) +* Allow to configure multiple disks/volumes for storing data for send in `Distributed` engine [#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([Azat Khuzhin](https://github.com/azat)) +* Support storage policy (``) for storing temporary data. [#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([Azat Khuzhin](https://github.com/azat)) +* Added `X-ClickHouse-Exception-Code` HTTP header that is set if exception was thrown before sending data. This implements [#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([Mikhail Korotov](https://github.com/millb)) +* Added function `ifNotFinite`. It is just a syntactic sugar: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added `last_successful_update_time` column in `system.dictionaries` table [#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Add `blockSerializedSize` function (size on disk without compression) [#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([Azat Khuzhin](https://github.com/azat)) +* Add function `moduloOrZero` [#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([hcz](https://github.com/hczhcz)) +* Added system tables `system.zeros` and `system.zeros_mt` as well as tale functions `zeros()` and `zeros_mt()`. Tables (and table functions) contain single column with name `zero` and type `UInt8`. This column contains zeros. It is needed for test purposes as the fastest method to generate many rows. This fixes [#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Experimental Feature +* Add new compact format of parts in `MergeTree`-family tables in which all columns are stored in one file. It helps to increase performance of small and frequent inserts. The old format (one file per column) is now called wide. Data storing format is controlled by settings `min_bytes_for_wide_part` and `min_rows_for_wide_part`. [#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([Anton Popov](https://github.com/CurtizJ)) +* Support for S3 storage for `Log`, `TinyLog` and `StripeLog` tables. [#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([Pavel Kovalenko](https://github.com/Jokser)) + +#### Bug Fix +* Fixed inconsistent whitespaces in log messages. [#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix bug in which arrays of unnamed tuples were flattened as Nested structures on table creation. [#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([achulkov2](https://github.com/achulkov2)) +* Fixed the issue when "Too many open files" error may happen if there are too many files matching glob pattern in `File` table or `file` table function. Now files are opened lazily. This fixes [#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* DROP TEMPORARY TABLE now drops only temporary table. [#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([Vitaly Baranov](https://github.com/vitlibar)) +* Remove outdated partition when we shutdown the server or DETACH/ATTACH a table. [#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([Guillaume Tassery](https://github.com/YiuRULE)) +* For how the default disk calculates the free space from `data` subdirectory. Fixed the issue when the amount of free space is not calculated correctly if the `data` directory is mounted to a separate device (rare case). This fixes [#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([Mikhail Korotov](https://github.com/millb)) +* Allow comma (cross) join with IN () inside. [#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([Artem Zuikov](https://github.com/4ertus2)) +* Allow to rewrite CROSS to INNER JOIN if there's [NOT] LIKE operator in WHERE section. [#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix possible incorrect result after `GROUP BY` with enabled setting `distributed_aggregation_memory_efficient`. Fixes [#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Found keys were counted as missed in metrics of cache dictionaries. [#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix replication protocol incompatibility introduced in [#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) +* Fixed race condition on `queue_task_handle` at the startup of `ReplicatedMergeTree` tables. [#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* The token `NOT` didn't work in `SHOW TABLES NOT LIKE` query [#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added range check to function `h3EdgeLengthM`. Without this check, buffer overflow is possible. [#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed up a bug in batched calculations of ternary logical OPs on multiple arguments (more than 10). [#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) +* Fix error of PREWHERE optimization, which could lead to segfaults or `Inconsistent number of columns got from MergeTreeRangeReader` exception. [#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([Anton Popov](https://github.com/CurtizJ)) +* Fix unexpected `Timeout exceeded while reading from socket` exception, which randomly happens on secure connection before timeout actually exceeded and when query profiler is enabled. Also add `connect_timeout_with_failover_secure_ms` settings (default 100ms), which is similar to `connect_timeout_with_failover_ms`, but is used for secure connections (because SSL handshake is slower, than ordinary TCP connection) [#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([tavplubix](https://github.com/tavplubix)) +* Fix bug with mutations finalization, when mutation may hang in state with `parts_to_do=0` and `is_done=0`. [#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([alesapin](https://github.com/alesapin)) +* Use new ANY JOIN logic with `partial_merge_join` setting. It's possible to make `ANY|ALL|SEMI LEFT` and `ALL INNER` joins with `partial_merge_join=1` now. [#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([Artem Zuikov](https://github.com/4ertus2)) +* Shard now clamps the settings got from the initiator to the shard's constaints instead of throwing an exception. This fix allows to send queries to a shard with another constraints. [#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fixed memory management problem in `MergeTreeReadPool`. [#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix `toDecimal*OrNull()` functions family when called with string `e`. Fixes [#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([Artem Zuikov](https://github.com/4ertus2)) +* Make sure that `FORMAT Null` sends no data to the client. [#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Fix bug that timestamp in `LiveViewBlockInputStream` will not updated. `LIVE VIEW` is an experimental feature. [#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([vxider](https://github.com/Vxider)) [#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([vxider](https://github.com/Vxider)) +* Fixed `ALTER MODIFY TTL` wrong behavior which did not allow to delete old TTL expressions. [#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fixed UBSan report in MergeTreeIndexSet. This fixes [#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed the behaviour of `match` and `extract` functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. This fixes [#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid throwing from destructor in Apache Avro 3rd-party library. [#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([Andrew Onyshchuk](https://github.com/oandrew)) +* Don't commit a batch polled from `Kafka` partially as it can lead to holes in data. [#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([filimonov](https://github.com/filimonov)) +* Fix `joinGet` with nullable return types. https://github.com/ClickHouse/ClickHouse/issues/8919 [#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([Amos Bird](https://github.com/amosbird)) +* Fix data incompatibility when compressed with `T64` codec. [#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([Artem Zuikov](https://github.com/4ertus2)) Fix data type ids in `T64` compression codec that leads to wrong (de)compression in affected versions. [#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([Artem Zuikov](https://github.com/4ertus2)) +* Add setting `enable_early_constant_folding` and disable it in some cases that leads to errors. [#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix pushdown predicate optimizer with VIEW and enable the test [#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([Winter Zhang](https://github.com/zhang2014)) +* Fix segfault in `Merge` tables, that can happen when reading from `File` storages [#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([tavplubix](https://github.com/tavplubix)) +* Added a check for storage policy in `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. Otherwise it could make data of part inaccessible after restart and prevent ClickHouse to start. [#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix alters if there is TTL set for table. [#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([Anton Popov](https://github.com/CurtizJ)) +* Fix race condition that can happen when `SYSTEM RELOAD ALL DICTIONARIES` is executed while some dictionary is being modified/added/removed. [#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([Vitaly Baranov](https://github.com/vitlibar)) +* In previous versions `Memory` database engine use empty data path, so tables are created in `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([tavplubix](https://github.com/tavplubix)) +* Fixed wrong log messages about missing default disk or policy. [#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix not(has()) for the bloom_filter index of array types. [#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) +* Allow first column(s) in a table with `Log` engine be an alias [#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([Ivan](https://github.com/abyss7)) +* Fix order of ranges while reading from `MergeTree` table in one thread. It could lead to exceptions from `MergeTreeRangeReader` or wrong query results. [#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([Anton Popov](https://github.com/CurtizJ)) +* Make `reinterpretAsFixedString` to return `FixedString` instead of `String`. [#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([Andrew Onyshchuk](https://github.com/oandrew)) +* Avoid extremely rare cases when the user can get wrong error message (`Success` instead of detailed error description). [#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Do not crash when using `Template` format with empty row template. [#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Metadata files for system tables could be created in wrong place [#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([tavplubix](https://github.com/tavplubix)) Fixes [#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). +* Fix data race on exception_ptr in cache dictionary [#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Do not throw an exception for query `ATTACH TABLE IF NOT EXISTS`. Previously it was thrown if table already exists, despite the `IF NOT EXISTS` clause. [#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([Anton Popov](https://github.com/CurtizJ)) +* Fixed missing closing paren in exception message. [#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid message `Possible deadlock avoided` at the startup of clickhouse-client in interactive mode. [#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed the issue when padding at the end of base64 encoded value can be malformed. Update base64 library. This fixes [#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), closes [#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Prevent losing data in `Kafka` in rare cases when exception happens after reading suffix but before commit. Fixes [#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([filimonov](https://github.com/filimonov)) +* Fixed exception in `DROP TABLE IF EXISTS` [#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([Nikita Vasilev](https://github.com/nikvas0)) +* Fix crash when a user tries to `ALTER MODIFY SETTING` for old-formated `MergeTree` table engines family. [#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) +* Support for UInt64 numbers that don't fit in Int64 in JSON-related functions. Update SIMDJSON to master. This fixes [#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed execution of inversed predicates when non-strictly monotinic functional index is used. [#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([Alexander Kazakov](https://github.com/Akazz)) +* Don't try to fold `IN` constant in `GROUP BY` [#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([Amos Bird](https://github.com/amosbird)) +* Fix bug in `ALTER DELETE` mutations which leads to index corruption. This fixes [#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) and [#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). Additionally fix extremely rare race conditions in `ReplicatedMergeTree` `ALTER` queries. [#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([alesapin](https://github.com/alesapin)) +* When the setting `compile_expressions` is enabled, you can get `unexpected column` in `LLVMExecutableFunction` when we use `Nullable` type [#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Multiple fixes for `Kafka` engine: 1) fix duplicates that were appearing during consumer group rebalance. 2) Fix rare 'holes' appeared when data were polled from several partitions with one poll and committed partially (now we always process / commit the whole polled block of messages). 3) Fix flushes by block size (before that only flushing by timeout was working properly). 4) better subscription procedure (with assignment feedback). 5) Make tests work faster (with default intervals and timeouts). Due to the fact that data was not flushed by block size before (as it should according to documentation), that PR may lead to some performance degradation with default settings (due to more often & tinier flushes which are less optimal). If you encounter the performance issue after that change - please increase `kafka_max_block_size` in the table to the bigger value ( for example `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). Fixes [#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([filimonov](https://github.com/filimonov)) +* Fix `Parameter out of bound` exception in some queries after PREWHERE optimizations. [#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) +* Fixed the case of mixed-constness of arguments of function `arrayZip`. [#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* When executing `CREATE` query, fold constant expressions in storage engine arguments. Replace empty database name with current database. Fixes [#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([tavplubix](https://github.com/tavplubix)) +* Now it's not possible to create or add columns with simple cyclic aliases like `a DEFAULT b, b DEFAULT a`. [#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) +* Fixed a bug with double move which may corrupt original part. This is relevant if you use `ALTER TABLE MOVE` [#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Allow `interval` identifier to correctly parse without backticks. Fixed issue when a query cannot be executed even if the `interval` identifier is enclosed in backticks or double quotes. This fixes [#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed fuzz test and incorrect behaviour of `bitTestAll`/`bitTestAny` functions. [#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix possible crash/wrong number of rows in `LIMIT n WITH TIES` when there are a lot of rows equal to n'th row. [#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) +* Fix mutations with parts written with enabled `insert_quorum`. [#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([alesapin](https://github.com/alesapin)) +* Fix data race at destruction of `Poco::HTTPServer`. It could happen when server is started and immediately shut down. [#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) +* Fix bug in which a misleading error message was shown when running `SHOW CREATE TABLE a_table_that_does_not_exist`. [#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2](https://github.com/achulkov2)) +* Fixed `Parameters are out of bound` exception in some rare cases when we have a constant in the `SELECT` clause when we have an `ORDER BY` and a `LIMIT` clause. [#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Fix mutations finalization, when already done mutation can have status `is_done=0`. [#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([alesapin](https://github.com/alesapin)) +* Prevent from executing `ALTER ADD INDEX` for MergeTree tables with old syntax, because it doesn't work. [#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) +* During server startup do not access table, which `LIVE VIEW` depends on, so server will be able to start. Also remove `LIVE VIEW` dependencies when detaching `LIVE VIEW`. `LIVE VIEW` is an experimental feature. [#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) +* Fix possible segfault in `MergeTreeRangeReader`, while executing `PREWHERE`. [#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([Anton Popov](https://github.com/CurtizJ)) +* Fix possible mismatched checksums with column TTLs. [#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) +* Fixed a bug when parts were not being moved in background by TTL rules in case when there is only one volume. [#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fixed the issue `Method createColumn() is not implemented for data type Set`. This fixes [#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now we will try finalize mutations more frequently. [#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) +* Fix `intDiv` by minus one constant [#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([hcz](https://github.com/hczhcz)) +* Fix possible race condition in `BlockIO`. [#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix bug leading to server termination when trying to use / drop `Kafka` table created with wrong parameters. [#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([filimonov](https://github.com/filimonov)) +* Added workaround if OS returns wrong result for `timer_create` function. [#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed error in usage of `min_marks_for_seek` parameter. Fixed the error message when there is no sharding key in Distributed table and we try to skip unused shards. [#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([Azat Khuzhin](https://github.com/azat)) + +#### Improvement +* Implement `ALTER MODIFY/DROP` queries on top of mutations for `ReplicatedMergeTree*` engines family. Now `ALTERS` blocks only at the metadata update stage, and don't block after that. [#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([alesapin](https://github.com/alesapin)) +* Add ability to rewrite CROSS to INNER JOINs with `WHERE` section containing unqialified names. [#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([Artem Zuikov](https://github.com/4ertus2)) +* Make `SHOW TABLES` and `SHOW DATABASES` queries support the `WHERE` expressions and `FROM`/`IN` [#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([sundyli](https://github.com/sundy-li)) +* Added a setting `deduplicate_blocks_in_dependent_materialized_views`. [#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([urykhy](https://github.com/urykhy)) +* After recent changes MySQL client started to print binary strings in hex thereby making them not readable ([#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). The workaround in ClickHouse is to mark string columns as UTF-8, which is not always, but usually the case. [#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([Yuriy Baranov](https://github.com/yurriy)) +* Add support of String and FixedString keys for `sumMap` [#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) +* Support string keys in SummingMergeTree maps [#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) +* Signal termination of thread to the thread pool even if the thread has thrown exception [#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +* Allow to set `query_id` in `clickhouse-benchmark` [#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([Anton Popov](https://github.com/CurtizJ)) +* Don't allow strange expressions in `ALTER TABLE ... PARTITION partition` query. This addresses [#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* The table `system.table_engines` now provides information about feature support (like `supports_ttl` or `supports_sort_order`). [#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([Max Akhmedov](https://github.com/zlobober)) +* Enable `system.metric_log` by default. It will contain rows with values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval (one second by default). The table is very small (usually in order of megabytes) and collecting this data by default is reasonable. [#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([Ivan](https://github.com/abyss7)) +* Now temporary `LIVE VIEW` is created by `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` instead of `CREATE TEMPORARY LIVE VIEW ...`, because the previous syntax was not consistent with `CREATE TEMPORARY TABLE ...` [#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([tavplubix](https://github.com/tavplubix)) +* Add text_log.level configuration parameter to limit entries that goes to `system.text_log` table [#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([Azat Khuzhin](https://github.com/azat)) +* Allow to put downloaded part to a disks/volumes according to TTL rules [#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([Vladimir Chebotarev](https://github.com/excitoon)) +* For external MySQL dictionaries, allow to mutualize MySQL connection pool to "share" them among dictionaries. This option significantly reduces the number of connections to MySQL servers. [#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) +* Show nearest query execution time for quantiles in `clickhouse-benchmark` output instead of interpolated values. It's better to show values that correspond to the execution time of some queries. [#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Possibility to add key & timestamp for the message when inserting data to Kafka. Fixes [#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([filimonov](https://github.com/filimonov)) +* If server is run from terminal, highlight thread number, query id and log priority by colors. This is for improved readability of correlated log messages for developers. [#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better exception message while loading tables for `Ordinary` database. [#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Implement `arraySlice` for arrays with aggregate function states. This fixes [#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Allow constant functions and constant arrays to be used on the right side of IN operator. [#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([Anton Popov](https://github.com/CurtizJ)) +* If zookeeper exception has happened while fetching data for system.replicas, display it in a separate column. This implements [#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Atomically remove MergeTree data parts on destroy. [#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Support row-level security for Distributed tables. [#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([Ivan](https://github.com/abyss7)) +* Now we recognize suffix (like KB, KiB...) in settings values. [#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([Mikhail Korotov](https://github.com/millb)) +* Prevent out of memory while constructing result of a large JOIN. [#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([Artem Zuikov](https://github.com/4ertus2)) +* Added names of clusters to suggestions in interactive mode in `clickhouse-client`. [#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([Ivan](https://github.com/abyss7)) +* Added column `exception_code` in `system.query_log` table. [#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([Mikhail Korotov](https://github.com/millb)) +* Enabled MySQL compatibility server on port `9004` in the default server configuration file. Fixed password generation command in the example in configuration. [#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([Yuriy Baranov](https://github.com/yurriy)) +* Prevent abort on shutdown if the filesystem is readonly. This fixes [#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better exception message when length is required in HTTP POST query. [#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add `_path` and `_file` virtual columns to `HDFS` and `File` engines and `hdfs` and `file` table functions [#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([Olga Khvostikova](https://github.com/stavrolia)) +* Fix error `Cannot find column` while inserting into `MATERIALIZED VIEW` in case if new column was added to view's internal table. [#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix progress over native client-server protocol, by send progress after final update (like logs). This may be relevant only to some third-party tools that are using native protocol. [#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([Azat Khuzhin](https://github.com/azat)) +* Add a system metric tracking the number of client connections using MySQL protocol ([#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([Eugene Klimov](https://github.com/Slach)) +* From now on, HTTP responses will have `X-ClickHouse-Timezone` header set to the same timezone value that `SELECT timezone()` would report. [#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([Denis Glazachev](https://github.com/traceon)) + +#### Performance Improvement +* Improve performance of analysing index with IN [#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([Anton Popov](https://github.com/CurtizJ)) +* Simpler and more efficient code in Logical Functions + code cleanups. A followup to [#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([Alexander Kazakov](https://github.com/Akazz)) +* Overall performance improvement (in range of 5%..200% for affected queries) by ensuring even more strict aliasing with C++20 features. [#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([Amos Bird](https://github.com/amosbird)) +* More strict aliasing for inner loops of comparison functions. [#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* More strict aliasing for inner loops of arithmetic functions. [#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* A ~3 times faster implementation for ColumnVector::replicate(), via which ColumnConst::convertToFullColumn() is implemented. Also will be useful in tests when materializing constants. [#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([Alexander Kazakov](https://github.com/Akazz)) +* Another minor performance improvement to `ColumnVector::replicate()` (this speeds up the `materialize` function and higher order functions) an even further improvement to [#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([Alexander Kazakov](https://github.com/Akazz)) +* Improved performance of `stochasticLinearRegression` aggregate function. This patch is contributed by Intel. [#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improve performance of `reinterpretAsFixedString` function. [#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Do not send blocks to client for `Null` format in processors pipeline. [#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) + +#### Build/Testing/Packaging Improvement +* Exception handling now works correctly on Windows Subsystem for Linux. See https://github.com/ClickHouse-Extras/libunwind/pull/3 This fixes [#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) +* Replace `readline` with `replxx` for interactive line editing in `clickhouse-client` [#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([Ivan](https://github.com/abyss7)) +* Better build time and less template instantiations in FunctionsComparison. [#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added integration with `clang-tidy` in CI. See also [#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now we link ClickHouse in CI using `lld` even for `gcc`. [#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([alesapin](https://github.com/alesapin)) +* Allow to randomize thread scheduling and insert glitches when `THREAD_FUZZER_*` environment variables are set. This helps testing. [#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enable secure sockets in stateless tests [#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([tavplubix](https://github.com/tavplubix)) +* Make SPLIT_SHARED_LIBRARIES=OFF more robust [#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([Azat Khuzhin](https://github.com/azat)) +* Make "performance_introspection_and_logging" test reliable to random server stuck. This may happen in CI environment. See also [#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Validate XML in style check. [#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed race condition in test `00738_lock_for_inner_table`. This test relied on sleep. [#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove performance tests of type `once`. This is needed to run all performance tests in statistical comparison mode (more reliable). [#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added performance test for arithmetic functions. [#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added performance test for `sumMap` and `sumMapWithOverflow` aggregate functions. Follow-up for [#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Ensure style of ErrorCodes by style check. [#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add script for tests history. [#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([alesapin](https://github.com/alesapin)) +* Add GCC warning `-Wsuggest-override` to locate and fix all places where `override` keyword must be used. [#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +* Ignore weak symbol under Mac OS X because it must be defined [#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([Deleted user](https://github.com/ghost)) +* Normalize running time of some queries in performance tests. This is done in preparation to run all the performance tests in comparison mode. [#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix some tests to support pytest with query tests [#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([Ivan](https://github.com/abyss7)) +* Enable SSL in build with MSan, so server will not fail at startup when running stateless tests [#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([tavplubix](https://github.com/tavplubix)) +* Fix database substitution in test results [#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([Ilya Yatsishin](https://github.com/qoega)) +* Build fixes for miscellaneous platforms [#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([proller](https://github.com/proller)) [#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([proller](https://github.com/proller)) [#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([proller](https://github.com/proller)) +* Added disks section to stateless-with-coverage test docker image [#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([Pavel Kovalenko](https://github.com/Jokser)) +* Get rid of in-source-tree files when building with GRPC [#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([Amos Bird](https://github.com/amosbird)) +* Slightly faster build time by removing SessionCleaner from Context. Make the code of SessionCleaner more simple. [#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Updated checking for hung queries in clickhouse-test script [#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) +* Removed some useless files from repository. [#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Changed type of math perftests from `once` to `loop`. [#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) +* Suppress some test failures under MSan. [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Speedup "exception while insert" test. This test often time out in debug-with-coverage build. [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Updated `libcxx` and `libcxxabi` to master. In preparation to [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix flacky test `00910_zookeeper_test_alter_compression_codecs`. [#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Clean up duplicated linker flags. Make sure the linker won't look up an unexpected symbol. [#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([Amos Bird](https://github.com/amosbird)) +* Add `clickhouse-odbc` driver into test images. This allows to test interaction of ClickHouse with ClickHouse via its own ODBC driver. [#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov)) +* Fix several bugs in unit tests. [#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([alesapin](https://github.com/alesapin)) +* Enable `-Wmissing-include-dirs` GCC warning to eliminate all non-existing includes - mostly as a result of CMake scripting errors [#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +* Describe reasons if query profiler cannot work. This is intended for [#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update OpenSSL to upstream master. Fixed the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` and `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. The issue was present in version 20.1. [#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update Dockerfile for server [#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([Ilya Mazaev](https://github.com/ne-ray)) +* Minor fixes in build-gcc-from-sources script [#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([Michael Nacharov](https://github.com/mnach)) +* Replace `numbers` to `zeros` in perftests where `number` column is not used. This will lead to more clean test results. [#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix stack overflow issue when using initializer_list in Column constructors. [#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([Deleted user](https://github.com/ghost)) +* Upgrade librdkafka to v1.3.0. Enable bundled `rdkafka` and `gsasl` libraries on Mac OS X. [#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([Andrew Onyshchuk](https://github.com/oandrew)) +* build fix on GCC 9.2.0 [#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([vxider](https://github.com/Vxider)) + + +## ClickHouse release v20.1 + +### ClickHouse release v20.1.8.41, 2020-03-20 + +#### Bug Fix +* Fix possible permanent `Cannot schedule a task` error (due to unhandled exception in `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). This fixes [#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) +* Fix excessive memory consumption in `ALTER` queries (mutations). This fixes [#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) and [#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alesapin](https://github.com/alesapin)) +* Fix bug in backquoting in external dictionaries DDL. This fixes [#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) + +### ClickHouse release v20.1.7.38, 2020-03-18 + +#### Bug Fix +* Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. I lead to exception while using this functions in remote queries. [#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). This issue was in all ClickHouse releases. +* Allow `ALTER ON CLUSTER` of `Distributed` tables with internal replication. This fixes [#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)). This issue was in all ClickHouse releases. +* Fix possible exceptions `Size of filter doesn't match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. Fixes [#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) +* Fixed the issue: timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)). This issue was in all ClickHouse releases. +* Now it's not possible to create or add columns with simple cyclic aliases like `a DEFAULT b, b DEFAULT a`. [#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) +* Fixed the issue when padding at the end of base64 encoded value can be malformed. Update base64 library. This fixes [#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), closes [#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix data race at destruction of `Poco::HTTPServer`. It could happen when server is started and immediately shut down. [#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) +* Fix possible crash/wrong number of rows in `LIMIT n WITH TIES` when there are a lot of rows equal to n'th row. [#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) +* Fix possible mismatched checksums with column TTLs. [#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) +* Fix crash when a user tries to `ALTER MODIFY SETTING` for old-formated `MergeTree` table engines family. [#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) +* Now we will try finalize mutations more frequently. [#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) +* Fix replication protocol incompatibility introduced in [#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) +* Fix not(has()) for the bloom_filter index of array types. [#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) +* Fixed the behaviour of `match` and `extract` functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. This fixes [#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Build/Testing/Packaging Improvement + +* Exception handling now works correctly on Windows Subsystem for Linux. See https://github.com/ClickHouse-Extras/libunwind/pull/3 This fixes [#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) + + +### ClickHouse release v20.1.6.30, 2020-03-05 + +#### Bug Fix + +* Fix data incompatibility when compressed with `T64` codec. +[#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [(abyss7)](https://github.com/abyss7) +* Fix order of ranges while reading from MergeTree table in one thread. Fixes [#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). +[#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(CurtizJ)](https://github.com/CurtizJ) +* Fix possible segfault in `MergeTreeRangeReader`, while executing `PREWHERE`. Fixes [#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). +[#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(CurtizJ)](https://github.com/CurtizJ) +* Fix `reinterpretAsFixedString` to return `FixedString` instead of `String`. +[#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [(oandrew)](https://github.com/oandrew) +* Fix `joinGet` with nullable return types. Fixes [#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) +[#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [(amosbird)](https://github.com/amosbird) +* Fix fuzz test and incorrect behaviour of bitTestAll/bitTestAny functions. +[#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [(alexey-milovidov)](https://github.com/alexey-milovidov) +* Fix the behaviour of match and extract functions when haystack has zero bytes. The behaviour was wrong when haystack was constant. Fixes [#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) +[#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [(alexey-milovidov)](https://github.com/alexey-milovidov) +* Fixed execution of inversed predicates when non-strictly monotinic functional index is used. Fixes [#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) +[#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [(Akazz)](https://github.com/Akazz) +* Allow to rewrite `CROSS` to `INNER JOIN` if there's `[NOT] LIKE` operator in `WHERE` section. Fixes [#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) +[#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4ertus2)](https://github.com/4ertus2) +* Allow first column(s) in a table with Log engine be an alias. +[#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(abyss7)](https://github.com/abyss7) +* Allow comma join with `IN()` inside. Fixes [#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). +[#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4ertus2)](https://github.com/4ertus2) +* Improve `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn't change type of column and `MODIFY` type doesn't loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). +[#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) +* Fix mutations finalization, when already done mutation can have status is_done=0. +[#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) +* Support "Processors" pipeline for system.numbers and system.numbers_mt. This also fixes the bug when `max_execution_time` is not respected. +[#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [(KochetovNicolai)](https://github.com/KochetovNicolai) +* Fix wrong counting of `DictCacheKeysRequestedFound` metric. +[#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [(nikitamikhaylov)](https://github.com/nikitamikhaylov) +* Added a check for storage policy in `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` which otherwise could make data of part inaccessible after restart and prevent ClickHouse to start. +[#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [(excitoon)](https://github.com/excitoon) +* Fixed UBSan report in `MergeTreeIndexSet`. This fixes [#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) +[#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [(alexey-milovidov)](https://github.com/alexey-milovidov) +* Fix possible datarace in BlockIO. +[#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [(KochetovNicolai)](https://github.com/KochetovNicolai) +* Support for `UInt64` numbers that don't fit in Int64 in JSON-related functions. Update `SIMDJSON` to master. This fixes [#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) +[#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [(alexey-milovidov)](https://github.com/alexey-milovidov) +* Fix the issue when the amount of free space is not calculated correctly if the data directory is mounted to a separate device. For default disk calculate the free space from data subdirectory. This fixes [#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) +[#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [(millb)](https://github.com/millb) +* Fix the issue when TLS connections may fail with the message `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` Update OpenSSL to upstream master. +[#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [(alexey-milovidov)](https://github.com/alexey-milovidov) +* When executing `CREATE` query, fold constant expressions in storage engine arguments. Replace empty database name with current database. Fixes [#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Also fix check for local address in ClickHouseDictionarySource. +[#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) +* Fix segfault in `StorageMerge`, which can happen when reading from StorageFile. +[#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [(tabplubix)](https://github.com/tavplubix) +* Prevent losing data in `Kafka` in rare cases when exception happens after reading suffix but before commit. Fixes [#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). Related: [#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) +[#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) +* Fix bug leading to server termination when trying to use / drop `Kafka` table created with wrong parameters. Fixes [#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Incorporates [#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). +[#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) + +#### New Feature +* Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity. +[#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) + +### ClickHouse release v20.1.2.4, 2020-01-22 + +#### Backward Incompatible Change +* Make the setting `merge_tree_uniform_read_distribution` obsolete. The server still recognizes this setting but it has no effect. [#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Changed return type of the function `greatCircleDistance` to `Float32` because now the result of calculation is `Float32`. [#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now it's expected that query parameters are represented in "escaped" format. For example, to pass string `ab` you have to write `a\tb` or `a\b` and respectively, `a%5Ctb` or `a%5C%09b` in URL. This is needed to add the possibility to pass NULL as `\N`. This fixes [#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enable `use_minimalistic_part_header_in_zookeeper` setting for `ReplicatedMergeTree` by default. This will significantly reduce amount of data stored in ZooKeeper. This setting is supported since version 19.1 and we already use it in production in multiple services without any issues for more than half a year. Disable this setting if you have a chance to downgrade to versions older than 19.1. [#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Data skipping indices are production ready and enabled by default. The settings `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` and `allow_experimental_multiple_joins_emulation` are now obsolete and do nothing. [#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add new `ANY JOIN` logic for `StorageJoin` consistent with `JOIN` operation. To upgrade without changes in behaviour you need add `SETTINGS any_join_distinct_right_table_keys = 1` to Engine Join tables metadata or recreate these tables after upgrade. [#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) +* Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) + +#### New Feature +* Added information about part paths to `system.merges`. [#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Add ability to execute `SYSTEM RELOAD DICTIONARY` query in `ON CLUSTER` mode. [#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Add ability to execute `CREATE DICTIONARY` queries in `ON CLUSTER` mode. [#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) +* Now user's profile in `users.xml` can inherit multiple profiles. [#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +* Added `system.stack_trace` table that allows to look at stack traces of all server threads. This is useful for developers to introspect server state. This fixes [#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add `DateTime64` datatype with configurable sub-second precision. [#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) +* Add table function `clusterAllReplicas` which allows to query all the nodes in the cluster. [#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) +* Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature. [#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) +* Speed up parsing of data files in `CSV`, `TSV` and `JSONEachRow` format by doing it in parallel. [#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Add function `bankerRound` which performs banker's rounding. [#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) +* Support more languages in embedded dictionary for region names: 'ru', 'en', 'ua', 'uk', 'by', 'kz', 'tr', 'de', 'uz', 'lv', 'lt', 'et', 'pt', 'he', 'vi'. [#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improvements in consistency of `ANY JOIN` logic. Now `t1 ANY LEFT JOIN t2` equals `t2 ANY RIGHT JOIN t1`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Add setting `any_join_distinct_right_table_keys` which enables old behaviour for `ANY INNER JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Add new `SEMI` and `ANTI JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by asynchronous inserts into `Distributed` table. [#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Add optional reset column argument for `runningAccumulate` which allows to reset aggregation results for each new key value. [#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) +* Add ability to use ClickHouse as Prometheus endpoint. [#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) +* Add section `` in `config.xml` which restricts allowed hosts for remote table engines and table functions `URL`, `S3`, `HDFS`. [#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) +* Added function `greatCircleAngle` which calculates the distance on a sphere in degrees. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Changed Earth radius to be consistent with H3 library. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added `JSONCompactEachRow` and `JSONCompactEachRowWithNamesAndTypes` formats for input and output. [#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) +* Added feature for file-related table engines and table functions (`File`, `S3`, `URL`, `HDFS`) which allows to read and write `gzip` files based on additional engine parameter or file extension. [#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) +* Added the `randomASCII(length)` function, generating a string with a random set of [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) printable characters. [#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([BayoNet](https://github.com/BayoNet)) +* Added function `JSONExtractArrayRaw` which returns an array on unparsed json array elements from `JSON` string. [#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) +* Add `arrayZip` function which allows to combine multiple arrays of equal lengths into one array of tuples. [#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([Winter Zhang](https://github.com/zhang2014)) +* Add ability to move data between disks according to configured `TTL`-expressions for `*MergeTree` table engines family. [#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Added new aggregate function `avgWeighted` which allows to calculate weighted average. [#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) +* Now parallel parsing is enabled by default for `TSV`, `TSKV`, `CSV` and `JSONEachRow` formats. [#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Add several geo functions from `H3` library: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` and `h3kRing`. [#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) +* Added support for brotli (`br`) compression in file-related storages and table functions. This fixes [#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add `groupBit*` functions for the `SimpleAggregationFunction` type. [#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) + +#### Bug Fix +* Fix rename of tables with `Distributed` engine. Fixes issue [#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) +* Now dictionaries support `EXPRESSION` for attributes in arbitrary string in non-ClickHouse SQL dialect. [#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) +* Fix broken `INSERT SELECT FROM mysql(...)` query. This fixes [#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) and [#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) +* Fix error "Mismatch column sizes" when inserting default `Tuple` from `JSONEachRow`. This fixes [#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) +* Now an exception will be thrown in case of using `WITH TIES` alongside `LIMIT BY`. Also add ability to use `TOP` with `LIMIT BY`. This fixes [#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix unintendent dependency from fresh glibc version in `clickhouse-odbc-bridge` binary. [#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Bird](https://github.com/amosbird)) +* Fix bug in check function of `*MergeTree` engines family. Now it doesn't fail in case when we have equal amount of rows in last granule and last mark (non-final). [#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) +* Fix insert into `Enum*` columns after `ALTER` query, when underlying numeric type is equal to table specified type. This fixes [#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) +* Allowed non-constant negative "size" argument for function `substring`. It was not allowed by mistake. This fixes [#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix parsing bug when wrong number of arguments passed to `(O|J)DBC` table engine. [#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) +* Using command name of the running clickhouse process when sending logs to syslog. In previous versions, empty string was used instead of command name. [#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) +* Fix check of allowed hosts for `localhost`. This PR fixes the solution provided in [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix rare crash in `argMin` and `argMax` functions for long string arguments, when result is used in `runningAccumulate` function. This fixes [#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) +* Fix memory overcommit for tables with `Buffer` engine. [#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) +* Fixed potential bug in functions that can take `NULL` as one of the arguments and return non-NULL. [#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better metrics calculations in thread pool for background processes for `MergeTree` table engines. [#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix function `IN` inside `WHERE` statement when row-level table filter is present. Fixes [#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) +* Now an exception is thrown if the integral value is not parsed completely for settings values. [#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) +* Fix exception when aggregate function is used in query to distributed table with more than two local shards. [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) +* Now bloom filter can handle zero length arrays and doesn't perform redundant calculations. [#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) +* Fixed checking if a client host is allowed by matching the client host to `host_regexp` specified in `users.xml`. [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) +* Relax ambiguous column check that leads to false positives in multiple `JOIN ON` section. [#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed possible server crash (`std::terminate`) when the server cannot send or write data in `JSON` or `XML` format with values of `String` data type (that require `UTF-8` validation) or when compressing result data with Brotli algorithm or in some other rare cases. This fixes [#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix race condition in `StorageDistributedDirectoryMonitor` found by CI. This fixes [#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now background merges in `*MergeTree` table engines family preserve storage policy volume order more accurately. [#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Now table engine `Kafka` works properly with `Native` format. This fixes [#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +* Fixed formats with headers (like `CSVWithNames`) which were throwing exception about EOF for table engine `Kafka`. [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +* Fixed a bug with making set from subquery in right part of `IN` section. This fixes [#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) and [#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix possible crash while reading from storage `File`. [#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed reading of the files in `Parquet` format containing columns of type `list`. [#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) +* Fix error `Not found column` for distributed queries with `PREWHERE` condition dependent on sampling key if `max_parallel_replicas > 1`. [#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix error `Not found column` if query used `PREWHERE` dependent on table's alias and the result set was empty because of primary key condition. [#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed return type for functions `rand` and `randConstant` in case of `Nullable` argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Disabled predicate push-down for `WITH FILL` expression. This fixes [#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Winter Zhang](https://github.com/zhang2014)) +* Fixed incorrect `count()` result for `SummingMergeTree` when `FINAL` section is used. [#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix possible incorrect result for constant functions from remote servers. It happened for queries with functions like `version()`, `uptime()`, etc. which returns different constant values for different servers. This fixes [#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix complicated bug in push-down predicate optimization which leads to wrong results. This fixes a lot of issues on push-down predicate optimization. [#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Winter Zhang](https://github.com/zhang2014)) +* Fix crash in `CREATE TABLE .. AS dictionary` query. [#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) +* Several improvements ClickHouse grammar in `.g4` file. [#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) +* Fix bug that leads to crashes in `JOIN`s with tables with engine `Join`. This fixes [#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix redundant dictionaries reload on `CREATE DATABASE`. [#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) +* Limit maximum number of streams for read from `StorageFile` and `StorageHDFS`. Fixes https://github.com/ClickHouse/ClickHouse/issues/7650. [#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) +* Fix bug in `ALTER ... MODIFY ... CODEC` query, when user specify both default expression and codec. Fixes [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) +* Fix error in background merge of columns with `SimpleAggregateFunction(LowCardinality)` type. [#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed type check in function `toDateTime64`. [#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) +* Now server do not crash on `LEFT` or `FULL JOIN` with and Join engine and unsupported `join_use_nulls` settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) +* Now `DROP DICTIONARY IF EXISTS db.dict` query doesn't throw exception if `db` doesn't exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix possible crashes in table functions (`file`, `mysql`, `remote`) caused by usage of reference to removed `IStorage` object. Fix incorrect parsing of columns specified at insertion into table function. [#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) +* Ensure network be up before starting `clickhouse-server`. This fixes [#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) +* Fix timeouts handling for secure connections, so queries doesn't hang indefenitely. This fixes [#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `clickhouse-copier`'s redundant contention between concurrent workers. [#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +* Now mutations doesn't skip attached parts, even if their mutation version were larger than current mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) +* Ignore redundant copies of `*MergeTree` data parts after move to another disk and server restart. [#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix crash in `FULL JOIN` with `LowCardinality` in `JOIN` key. [#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) +* Forbidden to use column name more than once in insert query like `INSERT INTO tbl (x, y, x)`. This fixes [#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) +* Added fallback for detection the number of physical CPU cores for unknown CPUs (using the number of logical CPU cores). This fixes [#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `There's no column` error for materialized and alias columns. [#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed sever crash when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier. Just like `EXISTS t`. This fixes [#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). This bug was introduced in version 19.17. [#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix rare bug with error `"Sizes of columns doesn't match"` that might appear when using `SimpleAggregateFunction` column. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +* Fix bug where user with empty `allow_databases` got access to all databases (and same for `allow_dictionaries`). [#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) +* Fix client crash when server already disconnected from client. [#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) +* Fix `ORDER BY` behaviour in case of sorting by primary key prefix and non primary key suffix. [#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) +* Check if qualified column present in the table. This fixes [#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed behavior with `ALTER MOVE` ran immediately after merge finish moves superpart of specified. Fixes [#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix possible server crash while using `UNION` with different number of columns. Fixes [#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix size of result substring for function `substr` with negative size. [#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now server does not execute part mutation in `MergeTree` if there are not enough free threads in background pool. [#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) +* Fix a minor typo on formatting `UNION ALL` AST. [#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) +* Fixed incorrect bloom filter results for negative numbers. This fixes [#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([Winter Zhang](https://github.com/zhang2014)) +* Fixed potential buffer overflow in decompress. Malicious user can pass fabricated compressed data that will cause read after buffer. This issue was found by Eldar Zaitov from Yandex information security team. [#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix incorrect result because of integers overflow in `arrayIntersect`. [#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now `OPTIMIZE TABLE` query will not wait for offline replicas to perform the operation. [#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) +* Fixed `ALTER TTL` parser for `Replicated*MergeTree` tables. [#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix communication between server and client, so server read temporary tables info after query failure. [#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) +* Fix `bitmapAnd` function error when intersecting an aggregated bitmap and a scalar bitmap. [#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) +* Refine the definition of `ZXid` according to the ZooKeeper Programmer's Guide which fixes bug in `clickhouse-cluster-copier`. [#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +* `odbc` table function now respects `external_table_functions_use_nulls` setting. [#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) +* Fixed bug that lead to a rare data race. [#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) +* Now `SYSTEM RELOAD DICTIONARY` reloads a dictionary completely, ignoring `update_field`. This fixes [#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) +* Add ability to check if dictionary exists in create query. [#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alesapin](https://github.com/alesapin)) +* Fix `Float*` parsing in `Values` format. This fixes [#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) +* Fix crash when we cannot reserve space in some background operations of `*MergeTree` table engines family. [#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix crash of merge operation when table contains `SimpleAggregateFunction(LowCardinality)` column. This fixes [#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) +* Restore support of all ICU locales and add the ability to apply collations for constant expressions. Also add language name to `system.collations` table. [#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) +* Fix bug when external dictionaries with zero minimal lifetime (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) don't update in background. [#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alesapin](https://github.com/alesapin)) +* Fix crash when external dictionary with ClickHouse source has subquery in query. [#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix incorrect parsing of file extension in table with engine `URL`. This fixes [#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) +* Fix `CHECK TABLE` query for `*MergeTree` tables without key. Fixes [#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) +* Fixed conversion of `Float64` to MySQL type. [#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) +* Now if table was not completely dropped because of server crash, server will try to restore and load it. [#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) +* Fixed crash in table function `file` while inserting into file that doesn't exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +* Fix rare deadlock which can happen when `trace_log` is in enabled. [#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) +* Add ability to work with different types besides `Date` in `RangeHashed` external dictionary created from DDL query. Fixes [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) +* Fixes crash when `now64()` is called with result of another function. [#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) +* Fixed bug with detecting client IP for connections through mysql wire protocol. [#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) +* Fix empty array handling in `arraySplit` function. This fixes [#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) +* Fixed the issue when `pid-file` of another running `clickhouse-server` may be deleted. [#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) +* Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) +* Fixed error in function `arrayReduce` that may lead to "double free" and error in aggregate function combinator `Resample` that may lead to memory leak. Added aggregate function `aggThrow`. This function can be used for testing purposes. [#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Improvement +* Improved logging when working with `S3` table engine. [#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +* Printed help message when no arguments are passed when calling `clickhouse-local`. This fixes [#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) +* Add setting `mutations_sync` which allows to wait `ALTER UPDATE/DELETE` queries synchronously. [#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) +* Allow to set up relative `user_files_path` in `config.xml` (in the way similar to `format_schema_path`). [#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) +* Add exception for illegal types for conversion functions with `-OrZero` postfix. [#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) +* Simplify format of the header of data sending to a shard in a distributed query. [#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) +* `Live View` table engine refactoring. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) +* Add additional checks for external dictionaries created from DDL-queries. [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) +* Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now table the first argument of `joinGet` function can be table indentifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) +* Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) +* Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) +* `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) +* Stack traces now display physical addresses (offsets in object file) instead of virtual memory addresses (where the object file was loaded). That allows the use of `addr2line` when binary is position independent and ASLR is active. This fixes [#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Support new syntax for row-level security filters: `
…
`. Fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) +* Now `cityHash` function can work with `Decimal` and `UUID` types. Fixes [#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) +* Removed fixed index granularity (it was 1024) from system logs because it's obsolete after implementation of adaptive granularity. [#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enabled MySQL compatibility server when ClickHouse is compiled without SSL. [#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([Yuriy Baranov](https://github.com/yurriy)) +* Now server checksums distributed batches, which gives more verbose errors in case of corrupted data in batch. [#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) +* Support `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` and `ATTACH TABLE` for `MySQL` database engine. [#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([Winter Zhang](https://github.com/zhang2014)) +* Add authentication in S3 table function and table engine. [#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Added check for extra parts of `MergeTree` at different disks, in order to not allow to miss data parts at undefined disks. [#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Enable SSL support for Mac client and server. [#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([Ivan](https://github.com/abyss7)) +* Now ClickHouse can work as MySQL federated server (see https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) +* `clickhouse-client` now only enable `bracketed-paste` when multiquery is on and multiline is off. This fixes (#7757)[https://github.com/ClickHouse/ClickHouse/issues/7757]. [#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Bird](https://github.com/amosbird)) +* Support `Array(Decimal)` in `if` function. [#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) +* Support Decimals in `arrayDifference`, `arrayCumSum` and `arrayCumSumNegative` functions. [#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) +* Added `lifetime` column to `system.dictionaries` table. [#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) +* Improved check for existing parts on different disks for `*MergeTree` table engines. Addresses [#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Integration with `AWS SDK` for `S3` interactions which allows to use all S3 features out of the box. [#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) +* Added support for subqueries in `Live View` tables. [#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) +* Check for using `Date` or `DateTime` column from `TTL` expressions was removed. [#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Information about disk was added to `system.detached_parts` table. [#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Now settings `max_(table|partition)_size_to_drop` can be changed without a restart. [#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +* Slightly better usability of error messages. Ask user not to remove the lines below `Stack trace:`. [#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better reading messages from `Kafka` engine in various formats after [#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([Ivan](https://github.com/abyss7)) +* Better compatibility with MySQL clients which don't support `sha2_password` auth plugin. [#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) +* Support more column types in MySQL compatibility server. [#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) +* Implement `ORDER BY` optimization for `Merge`, `Buffer` and `Materilized View` storages with underlying `MergeTree` tables. [#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) +* Now we always use POSIX implementation of `getrandom` to have better compatibility with old kernels (< 3.17). [#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Bird](https://github.com/amosbird)) +* Better check for valid destination in a move TTL rule. [#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Better checks for broken insert batches for `Distributed` table engine. [#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) +* Add column with array of parts name which mutations must process in future to `system.mutations` table. [#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) +* Parallel merge sort optimization for processors. [#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* The settings `mark_cache_min_lifetime` is now obsolete and does nothing. In previous versions, mark cache can grow in memory larger than `mark_cache_size` to accomodate data within `mark_cache_min_lifetime` seconds. That was leading to confusion and higher memory usage than expected, that is especially bad on memory constrained systems. If you will see performance degradation after installing this release, you should increase the `mark_cache_size`. [#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Preparation to use `tid` everywhere. This is needed for [#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Performance Improvement +* Performance optimizations in processors pipeline. [#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Non-blocking updates of expired keys in cache dictionaries (with permission to read old ones). [#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Compile ClickHouse without `-fno-omit-frame-pointer` globally to spare one more register. [#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Bird](https://github.com/amosbird)) +* Speedup `greatCircleDistance` function and add performance tests for it. [#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) +* Improved performance of function `roundDown`. [#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improved performance of `max`, `min`, `argMin`, `argMax` for `DateTime64` data type. [#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) +* Improved performance of sorting without a limit or with big limit and external sorting. [#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improved performance of formatting floating point numbers up to 6 times. [#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improved performance of `modulo` function. [#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Bird](https://github.com/amosbird)) +* Optimized `ORDER BY` and merging with single column key. [#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better implementation for `arrayReduce`, `-Array` and `-State` combinators. [#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Bird](https://github.com/amosbird)) +* Now `PREWHERE` should be optimized to be at least as efficient as `WHERE`. [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) +* Improve the way `round` and `roundBankers` handling negative numbers. [#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) +* Improved decoding performance of `DoubleDelta` and `Gorilla` codecs by roughly 30-40%. This fixes [#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) +* Improved performance of `base64` related functions. [#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added a function `geoDistance`. It is similar to `greatCircleDistance` but uses approximation to WGS-84 ellipsoid model. The performance of both functions are near the same. [#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Faster `min` and `max` aggregation functions for `Decimal` data type. [#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) +* Vectorize processing `arrayReduce`. [#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) +* `if` chains are now optimized as `multiIf`. [#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) +* Fix performance regression of `Kafka` table engine introduced in 19.15. This fixes [#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) +* Removed "pie" code generation that `gcc` from Debian packages occasionally brings by default. [#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Parallel parsing data formats [#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Enable optimized parser of `Values` with expressions by default (`input_format_values_deduce_templates_of_expressions=1`). [#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) + +#### Build/Testing/Packaging Improvement +* Build fixes for `ARM` and in minimal mode. [#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) +* Add coverage file flush for `clickhouse-server` when std::atexit is not called. Also slightly improved logging in stateless tests with coverage. [#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) +* Update LLVM library in contrib. Avoid using LLVM from OS packages. [#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make bundled `curl` build fully quiet. [#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) +* Fix some `MemorySanitizer` warnings. [#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Use `add_warning` and `no_warning` macros in `CMakeLists.txt`. [#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([Ivan](https://github.com/abyss7)) +* Add support of Minio S3 Compatible object (https://min.io/) for better integration tests. [#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) +* Imported `libc` headers to contrib. It allows to make builds more consistent across various systems (only for `x86_64-linux-gnu`). [#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove `-fPIC` from some libraries. [#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Clean `CMakeLists.txt` for curl. See https://github.com/ClickHouse/ClickHouse/pull/8011#issuecomment-569478910 [#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Silent warnings in `CapNProto` library. [#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add performance tests for short string optimized hash tables. [#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Bird](https://github.com/amosbird)) +* Now ClickHouse will build on `AArch64` even if `MADV_FREE` is not available. This fixes [#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Bird](https://github.com/amosbird)) +* Update `zlib-ng` to fix memory sanitizer problems. [#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually doesn't work at all. This fixes [#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed build on some systems after enabling `libc++`. This supersedes [#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make `Field` methods more type-safe to find more errors. [#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Added missing files to the `libc-headers` submodule. [#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix wrong `JSON` quoting in performance test output. [#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now stack trace is displayed for `std::exception` and `Poco::Exception`. In previous versions it was available only for `DB::Exception`. This improves diagnostics. [#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Porting `clock_gettime` and `clock_nanosleep` for fresh glibc versions. [#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Bird](https://github.com/amosbird)) +* Enable `part_log` in example config for developers. [#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix async nature of reload in `01036_no_superfluous_dict_reload_on_create_database*`. [#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) +* Fixed codec performance tests. [#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) +* Add install scripts for `.tgz` build and documentation for them. [#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) +* Removed old `ZSTD` test (it was created in year 2016 to reproduce the bug that pre 1.0 version of ZSTD has had). This fixes [#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed build on Mac OS Catalina. [#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) +* Increased number of rows in codec performance tests to make results noticeable. [#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) +* In debug builds, treat `LOGICAL_ERROR` exceptions as assertion failures, so that they are easier to notice. [#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Make formats-related performance test more deterministic. [#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update `lz4` to fix a MemorySanitizer failure. [#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Suppress a known MemorySanitizer false positive in exception handling. [#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Update `gcc` and `g++` to version 9 in `build/docker/build.sh` [#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) +* Add performance test case to test that `PREWHERE` is worse than `WHERE`. [#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Bird](https://github.com/amosbird)) +* Progress towards fixing one flacky test. [#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid MemorySanitizer report for data from `libunwind`. [#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Updated `libc++` to the latest version. [#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Build ICU library from sources. This fixes [#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Switched from `libressl` to `openssl`. ClickHouse should support TLS 1.3 and SNI after this change. This fixes [#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed UBSan report when using `chacha20_poly1305` from SSL (happens on connect to https://yandex.ru/). [#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix mode of default password file for `.deb` linux distros. [#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) +* Improved expression for getting `clickhouse-server` PID in `clickhouse-test`. [#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) +* Updated contrib/googletest to v1.10.0. [#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) +* Fixed ThreadSaninitizer report in `base64` library. Also updated this library to the latest version, but it doesn't matter. This fixes [#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `00600_replace_running_query` for processors. [#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Remove support for `tcmalloc` to make `CMakeLists.txt` simpler. [#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Release gcc builds now use `libc++` instead of `libstdc++`. Recently `libc++` was used only with clang. This will improve consistency of build configurations and portability. [#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enable ICU library for build with MemorySanitizer. [#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Suppress warnings from `CapNProto` library. [#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Removed special cases of code for `tcmalloc`, because it's no longer supported. [#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* In CI coverage task, kill the server gracefully to allow it to save the coverage report. This fixes incomplete coverage reports we've been seeing lately. [#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alesapin](https://github.com/alesapin)) +* Performance tests for all codecs against `Float64` and `UInt64` values. [#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) +* `termcap` is very much deprecated and lead to various problems (f.g. missing "up" cap and echoing `^J` instead of multi line) . Favor `terminfo` or bundled `ncurses`. [#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Bird](https://github.com/amosbird)) +* Fix `test_storage_s3` integration test. [#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Support `StorageFile(, null) ` to insert block into given format file without actually write to disk. This is required for performance tests. [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) +* Added argument `--print-time` to functional tests which prints execution time per test. [#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Added asserts to `KeyCondition` while evaluating RPN. This will fix warning from gcc-9. [#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Dump cmake options in CI builds. [#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Don't generate debug info for some fat libraries. [#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make `log_to_console.xml` always log to stderr, regardless of is it interactive or not. [#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Removed some unused features from `clickhouse-performance-test` tool. [#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now we will also search for `lld-X` with corresponding `clang-X` version. [#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alesapin](https://github.com/alesapin)) +* Parquet build improvement. [#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) +* More GCC warnings [#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +* Package for Arch Linux now allows to run ClickHouse server, and not only client. [#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix test with processors. Tiny performance fixes. [#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Update contrib/protobuf. [#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) +* In preparation of switching to c++20 as a new year celebration. "May the C++ force be with ClickHouse." [#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Bird](https://github.com/amosbird)) + +#### Experimental Feature +* Added experimental setting `min_bytes_to_use_mmap_io`. It allows to read big files without copying data from kernel to userspace. The setting is disabled by default. Recommended threshold is about 64 MB, because mmap/munmap is slow. [#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Reworked quotas as a part of access control system. Added new table `system.quotas`, new functions `currentQuota`, `currentQuotaKey`, new SQL syntax `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) +* Allow skipping unknown settings with warnings instead of throwing exceptions. [#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) +* Reworked row policies as a part of access control system. Added new table `system.row_policies`, new function `currentRowPolicies()`, new SQL syntax `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) + +#### Security Fix +* Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) diff --git a/docs/ru/extended_roadmap.md b/docs/ru/whats_new/extended_roadmap.md similarity index 93% rename from docs/ru/extended_roadmap.md rename to docs/ru/whats_new/extended_roadmap.md index 193e2035b56..12668b74ef4 100644 --- a/docs/ru/extended_roadmap.md +++ b/docs/ru/whats_new/extended_roadmap.md @@ -15,9 +15,7 @@ Задача «normalized z-Order curve» в перспективе может быть полезна для БК и Метрики, так как позволяет смешивать OrderID и PageID и избежать дублирования данных. В задаче также вводится способ индексации путём обращения функции нескольких аргументов на интервале, что имеет смысл для дальнейшего развития. -Изначально делал [Андрей Чулков](https://github.com/achulkov2), ВШЭ, теперь (не) доделывает [Ольга Хвостикова](https://github.com/stavrolia), но сроки немного сдвинуты из-за задачи 25.9. Будем надеятся на лучшее. - -Upd. Доделывать будет другой человек. Приоритет не высокий. +[Андрей Чулков](https://github.com/achulkov2), ВШЭ. ### 1.2. Wait-free каталог баз данных. {#wait-free-katalog-baz-dannykh} @@ -28,6 +26,7 @@ Q2. Делает [Александр Токмаков](https://github.com/tavplu Предлагается реализовать альтернативный подход, в котором таблицы и базы данных являются всего лишь ссылками на persistent объекты. Подробное описание задачи: [\#6787](https://github.com/ClickHouse/ClickHouse/issues/6787) Upd. Сделана крупная часть задачи, но ориентироваться стоит уже на Q2. +Upd. Pull request готов для мержа. ### 1.3. Неблокирующие ALTER. {#neblokiruiushchie-alter} @@ -39,18 +38,20 @@ Upd. Большая часть задачи реализована и добав Требует 1.3. Будет делать [Александр Сапин](https://github.com/alesapin). Ура, сделано. -### 1.5. ALTER RENAME COLUMN. {#alter-rename-column} +### 1.5. + ALTER RENAME COLUMN. {#alter-rename-column} [\#6861](https://github.com/ClickHouse/ClickHouse/issues/6861) Требует 1.3. Будет делать [Александр Сапин](https://github.com/alesapin). -### 1.6. Полиморфные куски данных. {#polimorfnye-kuski-dannykh} +### 1.6. + Полиморфные куски данных. {#polimorfnye-kuski-dannykh} -Компактные куски - Q1, куски в оперативке Q1/Q2. +Компактные куски - Q1, куски в оперативке Q1/Q2 - пункт 1.7. Компактные куски реализованы, ещё не включены по-умолчанию. Первым шагом включаем по-умолчанию для системных таблиц. +Upd. Включено для системных таблиц. + Делает [Антон Попов](https://github.com/CurtizJ), первый рабочий вариант в декабре. Пререквизит чтобы снизить сложность мелких INSERT, что в свою очередь нужно для 1.12, иначе задача 1.12 не сможет нормально работать. Особенно нужно для Яндекс.Облака. Данные в таблицах типа MergeTree в ClickHouse хранятся в виде набора независимых «кусков». Внутри куска, каждый столбец, а также индекс, хранится в отдельных файлах. Это сделано для возможности быстрых манипуляций со столбцами (пример - запрос ALTER DROP COLUMN). При вставке данных (INSERT), создаётся новый кусок. Для таблиц с большим количеством столбцов, запросы INSERT с маленьким количеством строк являются неэффективными, так как требуют создания большого количества файлов в файловой системе. Это является врождённой особенностью ClickHouse - одной из первой проблем, с которыми сталкиваются пользователи. Пользователям приходится буферизовывать данные и собирать их в более крупные пачки перед вставкой в ClickHouse. @@ -61,7 +62,7 @@ Upd. Большая часть задачи реализована и добав ### 1.7. Буферизация и WAL в MergeTree. {#buferizatsiia-i-wal-v-mergetree} -Требует 1.6. +Требует 1.6. Антон Попов. Задача взята в работу. Q2. ### 1.8. + Перенос между разделами по TTL. {#perenos-mezhdu-razdelami-po-ttl} @@ -74,7 +75,7 @@ Q1. Закоммичено, но есть технический долг, ко Будет делать Сорокин Николай, ВШЭ и Яндекс. -Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table\_engines/mergetree/\#table\_engine-mergetree-ttl +Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/\#table_engine-mergetree-ttl Это может быть задано для всей таблицы (тогда строки целиком удаляются после указанного времени) или для отдельных столбцов (тогда данные столбца физически удаляются с диска, а строки в таблице остаются; при чтении значений столбца, они читаются как значения по-умолчанию). @@ -88,7 +89,7 @@ Q1. Закоммичено, но есть технический долг, ко А вот пункт 2 требуется продумать. Не очевидно даже, какой лучше использовать синтаксис для этого при создании таблицы. Но мы придумаем - сразу видно несколько вариантов. -Частный случай такой задачи уже есть в https://clickhouse.tech/docs/ru/operations/table\_engines/graphitemergetree/ Но это было сделано для конкретной задачи. А надо обобщить. +Частный случай такой задачи уже есть в https://clickhouse.tech/docs/ru/operations/table_engines/graphitemergetree/ Но это было сделано для конкретной задачи. А надо обобщить. ### 1.10. Пережатие старых данных в фоне. {#perezhatie-starykh-dannykh-v-fone} @@ -100,17 +101,15 @@ Q1. Закоммичено, но есть технический долг, ко Предлагается добавить в ClickHouse настройки по пережатию данных и фоновые потоки, выполняющие эту задачу. -### 1.11. Виртуальная файловая система. {#virtualnaia-failovaia-sistema} +### 1.11. + Виртуальная файловая система. {#virtualnaia-failovaia-sistema} -В процессе реализации, сейчас на VFS переведены Log, TinyLog, StripeLog, готовится MergeTree. +На VFS переведены Log, TinyLog, StripeLog, а также MergeTree, что доказывает состоятельность реализации. -Q2. - -Нужно для Яндекс.Облака. Делает Александр, Яндекс.Облако, а также Олег Ершов, ВШЭ и Яндекс. +Нужно для Яндекс.Облака. Делает Александр, Яндекс.Облако. ClickHouse использует для хранения данных локальную файловую систему. Существует сценарий работы, в котором размещение старых (архивных) данных было бы выгодно на удалённой файловой системе. Если файловая система POSIX совместимая, то это не составляет проблем: ClickHouse успешно работает с Ceph, GlusterFS, MooseFS. Также востребованным является сценарий использования S3 (из-за доступности в облаке) или HDFS (для интеграции с Hadoop). Но эти файловые системы не являются POSIX совместимыми. Хотя для них существуют FUSE драйверы, но скорость работы сильно страдает и поддержка неполная. -ClickHouse использует небольшое подмножество функций ФС, но в то же время, и некоторые специфические части: симлинки и хардлинки, O\_DIRECT. Предлагается выделить всё взаимодействие с файловой системой в отдельный интерфейс. +ClickHouse использует небольшое подмножество функций ФС, но в то же время, и некоторые специфические части: симлинки и хардлинки, O_DIRECT. Предлагается выделить всё взаимодействие с файловой системой в отдельный интерфейс. ### 1.12. Экспериментальная реализация VFS поверх S3 и HDFS. {#eksperimentalnaia-realizatsiia-vfs-poverkh-s3-i-hdfs} @@ -121,13 +120,15 @@ Q2. Upd. Олег будет делать только часть про HDFS. +Upd. Реализация поверх S3 является рабочей на уровне PoC. + ### 1.13. Ускорение запросов с FINAL. {#uskorenie-zaprosov-s-final} -Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. +Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. Q2. ### 1.14. Не писать столбцы, полностью состоящие из нулей. {#ne-pisat-stolbtsy-polnostiu-sostoiashchie-iz-nulei} -Антон Попов. Q1/Q2. +Антон Попов. Q2. В очереди. Простая задача, является небольшим пререквизитом для потенциальной поддержки полуструктурированных данных. ### 1.15. Возможность иметь разный первичный ключ в разных кусках. {#vozmozhnost-imet-raznyi-pervichnyi-kliuch-v-raznykh-kuskakh} @@ -146,6 +147,7 @@ Upd. Олег будет делать только часть про HDFS. Требует 1.3 и 1.6. Полная замена hard links на sym links, что будет лучше для 1.12. + ## 2. Крупные рефакторинги. {#krupnye-refaktoringi} Для обоснования необходимости смотрите ссылки в описании других задач. @@ -161,6 +163,8 @@ Upd. Включили по-умолчанию. Удаление старого Upd. Уже есть первый релиз, в котором это включено по-умолчанию. +Upd. Всё ещё ждём удаление старого кода, которое должно случиться после релиза 20.4. + ### 2.2. Инфраструктура событий/метрик/ограничений/квот/трассировки. {#infrastruktura-sobytiimetrikogranicheniikvottrassirovki} В очереди. https://gist.github.com/alexey-milovidov/d62d73222d83b9319dc519cbb13aeff6 @@ -188,11 +192,14 @@ Upd. Уже есть первый релиз, в котором это вклю Александр Токмаков исправил множество проблем с использованием Context и сейчас переносит каталог БД наружу. Upd. Каталог БД вынесен из Context. +Upd. SharedContext вынесен из Context. ### 2.8. Декларативный парсер запросов. {#deklarativnyi-parser-zaprosov} Средний приоритет. Нужно для YQL. +Upd. В очереди. Иван Лежанкин. + ### 2.9. Логгировние в format-стиле. {#loggirovnie-v-format-stile} Делает [Иван Лежанкин](https://github.com/abyss7). Низкий приоритет. @@ -212,10 +219,14 @@ Upd. Каталог БД вынесен из Context. Задачу делает Алексей Миловидов. Прогресс 50% и разработка временно приостановлена. +Upd. Разработка всё ещё приостановлена. + ### 2.13. Каждая функция в отдельном файле. {#kazhdaia-funktsiia-v-otdelnom-faile} Задачу делает Алексей Миловидов. Прогресс 80%. Потребуется помощь других разработчиков. +Upd. Поползновения наблюдаются. + ### 2.14. Все функции с состоянием переделать на FunctionBuilder. {#vse-funktsii-s-sostoianiem-peredelat-na-functionbuilder} Долг [Николай Кочетов](https://github.com/KochetovNicolai). Сейчас код находится в переходном состоянии, что неприемлемо. @@ -224,13 +235,14 @@ Upd. Каталог БД вынесен из Context. Для нормализации работы materialized views поверх Merge, Distributed, Kafka. + ## 3. Документация. {#dokumentatsiia} Здесь задачи только по инфраструктуре документации. ### 3.1. Перенос документации по функциям в код. {#perenos-dokumentatsii-po-funktsiiam-v-kod} -Требует 2.12 и 2.13. Хотим в Q1/Q2, средний приоритет. +Требует 2.12 и 2.13. Хотим в Q2, средний приоритет. ### 3.2. Перенос однородных частей документации в код. {#perenos-odnorodnykh-chastei-dokumentatsii-v-kod} @@ -246,11 +258,12 @@ Upd. Иван Блинков сделал эту задачу путём зам Эту задачу сделает [Иван Блинков](https://github.com/blinkov/), до конца декабря 2019. Сделано. + ## 4. Сетевое взаимодействие. {#setevoe-vzaimodeistvie} ### 4.1. Уменьшение числа потоков при распределённых запросах. {#umenshenie-chisla-potokov-pri-raspredelionnykh-zaprosakh} -[Никита Лапков](https://github.com/laplab), весна 2020. Upd. Есть прототип. Upd. Он не работает. +Весна 2020. Upd. Есть прототип. Upd. Он не работает. Upd. Человек отказался от задачи, теперь сроки не определены. ### 4.2. Спекулятивное выполнение запросов на нескольких репликах. {#spekuliativnoe-vypolnenie-zaprosov-na-neskolkikh-replikakh} @@ -262,6 +275,8 @@ Upd. Иван Блинков сделал эту задачу путём зам Сейчас для распределённых запросов используется по потоку на соединение. Это позволяет хорошо распараллелить вычисления над полученными данными и утилизировать сеть, но становится сильно избыточным для больших кластеров. Для примера, создание 1000 потоков для чтения данных из 1000 серверов кластера - лишь расходует ресурсы и увеличивает время выполнения запроса. Вместо этого необходимо использовать количество потоков не большее количества процессорных ядер, и мультиплексировать в одном потоке общение с серверами. Реализация нетривиальна, так как мультиплексировать необходимо каждую стадию общения по сети, включая установку соединения и обмен handshake. +Upd. Сейчас обсуждается, как сделать другую задачу вместо этой. + ### 4.3. Ограничение числа одновременных скачиваний с реплик. {#ogranichenie-chisla-odnovremennykh-skachivanii-s-replik} Дмитрий Григорьев, ВШЭ. @@ -284,14 +299,16 @@ Upd. Иван Блинков сделал эту задачу путём зам Дмитрий Григорьев, ВШЭ. В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3. + ## 5. Операции. {#operatsii} -### 5.1. Разделение задач на более мелкие куски в clickhouse-copier. {#razdelenie-zadach-na-bolee-melkie-kuski-v-clickhouse-copier} +### 5.1. + Разделение задач на более мелкие куски в clickhouse-copier. {#razdelenie-zadach-na-bolee-melkie-kuski-v-clickhouse-copier} [\#9075](https://github.com/ClickHouse/ClickHouse/pull/9075) Q1. Нужно для Метрики, в очереди. Никита Михайлов. Upd. Задача на финальной стадии разработки. +Upd. Сделано. Эффективность работы под вопросом. Есть варианты, как сделать лучше. ### 5.2. Автонастройка лимита на оперативку и размера кэшей. {#avtonastroika-limita-na-operativku-i-razmera-keshei} @@ -305,6 +322,8 @@ Upd. Задача на финальной стадии разработки. Требует 7.5. Задачу хочет Метрика, Облако, БК, Маркет и Altinity. Первой LTS версией уже стала версия 19.14. Метрика, БК, Маркет, Altinity уже используют более свежие версии чем LTS. +Upd. Появилась вторая версия LTS - 20.3. + ## 6. Инструментирование. {#instrumentirovanie} @@ -319,9 +338,9 @@ Upd. Задача на финальной стадии разработки. Сравнительно простая задача, но только для опытных разработчиков. Нужна всем. Иван Лежанкин. Q1. Сделано. -### 6.3. Учёт оперативки total расширить не только на запросы. {#uchiot-operativki-total-rasshirit-ne-tolko-na-zaprosy} +### 6.3. + Учёт оперативки total расширить не только на запросы. {#uchiot-operativki-total-rasshirit-ne-tolko-na-zaprosy} -Исправление долгоживущей проблемы с дрифтом учёта оперативки. Нужна для Метрики и БК. Иван Лежанкин. Q1. +Исправление долгоживущей проблемы с дрифтом учёта оперативки. Нужна для Метрики и БК. ### 6.4. Поддержка perf events как метрик запроса. {#podderzhka-perf-events-kak-metrik-zaprosa} @@ -339,7 +358,7 @@ Upd. Задача на финальной стадии разработки. Сейчас есть стек трейс для почти всех, но не всех исключений. Требует 7.4. -### 6.7. + Таблица system.stack\_trace. {#tablitsa-system-stack-trace} +### 6.7. + Таблица system.stack_trace. {#tablitsa-system-stack-trace} Сравнительно простая задача, но только для опытных разработчиков. @@ -351,6 +370,7 @@ Upd. Задача на финальной стадии разработки. ### 6.10. Сбор общих системных метрик. {#sbor-obshchikh-sistemnykh-metrik} + ## 7. Сопровождение разработки. {#soprovozhdenie-razrabotki} ### 7.1. + ICU в submodules. {#icu-v-submodules} @@ -361,7 +381,7 @@ Upd. Задача на финальной стадии разработки. Сделал Алексей Миловидов. -### 7.3. Обновление Poco. {#obnovlenie-poco} +### 7.3. + Обновление Poco. {#obnovlenie-poco} Алексанр Кузьменков. @@ -383,13 +403,18 @@ Upd. Задача на финальной стадии разработки. Уже есть ASan, TSan, UBSan. Не хватает тестов под MSan. Они уже добавлены в CI, но не проходят. [Александр Кузьменков](https://github.com/akuzm) и [Александр Токмаков](https://github.com/tavplubix). -### 7.8. Добавить clang-tidy. {#dobavit-clang-tidy} +Upd. Задача всё ещё медленно тащится. + +### 7.8. + Добавить clang-tidy. {#dobavit-clang-tidy} Уже есть PVS-Studio. Мы очень довольны, но этого недостаточно. Upd. Алексей Миловидов. Добавлено некоторое множество проверок, но нужно рассмотреть все проверки подряд и добавить всё, что можно. +Upd. Рассмотрели все проверки подряд. -### 7.9. Проверки на стиль имён с помощью clang-tidy. {#proverki-na-stil-imion-s-pomoshchiu-clang-tidy} +### 7.9. + Проверки на стиль имён с помощью clang-tidy. {#proverki-na-stil-imion-s-pomoshchiu-clang-tidy} + +Сделано. Только в .cpp файлах и только для имён локальных переменных. Остальное слишком сложно. ### 7.10. Включение UBSan и MSan в интеграционных тестах. {#vkliuchenie-ubsan-i-msan-v-integratsionnykh-testakh} @@ -399,6 +424,8 @@ UBSan включен в функциональных тестах, но не в У нас мало unit тестов по сравнению с функциональными тестами и их использование не обязательно. Но они всё-равно важны и нет причин не запускать их под всеми видами sanitizers. +Илья Яцишин. + ### 7.12. Показывать тестовое покрытие нового кода в PR. {#pokazyvat-testovoe-pokrytie-novogo-koda-v-pr} Пока есть просто показ тестового покрытия всего кода. @@ -413,6 +440,8 @@ UBSan включен в функциональных тестах, но не в Подключение replxx вместо readline сделал Иван Лежанкин. +Есть технический долг с лицензиями файлов консорциума Unicode. + ### 7.14.1. Улучшение возможностей интерактивного режима clickhouse-client. {#uluchshenie-vozmozhnostei-interaktivnogo-rezhima-clickhouse-client} Тагир Кускаров, ВШЭ. @@ -449,9 +478,9 @@ Upd. В рамках данной задачи добавляем подстве Нужно для Яндекс.Облака для поддержки TLS 1.3. -### 7.16. tzdata внутри бинарника. {#tzdata-vnutri-binarnika} +### 7.16. + tzdata внутри бинарника. {#tzdata-vnutri-binarnika} -Как в Аркадии, но только в качестве fallback. +Как в Аркадии, fallback на системные. ### 7.17. + Доделать tgz пакеты. {#dodelat-tgz-pakety} @@ -476,7 +505,7 @@ https://github.com/ClickHouse/ClickHouse/issues/8027\#issuecomment-566670282 Проверили на настоящем сервере Huawei, а также в специальном Docker контейнере, который содержит внутри qemu-user-static. Также можно проверить на Cavium, на Raspberry Pi а также на твоём Android телефоне. -### 7.20. Автосборка для FreeBSD x86\_64. {#avtosborka-dlia-freebsd-x86-64} +### 7.20. Автосборка для FreeBSD x86_64. {#avtosborka-dlia-freebsd-x86-64} [Иван Лежанкин](https://github.com/abyss7). @@ -535,6 +564,8 @@ Fuzzing тестирование - это тестирование случай Также можно сделать функции с детерминированным генератором случайных чисел (аргументом передаётся seed) для воспроизводимости тестовых кейсов. Upd. Сергей Штыков сделал функцию `randomPrintableASCII`. +Upd. Илья Яцишин сделал табличную функцию `generateRandom`. +Upd. Эльдар Заитов добавляет OSS Fuzz. ### 7.24. Fuzzing лексера и парсера запросов; кодеков и форматов. {#fuzzing-leksera-i-parsera-zaprosov-kodekov-i-formatov} @@ -546,7 +577,7 @@ Upd. Сергей Штыков сделал функцию `randomPrintableASCII 2. Генерация и выполнение случайных синтаксически корректных запросов на случайных данных. -### 7.25. Синхронизация релизов в Аркадию. {#sinkhronizatsiia-relizov-v-arkadiiu} +### 7.25. + Синхронизация релизов в Аркадию. {#sinkhronizatsiia-relizov-v-arkadiiu} Изначально занимался Олег Алексеенков. Сейчас он перешёл работать в дружественный отдел, но обещает продолжать синхронизацию. Затем, возможно, [Иван Лежанкин](https://github.com/abyss7). Но сейчас приостановлено, так как Максим из YT должен исправить регрессию производительности в анализе индекса. @@ -557,9 +588,10 @@ Upd. Сергей Штыков сделал функцию `randomPrintableASCII Нужно для CHYT и YQL. -UPD: Все патчи Максима отправлены в master. Задача взята в работу. - +Upd: Все патчи Максима отправлены в master. Задача взята в работу. Upd: Задача в процессе реализации. Синхронизироваться будет master. Делает [Иван Лежанкин](https://github.com/abyss7) +Upd: Есть собирающийся прототип, но сборка как будто ещё не в trunk Аркадии. +Upd: Добавлено в Аркадию, но не все файлы (не побайтово). ### 7.26. Побайтовая идентичность репозитория с Аркадией. {#pobaitovaia-identichnost-repozitoriia-s-arkadiei} @@ -617,6 +649,7 @@ Upd: Задача в процессе реализации. Синхронизи Upd. Иван Блинков настроил CDN repo.clickhouse.tech, что решает проблему с доступностью зарубежом. Вопрос с operations, visibility пока актуален. + ## 8. Интеграция с внешними системами. {#integratsiia-s-vneshnimi-sistemami} ### 8.1. Поддержка ALTER MODIFY SETTING для Kafka. {#podderzhka-alter-modify-setting-dlia-kafka} @@ -629,11 +662,11 @@ Altinity. Никто не делает эту задачу. [Александр Кузьменков](https://github.com/akuzm). -### 8.3. Доработки globs (правильная поддержка диапазонов, уменьшение числа одновременных stream-ов). {#dorabotki-globs-pravilnaia-podderzhka-diapazonov-umenshenie-chisla-odnovremennykh-stream-ov} +### 8.3. + Доработки globs (правильная поддержка диапазонов, уменьшение числа одновременных stream-ов). {#dorabotki-globs-pravilnaia-podderzhka-diapazonov-umenshenie-chisla-odnovremennykh-stream-ov} [Ольга Хвостикова](https://github.com/stavrolia). -Уменьшение числа stream-ов сделано, а вот правильная поддержка диапазонов - нет. Будем надеяться на Q1/Q2. +Уменьшение числа stream-ов сделано, а вот правильная поддержка диапазонов - нет. Будем надеяться на Q1/Q2. Сделано. ### 8.4. Унификация File, HDFS, S3 под URL. {#unifikatsiia-file-hdfs-s3-pod-url} @@ -690,19 +723,21 @@ Andrew Onyshchuk. Есть pull request. Q1. Сделано. Павел Круглов, ВШЭ и Яндекс. Есть pull request. -### 8.16.2. Поддержка формата Thrift. {#podderzhka-formata-thrift} +### 8.16.2. - Поддержка формата Thrift. {#podderzhka-formata-thrift} -Павел Круглов, ВШЭ и Яндекс. +Павел Круглов, ВШЭ и Яндекс. Задача отменена. -### 8.16.3. Поддержка формата MsgPack. {#podderzhka-formata-msgpack} +### 8.16.3. + Поддержка формата MsgPack. {#podderzhka-formata-msgpack} Павел Круглов, ВШЭ и Яндекс. Задача взята в работу. -### 8.16.4. Формат Regexp. {#format-regexp} +Upd. Почти готово - есть лишь небольшой технический долг. + +### 8.16.4. + Формат Regexp. {#format-regexp} Павел Круглов, ВШЭ и Яндекс. -Есть pull request. +Есть pull request. Готово. ### 8.17. ClickHouse как MySQL реплика. {#clickhouse-kak-mysql-replika} @@ -735,6 +770,7 @@ Maxim Fedotov, Wargaming + Yuri Baranov, Яндекс. Нужно для БК. Декабрь 2019. В декабре для БК сделан минимальный вариант этой задачи. Максимальный вариант, вроде, никому не нужен. +Upd. Всё ещё кажется, что задача не нужна. ### 8.22. Поддержка синтаксиса для переменных в стиле MySQL. {#podderzhka-sintaksisa-dlia-peremennykh-v-stile-mysql} @@ -746,6 +782,7 @@ Upd. Юрий Баранов работает в Google, там запрещен Желательно 2.15. + ## 9. Безопасность. {#bezopasnost} ### 9.1. + Ограничение на хосты в запросах ко внешним системам. {#ogranichenie-na-khosty-v-zaprosakh-ko-vneshnim-sistemam} @@ -760,7 +797,12 @@ ClickHouse предоставляет возможность обратитьс Вместо этого предлагается описывать необходимые данные в конфигурационном файле сервера или в отдельном сервисе и ссылаться на них по именам. -### 9.3. Поддержка TLS для ZooKeeper. {#podderzhka-tls-dlia-zookeeper} +### 9.3. + Поддержка TLS для ZooKeeper. {#podderzhka-tls-dlia-zookeeper} + +[#10174](https://github.com/ClickHouse/ClickHouse/issues/10174) + +Есть pull request. + ## 10. Внешние словари. {#vneshnie-slovari} @@ -777,6 +819,7 @@ ClickHouse предоставляет возможность обратитьс Нужно для БК и Метрики. Поиск причин - [Александр Сапин](https://github.com/alesapin). Дальшейшее исправление возможно на стороне YT. Upd. Одну причину устранили, но ещё что-то неизвестное осталось. +Upd. Нас заставляют переписать эту библиотеку с одного API на другое, так как старое внезапно устарело. Кажется, что переписывание случайно исправит все проблемы. ### 10.3. Возможность чтения данных из статических таблиц в YT словарях. {#vozmozhnost-chteniia-dannykh-iz-staticheskikh-tablits-v-yt-slovariakh} @@ -802,7 +845,7 @@ Upd. Одну причину устранили, но ещё что-то неи Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ. -### 10.9. Уменьшение блокировок для cache словарей за счёт одновременных запросов одного и того же. {#umenshenie-blokirovok-dlia-cache-slovarei-za-schiot-odnovremennykh-zaprosov-odnogo-i-togo-zhe} +### 10.9. - Уменьшение блокировок для cache словарей за счёт одновременных запросов одного и того же. {#umenshenie-blokirovok-dlia-cache-slovarei-za-schiot-odnovremennykh-zaprosov-odnogo-i-togo-zhe} Заменено в пользу 10.10, 10.11. @@ -825,8 +868,6 @@ Upd. Одну причину устранили, но ещё что-то неи ### 10.14. Поддержка всех типов в функции transform. {#podderzhka-vsekh-tipov-v-funktsii-transform} -Задачу взяла Ольга Хвостикова. - ### 10.15. Использование словарей как специализированного layout для Join. {#ispolzovanie-slovarei-kak-spetsializirovannogo-layout-dlia-join} ### 10.16. Словари на локальном SSD. {#slovari-na-lokalnom-ssd} @@ -843,6 +884,7 @@ Upd. Одну причину устранили, но ещё что-то неи ### 10.19. Возможность зарегистрировать некоторые функции, использующие словари, под пользовательскими именами. {#vozmozhnost-zaregistrirovat-nekotorye-funktsii-ispolzuiushchie-slovari-pod-polzovatelskimi-imenami} + ## 11. Интерфейсы. {#interfeisy} ### 11.1. Вставка состояний агрегатных функций в виде кортежа аргументов или массива кортежей аргументов. {#vstavka-sostoianii-agregatnykh-funktsii-v-vide-kortezha-argumentov-ili-massiva-kortezhei-argumentov} @@ -851,6 +893,8 @@ Upd. Одну причину устранили, но ещё что-то неи Нужно разобраться, как упаковывать Java в статический бинарник, возможно AppImage. Или предоставить максимально простую инструкцию по установке jdbc-bridge. Может быть будет заинтересован Александр Крашенинников, Badoo, так как он разработал jdbc-bridge. +Upd. Александр Крашенинников перешёл в другую компанию и больше не занимается этим. + ### 11.3. + Интеграционные тесты ODBC драйвера путём подключения ClickHouse к самому себе через ODBC. {#integratsionnye-testy-odbc-draivera-putiom-podkliucheniia-clickhouse-k-samomu-sebe-cherez-odbc} Михаил Филимонов, Altinity. Готово. @@ -871,7 +915,9 @@ Upd. Одну причину устранили, но ещё что-то неи В ClickHouse в прошлом году добавили поддержку wire-протокола MySQL. PostgreSQL, так же как MySQL, использует несложный протокол общения между клиентом и сервером, но свой собственный. Поддержка этого протокола является востребованной и откроет новые возможности для ClickHouse. -### 11.9. Доработки ODBC драйвера. {#dorabotki-odbc-draivera} +Задача в процессе разработки. + +### 11.9. + Доработки ODBC драйвера. {#dorabotki-odbc-draivera} Денис Глазачев, Altinity. Хороший прогресс по этой задаче. @@ -881,12 +927,13 @@ zhang2014, есть pull request. Возможность описать в конфигурационном файле handler (путь в URL) для HTTP запросов к серверу, которому соответствует некоторый параметризованный запрос. Пользователь может вызвать этот обработчик и не должен передавать SQL запрос. + ## 12. Управление пользователями и доступом. {#upravlenie-polzovateliami-i-dostupom} -### 12.1. Role Based Access Control. {#role-based-access-control} +### 12.1. + Role Based Access Control. {#role-based-access-control} -[Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в начале февраля 2019. -Q1. Сейчас сделаны все интерфейсы в коде и запросы, но не сделаны варианты хранения прав кроме прототипа. +[Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в начале апреля 2019. +Q2. Сейчас сделаны все интерфейсы в коде и запросы, но не сделаны варианты хранения прав кроме прототипа. Upd. Сделано хранение прав. До готового к использованию состояния осталось несколько доработок. ### 12.2. + Управление пользователями и правами доступа с помощью SQL запросов. {#upravlenie-polzovateliami-i-pravami-dostupa-s-pomoshchiu-sql-zaprosov} @@ -897,7 +944,7 @@ Q1. Сделано управление правами полностью, но ### 12.3. Подключение справочника пользователей и прав доступа из LDAP. {#podkliuchenie-spravochnika-polzovatelei-i-prav-dostupa-iz-ldap} [Виталий Баранов](https://github.com/vitlibar). Требует 12.1. -Q1/Q2. +Q2. ### 12.4. Подключение IDM системы Яндекса как справочника пользователей и прав доступа. {#podkliuchenie-idm-sistemy-iandeksa-kak-spravochnika-polzovatelei-i-prav-dostupa} @@ -910,6 +957,8 @@ Q1/Q2. ### 12.6. Информация о пользователях и квотах в системной таблице. {#informatsiia-o-polzovateliakh-i-kvotakh-v-sistemnoi-tablitse} [Виталий Баранов](https://github.com/vitlibar). Требует 12.1. +Есть pull request. Q2. + ## 13. Разделение ресурсов, multi-tenancy. {#razdelenie-resursov-multi-tenancy} @@ -926,6 +975,8 @@ Q1/Q2. Требует 13.2 или сможем сделать более неудобную реализацию раньше. Обсуждается вариант неудобной реализации. Пока средний приоритет, целимся на Q1/Q2. Вариант реализации выбрал Александр Казаков. +Upd. Не уследили, и задачу стали обсуждать менеджеры. + ## 14. Диалект SQL. {#dialekt-sql} @@ -936,15 +987,13 @@ Q1/Q2. ### 14.2. Поддержка WITH для подзапросов. {#podderzhka-with-dlia-podzaprosov} -Михаил Коротов. - ### 14.3. Поддержка подстановок для множеств в правой части IN. {#podderzhka-podstanovok-dlia-mnozhestv-v-pravoi-chasti-in} ### 14.4. Поддержка подстановок для идентификаторов (имён) в SQL запросе. {#podderzhka-podstanovok-dlia-identifikatorov-imion-v-sql-zaprose} zhang2014 -### 14.5. Поддержка задания множества как массива в правой части секции IN. {#podderzhka-zadaniia-mnozhestva-kak-massiva-v-pravoi-chasti-sektsii-in} +### 14.5. + Поддержка задания множества как массива в правой части секции IN. {#podderzhka-zadaniia-mnozhestva-kak-massiva-v-pravoi-chasti-sektsii-in} Василий Немков, Altinity, делал эту задачу, но забросил её в пользу других задач. В результате, сейчас доделывает Антон Попов. @@ -993,7 +1042,7 @@ zhang2014 ### 14.16. Синонимы для функций из MySQL. {#sinonimy-dlia-funktsii-iz-mysql} -### 14.17. Ввести понятие stateful функций. {#vvesti-poniatie-stateful-funktsii} +### 14.17. + Ввести понятие stateful функций. {#vvesti-poniatie-stateful-funktsii} zhang2014. Для runningDifference, neighbour - их учёт в оптимизаторе запросов. @@ -1018,13 +1067,15 @@ zhang2014. Павел Потёмкин, ВШЭ. + ## 15. Улучшение поддержки JOIN. {#uluchshenie-podderzhki-join} -### 15.1. Доведение merge JOIN до продакшена. {#dovedenie-merge-join-do-prodakshena} +### 15.1. + Доведение merge JOIN до продакшена. {#dovedenie-merge-join-do-prodakshena} Артём Зуйков. Сейчас merge JOIN включается вручную опцией и всегда замедляет запросы. Хотим, чтобы он замедлял запросы только когда это неизбежно. Кстати, смысл merge JOIN появляется только совместно с 15.2 и 15.3. Q1. Сделали адаптивный вариант, но вроде он что-то всё-ещё замедляет. +Задача сделана, но всё работает слишком медленно. ### 15.1.1. Алгоритм two-level merge JOIN. {#algoritm-two-level-merge-join} @@ -1052,6 +1103,7 @@ Q1. Сделали адаптивный вариант, но вроде он ч Артём Зуйков. + ## 16. Типы данных и функции. {#tipy-dannykh-i-funktsii} ### 16.1. + DateTime64. {#datetime64} @@ -1062,6 +1114,8 @@ Q1. Сделали адаптивный вариант, но вроде он ч zhang2014 +Есть PR, в процессе ревью. + ### 16.3. Поддержка неконстантных аргументов с регулярными выражениями в функциях. {#podderzhka-nekonstantnykh-argumentov-s-reguliarnymi-vyrazheniiami-v-funktsiiakh} Данила Кутенин, но только после секретного изменения в работе. @@ -1073,6 +1127,7 @@ Upd. Секретного изменения в работе не будет, з ### 16.6. Функции нормализации и хэширования SQL запросов. {#funktsii-normalizatsii-i-kheshirovaniia-sql-zaprosov} + ## 17. Работа с географическими данными. {#rabota-s-geograficheskimi-dannymi} ### 17.1. Гео-словари для определения региона по координатам. {#geo-slovari-dlia-opredeleniia-regiona-po-koordinatam} @@ -1105,12 +1160,15 @@ Upd. Андрей сделал прототип более оптимально Сейчас функция тихо не работает в случае полигонов с самопересечениями, надо кидать исключение. + ## 18. Машинное обучение и статистика. {#mashinnoe-obuchenie-i-statistika} ### 18.1. Инкрементальная кластеризация данных. {#inkrementalnaia-klasterizatsiia-dannykh} Александр Кожихов, Максим Кузнецов. Обнаружена фундаментальная проблема в реализации, доделывает предположительно [Николай Кочетов](https://github.com/KochetovNicolai). Он может делегировать задачу кому угодно. +Исправление фундаментальной проблемы - есть PR. + ### 18.2. Агрегатные функции для статистических тестов. {#agregatnye-funktsii-dlia-statisticheskikh-testov} Артём Цыганов, Руденский Константин Игоревич, Семёнов Денис, ВШЭ. @@ -1121,7 +1179,8 @@ Upd. Андрей сделал прототип более оптимально ### 18.3. Инфраструктура для тренировки моделей в ClickHouse. {#infrastruktura-dlia-trenirovki-modelei-v-clickhouse} -В очереди. Возможно, Александр Кожихов. У него сначала идёт задача 24.26. +В очереди. + ## 19. Улучшение работы кластера. {#uluchshenie-raboty-klastera} @@ -1148,12 +1207,13 @@ Upd. Андрей сделал прототип более оптимально В прошлом году, Алексей добавил модельную реализацию (mock) интерфейса ZooKeeper для тестирования. Сейчас предлагается сделать реализацию поверх Etcd, а также расширить возможности тестовой реализации. Upd. Алексей сделал какой-то вариант, но борется с тем, что ничего не работает. +Upd. Есть pull request на начальной стадии. ### 19.3. Подключение YT Cypress или YDB как альтернативы ZooKeeper. {#podkliuchenie-yt-cypress-ili-ydb-kak-alternativy-zookeeper} Hold. Полезно для заказчиков внутри Яндекса, но есть риски. Эту задачу никто не будет делать. -### 19.4. internal\_replication = ‘auto’. {#internal-replication-auto} +### 19.4. internal_replication = ‘auto’. {#internal-replication-auto} ### 19.5. Реплицируемые базы данных. {#replitsiruemye-bazy-dannykh} @@ -1165,7 +1225,7 @@ Hold. Полезно для заказчиков внутри Яндекса, н ### 19.6. Одновременный выбор кусков для слияния многими репликами, отказ от leader election в ZK. {#odnovremennyi-vybor-kuskov-dlia-sliianiia-mnogimi-replikami-otkaz-ot-leader-election-v-zk} -Обсуждается. Возможно, будет делать Александр Казаков. +Обсуждается. Возможно, будет делать Александр Сапин. ### 19.7. Возможность записи данных при недоступности ZK и отказ от линейного порядка кусков в большинстве случаев. {#vozmozhnost-zapisi-dannykh-pri-nedostupnosti-zk-i-otkaz-ot-lineinogo-poriadka-kuskov-v-bolshinstve-sluchaev} @@ -1173,22 +1233,26 @@ Hold. Полезно для заказчиков внутри Яндекса, н ### 19.9. Отказ от хранения в ZK лога вставок и мержей. Обмен данными о кусках напрямую. {#otkaz-ot-khraneniia-v-zk-loga-vstavok-i-merzhei-obmen-dannymi-o-kuskakh-napriamuiu} +Три задачи выше обсуждаются, есть варианты. + ### 19.10. Облачные таблицы. {#oblachnye-tablitsy} Требует 1.6, 19.1, 19.6, 19.7, 19.8, 19.9. + ## 20. Мутации данных. {#mutatsii-dannykh} Пока все задачи по точечным UPDATE/DELETE имеют низкий приоритет, но ожидаем взять в работу в середине 2020. ### 20.1. Поддержка DELETE путём запоминания множества затронутых кусков и ключей. {#podderzhka-delete-putiom-zapominaniia-mnozhestva-zatronutykh-kuskov-i-kliuchei} -### 20.2. Поддержка DELETE путём преобразования множества ключей в множество row\_numbers на реплике, столбца флагов и индекса по диапазонам. {#podderzhka-delete-putiom-preobrazovaniia-mnozhestva-kliuchei-v-mnozhestvo-row-numbers-na-replike-stolbtsa-flagov-i-indeksa-po-diapazonam} +### 20.2. Поддержка DELETE путём преобразования множества ключей в множество row_numbers на реплике, столбца флагов и индекса по диапазонам. {#podderzhka-delete-putiom-preobrazovaniia-mnozhestva-kliuchei-v-mnozhestvo-row-numbers-na-replike-stolbtsa-flagov-i-indeksa-po-diapazonam} ### 20.3. Поддержка ленивых DELETE путём запоминания выражений и преобразования к множеству ключей в фоне. {#podderzhka-lenivykh-delete-putiom-zapominaniia-vyrazhenii-i-preobrazovaniia-k-mnozhestvu-kliuchei-v-fone} ### 20.4. Поддержка UPDATE с помощью преобразования в DELETE и вставок. {#podderzhka-update-s-pomoshchiu-preobrazovaniia-v-delete-i-vstavok} + ## 21. Оптимизации производительности. {#optimizatsii-proizvoditelnosti} ### 21.1. + Параллельный парсинг форматов. {#parallelnyi-parsing-formatov} @@ -1201,7 +1265,7 @@ Hold. Полезно для заказчиков внутри Яндекса, н После 21.1, предположительно Никита Михайлов. Задача сильно проще чем 21.1. -### 21.3. Исправление низкой производительности анализа индекса в случае большого множества в секции IN. {#ispravlenie-nizkoi-proizvoditelnosti-analiza-indeksa-v-sluchae-bolshogo-mnozhestva-v-sektsii-in} +### 21.3. + Исправление низкой производительности анализа индекса в случае большого множества в секции IN. {#ispravlenie-nizkoi-proizvoditelnosti-analiza-indeksa-v-sluchae-bolshogo-mnozhestva-v-sektsii-in} Нужно всем (Zen, БК, DataLens, TestEnv…). Антон Попов, Q1/Q2. @@ -1276,6 +1340,8 @@ Amos Bird. - Удаление из GROUP BY функций от других ключей GROUP BY. - Удаление дублирующихся DISTINCT, ORDER BY из подзапросов. +Несколько оптимизаций есть в PR. + ### 21.13. Fusion агрегатных функций. {#fusion-agregatnykh-funktsii} После или совместно с 21.11. @@ -1309,23 +1375,23 @@ Constraints позволяют задать выражение, истиннос В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top. -1. Рекурсивный вариант сортировки по кортежам. +2. Рекурсивный вариант сортировки по кортежам. Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить… например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения. -1. RadixSort для сортировки. +3. RadixSort для сортировки. Один наш знакомый начал делать задачу по попытке использования RadixSort для сортировки столбцов. Был сделан вариант indirect сортировки (для `getPermutation`), но не оптимизирован до конца - есть лишние ненужные перекладывания элементов. Для того, чтобы его оптимизировать, придётся добавить немного шаблонной магии (на последнем шаге что-то не копировать, вместо перекладывания индексов - складывать их в готовое место). Также этот человек добавил метод MSD Radix Sort для реализации radix partial sort. Но даже не проверил производительность. Наиболее содержательная часть задачи может состоять в применении Radix Sort для сортировки кортежей, расположенных в оперативке в виде Structure Of Arrays неизвестного в compile-time размера. Это может работать хуже, чем то, что описано в пункте 2… Но попробовать не помешает. -1. Three-way comparison sort. +4. Three-way comparison sort. Виртуальный метод `compareAt` возвращает -1, 0, 1. Но алгоритмы сортировки сравнениями обычно рассчитаны на `operator<` и не могут получить преимущества от three-way comparison. А можно ли написать так, чтобы преимущество было? -1. pdq partial sort +5. pdq partial sort -Хороший алгоритм сортировки сравнениями `pdqsort` не имеет варианта partial sort. Заметим, что на практике, почти все сортировки в запросах ClickHouse являются partial\_sort, так как `ORDER BY` почти всегда идёт с `LIMIT`. Кстати, Данила Кутенин уже попробовал это и показал, что в тривиальном случае преимущества нет. Но не очевидно, что нельзя сделать лучше. +Хороший алгоритм сортировки сравнениями `pdqsort` не имеет варианта partial sort. Заметим, что на практике, почти все сортировки в запросах ClickHouse являются partial_sort, так как `ORDER BY` почти всегда идёт с `LIMIT`. Кстати, Данила Кутенин уже попробовал это и показал, что в тривиальном случае преимущества нет. Но не очевидно, что нельзя сделать лучше. ### 21.20. Использование материализованных представлений для оптимизации запросов. {#ispolzovanie-materializovannykh-predstavlenii-dlia-optimizatsii-zaprosov} @@ -1344,6 +1410,7 @@ Constraints позволяют задать выражение, истиннос zhang2014. Есть pull request. + ## 22. Долги и недоделанные возможности. {#dolgi-i-nedodelannye-vozmozhnosti} ### 22.1. + Исправление неработающих таймаутов, если используется TLS. {#ispravlenie-nerabotaiushchikh-taimautov-esli-ispolzuetsia-tls} @@ -1356,12 +1423,11 @@ N.Vartolomei. ### 22.3. Защита от абсурдно заданных пользователем кодеков. {#zashchita-ot-absurdno-zadannykh-polzovatelem-kodekov} -В очереди, скорее всего [Ольга Хвостикова](https://github.com/stavrolia). - -### 22.4. Исправление оставшихся deadlocks в табличных RWLock-ах. {#ispravlenie-ostavshikhsia-deadlocks-v-tablichnykh-rwlock-akh} +### 22.4. + Исправление оставшихся deadlocks в табличных RWLock-ах. {#ispravlenie-ostavshikhsia-deadlocks-v-tablichnykh-rwlock-akh} Александр Казаков. Нужно для Яндекс.Метрики и Datalens. Задача постепенно тащится и исправлениями в соседних местах стала менее актуальна. В Q1 будет сделана или отменена с учётом 1.2. и 1.3. +Upd. Добавили таймауты. ### 22.5. + Исправление редких срабатываний TSan в stress тестах в CI. {#ispravlenie-redkikh-srabatyvanii-tsan-v-stress-testakh-v-ci} @@ -1384,7 +1450,7 @@ N.Vartolomei. [Виталий Баранов](https://github.com/vitlibar). Возможно, это - фундаментальная проблема и следует её только документировать. Кажется, отменяем, но пока ещё не всё ясно. -### 22.10. Исправление дрифта при отслеживании потребления памяти запросами. {#ispravlenie-drifta-pri-otslezhivanii-potrebleniia-pamiati-zaprosami} +### 22.10. + Исправление дрифта при отслеживании потребления памяти запросами. {#ispravlenie-drifta-pri-otslezhivanii-potrebleniia-pamiati-zaprosami} Требует 6.3., но можно улучшить отдельными хаками. Нужно Метрике и БК. @@ -1431,7 +1497,7 @@ Altinity. [Николай Кочетов](https://github.com/KochetovNicolai), декабрь 2019. -### 22.21. Неправильное поведение DateTime в районе начала unix epoch. {#nepravilnoe-povedenie-datetime-v-raione-nachala-unix-epoch} +### 22.21. + Неправильное поведение DateTime в районе начала unix epoch. {#nepravilnoe-povedenie-datetime-v-raione-nachala-unix-epoch} Алексей Миловидов. @@ -1470,18 +1536,19 @@ Altinity. [Александр Сапин](https://github.com/alesapin) + ## 23. Default Festival. {#default-festival} -### 23.1. + Включение minimalistic\_part\_header в ZooKeeper. {#vkliuchenie-minimalistic-part-header-v-zookeeper} +### 23.1. + Включение minimalistic_part_header в ZooKeeper. {#vkliuchenie-minimalistic-part-header-v-zookeeper} Сильно уменьшает объём данных в ZooKeeper. Уже год в продакшене в Яндекс.Метрике. Алексей Миловидов, ноябрь 2019. -### 23.2. Включение distributed\_aggregation\_memory\_efficient. {#vkliuchenie-distributed-aggregation-memory-efficient} +### 23.2. Включение distributed_aggregation_memory_efficient. {#vkliuchenie-distributed-aggregation-memory-efficient} Есть риски меньшей производительности лёгких запросов, хотя производительность тяжёлых запросов всегда увеличивается. -### 23.3. Включение min\_bytes\_to\_external\_sort и min\_bytes\_to\_external\_group\_by. {#vkliuchenie-min-bytes-to-external-sort-i-min-bytes-to-external-group-by} +### 23.3. Включение min_bytes_to_external_sort и min_bytes_to_external_group_by. {#vkliuchenie-min-bytes-to-external-sort-i-min-bytes-to-external-group-by} Желательно 5.2. и 13.1. @@ -1489,7 +1556,7 @@ Altinity. Есть гипотеза, что плохо работает на очень больших кластерах. -### 23.5. Включение compile\_expressions. {#vkliuchenie-compile-expressions} +### 23.5. Включение compile_expressions. {#vkliuchenie-compile-expressions} Требует 7.2. Задачу изначально на 99% сделал Денис Скоробогатов, ВШЭ и Яндекс. Остальной процент доделывал Алексей Миловидов, а затем [Александр Сапин](https://github.com/alesapin). @@ -1514,6 +1581,7 @@ Q1. [Николай Кочетов](https://github.com/KochetovNicolai). Возможность mlock бинарника сделал Олег Алексеенков [\#3553](https://github.com/ClickHouse/ClickHouse/pull/3553) . Поможет, когда на серверах кроме ClickHouse работает много посторонних программ (мы иногда называем их в шутку «треш-программами»). + ## 24. Экспериментальные задачи. {#eksperimentalnye-zadachi} ### 24.1. Веб-интерфейс для просмотра состояния кластера и профилирования запросов. {#veb-interfeis-dlia-prosmotra-sostoianiia-klastera-i-profilirovaniia-zaprosov} @@ -1549,11 +1617,13 @@ ClickHouse поддерживает LZ4 и ZSTD для сжатия данных 1. Шифрование на уровне VFS. +Обсуждаются детали реализации. Q3/Q4. + ### 24.5. Поддержка функций шифрования для отдельных значений. {#podderzhka-funktsii-shifrovaniia-dlia-otdelnykh-znachenii} Смотрите также 24.5. -1. Шифрование отдельных значений. +2. Шифрование отдельных значений. Для этого требуется реализовать функции шифрования и расшифрования, доступные из SQL. Для шифрования реализовать возможность добавления нужного количества случайных бит для исключения одинаковых зашифрованных значений на одинаковых данных. Это позволит реализовать возможность «забывания» данных без удаления строк таблицы: можно шифровать данные разных клиентов разными ключами, и для того, чтобы забыть данные одного клиента, потребуется всего лишь удалить ключ. ### 24.6. Userspace RAID. {#userspace-raid} @@ -1586,7 +1656,7 @@ RAID позволяет одновременно увеличить надёжн Дмитрий Ковальков, ВШЭ и Яндекс. -Подавляющее большинство кода ClickHouse написана для x86\_64 с набором инструкций до SSE 4.2 включительно. Лишь отдельные редкие функции поддерживают AVX/AVX2/AVX512 с динамической диспетчеризацией. +Подавляющее большинство кода ClickHouse написана для x86_64 с набором инструкций до SSE 4.2 включительно. Лишь отдельные редкие функции поддерживают AVX/AVX2/AVX512 с динамической диспетчеризацией. В первой части задачи, следует добавить в ClickHouse реализации некоторых примитивов, оптимизированные под более новый набор инструкций. Например, AVX2 реализацию генератора случайных чисел pcg: https://github.com/lemire/simdpcg @@ -1598,6 +1668,8 @@ RAID позволяет одновременно увеличить надёжн Продолжение 24.8. +Upd. Есть pull request. + ### 24.10. Поддержка типов half/bfloat16/unum. {#podderzhka-tipov-halfbfloat16unum} [\#7657](https://github.com/ClickHouse/ClickHouse/issues/7657) @@ -1633,6 +1705,8 @@ ClickHouse предоставляет достаточно богатый наб В компании nVidia сделали прототип offloading вычисления GROUP BY с некоторыми из агрегатных функций в ClickHouse и обещат предоставить исходники в публичный доступ для дальнейшего развития. Предлагается изучить этот прототип и расширить его применимость для более широкого сценария использования. В качестве альтернативы, предлагается изучить исходные коды системы `OmniSci` или `Alenka` или библиотеку `CUB` https://nvlabs.github.io/cub/ и применить некоторые из алгоритмов в ClickHouse. Upd. В компании nVidia выложили прототип, теперь нужна интеграция в систему сборки. +Upd. Интеграция в систему сборки - Иван Лежанкин. +Upd. Есть прототип bitonic sort. ### 24.13. Stream запросы. {#stream-zaprosy} @@ -1656,6 +1730,8 @@ Upd. В компании nVidia выложили прототип, теперь Максим Серебряков +Задача в работе. + ### 24.18. Не TCP протокол передачи файлов при репликации. {#ne-tcp-protokol-peredachi-failov-pri-replikatsii} ### 24.19. Промежуточное состояние GROUP BY как структура данных для key-value доступа. {#promezhutochnoe-sostoianie-group-by-kak-struktura-dannykh-dlia-key-value-dostupa} @@ -1750,6 +1826,8 @@ Amos Bird, но его решение слишком громоздкое и п Рассматривается вариант - поддержка GRPC в ClickHouse. Здесь есть неочевидные моменты, такие как - эффективная передача массивов данных в column-oriented формате - насколько удобно будет обернуть это в GRPC. +Задача в работе, есть pull request. + ## 25. DevRel {#devrel} ### 25.1. + Перевод инструкции для начинающих разработчиков. {#perevod-instruktsii-dlia-nachinaiushchikh-razrabotchikov} @@ -1791,7 +1869,7 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.10. Митапы в России и Беларуси: Москва x2 + митап для разработчиков или хакатон, Санкт-Петербург, Минск, Нижний Новгород, Екатеринбург, Новосибирск и/или Академгородок, Иннополис или Казань. {#mitapy-v-rossii-i-belarusi-moskva-x2-mitap-dlia-razrabotchikov-ili-khakaton-sankt-peterburg-minsk-nizhnii-novgorod-ekaterinburg-novosibirsk-iili-akademgorodok-innopolis-ili-kazan} -Екатерина - организация +Екатерина - организация. Upd. Проведено два онлайн митапа на русском. ### 25.11. Митапы зарубежные: восток США (Нью Йорк, возможно Raleigh), возможно северо-запад (Сиэтл), Китай (Пекин снова, возможно митап для разработчиков или хакатон), Лондон. {#mitapy-zarubezhnye-vostok-ssha-niu-iork-vozmozhno-raleigh-vozmozhno-severo-zapad-sietl-kitai-pekin-snova-vozmozhno-mitap-dlia-razrabotchikov-ili-khakaton-london} @@ -1807,7 +1885,8 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.14. Конференции в России: все HighLoad, возможно CodeFest, DUMP или UWDC, возможно C++ Russia. {#konferentsii-v-rossii-vse-highload-vozmozhno-codefest-dump-ili-uwdc-vozmozhno-c-russia} -Алексей Миловидов и все подготовленные докладчики +Алексей Миловидов и все подготовленные докладчики. +Upd. Есть Saint HighLoad online. ### 25.15. Конференции зарубежные: Percona, DataOps, попытка попасть на более крупные. {#konferentsii-zarubezhnye-percona-dataops-popytka-popast-na-bolee-krupnye} @@ -1826,13 +1905,13 @@ Amos Bird, но его решение слишком громоздкое и п Требуется проработать вопрос безопасности и изоляции инстансов (поднятие в контейнерах с ограничениями по сети), подключение тестовых датасетов с помощью copy-on-write файловой системы; органичения ресурсов. -Есть минимальный прототип. Сделал Илья Яцишин. Этот прототип работает только с одной версией ClickHouse и не позволяет делиться ссылками на результаты запросов. +Есть минимальный прототип. Сделал Илья Яцишин. Этот прототип не позволяет делиться ссылками на результаты запросов. ### 25.17. Взаимодействие с ВУЗами: ВШЭ, УрФУ, ICT Beijing. {#vzaimodeistvie-s-vuzami-vshe-urfu-ict-beijing} Алексей Миловидов и вся группа разработки -### 25.18. Лекция в ШАД. {#lektsiia-v-shad} +### 25.18. - Лекция в ШАД. {#lektsiia-v-shad} Алексей Миловидов @@ -1848,7 +1927,7 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.22. On-site помощь с ClickHouse компаниям в дни рядом с мероприятиями. {#on-site-pomoshch-s-clickhouse-kompaniiam-v-dni-riadom-s-meropriiatiiami} -[Иван Блинков](https://github.com/blinkov/) - организация +[Иван Блинков](https://github.com/blinkov/) - организация. Проверил мероприятие для турецкой компании. ### 25.23. Новый мерч для ClickHouse. {#novyi-merch-dlia-clickhouse} diff --git a/docs/ru/whats_new/index.md b/docs/ru/whats_new/index.md new file mode 100644 index 00000000000..b97b3623a9e --- /dev/null +++ b/docs/ru/whats_new/index.md @@ -0,0 +1,6 @@ +--- +toc_folder_title: Что нового? +toc_priority: 72 +--- + + diff --git a/docs/ru/whats_new/roadmap.md b/docs/ru/whats_new/roadmap.md new file mode 120000 index 00000000000..5ef0ebdb1bb --- /dev/null +++ b/docs/ru/whats_new/roadmap.md @@ -0,0 +1 @@ +../../en/whats_new/roadmap.md \ No newline at end of file diff --git a/docs/ru/security_changelog.md b/docs/ru/whats_new/security_changelog.md similarity index 99% rename from docs/ru/security_changelog.md rename to docs/ru/whats_new/security_changelog.md index c8f66bf0475..9a2dab8ba14 100644 --- a/docs/ru/security_changelog.md +++ b/docs/ru/whats_new/security_changelog.md @@ -1,3 +1,5 @@ +# Security Changelog + ## Исправлено в релизе 19.14.3.3, 2019-09-10 {#ispravleno-v-relize-19-14-3-3-2019-09-10} ### CVE-2019-15024 {#cve-2019-15024} diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml deleted file mode 100644 index 0df04f892cf..00000000000 --- a/docs/toc_ru.yml +++ /dev/null @@ -1,253 +0,0 @@ -nav: - -- 'Введение': - - 'Обзор': 'index.md' - - 'Отличительные возможности ClickHouse': 'introduction/distinctive_features.md' - - 'Особенности ClickHouse, которые могут считаться недостатками': 'introduction/features_considered_disadvantages.md' - - 'Производительность': 'introduction/performance.md' - - 'История': 'introduction/history.md' - - 'Информационная поддержка': 'introduction/info.md' - - 'Пользователи': 'introduction/adopters.md' - -- 'Начало работы': - - 'hidden': 'getting_started/index.md' - - 'Установка': 'getting_started/install.md' - - 'Руководство для начинающих': 'getting_started/tutorial.md' - - 'Тестовые наборы данных': - - 'Введение': 'getting_started/example_datasets/index.md' - - 'OnTime': 'getting_started/example_datasets/ontime.md' - - 'Данные о такси в Нью-Йорке': 'getting_started/example_datasets/nyc_taxi.md' - - 'AMPLab Big Data Benchmark': 'getting_started/example_datasets/amplab_benchmark.md' - - 'WikiStat': 'getting_started/example_datasets/wikistat.md' - - 'Терабайт логов кликов от Criteo': 'getting_started/example_datasets/criteo.md' - - 'Схема «Звезда»': 'getting_started/example_datasets/star_schema.md' - - 'Данные Яндекс.Метрики': 'getting_started/example_datasets/metrica.md' - - 'Playground': 'getting_started/playground.md' - -- 'Интерфейсы': - - 'Введение': 'interfaces/index.md' - - 'Клиент командной строки': 'interfaces/cli.md' - - 'Нативный интерфейс (TCP)': 'interfaces/tcp.md' - - 'HTTP-интерфейс': 'interfaces/http.md' - - 'MySQL-интерфейс': 'interfaces/mysql.md' - - 'Форматы входных и выходных данных': 'interfaces/formats.md' - - 'JDBC-драйвер': 'interfaces/jdbc.md' - - 'ODBC-драйвер': 'interfaces/odbc.md' - - 'C++ клиентская библиотека': 'interfaces/cpp.md' - - 'От сторонних разработчиков': - - 'Клиентские библиотеки': 'interfaces/third-party/client_libraries.md' - - 'Интеграции': 'interfaces/third-party/integrations.md' - - 'Визуальные интерфейсы': 'interfaces/third-party/gui.md' - - 'Прокси': 'interfaces/third-party/proxy.md' - -- 'Движки баз данных': - - 'Введение': 'database_engines/index.md' - - 'MySQL': 'database_engines/mysql.md' - - 'Lazy': 'database_engines/lazy.md' - -- 'Движки таблиц': - - 'Введение': 'operations/table_engines/index.md' - - 'Семейство MergeTree': - - 'MergeTree': 'operations/table_engines/mergetree.md' - - 'Репликация данных': 'operations/table_engines/replication.md' - - 'Произвольный ключ партиционирования': 'operations/table_engines/custom_partitioning_key.md' - - 'ReplacingMergeTree': 'operations/table_engines/replacingmergetree.md' - - 'SummingMergeTree': 'operations/table_engines/summingmergetree.md' - - 'AggregatingMergeTree': 'operations/table_engines/aggregatingmergetree.md' - - 'CollapsingMergeTree': 'operations/table_engines/collapsingmergetree.md' - - 'VersionedCollapsingMergeTree': 'operations/table_engines/versionedcollapsingmergetree.md' - - 'GraphiteMergeTree': 'operations/table_engines/graphitemergetree.md' - - 'Семейство Log': - - 'Введение': 'operations/table_engines/log_family.md' - - 'StripeLog': 'operations/table_engines/stripelog.md' - - 'Log': 'operations/table_engines/log.md' - - 'TinyLog': 'operations/table_engines/tinylog.md' - - 'Интеграции': - - 'Kafka': 'operations/table_engines/kafka.md' - - 'MySQL': 'operations/table_engines/mysql.md' - - 'JDBC': 'operations/table_engines/jdbc.md' - - 'ODBC': 'operations/table_engines/odbc.md' - - 'HDFS': 'operations/table_engines/hdfs.md' - - 'Особые': - - 'Distributed': 'operations/table_engines/distributed.md' - - 'Внешние данные': 'operations/table_engines/external_data.md' - - 'Dictionary': 'operations/table_engines/dictionary.md' - - 'Merge': 'operations/table_engines/merge.md' - - 'File': 'operations/table_engines/file.md' - - 'Null': 'operations/table_engines/null.md' - - 'Set': 'operations/table_engines/set.md' - - 'Join': 'operations/table_engines/join.md' - - 'URL': 'operations/table_engines/url.md' - - 'View': 'operations/table_engines/view.md' - - 'MaterializedView': 'operations/table_engines/materializedview.md' - - 'Memory': 'operations/table_engines/memory.md' - - 'Buffer': 'operations/table_engines/buffer.md' - - 'GenerateRandom': 'operations/table_engines/generate.md' - -- 'Справка по SQL': - - 'hidden': 'query_language/index.md' - - 'Общий синтаксис': 'query_language/syntax.md' - - 'Запросы': - - 'SELECT': 'query_language/select.md' - - 'INSERT INTO': 'query_language/insert_into.md' - - 'CREATE': 'query_language/create.md' - - 'ALTER': 'query_language/alter.md' - - 'SYSTEM': 'query_language/system.md' - - 'SHOW': 'query_language/show.md' - - 'Прочие': 'query_language/misc.md' - - 'Функции': - - 'Введение': 'query_language/functions/index.md' - - 'Арифметические функции': 'query_language/functions/arithmetic_functions.md' - - 'Функции сравнения': 'query_language/functions/comparison_functions.md' - - 'Логические функции': 'query_language/functions/logical_functions.md' - - 'Функции преобразования типов': 'query_language/functions/type_conversion_functions.md' - - 'Функции для работы с датами и временем': 'query_language/functions/date_time_functions.md' - - 'Функции для работы со строками': 'query_language/functions/string_functions.md' - - 'Функции поиска в строках': 'query_language/functions/string_search_functions.md' - - 'Функции поиска и замены в строках': 'query_language/functions/string_replace_functions.md' - - 'Условные функции': 'query_language/functions/conditional_functions.md' - - 'Математические функции': 'query_language/functions/math_functions.md' - - 'Функции округления': 'query_language/functions/rounding_functions.md' - - 'Функции по работе с массивами': 'query_language/functions/array_functions.md' - - 'Функции разбиения и слияния строк и массивов': 'query_language/functions/splitting_merging_functions.md' - - 'Битовые функции': 'query_language/functions/bit_functions.md' - - 'Функции для битмапов': 'query_language/functions/bitmap_functions.md' - - 'Функции хэширования': 'query_language/functions/hash_functions.md' - - 'Функции генерации псевдослучайных чисел': 'query_language/functions/random_functions.md' - - 'Функции для работы с UUID': 'query_language/functions/uuid_functions.md' - - 'Функции кодирования': 'query_language/functions/encoding_functions.md' - - 'Функции для работы с URL': 'query_language/functions/url_functions.md' - - 'Функции для работы с IP-адресами': 'query_language/functions/ip_address_functions.md' - - 'Функции для работы с JSON.': 'query_language/functions/json_functions.md' - - 'Функции высшего порядка': 'query_language/functions/higher_order_functions.md' - - 'Функции для работы с внешними словарями': 'query_language/functions/ext_dict_functions.md' - - 'Функции для работы со словарями Яндекс.Метрики': 'query_language/functions/ym_dict_functions.md' - - 'Функции для реализации оператора IN.': 'query_language/functions/in_functions.md' - - 'Функция arrayJoin': 'query_language/functions/array_join.md' - - 'Функции для работы с географическими координатами': 'query_language/functions/geo.md' - - 'Функции c Nullable аргументами': 'query_language/functions/functions_for_nulls.md' - - 'Функции машинного обучения': 'query_language/functions/machine_learning_functions.md' - - 'Функции для интроспекции': 'query_language/functions/introspection.md' - - 'Прочие функции': 'query_language/functions/other_functions.md' - - 'Агрегатные функции': - - 'Введение': 'query_language/agg_functions/index.md' - - 'Справочник функций': 'query_language/agg_functions/reference.md' - - 'Комбинаторы агрегатных функций': 'query_language/agg_functions/combinators.md' - - 'Параметрические агрегатные функции': 'query_language/agg_functions/parametric_functions.md' - - 'Табличные функции': - - 'Введение': 'query_language/table_functions/index.md' - - 'file': 'query_language/table_functions/file.md' - - 'merge': 'query_language/table_functions/merge.md' - - 'numbers': 'query_language/table_functions/numbers.md' - - 'remote': 'query_language/table_functions/remote.md' - - 'url': 'query_language/table_functions/url.md' - - 'mysql': 'query_language/table_functions/mysql.md' - - 'jdbc': 'query_language/table_functions/jdbc.md' - - 'odbc': 'query_language/table_functions/odbc.md' - - 'hdfs': 'query_language/table_functions/hdfs.md' - - 'input': 'query_language/table_functions/input.md' - - 'generateRandom': 'query_language/table_functions/generate.md' - - 'Словари': - - 'Введение': 'query_language/dicts/index.md' - - 'Внешние словари': - - 'Общее описание': 'query_language/dicts/external_dicts.md' - - 'Настройка внешнего словаря': 'query_language/dicts/external_dicts_dict.md' - - 'Хранение словарей в памяти': 'query_language/dicts/external_dicts_dict_layout.md' - - 'Обновление словарей': 'query_language/dicts/external_dicts_dict_lifetime.md' - - 'Источники внешних словарей': 'query_language/dicts/external_dicts_dict_sources.md' - - 'Ключ и поля словаря': 'query_language/dicts/external_dicts_dict_structure.md' - - 'Иерархические словари': 'query_language/dicts/external_dicts_dict_hierarchical.md' - - 'Встроенные словари': 'query_language/dicts/internal_dicts.md' - - 'Операторы': 'query_language/operators.md' - - 'Типы данных': - - 'Введение': 'data_types/index.md' - - 'UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64': 'data_types/int_uint.md' - - 'Float32, Float64': 'data_types/float.md' - - 'Decimal': 'data_types/decimal.md' - - 'Булевы значения': 'data_types/boolean.md' - - 'String': 'data_types/string.md' - - 'FixedString(N)': 'data_types/fixedstring.md' - - 'UUID': 'data_types/uuid.md' - - 'Date': 'data_types/date.md' - - 'DateTime': 'data_types/datetime.md' - - 'DateTime64': 'data_types/datetime64.md' - - 'Enum': 'data_types/enum.md' - - 'Array(T)': 'data_types/array.md' - - 'AggregateFunction(name, types_of_arguments...)': 'data_types/nested_data_structures/aggregatefunction.md' - - 'Tuple(T1, T2, ...)': 'data_types/tuple.md' - - 'Nullable': 'data_types/nullable.md' - - 'Вложенные структуры данных': - - 'hidden': 'data_types/nested_data_structures/index.md' - - 'Nested(Name1 Type1, Name2 Type2, ...)': 'data_types/nested_data_structures/nested.md' - - 'Служебные типы данных': - - 'hidden': 'data_types/special_data_types/index.md' - - 'Expression': 'data_types/special_data_types/expression.md' - - 'Set': 'data_types/special_data_types/set.md' - - 'Nothing': 'data_types/special_data_types/nothing.md' - - 'Interval': 'data_types/special_data_types/interval.md' - - 'Domains': - - 'Overview': 'data_types/domains/overview.md' - - 'IPv4': 'data_types/domains/ipv4.md' - - 'IPv6': 'data_types/domains/ipv6.md' - -- 'Руководства': - - 'Обзор': 'guides/index.md' - - 'Применение CatBoost моделей': 'guides/apply_catboost_model.md' - -- 'Эксплуатация': - - 'Введение': 'operations/index.md' - - 'Требования': 'operations/requirements.md' - - 'Мониторинг': 'operations/monitoring.md' - - 'Устранение неисправностей': 'operations/troubleshooting.md' - - 'Советы по эксплуатации': 'operations/tips.md' - - 'Обновление ClickHouse': 'operations/update.md' - - 'Права доступа': 'operations/access_rights.md' - - 'Резервное копирование': 'operations/backup.md' - - 'Конфигурационные файлы': 'operations/configuration_files.md' - - 'Квоты': 'operations/quotas.md' - - 'Системные таблицы': 'operations/system_tables.md' - - 'Оптимизация производительности': - - 'Профилирование запросов': 'operations/performance/sampling_query_profiler.md' - - 'Тестирование оборудования': 'operations/performance_test.md' - - 'Конфигурационные параметры сервера': - - 'Введение': 'operations/server_settings/index.md' - - 'Серверные настройки': 'operations/server_settings/settings.md' - - 'Настройки': - - 'Введение': 'operations/settings/index.md' - - 'Разрешения на выполнение запросов': 'operations/settings/permissions_for_queries.md' - - 'Ограничения на сложность запроса': 'operations/settings/query_complexity.md' - - 'Настройки': 'operations/settings/settings.md' - - 'Ограничения на изменение настроек': 'operations/settings/constraints_on_settings.md' - - 'Профили настроек': 'operations/settings/settings_profiles.md' - - 'Настройки пользователей': 'operations/settings/settings_users.md' - - 'Утилиты': - - 'Введение': 'operations/utils/index.md' - - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - - 'clickhouse-local': 'operations/utils/clickhouse-local.md' - - 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md' - -- 'Разработка': - - 'hidden': 'development/index.md' - - 'Инструкция для начинающего разработчика ClickHouse': 'development/developer_instruction.md' - - 'Обзор архитектуры ClickHouse': 'development/architecture.md' - - 'Навигация по коду ClickHouse': 'development/browse_code.md' - - 'Как собрать ClickHouse на Linux': 'development/build.md' - - 'Как собрать ClickHouse на Mac OS X': 'development/build_osx.md' - - 'Как собрать ClickHouse на Linux для Mac OS X': 'development/build_cross_osx.md' - - 'Как собрать ClickHouse на Linux для AARCH64 (ARM64)': 'development/build_cross_arm.md' - - 'Как писать код на C++': 'development/style.md' - - 'Как запустить тесты': 'development/tests.md' - - 'Сторонние библиотеки': 'development/contrib.md' - -- 'Что нового': - - 'Changelog': - - '2020': 'changelog/index.md' - - '2019': 'changelog/2019.md' - - '2018': 'changelog/2018.md' - - '2017': 'changelog/2017.md' - - 'Security changelog': 'security_changelog.md' - - 'Roadmap': 'roadmap.md' - - 'Подробный roadmap 2020': 'extended_roadmap.md' - -- 'F.A.Q.': - - 'Общие вопросы': 'faq/general.md' diff --git a/docs/tools/README.md b/docs/tools/README.md index 11e04de4b6a..56ca016ad9e 100644 --- a/docs/tools/README.md +++ b/docs/tools/README.md @@ -38,5 +38,5 @@ The easiest way to see the result is to use `--livereload=8888` argument of buil At the moment there’s no easy way to do just that, but you can consider: -- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng). +- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/enQtOTUzMjM4ODQwNTc5LWJmMjE3Yjc2YmI1ZDBlZmI4ZTc3OWY3ZTIwYTljYzY4MzBlODM3YzBjZTc1YmYyODRlZTJkYTgzYzBiNTA2Yjk). - Some search engines allow to subscribe on specific website changes via email and you can opt-in for that for https://clickhouse.tech. diff --git a/docs/tools/build.py b/docs/tools/build.py index 65b9f9f8c04..9749976d13c 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -82,6 +82,7 @@ def build_for_lang(lang, args): 'fr': 'Français', 'ru': 'Русский', 'ja': '日本語', + 'tr': 'Türkçe', 'fa': 'فارسی' } @@ -92,6 +93,7 @@ def build_for_lang(lang, args): 'fr': 'Documentation ClickHouse %s', 'ru': 'Документация ClickHouse %s', 'ja': 'ClickHouseドキュメント %s', + 'tr': 'ClickHouse Belgeleri %s', 'fa': 'مستندات %sClickHouse' } @@ -109,6 +111,7 @@ def build_for_lang(lang, args): 'codehilite', 'nl2br', 'sane_lists', + 'pymdownx.details', 'pymdownx.magiclink', 'pymdownx.superfences', 'extra', @@ -124,8 +127,10 @@ def build_for_lang(lang, args): if args.htmlproofer: plugins.append('htmlproofer') + site_name = site_names.get(lang, site_names['en']) % args.version_prefix + site_name = site_name.replace(' ', ' ') raw_config = dict( - site_name=site_names.get(lang, site_names['en']) % args.version_prefix, + site_name=site_name, site_url=f'https://clickhouse.tech/docs/{lang}/', docs_dir=os.path.join(args.docs_dir, lang), site_dir=site_dir, @@ -219,7 +224,20 @@ def build_single_page_version(lang, args, nav, cfg): os.path.join(site_temp, 'single'), single_page_output_path ) - + + single_page_index_html = os.path.join(single_page_output_path, 'index.html') + single_page_content_js = os.path.join(single_page_output_path, 'content.js') + with open(single_page_index_html, 'r') as f: + sp_prefix, sp_js, sp_suffix = f.read().split('') + with open(single_page_index_html, 'w') as f: + f.write(sp_prefix) + f.write(sp_suffix) + with open(single_page_content_js, 'w') as f: + if args.minify: + import jsmin + sp_js = jsmin.jsmin(sp_js) + f.write(sp_js) + logging.info(f'Re-building single page for {lang} pdf/test') with util.temp_dir() as test_dir: extra['single_page'] = False @@ -288,13 +306,14 @@ def write_redirect_html(out_path, to_url): except OSError: pass with open(out_path, 'w') as f: - f.write(f''' + f.write(f''' + Page Redirection @@ -305,9 +324,9 @@ def write_redirect_html(out_path, to_url): def build_redirect_html(args, from_path, to_path): - for lang in ['en', 'es', 'fr', 'ja', 'fa']: # TODO: args.lang.split(','): + for lang in args.lang.split(','): out_path = os.path.join(args.docs_output_dir, lang, from_path.replace('.md', '/index.html')) - version_prefix = args.version_prefix + '/' if args.version_prefix else '/' + version_prefix = f'/{args.version_prefix}/' if args.version_prefix else '/' target_path = to_path.replace('.md', '/') to_url = f'/docs{version_prefix}{lang}/{target_path}' to_url = to_url.strip() @@ -343,9 +362,12 @@ def build(args): build_releases(args, build_docs) if not args.skip_website: + website.process_benchmark_results(args) website.minify_website(args) for static_redirect in [ + ('benchmark.html', '/benchmark/dbms/'), + ('benchmark_hardware.html', '/benchmark/hardware/'), ('tutorial.html', '/docs/en/getting_started/tutorial/',), ('reference_en.html', '/docs/en/single/', ), ('reference_ru.html', '/docs/ru/single/',), @@ -361,13 +383,14 @@ if __name__ == '__main__': os.chdir(os.path.join(os.path.dirname(__file__), '..')) website_dir = os.path.join('..', 'website') arg_parser = argparse.ArgumentParser() - arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja,fa') + arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja,tr,fa') arg_parser.add_argument('--docs-dir', default='.') arg_parser.add_argument('--theme-dir', default=website_dir) arg_parser.add_argument('--website-dir', default=website_dir) arg_parser.add_argument('--output-dir', default='build') arg_parser.add_argument('--enable-stable-releases', action='store_true') - arg_parser.add_argument('--stable-releases-limit', type=int, default='10') + arg_parser.add_argument('--stable-releases-limit', type=int, default='4') + arg_parser.add_argument('--lts-releases-limit', type=int, default='2') arg_parser.add_argument('--version-prefix', type=str, default='') arg_parser.add_argument('--is-stable-release', action='store_true') arg_parser.add_argument('--skip-single-page', action='store_true') @@ -400,7 +423,7 @@ if __name__ == '__main__': from build import build build(args) - + if args.livereload: new_args = [arg for arg in sys.argv if not arg.startswith('--livereload')] new_args = sys.executable + ' ' + ' '.join(new_args) diff --git a/docs/tools/convert_toc.py b/docs/tools/convert_toc.py deleted file mode 100755 index 9bfc347d244..00000000000 --- a/docs/tools/convert_toc.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import subprocess - -import yaml - -import util - -lang = 'zh' -base_dir = os.path.join(os.path.dirname(__file__), '..') -en_dir = os.path.join(base_dir, 'en') -docs_dir = os.path.join(base_dir, lang) -redirects_file = os.path.join(base_dir, 'redirects.txt') -redirects = {} - - -def make_key(path): - path = path.lower().replace(' ', '_').replace('c++', 'cpp') - if path.startswith('clickhouse_'): - path = path.replace('clickhouse_', '', 1) - if path != 'data_types' and path.startswith('data_'): - path = path.replace('data_', '', 1) - to_remove = [ - '\'', '"', '.', ',', '(', ')', - 'how_to_', '_of_clickhouse' - ] - for token in to_remove: - path = path.replace(token, '') - return path - - -def process_md_file(title, idx, original_path, proper_path): - proper_md_path = '/'.join(proper_path + [original_path.rsplit('/', 1)[-1]]) - if proper_md_path == 'introduction/index.md': - proper_md_path = 'index.md' - print(locals()) - if original_path != proper_md_path: - redirects[original_path] = proper_md_path - original_path = os.path.join(docs_dir, original_path) - proper_md_path = os.path.join(docs_dir, proper_md_path) - if os.path.exists(original_path): - meta, content = util.read_md_file(original_path) - else: - meta, content = util.read_md_file(proper_md_path) - meta['toc_title'] = title - meta['toc_priority'] = idx - if title == 'hidden': - meta['toc_hidden'] = True - - for src, dst in redirects.items(): - content = content.replace('(' + src, '(' + dst) - content = content.replace('../' + src, '../' + dst) - - util.write_md_file(proper_md_path, meta, content) - if original_path != proper_md_path: - subprocess.check_call(f'git add {proper_md_path}', shell=True) - if os.path.exists(original_path): - subprocess.check_call(f'rm {original_path}', shell=True) - - -def process_toc_entry(entry, path, idx): - if isinstance(entry, list): - for e in entry: - idx = process_toc_entry(e, path, idx) - elif isinstance(entry, dict): - for key, value in entry.items(): - next_path = path + [make_key(key)] - index_md_idx = idx - idx += 1 - - if isinstance(value, list): - for v in value: - process_toc_entry(v, next_path, idx) - idx += 1 - else: - process_md_file(key, idx, value, path) - idx += 1 - - index_md_path = os.path.join(docs_dir, '/'.join(next_path), 'index.md') - if os.path.exists(os.path.dirname(index_md_path)): - index_meta, index_content = util.read_md_file(index_md_path) - if not index_meta.get('toc_folder_title'): - index_meta['toc_folder_title'] = key - index_meta['toc_priority'] = index_md_idx - util.write_md_file(index_md_path, index_meta, index_content) - subprocess.check_call(f'git add {index_md_path}', shell=True) - return idx - - -def process_toc_yaml(path): - with util.cd(docs_dir): - init_redirects() - with open(path, 'r') as f: - data = yaml.full_load(f.read()) - process_toc_entry(data['nav'], [], 1) - update_redirects() - - -def init_redirects(): - with open(redirects_file, 'r') as f: - for line in f: - src, dst = line.strip().split(' ', 1) - redirects[src] = dst - - -def update_redirects(): - with open(redirects_file, 'w') as f: - for src, dst in sorted(redirects.items()): - print(f'{src} {dst}', file=f) - - -def sync_translation(): - init_redirects() - for src, dst in redirects.items(): - en_src = os.path.join(en_dir, src) - lang_src = os.path.join(docs_dir, src) - lang_dst = os.path.join(docs_dir, dst) - if os.path.exists(lang_src): - if os.path.islink(lang_src): - pass - else: - en_meta, en_content = util.read_md_file(en_src) - lang_meta, lang_content = util.read_md_file(lang_src) - en_meta.update(lang_meta) - - for src_link, dst_link in redirects.items(): - lang_content = lang_content.replace('(' + src_link, '(' + dst) - lang_content = lang_content.replace('../' + src_link, '../' + dst) - - util.write_md_file(lang_dst, en_meta, lang_content) - subprocess.check_call(f'git add {lang_dst}', shell=True) - subprocess.check_call(f'rm {lang_src}', shell=True) - - -if __name__ == '__main__': - sync_translation() - # if len(sys.argv) == 1: - # process_toc_yaml(os.path.join(base_dir, f'toc_{lang}.yml')) - # else: - # process_toc_yaml(sys.argv[1]) diff --git a/docs/tools/github.py b/docs/tools/github.py index d32c78abb14..abe41cc79fe 100644 --- a/docs/tools/github.py +++ b/docs/tools/github.py @@ -11,38 +11,58 @@ import requests import util +def yield_candidates(): + for page in range(1, 100): + url = 'https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100&page=%d' % page + for candidate in requests.get(url).json(): + yield candidate + + def choose_latest_releases(args): logging.info('Collecting release candidates') seen = collections.OrderedDict() candidates = [] - for page in range(1, args.stable_releases_limit): - url = 'https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100&page=%d' % page - candidates += requests.get(url).json() - logging.info('Collected all release candidates') + stable_count = 0 + lts_count = 0 - for tag in candidates: + for tag in yield_candidates(): if isinstance(tag, dict): name = tag.get('name', '') - is_unstable = ('stable' not in name) and ('lts' not in name) + is_stable = 'stable' in name + is_lts = 'lts' in name + is_unstable = not (is_stable or is_lts) is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name) if is_unstable or is_in_blacklist: continue major_version = '.'.join((name.split('.', 2))[:2]) if major_version not in seen: - seen[major_version] = (name, tag.get('tarball_url'),) - if len(seen) > args.stable_releases_limit: + if (stable_count >= args.stable_releases_limit) and (lts_count >= args.lts_releases_limit): break + + payload = (name, tag.get('tarball_url'), is_lts,) + if is_lts: + if lts_count < args.lts_releases_limit: + seen[major_version] = payload + lts_count += 1 + else: + if stable_count < args.stable_releases_limit: + seen[major_version] = payload + stable_count += 1 + + logging.debug( + f'Stables: {stable_count}/{args.stable_releases_limit} LTS: {lts_count}/{args.lts_releases_limit}' + ) else: logging.fatal('Unexpected GitHub response: %s', str(candidates)) sys.exit(1) - logging.info('Found stable releases: %s', str(seen.keys())) + logging.info('Found stable releases: %s', ', '.join(seen.keys())) return seen.items() def process_release(args, callback, release): - name, (full_name, tarball_url,) = release - logging.info('Building docs for %s', full_name) + name, (full_name, tarball_url, is_lts,) = release + logging.info(f'Building docs for {full_name}') buf = io.BytesIO(requests.get(tarball_url).content) tar = tarfile.open(mode='r:gz', fileobj=buf) with util.temp_dir() as base_dir: @@ -79,3 +99,15 @@ def get_events(args): 'event_date': tail[1].replace('on ', '').replace('.', '') }) return events + + +if __name__ == '__main__': + class DummyArgs(object): + lts_releases_limit = 1 + stable_releases_limit = 3 + logging.basicConfig( + level=logging.DEBUG, + stream=sys.stderr + ) + for item in choose_latest_releases(DummyArgs()): + print(item) diff --git a/docs/tools/make_links.sh b/docs/tools/make_links.sh index dfe9c56615d..2a27990f445 100755 --- a/docs/tools/make_links.sh +++ b/docs/tools/make_links.sh @@ -6,7 +6,7 @@ function do_make_links() { - langs=(en es zh fr ru ja fa) + langs=(en es zh fr ru ja tr fa) src_file="$1" for lang in "${langs[@]}" do diff --git a/docs/tools/nav.py b/docs/tools/nav.py index 56d47d58d07..fe11b21d8e7 100644 --- a/docs/tools/nav.py +++ b/docs/tools/nav.py @@ -35,6 +35,8 @@ def build_nav_entry(root): title = meta.get('toc_folder_title', 'hidden') prio = meta.get('toc_priority', 9999) logging.debug(f'Nav entry: {prio}, {title}, {path}') + if not content.strip(): + title = 'hidden' result_items.append((prio, title, path)) result_items = sorted(result_items, key=lambda x: (x[0], x[1])) result = collections.OrderedDict([(item[1], item[2]) for item in result_items]) @@ -45,8 +47,16 @@ def build_nav(lang, args): docs_dir = os.path.join(args.docs_dir, lang) _, _, nav = build_nav_entry(docs_dir) result = [] + index_key = None for key, value in nav.items(): if key and value: + if value == 'index.md': + index_key = key + continue result.append({key: value}) + if index_key: + key = list(result[0].keys())[0] + result[0][key][index_key] = 'index.md' + result[0][key].move_to_end(index_key, last=False) print('result', result) return result diff --git a/docs/tools/release.sh b/docs/tools/release.sh index e0f580c383b..8c61d16966f 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -4,9 +4,9 @@ set -ex BASE_DIR=$(dirname $(readlink -f $0)) BUILD_DIR="${BASE_DIR}/../build" PUBLISH_DIR="${BASE_DIR}/../publish" -BASE_DOMAIN="${BASE_DOMAIN:-clickhouse.tech}" -GIT_TEST_URI="${GIT_TEST_URI:-git@github.com:ClickHouse/clickhouse.github.io.git}" -GIT_PROD_URI="git@github.com:ClickHouse/clickhouse.github.io.git" +BASE_DOMAIN="${BASE_DOMAIN:-content.clickhouse.tech}" +GIT_TEST_URI="${GIT_TEST_URI:-git@github.com:ClickHouse/clickhouse-website-content.git}" +GIT_PROD_URI="git@github.com:ClickHouse/clickhouse-website-content.git" EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---enable-stable-releases --minify}" HISTORY_SIZE="${HISTORY_SIZE:-5}" @@ -44,7 +44,7 @@ then if [[ ! -z "${CLOUDFLARE_TOKEN}" ]] then sleep 1m - git diff --stat="9999,9999" --diff-filter=M HEAD~1 | grep '|' | awk '$1 ~ /\.html$/ { if ($3>4) { url="https://'${BASE_DOMAIN}'/"$1; sub(/\/index.html/, "/", url); print "\""url"\""; }}' | split -l 25 /dev/stdin PURGE + git diff --stat="9999,9999" --diff-filter=M HEAD~1 | grep '|' | awk '$1 ~ /\.html$/ { if ($3>4) { url="https://clickhouse.tech/"$1; sub(/\/index.html/, "/", url); print "\""url"\""; }}' | split -l 25 /dev/stdin PURGE for FILENAME in $(ls PURGE*) do POST_DATA=$(cat "${FILENAME}" | sed -n -e 'H;${x;s/\n/,/g;s/^,//;p;}' | awk '{print "{\"files\":["$0"]}";}') diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 587bcabb8fb..cc63ea71d8c 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -10,22 +10,21 @@ cssmin==0.2.0 future==0.18.2 htmlmin==0.1.12 idna==2.9 -Jinja2==2.11.1 +Jinja2==2.11.2 jinja2-highlight==0.6.1 jsmin==2.2.2 livereload==2.6.1 -lunr==0.5.6 Markdown==3.2.1 MarkupSafe==1.1.1 mkdocs==1.1 mkdocs-htmlproofer-plugin==0.0.3 -mkdocs-macros-plugin==0.4.4 -nltk==3.4.5 +mkdocs-macros-plugin==0.4.6 +nltk==3.5 nose==1.3.7 protobuf==3.11.3 -numpy==1.18.2 +numpy==1.18.3 Pygments==2.5.2 -pymdown-extensions==7.0 +pymdown-extensions==7.1 python-slugify==1.2.6 PyYAML==5.3.1 repackage==0.7.3 @@ -36,4 +35,4 @@ soupsieve==2.0 termcolor==1.1.0 tornado==5.1.1 Unidecode==1.1.1 -urllib3==1.25.8 +urllib3==1.25.9 diff --git a/docs/tools/translate/requirements.txt b/docs/tools/translate/requirements.txt index d49220602a0..3c212ee8bc2 100644 --- a/docs/tools/translate/requirements.txt +++ b/docs/tools/translate/requirements.txt @@ -3,10 +3,10 @@ certifi==2020.4.5.1 chardet==3.0.4 googletrans==2.4.0 idna==2.9 -Jinja2==2.11.1 +Jinja2==2.11.2 pandocfilters==1.4.2 python-slugify==4.0.0 PyYAML==5.3.1 requests==2.23.0 text-unidecode==1.3 -urllib3==1.25.8 +urllib3==1.25.9 diff --git a/docs/tools/translate/translate.py b/docs/tools/translate/translate.py index 5b4dc3f5486..759e5b849d2 100755 --- a/docs/tools/translate/translate.py +++ b/docs/tools/translate/translate.py @@ -63,8 +63,8 @@ def translate_toc(root, lang): def translate_po(): import babel.messages.pofile - base_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'website', 'locale') - for lang in ['en', 'zh', 'es', 'fr', 'ru', 'ja', 'fa']: + base_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'website', 'locale') + for lang in ['en', 'zh', 'es', 'fr', 'ru', 'ja', 'tr', 'fa']: po_path = os.path.join(base_dir, lang, 'LC_MESSAGES', 'messages.po') with open(po_path, 'r') as f: po_file = babel.messages.pofile.read_po(f, locale=lang, domain='messages') diff --git a/docs/tools/translate/update-po.sh b/docs/tools/translate/update-po.sh index bf9d1d4b7fb..f2f4039bcb8 100755 --- a/docs/tools/translate/update-po.sh +++ b/docs/tools/translate/update-po.sh @@ -2,11 +2,11 @@ # Usage: update-po.sh set -ex BASE_DIR=$(dirname $(readlink -f $0)) -WEBSITE_DIR="${BASE_DIR}/../../website" +WEBSITE_DIR="${BASE_DIR}/../../../website" LOCALE_DIR="${WEBSITE_DIR}/locale" MESSAGES_POT="${LOCALE_DIR}/messages.pot" BABEL_INI="${BASE_DIR}/babel-mapping.ini" -LANGS="en zh es fr ru ja fa" +LANGS="en zh es fr ru ja tr fa" source "${BASE_DIR}/venv/bin/activate" cd "${WEBSITE_DIR}" pybabel extract "." -o "${MESSAGES_POT}" -F "${BABEL_INI}" diff --git a/docs/tools/website.py b/docs/tools/website.py index 9704cf7d5a4..7a45d22da5c 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -1,4 +1,5 @@ import hashlib +import json import logging import os import shutil @@ -13,21 +14,6 @@ import jsmin import mdx_clickhouse -def copy_icons(args): - logging.info('Copying icons') - icons_dir = os.path.join(args.output_dir, 'images', 'icons') - os.makedirs(icons_dir) - for icon in [ - 'github', - 'edit', - 'external-link' - ]: - icon = '%s.svg' % icon - icon_src = os.path.join(args.website_dir, 'images', 'feathericons', 'icons', icon) - icon_dst = os.path.join(icons_dir, icon) - shutil.copy2(icon_src, icon_dst) - - def build_website(args): logging.info('Building website') env = jinja2.Environment( @@ -51,13 +37,14 @@ def build_website(args): '*.md', '*.sh', '*.css', - '*.js', + 'js/*.js', 'build', 'docs', 'public', 'node_modules', 'templates', - 'feathericons' + 'feathericons', + 'locale' ) ) @@ -67,7 +54,7 @@ def build_website(args): continue path = os.path.join(root, filename) - if not (filename.endswith('.html') or filename.endswith('.js')): + if not filename.endswith('.html'): continue logging.info('Processing %s', path) with open(path, 'rb') as f: @@ -130,11 +117,12 @@ def minify_website(args): logging.info(closure_args) if closure.run(*closure_args): raise RuntimeError('failed to run closure compiler') - + else: - logging.info(command) js_in = ' '.join(js_in) - output = subprocess.check_output(f'cat {js_in} > {js_out}', shell=True) + command = f'cat {js_in} > {js_out}' + logging.info(command) + output = subprocess.check_output(command, shell=True) logging.debug(output) with open(js_out, 'rb') as f: js_digest = hashlib.sha3_224(f.read()).hexdigest()[0:8] @@ -164,3 +152,19 @@ def minify_website(args): content = jsmin.jsmin(content) with open(path, 'wb') as f: f.write(content.encode('utf-8')) + + +def process_benchmark_results(args): + benchmark_root = os.path.join(args.website_dir, 'benchmark') + for benchmark_kind in ['dbms', 'hardware']: + results = [] + results_root = os.path.join(benchmark_root, benchmark_kind, 'results') + for result in sorted(os.listdir(results_root)): + result_file = os.path.join(results_root, result) + logging.debug(f'Reading benchmark result from {result_file}') + with open(result_file, 'r') as f: + results += json.loads(f.read()) + results_js = os.path.join(args.output_dir, 'benchmark', benchmark_kind, 'results.js') + with open(results_js, 'w') as f: + data = json.dumps(results) + f.write(f'var results = {data};') diff --git a/docs/tr/commercial/cloud.md b/docs/tr/commercial/cloud.md new file mode 100644 index 00000000000..f20656cc428 --- /dev/null +++ b/docs/tr/commercial/cloud.md @@ -0,0 +1,21 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +--- + +# ClickHouse Bulut Servis Sağlayıcıları {#clickhouse-cloud-service-providers} + +!!! info "Bilgin" + Yönetilen ClickHouse hizmeti ile genel bir bulut başlattıysanız, çekinmeyin [çekme isteği aç](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/cloud.md) aşağıdaki listeye ekleyerek. + +## Yandex Bulut {#yandex-cloud} + +[Yandex Clickhouse için yönetilen hizmet](https://cloud.yandex.com/services/managed-clickhouse?utm_source=referrals&utm_medium=clickhouseofficialsite&utm_campaign=link3) aşağıdaki temel özellikleri sağlar: + +- İçin tamamen yönetilen ZooKeeper hizmeti [ClickHouse çoğaltma](../engines/table_engines/mergetree_family/replication.md) +- Çoklu depolama tipi seçenekleri +- Farklı kullanılabilirlik bölgelerinde yinelemeler +- Şifreleme ve izolasyon +- Otomatik bakım + +{## [Orijinal makale](https://clickhouse.tech/docs/en/commercial/cloud/) ##} diff --git a/docs/tr/commercial/index.md b/docs/tr/commercial/index.md new file mode 100644 index 00000000000..104a3427146 --- /dev/null +++ b/docs/tr/commercial/index.md @@ -0,0 +1,9 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Ticari +toc_priority: 70 +toc_title: Ticari +--- + + diff --git a/docs/tr/development/architecture.md b/docs/tr/development/architecture.md new file mode 100644 index 00000000000..4713c2e5409 --- /dev/null +++ b/docs/tr/development/architecture.md @@ -0,0 +1,203 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 62 +toc_title: "ClickHouse mimarisine genel bak\u0131\u015F" +--- + +# ClickHouse mimarisine genel bakış {#overview-of-clickhouse-architecture} + +ClickHouse gerçek bir sütun yönelimli DBMS olduğunu. Veriler sütunlar tarafından ve dizilerin yürütülmesi sırasında (vektörler veya sütun parçaları) saklanır. Mümkün olduğunda, işlemler tek tek değerler yerine dizilere gönderilir. Buna denir “vectorized query execution,” ve gerçek veri işleme maliyetini düşürmeye yardımcı olur. + +> Bu fikir yeni bir şey değil. Bu kadar uzanır `APL` programlama dili ve Tor andunları: `A +`, `J`, `K`, ve `Q`. Dizi programlama bilimsel veri işlemede kullanılır. Bu fikir ilişkisel veritabanlarında yeni bir şey değildir: örneğin, `Vectorwise` sistem. + +Sorgu işlemeyi hızlandırmak için iki farklı yaklaşım vardır: vektörize sorgu yürütme ve çalışma zamanı kodu oluşturma. İkincisi, tüm Yönlendirme ve dinamik gönderimi kaldırır. Bu yaklaşımların hiçbiri diğerinden kesinlikle daha iyi değildir. Çalışma zamanı kodu üretimi, birçok işlemi birleştirdiğinde daha iyi olabilir, böylece CPU yürütme birimlerini ve boru hattını tam olarak kullanır. Vectorized sorgu yürütme daha az pratik olabilir, çünkü önbelleğe yazılması ve geri okunması gereken geçici vektörler içerir. Geçici veri L2 önbelleğinde uymuyorsa, bu bir sorun haline gelir. Ancak vektörize sorgu yürütme, CPU'nun SIMD yeteneklerini daha kolay kullanır. Bir [araştırma öd paperevi](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) arkadaşlarımız tarafından yazıldı, her iki yaklaşımı birleştirmenin daha iyi olduğunu gösteriyor. ClickHouse vectorized sorgu yürütme kullanır ve çalışma zamanı kodu üretimi için başlangıç desteği sınırlıdır. + +## Sütun {#columns} + +`IColumn` arabirim, bellekteki sütunları temsil etmek için kullanılır (aslında, sütunların parçaları). Bu arayüz, çeşitli ilişkisel operatörlerin uygulanması için yardımcı yöntemler sağlar. Hemen hemen tüm işlemler değişmez: orijinal sütunu değiştirmezler, ancak yeni bir değiştirilmiş bir tane oluştururlar. Örneğin, `IColumn :: filter` yöntem bir filtre bayt maskesi kabul eder. Bu için kullanılır `WHERE` ve `HAVING` ilişkisel operatörler. Ek örnekler: `IColumn :: permute` desteklemek için yöntem `ORDER BY`, bu `IColumn :: cut` desteklemek için yöntem `LIMIT`. + +Çeşitli `IColumn` uygulanışlar (`ColumnUInt8`, `ColumnString` ve benzeri) sütunların bellek düzeninden sorumludur. Bellek düzeni genellikle bitişik bir dizidir. Sütunların tamsayı türü için, sadece bir bitişik dizidir, örneğin `std :: vector`. İçin `String` ve `Array` sütunlar, iki vektördür: biri bitişik olarak yerleştirilmiş tüm dizi elemanları için ve her dizinin başlangıcına ait ofsetler için ikinci bir tane. Ayrıca var `ColumnConst` bu bellekte sadece bir değer depolar, ancak bir sütuna benziyor. + +## Alan {#field} + +Bununla birlikte, bireysel değerlerle de çalışmak mümkündür. Bireysel bir değeri temsil etmek için, `Field` kullanılır. `Field` sadece ayrımcılığa uğramış bir birlik mi `UInt64`, `Int64`, `Float64`, `String` ve `Array`. `IColumn` has the `operator[]` n - inci değerini bir olarak alma yöntemi `Field` ve... `insert` bir ekleme yöntemi `Field` bir sütunun sonuna. Bu yöntemler çok verimli değildir, çünkü geçici olarak uğraşmayı gerektirirler `Field` tek bir değeri temsil eden nesneler. Daha etkili yöntemleri vardır, mesela: `insertFrom`, `insertRangeFrom` ve bu yüzden. + +`Field` bir tablo için belirli bir veri türü hakkında yeterli bilgiye sahip değildir. Mesela, `UInt8`, `UInt16`, `UInt32`, ve `UInt64` hepsi olarak temsil edilir `UInt64` in a `Field`. + +## Sızdıran Soyutlamalar {#leaky-abstractions} + +`IColumn` verilerin ortak ilişkisel dönüşümleri için yöntemler vardır, ancak tüm ihtiyaçları karşılamazlar. Mesela, `ColumnUInt64` iki sütunun toplamını hesaplamak için bir yöntem yoktur ve `ColumnString` bir alt dize araması çalıştırmak için bir yöntem yok. Bu sayısız rutinleri dışında uygulanmaktadır `IColumn`. + +Sütunlar üzerinde çeşitli işlevler kullanarak genel, verimli olmayan bir şekilde uygulanabilir `IColumn` çıkarma yöntemleri `Field` değerleri veya belirli bir veri iç bellek düzeni bilgisini kullanarak özel bir şekilde `IColumn` uygulanış. Döküm fonksiyonları ile belirli bir `IColumn` yazın ve doğrudan iç temsil ile anlaşma. Mesela, `ColumnUInt64` has the `getData` bir iç diziye başvuru döndüren yöntem, daha sonra ayrı bir yordam okur veya bu diziyi doğrudan doldurur. Sahibiz “leaky abstractions” çeşitli rutinlerin verimli uzmanlıklarına izin vermek. + +## Veri Türleri {#data_types} + +`IDataType` seri hale getirme ve serileştirmeden sorumludur: ikili veya metin biçiminde sütunların veya bireysel değerlerin parçalarını okumak ve yazmak için. `IDataType` tablolardaki veri türlerine doğrudan karşılık gelir. Örneğin, `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` ve böyle devam eder. + +`IDataType` ve `IColumn` sadece gevşek birbirleriyle ilişkilidir. Farklı veri türleri bellekte aynı tarafından temsil edilebilir `IColumn` uygulanışlar. Mesela, `DataTypeUInt32` ve `DataTypeDateTime` her ikisi de tarafından temsil edilir `ColumnUInt32` veya `ColumnConstUInt32`. Buna ek olarak, aynı veri türü farklı tarafından temsil edilebilir `IColumn` uygulanışlar. Mesela, `DataTypeUInt8` tarafından temsil edilebilir `ColumnUInt8` veya `ColumnConstUInt8`. + +`IDataType` yalnızca meta verileri depolar. Mesela, `DataTypeUInt8` hiçbir şey saklamıyor (vptr hariç) ve `DataTypeFixedString` mağazalar sadece `N` (sabit boyutlu dizelerin boyutu). + +`IDataType` çeşitli veri formatları için yardımcı yöntemlere sahiptir. Örnekler, Olası Alıntı ile bir değeri serileştirmek, json için bir değeri serileştirmek ve XML formatının bir parçası olarak bir değeri serileştirmek için kullanılan yöntemlerdir. Veri formatlarına doğrudan yazışma yoktur. Örneğin, farklı veri biçimleri `Pretty` ve `TabSeparated` aynı kullanabilirsiniz `serializeTextEscaped` hel methodper yöntemi `IDataType` Arabirim. + +## Blok {#block} + +A `Block` bellekteki bir tablonun bir alt kümesini (yığın) temsil eden bir kapsayıcıdır. Bu sadece üçlü bir dizi: `(IColumn, IDataType, column name)`. Sorgu yürütme sırasında veri tarafından işlenir `Block`s. Eğer bir `Block`(bu yaptığımız verileri `IColumn` nesne), biz onun türü hakkında bilgi var (içinde `IDataType`) bu bize bu sütunla nasıl başa çıkacağımızı söyler ve sütun adına sahibiz. Tablodan orijinal sütun adı veya hesaplamaların geçici sonuçlarını almak için atanan bazı yapay ad olabilir. + +Bir bloktaki sütunlar üzerinde bazı işlevleri hesapladığımızda, bloğa sonucu olan başka bir sütun ekleriz ve işlemler değişmez olduğu için işlevin argümanları için sütunlara dokunmayız. Daha sonra, gereksiz sütunlar bloktan kaldırılabilir, ancak değiştirilemez. Ortak alt ifadelerin ortadan kaldırılması için uygundur. + +İşlenen her veri yığını için bloklar oluşturulur. Aynı hesaplama türü için, sütun adları ve türleri farklı bloklar için aynı kalır ve yalnızca sütun verileri değişir unutmayın. Küçük blok boyutları shared\_ptrs ve sütun adlarını kopyalamak için geçici dizeleri yüksek bir ek yükü olduğundan blok üstbilgisinden blok verileri bölmek daha iyidir. + +## Blok Akışları {#block-streams} + +Blok akışları veri işleme içindir. Bir yerden veri okumak, veri dönüşümleri gerçekleştirmek veya bir yere veri yazmak için blok akışları kullanıyoruz. `IBlockInputStream` has the `read` mevcut iken bir sonraki bloğu getirme yöntemi. `IBlockOutputStream` has the `write` bloğu bir yere itmek için yöntem. + +Akar responsibles areular sorumludur: + +1. Bir tabloya okuma veya yazma. Tablo sadece okuma veya yazma blokları için bir akış döndürür. +2. Veri formatlarının uygulanması. Örneğin, bir terminale veri çıkışı yapmak istiyorsanız `Pretty` biçim, blokları ittiğiniz bir blok çıkış akışı oluşturursunuz ve bunları biçimlendirir. +3. Veri dönüşümleri gerçekleştirme. Diyelim ki var `IBlockInputStream` ve filtrelenmiş bir akış oluşturmak istiyorum. Yarat createtığınız `FilterBlockInputStream` ve akışı ile başlatın. Sonra bir blok çektiğinizde `FilterBlockInputStream`, akışınızdan bir blok çeker, filtreler ve filtrelenmiş bloğu size döndürür. Sorgu yürütme boru hatları bu şekilde temsil edilir. + +Daha sofistike dönüşümler var. Örneğin, çektiğiniz zaman `AggregatingBlockInputStream`, kaynağındaki tüm verileri okur, toplar ve sizin için toplanmış bir veri akışı döndürür. Başka bir örnek: `UnionBlockInputStream` yapıcıdaki birçok giriş kaynağını ve ayrıca bir dizi iş parçacığını kabul eder. Birden çok iş parçacığı başlatır ve paralel olarak birden fazla kaynaktan okur. + +> Blok akışları “pull” akışı kontrol etme yaklaşımı: ilk akıştan bir blok çektiğinizde, gerekli blokları iç içe geçmiş akışlardan çeker ve tüm yürütme boru hattı çalışır. Ne “pull” ne “push” en iyi çözümdür, çünkü kontrol akışı örtükdür ve bu, birden fazla sorgunun eşzamanlı yürütülmesi (birçok boru hattının birlikte birleştirilmesi) gibi çeşitli özelliklerin uygulanmasını sınırlar. Bu sınırlama, coroutines ile veya sadece birbirlerini bekleyen ekstra iş parçacıkları çalıştırarak aşılabilir. Kontrol akışını açık hale getirirsek daha fazla olasılığa sahip olabiliriz: verileri bir hesaplama biriminden diğerine bu hesaplama birimlerinin dışında geçirme mantığını bulursak. Re thisad this [makale](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) daha fazla düşünce için. + +Sorgu yürütme boru hattının her adımda geçici veri oluşturduğuna dikkat etmeliyiz. Blok boyutunu yeterince küçük tutmaya çalışıyoruz, böylece geçici veriler CPU önbelleğine sığıyor. Bu varsayımla, geçici veri yazmak ve okumak, diğer hesaplamalarla karşılaştırıldığında neredeyse ücretsizdir. Boru hattındaki birçok operasyonu bir araya getirmek için bir alternatif düşünebiliriz. Boru hattını mümkün olduğunca kısa hale getirebilir ve geçici verilerin çoğunu kaldırabilir, bu da bir avantaj olabilir, ancak dezavantajları da vardır. Örneğin, bölünmüş bir boru hattı, Ara verileri önbelleğe almayı, aynı anda çalışan benzer sorgulardan Ara verileri çalmayı ve benzer sorgular için boru hatlarını birleştirmeyi kolaylaştırır. + +## Biçimliler {#formats} + +Veri formatları blok akışları ile uygulanır. Var “presentational” sadece müşteriye veri çıkışı için uygun biçimler, örneğin `Pretty` sadece sağlayan biçim `IBlockOutputStream`. Ve gibi giriş / çıkış biçimleri vardır `TabSeparated` veya `JSONEachRow`. + +Satır akışları da vardır: `IRowInputStream` ve `IRowOutputStream`. Verileri bloklarla değil, tek tek satırlarla çekmenize/itmenize izin verirler. Ve sadece satır yönelimli formatların uygulanmasını basitleştirmek için gereklidir. Sarıcı `BlockInputStreamFromRowInputStream` ve `BlockOutputStreamFromRowOutputStream` satır yönelimli akışları normal blok yönelimli akışlara dönüştürmenize izin verin. + +## I/O {#io} + +Bayt yönelimli giriş / çıkış için, `ReadBuffer` ve `WriteBuffer` soyut sınıflar. C++yerine kullanılırlar `iostream`s. merak etmeyin: her olgun C++ projesi başka bir şey kullanıyor `iostream`s iyi nedenlerden dolayı. + +`ReadBuffer` ve `WriteBuffer` sadece bitişik bir tampon ve bu tampondaki konuma işaret eden bir imleç. Uygulamalar, arabellek belleğine sahip olabilir veya sahip olmayabilir. Arabelleği aşağıdaki verilerle doldurmak için sanal bir yöntem vardır (for `ReadBuffer`) veya tamponu bir yere yıkamak için (için `WriteBuffer`). Sanal yöntemler nadiren denir. + +Uygulamaları `ReadBuffer`/`WriteBuffer` sıkıştırma uygulamak için dosyalar ve dosya tanımlayıcıları ve ağ soketleri ile çalışmak için kullanılır (`CompressedWriteBuffer` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names `ConcatReadBuffer`, `LimitReadBuffer`, ve `HashingWriteBuffer` kendileri için konuşuyoruz. + +Read / WriteBuffers sadece baytlarla ilgilenir. Fonksiyonları vardır `ReadHelpers` ve `WriteHelpers` başlık dosyaları biçimlendirme giriş/çıkış ile yardımcı olmak için. Örneğin, ondalık biçimde bir sayı yazmak için yardımcılar vardır. + +Bir sonuç kümesi yazmak istediğinizde neler olduğuna bakalım `JSON` stdout için biçimlendirin. Eğer bir sonuç getirilecek hazır set var `IBlockInputStream`. Yarat createtığınız `WriteBufferFromFileDescriptor(STDOUT_FILENO)` STDOUT için bayt yazmak için. Yarat createtığınız `JSONRowOutputStream` bununla başlatıldı `WriteBuffer`, satır yazmak için `JSON` stdout. Yarat createtığınız `BlockOutputStreamFromRowOutputStream` bu da yetmiyormuş gibi göstermek için `IBlockOutputStream`. Sonra Ara `copyData` veri aktarmak için `IBlockInputStream` -e doğru `IBlockOutputStream` ve her şey çalışıyor. İçten, `JSONRowOutputStream` çeşitli json sınırlayıcıları yazacak ve `IDataType::serializeTextJSON` bir referans ile yöntem `IColumn` ve satır numarası argüman olarak. Sonuç olarak, `IDataType::serializeTextJSON` bir yöntem çağırır `WriteHelpers.h`: mesela, `writeText` sayısal türler ve `writeJSONString` için `DataTypeString`. + +## Tablolar {#tables} + +Bu `IStorage` arayüz tabloları temsil eder. Bu arayüzün farklı uygulamaları farklı tablo motorlarıdır. Örnekler şunlardır `StorageMergeTree`, `StorageMemory` ve bu yüzden. Bu sınıfların örnekleri sadece tablolardır. + +Anahtar `IStorage` yöntemler şunlardır `read` ve `write`. Ayrıca vardır `alter`, `rename`, `drop` ve bu yüzden. Bu `read` yöntem aşağıdaki bağımsız değişkenleri kabul eder: bir tablodan okunacak sütun kümesi, `AST` dikkate alınması gereken sorgu ve döndürülmesi gereken akış sayısı. Bir veya birden fazla döndürür `IBlockInputStream` nesneler ve sorgu yürütme sırasında bir tablo altyapısı içinde tamamlanan veri işleme aşaması hakkında bilgi. + +Çoğu durumda, read yöntemi yalnızca belirtilen sütunları bir tablodan okumaktan sorumludur, daha fazla veri işleme için değil. Tüm diğer veri işleme sorgu yorumlayıcısı tarafından yapılır ve sorumluluk dışında `IStorage`. + +Ancak önemli istisnalar var: + +- AST sorgusu için geçirilir `read` yöntemi ve tablo altyapısı dizin kullanımını türetmek ve bir tablodan daha az veri okumak için kullanabilirsiniz. +- Bazen tablo motoru verileri belirli bir aşamaya kadar işleyebilir. Mesela, `StorageDistributed` uzak sunuculara sorgu gönderebilir, farklı uzak sunuculardan gelen verilerin birleştirilebileceği bir aşamaya veri işlemelerini isteyebilir ve bu önceden işlenmiş verileri döndürebilir. Sorgu yorumlayıcısı daha sonra verileri işlemeyi tamamlar. + +Tablo `read` yöntem birden çok döndürebilir `IBlockInputStream` nesneleri paralel veri işleme izin vermek için. Bu çoklu blok giriş akışları bir tablodan paralel olarak okuyabilir. Ardından, bu akışları bağımsız olarak hesaplanabilen çeşitli dönüşümlerle (ifade değerlendirme veya filtreleme gibi) sarabilir ve bir `UnionBlockInputStream` bunların üzerine, paralel olarak birden fazla akıştan okumak için. + +Ayrıca vardır `TableFunction`s. bunlar geçici olarak dönen işlevlerdir `IStorage` içinde kullanılacak nesne `FROM` bir sorgu yan tümcesi. + +Tablo motorunuzu nasıl uygulayacağınıza dair hızlı bir fikir edinmek için, basit bir şeye bakın `StorageMemory` veya `StorageTinyLog`. + +> Bu sonucu `read` yöntem, `IStorage` dönüşler `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. + +## Ayrıştırıcılar {#parsers} + +Elle yazılmış özyinelemeli iniş ayrıştırıcı bir sorgu ayrıştırır. Mesela, `ParserSelectQuery` sorgunun çeşitli bölümleri için temel ayrıştırıcıları yinelemeli olarak çağırır. Ayrıştırıcılar bir `AST`. Bu `AST` örnekleri olan düğüm bylerle temsil edilir `IAST`. + +> Ayrıştırıcı jeneratörler tarihsel nedenlerle kullanılmaz. + +## Tercümanlar {#interpreters} + +Sorgu yürütme kanalının oluşturulmasından tercümanlar sorumludur. `AST`. Gibi basit tercümanlar vardır `InterpreterExistsQuery` ve `InterpreterDropQuery` veya daha sofistike `InterpreterSelectQuery`. Sorgu yürütme boru hattı, blok giriş veya çıkış akışlarının birleşimidir. Örneğin, yorumlama sonucu `SELECT` sorgu olduğunu `IBlockInputStream` sonuç kümesini okumak için; INSERT sorgusunun sonucu `IBlockOutputStream` ekleme için veri yazmak ve yorumlama sonucu `INSERT SELECT` sorgu olduğunu `IBlockInputStream` bu, ilk okumada boş bir sonuç kümesi döndürür, ancak verileri kopyalar `SELECT` -e doğru `INSERT` aynı zamanda. + +`InterpreterSelectQuery` kullanma `ExpressionAnalyzer` ve `ExpressionActions` sorgu analizi ve dönüşümler için makine. Bu, kural tabanlı sorgu iyileştirmelerinin çoğunun yapıldığı yerdir. `ExpressionAnalyzer` oldukça dağınık ve yeniden yazılmalıdır: modüler dönüşümlere veya sorguya izin vermek için ayrı sınıflara çeşitli sorgu dönüşümleri ve optimizasyonlar çıkarılmalıdır. + +## İşlevler {#functions} + +Sıradan fonksiyonlar ve toplam fonksiyonlar vardır. Toplama işlevleri için bir sonraki bölüme bakın. + +Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`'s vectorized sorgu yürütme uygulamak için veri. + +Gibi bazı çeşitli fonksiyonlar vardır [blockSize](../sql_reference/functions/other_functions.md#function-blocksize), [rowNumberİnBlock](../sql_reference/functions/other_functions.md#function-rownumberinblock), ve [runningAccumulate](../sql_reference/functions/other_functions.md#function-runningaccumulate), blok işlemeyi istismar eden ve satırların bağımsızlığını ihlal eden. + +Clickhouse'un güçlü yazımı var, bu yüzden örtük tür dönüşümü yok. Bir işlev belirli bir tür kombinasyonunu desteklemiyorsa, bir istisna atar. Ancak, birçok farklı tür kombinasyonu için işlevler çalışabilir (aşırı yüklenebilir). Örneğin, `plus` fonksiyonu (uygulamak için `+` operatör) sayısal türlerin herhangi bir kombinasyonu için çalışır: `UInt8` + `Float32`, `UInt16` + `Int8` ve bu yüzden. Ayrıca, bazı variadic işlevleri gibi bağımsız değişkenlerin herhangi bir sayıda kabul edebilir `concat` işlev. + +Bir işlev açıkça desteklenen veri türlerini gönderir ve desteklenen çünkü bir işlev uygulamak biraz rahatsız edici olabilir `IColumns`. Örneğin, `plus` işlev, sayısal türlerin ve sabit veya sabit olmayan sol ve sağ bağımsız değişkenlerin her birleşimi için bir C++ şablonunun örneklendirilmesiyle oluşturulan koda sahiptir. + +Bu şablon kodu kabartmak önlemek için çalışma zamanı kodu nesil uygulamak için mükemmel bir yerdir. Ayrıca, kaynaşmış çarpma-ekleme gibi kaynaşmış işlevler eklemeyi veya bir döngü yinelemesinde birden fazla karşılaştırma yapmayı mümkün kılar. + +Vektörize sorgu yürütme nedeniyle, işlevler kısa devre değildir. Örneğin, yazarsanız `WHERE f(x) AND g(y)`, her iki taraf da satırlar için bile hesaplanır `f(x)` sıfırdır (hariç `f(x)` sıfır sabit bir ifadedir). Ama eğer seçicilik `f(x)` durum yüksek ve hesaplama `f(x)` çok daha ucuzdur `g(y)`, çok geçişli hesaplama uygulamak daha iyidir. İlk önce hesaplayacaktı `f(x)`, daha sonra sonucu sütunları süzün ve sonra hesaplayın `g(y)` sadece daha küçük, filtrelenmiş veri parçaları için. + +## Toplama Fonksiyonları {#aggregate-functions} + +Toplama işlevleri durumsal işlevlerdir. Geçirilen değerleri bir duruma biriktirir ve bu durumdan sonuç almanıza izin verir. İle Yönet theyil theirler. `IAggregateFunction` Arabirim. Devletler oldukça basit olabilir (devlet `AggregateFunctionCount` sadece bir tek `UInt64` değeri) veya oldukça karmaşık (devlet `AggregateFunctionUniqCombined` doğrusal bir dizi, bir karma tablo ve bir kombinasyonudur `HyperLogLog` olasılıksal veri yapısı). + +Devletler tahsis edilir `Arena` (bir bellek havuzu) yüksek önemlilik yürütürken birden çok durumla başa çıkmak için `GROUP BY` sorgu. Devletler önemsiz olmayan bir yapıcı ve yıkıcı olabilir: örneğin, karmaşık toplama durumları ek belleği kendileri tahsis edebilir. Devletlerin yaratılmasına ve yok edilmesine ve mülkiyet ve yıkım düzeninin doğru bir şekilde geçmesine biraz dikkat gerektirir. + +Toplama durumları serileştirilmiş ve dağıtılmış sorgu yürütme sırasında ağ üzerinden geçmek veya bunları yeterli RAM olmadığı diskte yazmak için serileştirilmiş. Hatta bir tablo ile saklanabilir `DataTypeAggregateFunction` verilerin artımlı toplanmasına izin vermek için. + +> Toplu işlev durumları için seri hale getirilmiş veri biçimi şu anda sürümlendirilmemiş. Toplama durumları yalnızca geçici olarak saklanırsa sorun olmaz. Ama biz var `AggregatingMergeTree` artan toplama için tablo motoru ve insanlar zaten üretimde kullanıyor. Gelecekte herhangi bir toplama işlevi için seri hale getirilmiş biçimi değiştirirken geriye dönük uyumluluğun gerekli olmasının nedeni budur. + +## Hizmetçi {#server} + +Sunucu birkaç farklı arayüz uygular: + +- Herhangi bir yabancı istemciler için bir HTTP arabirimi. +- Dağıtılmış sorgu yürütme sırasında yerel ClickHouse istemcisi ve sunucular arası iletişim için bir TCP arabirimi. +- Çoğaltma için veri aktarımı için bir arabirim. + +DAHİLİ olarak, coroutines veya lifler olmadan sadece ilkel bir çok iş parçacıklı sunucudur. Sunucu, yüksek oranda basit sorguları işlemek için değil, nispeten düşük oranda karmaşık sorguları işlemek için tasarlandığından, her biri analitik için çok miktarda veri işleyebilir. + +Sunucu başlatır `Context` sorgu yürütme için gerekli ortama sahip sınıf: kullanılabilir veritabanlarının, kullanıcıların ve erişim haklarının, ayarların, kümelerin, işlem listesinin, sorgu günlüğünün vb. listesi. Tercümanlar bu ortamı kullanır. + +Sunucu TCP protokolü için tam geriye ve ileriye dönük uyumluluk sağlıyoruz: eski istemciler yeni sunucularla konuşabilir ve yeni istemciler eski sunucularla konuşabilir. Ancak sonsuza dek korumak istemiyoruz ve yaklaşık bir yıl sonra eski sürümler için destek kaldırıyoruz. + +!!! note "Not" + Çoğu harici uygulama için, HTTP arayüzünü kullanmanızı öneririz, çünkü basit ve kullanımı kolaydır. TCP protokolü, iç veri yapılarına daha sıkı bir şekilde bağlanır: veri bloklarını geçirmek için bir iç biçim kullanır ve sıkıştırılmış veriler için özel çerçeveleme kullanır. Bu protokol için bir C kütüphanesi yayınlamadık çünkü pratik olmayan ClickHouse kod tabanının çoğunu bağlamayı gerektiriyor. + +## Dağıtılmış Sorgu Yürütme {#distributed-query-execution} + +Bir küme kurulumundaki sunucular çoğunlukla bağımsızdır. Sen-ebilmek yaratmak a `Distributed` bir kümedeki bir veya tüm sunucularda tablo. Bu `Distributed` table does not store data itself – it only provides a “view” Bir kümenin birden çok düğümündeki tüm yerel tablolara. Bir seçtiğinizde `Distributed` tablo, bu sorguyu yeniden yazar, Yük Dengeleme ayarlarına göre uzak düğümleri seçer ve sorguyu onlara gönderir. Bu `Distributed` tablo, farklı sunuculardan gelen Ara sonuçların birleştirilebileceği bir aşamaya kadar bir sorguyu işlemek için uzak sunuculardan ister. Sonra Ara sonuçları alır ve onları birleştirir. Dağıtılmış tablo, uzak sunuculara mümkün olduğunca fazla çalışma dağıtmaya çalışır ve ağ üzerinden çok fazla ara veri göndermez. + +In veya JOIN yan tümcelerinde alt sorgular olduğunda işler daha karmaşık hale gelir ve her biri bir `Distributed` Tablo. Bu sorguların yürütülmesi için farklı stratejilerimiz var. + +Dağıtılmış sorgu yürütme için genel bir sorgu planı yoktur. Her düğüm, işin kendi kısmı için yerel sorgu planına sahiptir. Biz sadece basit tek geçişli dağıtılmış sorgu yürütme var: biz uzak düğümler için sorgular göndermek ve sonra sonuçları birleştirmek. Ancak bu, yüksek önemlilik grubu BYs ile veya JOIN için büyük miktarda geçici veri içeren karmaşık sorgular için mümkün değildir. Bu gibi durumlarda, gerek “reshuffle” ek koordinasyon gerektiren sunucular arasındaki veriler. ClickHouse bu tür bir sorgu yürütülmesini desteklemiyor ve üzerinde çalışmamız gerekiyor. + +## Ağaç Birleştirme {#merge-tree} + +`MergeTree` birincil anahtarla dizin oluşturmayı destekleyen bir depolama altyapısı ailesidir. Birincil anahtar, isteğe bağlı bir sütun veya ifade kümesi olabilir. Veri `MergeTree` tablo saklanır “parts”. Her bölüm verileri birincil anahtar sırasına göre saklar, böylece veriler birincil anahtar tuple tarafından lexicographically sıralanır. Tüm tablo sütunları ayrı olarak saklanır `column.bin` bu kısımlardaki dosyalar. Dosyalar sıkıştırılmış bloklardan oluşur. Her blok, ortalama değer boyutuna bağlı olarak genellikle 64 KB ila 1 MB sıkıştırılmamış veridir. Bloklar, birbiri ardına bitişik olarak yerleştirilmiş sütun değerlerinden oluşur. Sütun değerleri her sütun için aynı sıradadır (birincil anahtar siparişi tanımlar), bu nedenle birçok sütun tarafından yineleme yaptığınızda, karşılık gelen satırlar için değerler alırsınız. + +Birincil anahtarın kendisi “sparse”. Her satır Adres yok ama verilerin sadece biraz değişir. Ayıran `primary.idx` dosya, n'nin çağrıldığı her N-inci satır için birincil anahtarın değerine sahiptir `index_granularity` (genellikle, n = 8192). Ayrıca, her sütun için, biz var `column.mrk` dosyaları ile “marks,” veri dosyasındaki her N-inci satıra ofset olan. Her işaret bir çifttir: dosyadaki ofset sıkıştırılmış bloğun başlangıcına ve sıkıştırılmış bloktaki ofset verilerin başlangıcına. Genellikle, sıkıştırılmış bloklar işaretlerle hizalanır ve sıkıştırılmış bloktaki ofset sıfırdır. İçin veri `primary.idx` her zaman bellekte bulunur ve veri `column.mrk` dosyalar önbelleğe alınır. + +Bir kısm aından bir şey okuy readacağımız zaman `MergeTree` bak biz `primary.idx` veri ve istenen verileri içerebilecek aralıkları bulun, ardından `column.mrk` veri ve bu aralıkları okumaya başlamak için nerede için uzaklıklar hesaplayın. Çünkü seyrek, fazla veri okunabilir. ClickHouse, basit nokta sorgularının yüksek bir yükü için uygun değildir, çünkü tüm Aralık `index_granularity` her anahtar için satırlar okunmalı ve her sütun için sıkıştırılmış bloğun tamamı sıkıştırılmalıdır. Dizin için fark edilebilir bellek tüketimi olmadan tek bir sunucu başına trilyonlarca satır tutabilmemiz gerektiğinden dizini seyrek yaptık. Ayrıca, birincil anahtar seyrek olduğundan, benzersiz değildir: ekleme zamanında tablodaki anahtarın varlığını denetleyemez. Bir tabloda aynı anahtara sahip birçok satır olabilir. + +Ne zaman sen `INSERT` içine veri bir demet `MergeTree`, bu grup birincil anahtar sırasına göre sıralanır ve yeni bir bölüm oluşturur. Bazı parçaları periyodik olarak seçen ve parça sayısını nispeten düşük tutmak için bunları tek bir sıralanmış parçaya birleştiren arka plan iş parçacıkları vardır. Bu yüzden denir `MergeTree`. Tabii ki, birleştirme yol açar “write amplification”. Tüm parçalar değişmez: sadece oluşturulur ve silinir, ancak değiştirilmez. SELECT yürütüldüğünde, tablonun bir anlık görüntüsünü (bir parça kümesi) tutar. Birleştirildikten sonra, arızadan sonra iyileşmeyi kolaylaştırmak için eski parçaları bir süre tutuyoruz, bu nedenle birleştirilmiş bir parçanın muhtemelen kırıldığını görürsek, kaynak parçalarıyla değiştirebiliriz. + +`MergeTree` içermediği için bir lsm ağacı değildir “memtable” ve “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. + +> MergeTree tabloları yalnızca bir (birincil) dizine sahip olabilir: herhangi bir ikincil dizin yoktur. Bir mantıksal tablo altında birden fazla fiziksel gösterime izin vermek, örneğin verileri birden fazla fiziksel sırayla depolamak veya hatta orijinal verilerle birlikte önceden toplanmış verilerle gösterimlere izin vermek güzel olurdu. + +Arka plan birleştirmeleri sırasında ek iş yapan MergeTree motorları vardır. Örnekler şunlardır `CollapsingMergeTree` ve `AggregatingMergeTree`. Bu, güncellemeler için özel destek olarak kabul edilebilir. Kullanıcıların genellikle arka plan birleştirmeleri yürütüldüğünde zaman üzerinde hiçbir kontrole sahip çünkü bu gerçek güncellemeler olmadığını unutmayın, ve bir veri `MergeTree` tablo hemen hemen her zaman tamamen birleştirilmiş formda değil, birden fazla bölümde saklanır. + +## Çoğalma {#replication} + +Clickhouse çoğaltma başına tablo bazında yapılandırılabilir. Aynı sunucuda bazı çoğaltılmış ve bazı çoğaltılmamış tablolar olabilir. Ayrıca, iki faktörlü çoğaltmaya sahip bir tablo ve üç faktörlü bir tablo gibi farklı şekillerde çoğaltılmış tablolar da olabilir. + +Çoğaltma uygulanır `ReplicatedMergeTree` depolama motoru. The path in `ZooKeeper` depolama altyapısı için bir parametre olarak belirtilir. Aynı yolu olan tüm tablolar `ZooKeeper` birbirlerinin kopyaları haline gelir: verilerini senkronize eder ve tutarlılığı korurlar. Yinelemeler, bir tablo oluşturarak veya bırakarak dinamik olarak eklenebilir ve kaldırılabilir. + +Çoğaltma, zaman uyumsuz bir çoklu ana düzeni kullanır. Bir oturum olan herhangi bir yinelemeye veri ekleyebilirsiniz. `ZooKeeper` ve veriler diğer tüm yinelemelere zaman uyumsuz olarak çoğaltılır. ClickHouse güncelleştirmeleri desteklemediğinden, çoğaltma çakışmaz. Eklerin çekirdek onayı olmadığından, bir düğüm başarısız olursa, yalnızca eklenen veriler kaybolabilir. + +Çoğaltma için meta veri zookeeper saklanır. Hangi eylemlerin yapılacağını listeleyen bir çoğaltma günlüğü vardır. Eylemler şunlardır: parça al; parçaları Birleştir; bir bölüm bırak vb. Her çoğaltma, çoğaltma günlüğünü kendi kuyruğuna kopyalar ve sonra da sıradaki eylemleri yürütür. Örneğin, ekleme, “get the part” eylem günlüğüne oluşturulur ve her çoğaltma bu bölümü indirir. Birleştirmeler, baytla aynı sonuçları elde etmek için yinelemeler arasında koordine edilir. Tüm parçalar tüm kopyalarda aynı şekilde birleştirilir. Bir kopyayı lider olarak seçerek elde edilir ve bu çoğaltma birleştirir ve yazar “merge parts” günlük eylemler. + +Çoğaltma fiziksel: yalnızca sıkıştırılmış parçalar sorgular değil düğümler arasında aktarılır. Birleştirmeler, çoğu durumda ağ amplifikasyonundan kaçınarak ağ maliyetlerini düşürmek için her yinelemede bağımsız olarak işlenir. Büyük birleştirilmiş parçalar, yalnızca önemli çoğaltma gecikmesi durumunda ağ üzerinden gönderilir. + +Ayrıca, her bir kopya, ZooKeeper içindeki durumunu parça seti ve sağlama toplamı olarak depolar. Yerel dosya sistemindeki durum ZooKeeper referans durumundan ayrıldığında, kopya diğer kopyalardan eksik ve bozuk parçaları indirerek tutarlılığını geri yükler. Yerel dosya sisteminde beklenmeyen veya bozuk bazı veriler olduğunda, ClickHouse onu kaldırmaz, ancak ayrı bir dizine taşır ve unutur. + +!!! note "Not" + ClickHouse kümesi bağımsız parçalardan oluşur ve her parça kopyalardan oluşur. Küme **elastik değil**, böylece yeni bir parça ekledikten sonra, veriler otomatik olarak kırıklar arasında yeniden dengelenmez. Bunun yerine, küme yükünün eşit olmayan şekilde ayarlanması gerekiyor. Bu uygulama size daha fazla kontrol sağlar ve onlarca düğüm gibi nispeten küçük kümeler için uygundur. Ancak üretimde kullandığımız yüzlerce düğüm içeren kümeler için bu yaklaşım önemli bir dezavantaj haline gelir. Kümeler arasında otomatik olarak bölünebilen ve dengelenebilen dinamik olarak çoğaltılmış bölgelerle kümeye yayılan bir tablo altyapısı uygulamalıyız. + +{## [Orijinal makale](https://clickhouse.tech/docs/en/development/architecture/) ##} diff --git a/docs/tr/development/browse_code.md b/docs/tr/development/browse_code.md new file mode 100644 index 00000000000..dacebf01050 --- /dev/null +++ b/docs/tr/development/browse_code.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 63 +toc_title: "ClickHouse Kaynak Koduna G\xF6z At\u0131n" +--- + +# ClickHouse Kaynak Koduna Göz Atın {#browse-clickhouse-source-code} + +Kullanabilirsiniz **Woboq** online kod tarayıcı mevcut [burada](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). Bu kod navigasyon ve semantik vurgulama, arama ve indeksleme sağlar. Kod anlık görüntüsü günlük olarak güncellenir. + +Ayrıca, kaynaklara göz atabilirsiniz [GitHub](https://github.com/ClickHouse/ClickHouse) herzamanki. + +IDE'NİN ne kullanacağı ile ilgileniyorsanız, CLion, QT Creator, vs Code ve KDevelop (uyarılar ile) öneririz. Herhangi bir favori IDE kullanabilirsiniz. Vim ve Emacs da sayılır. diff --git a/docs/tr/development/build.md b/docs/tr/development/build.md new file mode 100644 index 00000000000..3a3cda15f1b --- /dev/null +++ b/docs/tr/development/build.md @@ -0,0 +1,141 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 64 +toc_title: "Linux \xFCzerinde ClickHouse nas\u0131l olu\u015Fturulur" +--- + +# Geliştirme için ClickHouse nasıl inşa edilir {#how-to-build-clickhouse-for-development} + +Aşağıdaki öğretici Ubuntu Linux sistemine dayanmaktadır. +Uygun değişikliklerle, başka herhangi bir Linux dağıtımı üzerinde de çalışması gerekir. +Desteklenen platformlar: x86\_64 ve AArch64. Power9 için destek deneyseldir. + +## Git, Cmake, Python ve Ninja'yı yükleyin {#install-git-cmake-python-and-ninja} + +``` bash +$ sudo apt-get install git cmake python ninja-build +``` + +Veya eski sistemlerde cmake yerine cmake3. + +## Gcc 9'u yükle {#install-gcc-9} + +Bunu yapmak için çeşitli yollar vardır. + +### Bir PPA paketinden yükleme {#install-from-a-ppa-package} + +``` bash +$ sudo apt-get install software-properties-common +$ sudo apt-add-repository ppa:ubuntu-toolchain-r/test +$ sudo apt-get update +$ sudo apt-get install gcc-9 g++-9 +``` + +### Kaynaklardan yükleyin {#install-from-sources} + +Bakmak [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) + +## Yapılar için GCC 9 kullanın {#use-gcc-9-for-builds} + +``` bash +$ export CC=gcc-9 +$ export CXX=g++-9 +``` + +## Checkout ClickHouse Kaynakları {#checkout-clickhouse-sources} + +``` bash +$ git clone --recursive git@github.com:ClickHouse/ClickHouse.git +``` + +veya + +``` bash +$ git clone --recursive https://github.com/ClickHouse/ClickHouse.git +``` + +## ClickHouse İnşa {#build-clickhouse} + +``` bash +$ cd ClickHouse +$ mkdir build +$ cd build +$ cmake .. +$ ninja +$ cd .. +``` + +Bir yürütülebilir dosya oluşturmak için çalıştırın `ninja clickhouse`. +Bu yaratacak `programs/clickhouse` ile kullanılabilecek çalıştırılabilir `client` veya `server` değişkenler. + +# Herhangi bir Linux üzerinde ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-any-linux} + +Yapı aşağıdaki bileşenleri gerektirir: + +- Git (yalnızca kaynakları kontrol etmek için kullanılır, yapı için gerekli değildir) +- Cmake 3.10 veya daha yeni +- Ninja (önerilir) veya yapmak +- C ++ derleyici: gcc 9 veya clang 8 veya daha yeni +- Linker :lld veya altın (klasik GNU ld çalışmaz) +- Python (sadece LLVM yapısında kullanılır ve isteğe bağlıdır) + +Tüm bileşenler yüklüyse, yukarıdaki adımlarla aynı şekilde oluşturabilirsiniz. + +Ubuntu Eoan için örnek: + + sudo apt update + sudo apt install git cmake ninja-build g++ python + git clone --recursive https://github.com/ClickHouse/ClickHouse.git + mkdir build && cd build + cmake ../ClickHouse + ninja + +OpenSUSE Tumbleweed için örnek: + + sudo zypper install git cmake ninja gcc-c++ python lld + git clone --recursive https://github.com/ClickHouse/ClickHouse.git + mkdir build && cd build + cmake ../ClickHouse + ninja + +Fedora Rawhide için örnek: + + sudo yum update + yum --nogpg install git cmake make gcc-c++ python2 + git clone --recursive https://github.com/ClickHouse/ClickHouse.git + mkdir build && cd build + cmake ../ClickHouse + make -j $(nproc) + +# ClickHouse inşa etmek zorunda değilsiniz {#you-dont-have-to-build-clickhouse} + +ClickHouse önceden oluşturulmuş ikili ve paketlerde mevcuttur. İkili dosyalar taşınabilir ve herhangi bir Linux lezzet üzerinde çalıştırılabilir. + +Onlar sürece her Master taahhüt ve her çekme isteği için kararlı, prestable ve test bültenleri için inşa edilmiştir. + +En taze yapıyı bulmak için `master`, go to [taahhüt sayfası](https://github.com/ClickHouse/ClickHouse/commits/master), commit yakınındaki ilk yeşil onay işaretini veya kırmızı çarpı işaretini tıklayın ve “Details” hemen sonra bağlantı “ClickHouse Build Check”. + +# ClickHouse Debian paketi nasıl oluşturulur {#how-to-build-clickhouse-debian-package} + +## Git ve Pbuilder'ı yükleyin {#install-git-and-pbuilder} + +``` bash +$ sudo apt-get update +$ sudo apt-get install git python pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring +``` + +## Checkout ClickHouse Kaynakları {#checkout-clickhouse-sources-1} + +``` bash +$ git clone --recursive --branch master https://github.com/ClickHouse/ClickHouse.git +$ cd ClickHouse +``` + +## Run Release Script {#run-release-script} + +``` bash +$ ./release +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/development/build/) diff --git a/docs/tr/development/build_cross_arm.md b/docs/tr/development/build_cross_arm.md new file mode 100644 index 00000000000..d0d18162d6f --- /dev/null +++ b/docs/tr/development/build_cross_arm.md @@ -0,0 +1,43 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 67 +toc_title: "AARCH64 (ARM64) i\xE7in Linux'ta ClickHouse nas\u0131l olu\u015Fturulur)" +--- + +# AARCH64 (ARM64) mimarisi için Linux'ta ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture} + +Bu, Linux makineniz olduğunda ve onu oluşturmak için kullanmak istediğinizde geçerlidir `clickhouse` AARCH64 CPU mimarisi ile başka bir Linux makinede çalışacak ikili. Bu, Linux sunucularında çalışan sürekli entegrasyon kontrolleri için tasarlanmıştır. + +AARCH64 için çapraz yapı, [Inşa talimatları](build.md) önce onları takip et. + +# Clang-8'i Yükle {#install-clang-8} + +Yönergeleri izleyin https://apt.llvm.org / Ubuntu veya Debian kurulumunuz için. +Örneğin, Ubuntu Bionic'te aşağıdaki komutları kullanabilirsiniz: + +``` bash +echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee /etc/apt/sources.list.d/llvm.list +sudo apt-get update +sudo apt-get install clang-8 +``` + +# Çapraz Derleme Araç Setini Yükle {#install-cross-compilation-toolset} + +``` bash +cd ClickHouse +mkdir -p build-aarch64/cmake/toolchain/linux-aarch64 +wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en' -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz +tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cmake/toolchain/linux-aarch64 --strip-components=1 +``` + +# ClickHouse İnşa {#build-clickhouse} + +``` bash +cd ClickHouse +mkdir build-arm64 +CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake +ninja -C build-arm64 +``` + +Ortaya çıkan ikili, yalnızca AARCH64 CPU mimarisi ile Linux'ta çalışacaktır. diff --git a/docs/tr/development/build_cross_osx.md b/docs/tr/development/build_cross_osx.md new file mode 100644 index 00000000000..2001ad985e2 --- /dev/null +++ b/docs/tr/development/build_cross_osx.md @@ -0,0 +1,64 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 66 +toc_title: "Mac OS X i\xE7in Linux'ta ClickHouse nas\u0131l olu\u015Fturulur" +--- + +# Mac OS X için Linux'ta ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-linux-for-mac-os-x} + +Bu, Linux makineniz olduğunda ve onu oluşturmak için kullanmak istediğinizde geçerlidir `clickhouse` OS X üzerinde çalışacak ikili. bu, Linux sunucularında çalışan sürekli entegrasyon kontrolleri için tasarlanmıştır. Clickhouse'u doğrudan Mac OS X'te oluşturmak istiyorsanız, devam edin [başka bir talimat](build_osx.md). + +Mac OS X için çapraz yapı, [Inşa talimatları](build.md) önce onları takip et. + +# Clang-8'i Yükle {#install-clang-8} + +Yönergeleri izleyin https://apt.llvm.org / Ubuntu veya Debian kurulumunuz için. +Örneğin biyonik için komutlar gibidir: + +``` bash +sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list +sudo apt-get install clang-8 +``` + +# Çapraz Derleme Araç Setini Yükle {#install-cross-compilation-toolset} + +Yüklediğimiz yolu hatırlayalım `cctools` olarak $ {CCTOOLS} + +``` bash +mkdir ${CCTOOLS} + +git clone https://github.com/tpoechtrager/apple-libtapi.git +cd apple-libtapi +INSTALLPREFIX=${CCTOOLS} ./build.sh +./install.sh +cd .. + +git clone https://github.com/tpoechtrager/cctools-port.git +cd cctools-port/cctools +./configure --prefix=${CCTOOLS} --with-libtapi=${CCTOOLS} --target=x86_64-apple-darwin +make install +``` + +Ayrıca, MacOS X SDK'YI çalışma ağacına indirmemiz gerekiyor. + +``` bash +cd ClickHouse +wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' +mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 +tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 +``` + +# ClickHouse İnşa {#build-clickhouse} + +``` bash +cd ClickHouse +mkdir build-osx +CC=clang-8 CXX=clang++-8 cmake . -Bbuild-osx -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake \ + -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar \ + -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib \ + -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld +ninja -C build-osx +``` + +Ortaya çıkan ikili bir Mach-O yürütülebilir biçimine sahip olacak ve Linux üzerinde çalıştırılamaz. diff --git a/docs/tr/development/build_osx.md b/docs/tr/development/build_osx.md new file mode 100644 index 00000000000..af750725070 --- /dev/null +++ b/docs/tr/development/build_osx.md @@ -0,0 +1,93 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 65 +toc_title: "Mac OS X \xFCzerinde ClickHouse nas\u0131l olu\u015Fturulur" +--- + +# Mac OS X üzerinde ClickHouse nasıl oluşturulur {#how-to-build-clickhouse-on-mac-os-x} + +Build Mac OS X 10.15 (Catalina) üzerinde çalışmalıdır) + +## Homebrew Yüklemek {#install-homebrew} + +``` bash +$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +``` + +## Gerekli derleyicileri, araçları ve kitaplıkları yükleyin {#install-required-compilers-tools-and-libraries} + +``` bash +$ brew install cmake ninja libtool gettext +``` + +## Checkout ClickHouse Kaynakları {#checkout-clickhouse-sources} + +``` bash +$ git clone --recursive git@github.com:ClickHouse/ClickHouse.git +``` + +veya + +``` bash +$ git clone --recursive https://github.com/ClickHouse/ClickHouse.git + +$ cd ClickHouse +``` + +## ClickHouse İnşa {#build-clickhouse} + +``` bash +$ mkdir build +$ cd build +$ cmake .. -DCMAKE_CXX_COMPILER=`which clang++` -DCMAKE_C_COMPILER=`which clang` +$ ninja +$ cd .. +``` + +## Uyarılar {#caveats} + +Clickhouse-server çalıştırmak istiyorsanız, sistemin maxfiles değişken artırmak için emin olun. + +!!! info "Not" + Sudo kullanmanız gerekecek. + +Bunu yapmak için aşağıdaki dosyayı oluşturun: + +/ Kütüphane / LaunchDaemons / sınırı.maxfiles.plist: + +``` xml + + + + + Label + limit.maxfiles + ProgramArguments + + launchctl + limit + maxfiles + 524288 + 524288 + + RunAtLoad + + ServiceIPC + + + +``` + +Aşağıdaki komutu çalıştırın: + +``` bash +$ sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist +``` + +Başlatmak. + +Çalışıp çalışmadığını kontrol etmek için şunları kullanabilirsiniz `ulimit -n` komut. + +[Orijinal makale](https://clickhouse.tech/docs/en/development/build_osx/) diff --git a/docs/tr/development/contrib.md b/docs/tr/development/contrib.md new file mode 100644 index 00000000000..7f09d9bc49b --- /dev/null +++ b/docs/tr/development/contrib.md @@ -0,0 +1,42 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 70 +toc_title: "Kullan\u0131lan \xDC\xE7\xFCnc\xFC Taraf K\xFCt\xFCphaneleri" +--- + +# Kullanılan Üçüncü Taraf Kütüphaneleri {#third-party-libraries-used} + +| Kitaplık | Lisans | +|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| base64 | [BSD 2-Clause Lisansı](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) | +| artırmak | [Bo Boostost Software Lic 1.0ense 1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) | +| brotli | [MIT](https://github.com/google/brotli/blob/master/LICENSE) | +| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) | +| cctz | [Apache Lic 2.0ense 2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) | +| çift dönüşüm | [BSD 3-Clause Lisansı](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | +| FastMemcpy | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | +| googletest | [BSD 3-Clause Lisansı](https://github.com/google/googletest/blob/master/LICENSE) | +| h33 | [Apache Lic 2.0ense 2.0](https://github.com/uber/h3/blob/master/LICENSE) | +| hyperscan | [BSD 3-Clause Lisansı](https://github.com/intel/hyperscan/blob/master/LICENSE) | +| libbtrie | [BSD 2-Clause Lisansı](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | +| libcxxabi | [BSD + MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | +| libdivide | [Zlib Lisansı](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | +| libgsasl | [LGPL v2. 1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) | +| libhdfs3 | [Apache Lic 2.0ense 2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) | +| libmetrohash | [Apache Lic 2.0ense 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | +| libpcg-rastgele | [Apache Lic 2.0ense 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) | +| libressl | [OpenSSL Lisansı](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) | +| librdkafka | [BSD 2-Clause Lisansı](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) | +| libwidechar\_width | [CC0 1.0 Evrensel](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) | +| llvm | [BSD 3-Clause Lisansı](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) | +| lz4 | [BSD 2-Clause Lisansı](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) | +| mariadb-bağlayıcı-c | [LGPL v2. 1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) | +| murmurhash | [Kamu Malı](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) | +| pdqsort | [Zlib Lisansı](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/pdqsort/license.txt) | +| az | [Boost Yazılım Lisansı-Sürüm 1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) | +| protobuf | [BSD 3-Clause Lisansı](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) | +| re2 | [BSD 3-Clause Lisansı](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) | +| UnixODBC | [LGPL v2. 1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) | +| zlib-ng | [Zlib Lisansı](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) | +| zstd | [BSD 3-Clause Lisansı](https://github.com/facebook/zstd/blob/dev/LICENSE) | diff --git a/docs/tr/development/developer_instruction.md b/docs/tr/development/developer_instruction.md new file mode 100644 index 00000000000..f9984d9bf20 --- /dev/null +++ b/docs/tr/development/developer_instruction.md @@ -0,0 +1,285 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 61 +toc_title: "Acemi ClickHouse Geli\u015Ftirici Talimat" +--- + +ClickHouse binası Linux, FreeBSD ve Mac OS X üzerinde desteklenmektedir. + +# Windows Kullanıyorsanız {#if-you-use-windows} + +Windows kullanıyorsanız, Ubuntu ile bir sanal makine oluşturmanız gerekir. Bir sanal makine ile çalışmaya başlamak için VirtualBox yükleyin. UB :unt :u'yu web sitesinden indirebilirsiniz: https://www.ubuntu.com/\#download. lütfen indirilen görüntüden bir sanal makine oluşturun (bunun için en az 4GB RAM ayırmalısınız). Ubuntu'da bir komut satırı terminali çalıştırmak için lütfen kelimeyi içeren bir program bulun “terminal” adına (gnome-terminal, konsole vb.)) veya sadece Ctrl+Alt+T tuşlarına basın. + +# 32 bit sistem kullanıyorsanız {#if-you-use-a-32-bit-system} + +ClickHouse çalışamaz veya 32-bit bir sistem üzerinde oluşturun. 64-bit bir sisteme erişim kazanmanız gerekir ve okumaya devam edebilirsiniz. + +# Github'da bir depo oluşturma {#creating-a-repository-on-github} + +ClickHouse repository ile çalışmaya başlamak için bir GitHub hesabına ihtiyacınız olacaktır. + +Muhtemelen zaten bir tane var, ama yapmazsanız, lütfen kayıt olun https://github.com. SSH anahtarlarınız yoksa, bunları üretmeli ve daha sonra Github'a yüklemelisiniz. Bu yamalar üzerinden göndermek için gereklidir. Diğer SSH sunucularıyla kullandığınız aynı SSH anahtarlarını kullanmak da mümkündür - muhtemelen zaten bunlara sahipsiniz. + +ClickHouse deposunun bir çatalı oluşturun. Bunu yapmak için lütfen tıklayın “fork” sağ üst köşedeki düğme https://github.com/ClickHouse/ClickHouse. bu hesabınıza ClickHouse / ClickHouse kendi kopyasını çatal olacaktır. + +Geliştirme süreci ilk ClickHouse sizin çatal içine amaçlanan değişiklikleri işlemekle ve daha sonra bir oluşturma oluşur “pull request” bu değişikliklerin ana depoya kabul edilmesi için (ClickHouse/ClickHouse). + +Git depoları ile çalışmak için lütfen yükleyin `git`. + +Bunu Ubuntu'da yapmak için komut satırı terminalinde çalışırsınız: + + sudo apt update + sudo apt install git + +Git kullanımı ile ilgili kısa bir el kitabı burada bulunabilir: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf. +Git ile ilgili ayrıntılı bir el kitabı için bkz. https://git-scm.com/book/en/v2. + +# Geliştirme Makinenize bir depo klonlama {#cloning-a-repository-to-your-development-machine} + +Ardından, kaynak dosyaları çalışma makinenize indirmeniz gerekir. Bu denir “to clone a repository” çünkü çalışma makinenizde deponun yerel bir kopyasını oluşturur. + +Komut satırında terminal Çalıştır: + + git clone --recursive git@guthub.com:your_github_username/ClickHouse.git + cd ClickHouse + +Not: lütfen, yerine *your\_github\_username* uygun olanı ile! + +Bu komut bir dizin oluşturacaktır `ClickHouse` projenin çalışma kopyasını içeren. + +Yapı sistemini çalıştırmakla ilgili sorunlara yol açabileceğinden, çalışma dizininin yolunun hiçbir boşluk içermemesi önemlidir. + +ClickHouse deposunun kullandığını lütfen unutmayın `submodules`. That is what the references to additional repositories are called (i.e. external libraries on which the project depends). It means that when cloning the repository you need to specify the `--recursive` yukarıdaki örnekte olduğu gibi bayrak. Depo alt modüller olmadan klonlanmışsa, bunları indirmek için aşağıdakileri çalıştırmanız gerekir: + + git submodule init + git submodule update + +Komutu ile durumunu kontrol edebilirsiniz: `git submodule status`. + +Aşağıdaki hata iletisini alırsanız: + + Permission denied (publickey). + fatal: Could not read from remote repository. + + Please make sure you have the correct access rights + and the repository exists. + +Genellikle Github'a bağlanmak için SSH anahtarlarının eksik olduğu anlamına gelir. Bu anahtarlar normalde `~/.ssh`. SSH anahtarlarının kabul edilmesi için bunları GitHub kullanıcı arayüzünün ayarlar bölümüne yüklemeniz gerekir. + +Depoyu https protokolü aracılığıyla da klonlayabilirsiniz: + + git clone https://github.com/ClickHouse/ClickHouse.git + +Ancak bu, değişikliklerinizi sunucuya göndermenize izin vermez. Yine de geçici olarak kullanabilir ve SSH anahtarlarını daha sonra deponun uzak adresini değiştirerek ekleyebilirsiniz `git remote` komut. + +Oradan güncellemeleri çekmek için orijinal ClickHouse repo'nun adresini yerel deponuza da ekleyebilirsiniz: + + git remote add upstream git@github.com:ClickHouse/ClickHouse.git + +Başarıyla bu komutu çalıştırdıktan sonra çalıştırarak ana ClickHouse repo güncellemeleri çekmek mümkün olacak `git pull upstream master`. + +## Alt modüllerle çalışma {#working-with-submodules} + +Git'teki alt modüllerle çalışmak acı verici olabilir. Sonraki komutlar onu yönetmeye yardımcı olacaktır: + + # ! each command accepts --recursive + # Update remote URLs for submodules. Barely rare case + git submodule sync + # Add new submodules + git submodule init + # Update existing submodules to the current state + git submodule update + # Two last commands could be merged together + git submodule update --init + +Bir sonraki komutlar, tüm alt modülleri başlangıç durumuna sıfırlamanıza yardımcı olacaktır (!UYARI! - herhangi bir değişiklik içinde silinecektir): + + # Synchronizes submodules' remote URL with .gitmodules + git submodule sync --recursive + # Update the registered submodules with initialize not yet initialized + git submodule update --init --recursive + # Reset all changes done after HEAD + git submodule foreach git reset --hard + # Clean files from .gitignore + git submodule foreach git clean -xfd + # Repeat last 4 commands for all submodule + git submodule foreach git submodule sync --recursive + git submodule foreach git submodule update --init --recursive + git submodule foreach git submodule foreach git reset --hard + git submodule foreach git submodule foreach git clean -xfd + +# Yapı Sistemi {#build-system} + +ClickHouse bina için Cmake ve Ninja kullanır. + +Cmake-ninja dosyaları (yapı görevleri) üretebilir bir meta-yapı sistemi. +Ninja-bu cmake oluşturulan görevleri yürütmek için kullanılan hıza odaklanarak daha küçük bir yapı sistemi. + +Ubuntu, Debian veya Mint run'a yüklemek için `sudo apt install cmake ninja-build`. + +Centos'ta, RedHat koşusu `sudo yum install cmake ninja-build`. + +Arch veya Gentoo kullanıyorsanız, muhtemelen cmake'i nasıl kuracağınızı kendiniz biliyorsunuz. + +Mac OS X üzerinde cmake ve Ninja yüklemek için ilk homebrew yüklemek ve daha sonra demlemek yoluyla her şeyi yüklemek: + + /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + brew install cmake ninja + +Ardından, cmake sürümünü kontrol edin: `cmake --version`. 3.3'ün altındaysa, web sitesinden daha yeni bir sürüm yüklemelisiniz: https://cmake.org/download/. + +# İsteğe Bağlı Harici Kütüphaneler {#optional-external-libraries} + +ClickHouse, bina için birkaç dış kütüphane kullanır. Alt modüllerde bulunan kaynaklardan ClickHouse ile birlikte oluşturuldukları için hepsinin ayrı olarak kurulması gerekmez. Listeyi kontrol edebilirsiniz `contrib`. + +# C++ Derleyici {#c-compiler} + +Derleyiciler gcc sürüm 9 ve Clang sürüm 8 veya üzeri başlayarak ClickHouse bina için desteklenmektedir. + +Resmi Yandex şu anda GCC'Yİ kullanıyor çünkü biraz daha iyi performansa sahip makine kodu üretiyor (kriterlerimize göre yüzde birkaçına kadar bir fark yaratıyor). Ve Clang genellikle geliştirme için daha uygundur. Yine de, sürekli entegrasyon (CI) platformumuz yaklaşık bir düzine yapı kombinasyonunu denetler. + +Ubuntu run GCC yüklemek için: `sudo apt install gcc g++` + +Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech / docs/TR/development / build / \#ınstall-gcc-9. + +Mac OS X build sadece Clang için desteklenir. Sadece koş `brew install llvm` + +Eğer Clang kullanmaya karar verirseniz, ayrıca yükleyebilirsiniz `libc++` ve `lld` eğer ne olduğunu biliyorsan. Kullanım `ccache` ayrıca tavsiye edilir. + +# İnşaat Süreci {#the-building-process} + +Artık ClickHouse oluşturmaya hazır olduğunuza göre ayrı bir dizin oluşturmanızı öneririz `build` için `ClickHouse` bu, tüm yapı eserlerini içerecek: + + mkdir build + cd build + +Birkaç farklı dizine (build\_release, build\_debug, vb.) sahip olabilirsiniz.) farklı yapı türleri için. + +İçinde iken `build` dizin, cmake çalıştırarak yapı yapılandırın. İlk çalıştırmadan önce, derleyici belirten ortam değişkenlerini tanımlamanız gerekir (bu örnekte sürüm 9 gcc derleyicisi). + +Linux: + + export CC=gcc-9 CXX=g++-9 + cmake .. + +Mac OS X: + + export CC=clang CXX=clang++ + cmake .. + +Bu `CC` değişken C için derleyiciyi belirtir (C derleyicisi için kısa) ve `CXX` değişken, hangi C++ derleyicisinin bina için kullanılacağını bildirir. + +Daha hızlı bir yapı için, `debug` yapı türü-hiçbir optimizasyonları ile bir yapı. Bunun için aşağıdaki parametreyi sağlayın `-D CMAKE_BUILD_TYPE=Debug`: + + cmake -D CMAKE_BUILD_TYPE=Debug .. + +Bu komutu çalıştırarak yapı türünü değiştirebilirsiniz. `build` dizin. + +İnşa etmek için ninja çalıştırın: + + ninja clickhouse-server clickhouse-client + +Bu örnekte yalnızca gerekli ikili dosyalar oluşturulacaktır. + +Tüm ikili dosyaları (Yardımcı Programlar ve testler) oluşturmanız gerekiyorsa, ninja'yı parametre olmadan çalıştırmalısınız: + + ninja + +Tam yapı, ana ikili dosyaları oluşturmak için yaklaşık 30GB boş disk alanı veya 15GB gerektirir. + +Yapı makinesinde büyük miktarda RAM mevcut olduğunda, paralel olarak çalışan yapı görevlerinin sayısını sınırlamanız gerekir `-j` param: + + ninja -j 1 clickhouse-server clickhouse-client + +4GB RAM'Lİ makinelerde, 8GB RAM için 1 belirtmeniz önerilir `-j 2` tavsiye edilir. + +Mesajı alırsanız: `ninja: error: loading 'build.ninja': No such file or directory` bu, bir yapı yapılandırması oluşturmanın başarısız olduğu ve yukarıdaki mesajı incelemeniz gerektiği anlamına gelir. + +Bina işleminin başarılı bir şekilde başlatılmasının ardından, yapı ilerlemesini görürsünüz-işlenmiş görevlerin sayısı ve toplam görev sayısı. + +Libhdfs2 kütüphanesinde protobuf dosyaları hakkında mesajlar oluştururken `libprotobuf WARNING` ortaya çıkabilir. Hiçbir şeyi etkilemezler ve göz ardı edilmeleri güvenlidir. + +Başarılı bir yapı üzerine yürütülebilir bir dosya alırsınız `ClickHouse//programs/clickhouse`: + + ls -l programs/clickhouse + +# Clickhouse'un yerleşik yürütülebilir dosyasını çalıştırma {#running-the-built-executable-of-clickhouse} + +Sunucuyu geçerli kullanıcı altında çalıştırmak için aşağıdakilere gitmeniz gerekir `ClickHouse/programs/server/` (dışında bulunan `build`) ve koş: + + ../../../build/programs/clickhouse server + +Bu durumda, ClickHouse geçerli dizinde bulunan yapılandırma dosyalarını kullanır. Koş youabilirsiniz `clickhouse server` komut satırı parametresi olarak bir yapılandırma dosyasının yolunu belirten herhangi bir dizinden `--config-file`. + +Başka bir terminalde clickhouse-client ile Clickhouse'a bağlanmak için `ClickHouse/build/programs/` ve koş `clickhouse client`. + +Eğer alırsanız `Connection refused` Mac OS X veya Freebsd'de mesaj, ana bilgisayar adresi 127.0.0.1 belirtmeyi deneyin: + + clickhouse client --host 127.0.0.1 + +Sisteminizde yüklü olan ClickHouse binary'nin üretim sürümünü özel olarak oluşturulmuş ClickHouse binaryinizle değiştirebilirsiniz. Bunu yapmak için resmi web sitesinden talimatları izleyerek Makinenize ClickHouse yükleyin. Ardından, aşağıdakileri çalıştırın: + + sudo service clickhouse-server stop + sudo cp ClickHouse/build/programs/clickhouse /usr/bin/ + sudo service clickhouse-server start + +Not thate that `clickhouse-client`, `clickhouse-server` ve diğerleri yaygın olarak paylaşılan sembolik bağlardır `clickhouse` ikilik. + +Ayrıca sisteminizde yüklü ClickHouse paketinden yapılandırma dosyası ile özel inşa ClickHouse ikili çalıştırabilirsiniz: + + sudo service clickhouse-server stop + sudo -u clickhouse ClickHouse/build/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml + +# IDE (entegre geliştirme ortamı) {#ide-integrated-development-environment} + +Hangi IDE kullanmak bilmiyorsanız, clion kullanmanızı öneririz. CLion ticari bir yazılımdır, ancak 30 günlük ücretsiz deneme süresi sunar. Öğrenciler için de ücretsizdir. CLion Linux ve Mac OS X hem de kullanılabilir. + +KDevelop ve QTCreator, ClickHouse geliştirmek için bir IDE'NİN diğer harika alternatifleridir. KDevelop kararsız olmasına rağmen çok kullanışlı bir IDE olarak geliyor. KDevelop projeyi açtıktan sonra bir süre sonra çökerse, tıklamanız gerekir “Stop All” proje dosyalarının listesini açar açmaz düğme. Bunu yaptıktan sonra KDevelop ile çalışmak iyi olmalıdır. + +Basit kod editörleri olarak, Yüce metin veya Visual Studio kodunu veya Kate'i (hepsi Linux'ta kullanılabilir) kullanabilirsiniz. + +Her ihtimale karşı, Clion'un yarattığını belirtmek gerekir `build` kendi başına yol, aynı zamanda kendi seçtikleri `debug` yapı türü için, yapılandırma için Clion'da tanımlanan ve sizin tarafınızdan yüklenmeyen bir cmake sürümünü kullanır ve son olarak CLion kullanacaktır `make` yerine yapı görevlerini çalıştırmak için `ninja`. Bu normal bir davranıştır, sadece karışıklığı önlemek için bunu aklınızda bulundurun. + +# Kod Yazma {#writing-code} + +Açıklaması ClickHouse mimarisi burada bulabilirsiniz: https://clickhouse.tech / doscs/TR / development / Arch /it /ec /ture/ + +Kod stili Kılavuzu: https://clickhouse.tech / doscs / TR / development / style/ + +Yazma testleri: https://clickhouse.teknoloji / doscs / TR / geliştirme / testler/ + +Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md + +# Test Verileri {#test-data} + +Clickhouse'un geliştirilmesi genellikle gerçekçi veri kümelerinin yüklenmesini gerektirir. Performans testi için özellikle önemlidir. Yandex'ten özel olarak hazırlanmış anonim veri setimiz var.Metrica. Ayrıca bazı 3GB boş disk alanı gerektirir. Bu verilerin geliştirme görevlerinin çoğunu gerçekleştirmek için gerekli olmadığını unutmayın. + + sudo apt install wget xz-utils + + wget https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz + wget https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz + + xz -v -d hits_v1.tsv.xz + xz -v -d visits_v1.tsv.xz + + clickhouse-client + + CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); + + CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); + + clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv + clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv + +# Çekme İsteği Oluşturma {#creating-pull-request} + +Github'un kullanıcı arayüzünde çatal deposuna gidin. Bir dalda gelişiyorsanız, o Dalı seçmeniz gerekir. Bir olacak “Pull request” ekranda bulunan düğme. Özünde, bu demektir “create a request for accepting my changes into the main repository”. + +Çalışma henüz tamamlanmamış olsa bile bir çekme isteği oluşturulabilir. Bu durumda lütfen kelimeyi koyun “WIP” (devam eden çalışma) başlığın başında, daha sonra değiştirilebilir. Bu, kooperatif Gözden geçirme ve değişikliklerin tartışılması ve mevcut tüm testlerin çalıştırılması için kullanışlıdır. Değişikliklerinizin kısa bir açıklamasını sağlamanız önemlidir, daha sonra sürüm değişiklikleri oluşturmak için kullanılacaktır. + +Yandex çalışanları PR'NİZİ bir etiketle etiketlediğinde testler başlayacaktır “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. + +Sistem, çekme isteğiniz için ayrı ayrı ClickHouse ikili yapıları hazırlayacaktır. Bu yapıları almak için tıklayın “Details” yanındaki bağlantı “ClickHouse build check” çekler listesinde giriş. Orada inşa doğrudan bağlantılar bulacaksınız .eğer üretim sunucularında bile dağıtabilirsiniz ClickHouse DEB paketleri (eğer hiçbir korku varsa). + +Büyük olasılıkla bazı yapılar ilk kez başarısız olur. Bunun nedeni, hem gcc hem de clang ile, hemen hemen tüm mevcut uyarılarla (her zaman `-Werror` bayrak) clang için etkin. Aynı sayfada, tüm yapı günlüklerini bulabilirsiniz, böylece tüm olası yollarla ClickHouse oluşturmak zorunda kalmazsınız. diff --git a/docs/tr/development/index.md b/docs/tr/development/index.md new file mode 100644 index 00000000000..fdd4c0c0805 --- /dev/null +++ b/docs/tr/development/index.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Geli\u015Fme" +toc_hidden: true +toc_priority: 58 +toc_title: "gizlenmi\u015F" +--- + +# ClickHouse Geliştirme {#clickhouse-development} + +[Orijinal makale](https://clickhouse.tech/docs/en/development/) diff --git a/docs/tr/development/style.md b/docs/tr/development/style.md new file mode 100644 index 00000000000..713f95cf053 --- /dev/null +++ b/docs/tr/development/style.md @@ -0,0 +1,841 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 68 +toc_title: "C++ kodu nas\u0131l yaz\u0131l\u0131r" +--- + +# C++ kodu nasıl yazılır {#how-to-write-c-code} + +## Genel Öneriler {#general-recommendations} + +**1.** Aşağıdakiler önerilerdir, gereksinimler değildir. + +**2.** Kod düzenliyorsanız, varolan kodun biçimlendirmesini takip etmek mantıklıdır. + +**3.** Kod stili tutarlılık için gereklidir. Tutarlılık, kodu okumayı kolaylaştırır ve aynı zamanda kodu aramayı da kolaylaştırır. + +**4.** Kuralların çoğunun mantıklı nedenleri yoktur, yerleşik uygulamalar tarafından belirlenir. + +## Biçimlendirir {#formatting} + +**1.** Biçimlendirme çoğu tarafından otomatik olarak yapılacaktır `clang-format`. + +**2.** Girintiler 4 boşluk vardır. Bir sekme dört boşluk ekleyecek şekilde geliştirme ortamınızı yapılandırın. + +**3.** Kıvırcık parantezlerin açılması ve kapatılması ayrı bir satırda olmalıdır. + +``` cpp +inline void readBoolText(bool & x, ReadBuffer & buf) +{ + char tmp = '0'; + readChar(tmp, buf); + x = tmp != '0'; +} +``` + +**4.** Tüm fonksiyon gövdesi tek ise `statement`, tek bir satır üzerine yerleştirilebilir. Yer (satır sonunda boşluk dışında) etrafında ayraç boşluk. + +``` cpp +inline size_t mask() const { return buf_size() - 1; } +inline size_t place(HashValue x) const { return x & mask(); } +``` + +**5.** Fonksiyonlar için. Parantezlerin etrafına boşluk koymayın. + +``` cpp +void reinsert(const Value & x) +``` + +``` cpp +memcpy(&buf[place_value], &x, sizeof(x)); +``` + +**6.** İçinde `if`, `for`, `while` ve diğer ifadeler, açılış braketinin önüne bir boşluk yerleştirilir (işlev çağrılarının aksine). + +``` cpp +for (size_t i = 0; i < rows; i += storage.index_granularity) +``` + +**7.** İkili operatörlerin etrafına boşluk Ekle (`+`, `-`, `*`, `/`, `%`, …) and the ternary operator `?:`. + +``` cpp +UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0'); +UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); +UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); +``` + +**8.** Bir satır beslemesi girilirse, operatörü yeni bir satıra koyun ve girintiyi ondan önce artırın. + +``` cpp +if (elapsed_ns) + message << " (" + << rows_read_on_server * 1000000000 / elapsed_ns << " rows/s., " + << bytes_read_on_server * 1000.0 / elapsed_ns << " MB/s.) "; +``` + +**9.** İsterseniz, bir çizgi içinde hizalama için boşluk kullanabilirsiniz. + +``` cpp +dst.ClickLogID = click.LogID; +dst.ClickEventID = click.EventID; +dst.ClickGoodEvent = click.GoodEvent; +``` + +**10.** Operatörler etrafında boşluk kullanmayın `.`, `->`. + +Gerekirse, operatör bir sonraki satıra sarılabilir. Bu durumda, önündeki ofset artar. + +**11.** Tekli operatörleri ayırmak için boşluk kullanmayın (`--`, `++`, `*`, `&`, …) from the argument. + +**12.** Virgülden sonra bir boşluk koyun, ancak ondan önce değil. Aynı kural, bir içindeki noktalı virgül için de geçerlidir `for` ifade. + +**13.** Ayırmak için boşluk kullanmayın `[]` operatör. + +**14.** İn a `template <...>` ifade, arasında bir boşluk kullanın `template` ve `<`; sonra boşluk yok `<` ya da önce `>`. + +``` cpp +template +struct AggregatedStatElement +{} +``` + +**15.** Sınıflarda ve yapılarda, yazın `public`, `private`, ve `protected` aynı seviyede `class/struct` ve kodun geri kalanını girinti. + +``` cpp +template +class MultiVersion +{ +public: + /// Version of object for usage. shared_ptr manage lifetime of version. + using Version = std::shared_ptr; + ... +} +``` + +**16.** Eğer aynı `namespace` tüm dosya için kullanılır ve başka önemli bir şey yoktur, içinde bir ofset gerekli değildir `namespace`. + +**17.** Eğer blok için bir `if`, `for`, `while` veya başka bir ifade tek bir `statement`, kıvırcık parantez isteğe bağlıdır. Place the `statement` bunun yerine ayrı bir satırda. Bu kural iç içe geçmiş için de geçerlidir `if`, `for`, `while`, … + +Ama eğer iç `statement` kıvırcık parantez içerir veya `else`, dış blok kıvırcık parantez içinde yazılmalıdır. + +``` cpp +/// Finish write. +for (auto & stream : streams) + stream.second->finalize(); +``` + +**18.** Çizgilerin uçlarında boşluk olmamalıdır. + +**19.** Kaynak dosyalar UTF-8 kodlanmıştır. + +**20.** ASCII olmayan karakterler dize değişmezlerinde kullanılabilir. + +``` cpp +<< ", " << (timer.elapsed() / chunks_stats.hits) << " μsec/hit."; +``` + +**21.** Tek bir satırda birden çok ifade yazmayın. + +**22.** Fonksiyonların içindeki kod bölümlerini gruplandırın ve bunları birden fazla boş satırla ayırın. + +**23.** Bir veya iki boş satırla ayrı işlevler, sınıflar vb. + +**24.** `A const` (bir değerle ilgili) tür adından önce yazılmalıdır. + +``` cpp +//correct +const char * pos +const std::string & s +//incorrect +char const * pos +``` + +**25.** Bir işaretçi veya başvuru bildirirken, `*` ve `&` semboller her iki taraftaki boşluklarla ayrılmalıdır. + +``` cpp +//correct +const char * pos +//incorrect +const char* pos +const char *pos +``` + +**26.** Şablon türlerini kullanırken, `using` anahtar kelime (en basit durumlar hariç). + +Başka bir deyişle, şablon parametreleri yalnızca `using` ve kodda tekrarlanmıyor. + +`using` bir işlevin içinde olduğu gibi yerel olarak bildirilebilir. + +``` cpp +//correct +using FileStreams = std::map>; +FileStreams streams; +//incorrect +std::map> streams; +``` + +**27.** Bir ifadede farklı türde birkaç değişken bildirmeyin. + +``` cpp +//incorrect +int x, *y; +``` + +**28.** C tarzı yayınları kullanmayın. + +``` cpp +//incorrect +std::cerr << (int)c <<; std::endl; +//correct +std::cerr << static_cast(c) << std::endl; +``` + +**29.** Sınıflarda ve yapılarda, grup üyeleri ve işlevleri her görünürlük kapsamı içinde ayrı ayrı. + +**30.** Küçük sınıflar ve yapılar için, yöntem bildirimini uygulamadan ayırmak gerekli değildir. + +Aynı şey, herhangi bir sınıf veya yapıdaki küçük yöntemler için de geçerlidir. + +Templated sınıflar ve yapılar için, yöntem bildirimlerini uygulamadan ayırmayın(aksi takdirde aynı çeviri biriminde tanımlanmaları gerekir). + +**31.** Satırları 80 yerine 140 karakterle sarabilirsiniz. + +**32.** Postfix gerekli değilse, her zaman önek artış / azaltma işleçlerini kullanın. + +``` cpp +for (Names::const_iterator it = column_names.begin(); it != column_names.end(); ++it) +``` + +## Yorumlar {#comments} + +**1.** Kodun önemsiz olmayan tüm bölümleri için yorum eklediğinizden emin olun. + +Bu çok önemli. Yorumu yazmak, kodun gerekli olmadığını veya yanlış tasarlandığını anlamanıza yardımcı olabilir. + +``` cpp +/** Part of piece of memory, that can be used. + * For example, if internal_buffer is 1MB, and there was only 10 bytes loaded to buffer from file for reading, + * then working_buffer will have size of only 10 bytes + * (working_buffer.end() will point to position right after those 10 bytes available for read). + */ +``` + +**2.** Yorumlar gerektiği kadar ayrıntılı olabilir. + +**3.** Açıklama yaptıkları koddan önce yorumları yerleştirin. Nadir durumlarda, yorumlar aynı satırda koddan sonra gelebilir. + +``` cpp +/** Parses and executes the query. +*/ +void executeQuery( + ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable) + WriteBuffer & ostr, /// Where to write the result + Context & context, /// DB, tables, data types, engines, functions, aggregate functions... + BlockInputStreamPtr & query_plan, /// Here could be written the description on how query was executed + QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// Up to which stage process the SELECT query + ) +``` + +**4.** Yorumlar sadece İngilizce olarak yazılmalıdır. + +**5.** Bir kitaplık yazıyorsanız, ana başlık dosyasına açıklayan ayrıntılı yorumları ekleyin. + +**6.** Ek bilgi vermeyen yorumlar eklemeyin. Özellikle, bu gibi boş yorumlar bırakmayın: + +``` cpp +/* +* Procedure Name: +* Original procedure name: +* Author: +* Date of creation: +* Dates of modification: +* Modification authors: +* Original file name: +* Purpose: +* Intent: +* Designation: +* Classes used: +* Constants: +* Local variables: +* Parameters: +* Date of creation: +* Purpose: +*/ +``` + +Örnek kaynaktan ödünç alınmıştır http://home.tamk.fi / ~ jaalto / kurs / kodlama stili / doc/ulaşılamaz-kod/. + +**7.** Çöp yorum yazmayın (yazar, oluşturma tarihi ..) her dosyanın başında. + +**8.** Tek satırlı yorumlar üç eğik çizgi ile başlar: `///` ve çok satırlı yorumlar ile başlar `/**`. Bu yorumlar dikkate alınır “documentation”. + +Not: bu yorumlardan belgeler oluşturmak için Doxygen kullanabilirsiniz. Ancak DOXYGEN genellikle kullanılmaz, çünkü IDE'DEKİ kodda gezinmek daha uygundur. + +**9.** Çok satırlı açıklamaların başında ve sonunda (çok satırlı bir açıklamayı kapatan satır hariç) boş satırları olmamalıdır. + +**10.** Kodu yorumlamak için temel yorumları kullanın, değil “documenting” yorumlar. + +**11.** İşlem yapmadan önce kodun yorumlanan kısımlarını silin. + +**12.** Yorumlarda veya kodda küfür kullanmayın. + +**13.** Büyük harf kullanmayın. Aşırı noktalama kullanmayın. + +``` cpp +/// WHAT THE FAIL??? +``` + +**14.** Sınırlayıcılar yapmak için yorum kullanmayın. + +``` cpp +///****************************************************** +``` + +**15.** Yorumlarda tartışmalara başlamayın. + +``` cpp +/// Why did you do this stuff? +``` + +**16.** Ne hakkında olduğunu açıklayan bir bloğun sonunda bir yorum yazmaya gerek yok. + +``` cpp +/// for +``` + +## Adlar {#names} + +**1.** Değişkenlerin ve sınıf üyelerinin adlarında alt çizgi içeren küçük harfler kullanın. + +``` cpp +size_t max_block_size; +``` + +**2.** İşlevlerin (yöntemlerin) adları için, küçük harfle başlayan camelCase kullanın. + +``` cpp +std::string getName() const override { return "Memory"; } +``` + +**3.** Sınıfların (yapıların) adları için büyük harfle başlayan CamelCase kullanın. Ben dışındaki önekler arayüzler için kullanılmaz. + +``` cpp +class StorageMemory : public IStorage +``` + +**4.** `using` sınıf aslarla aynı şekilde adlandırılır veya `_t` ucunda. + +**5.** Şablon Türü argümanlarının isimleri: basit durumlarda, kullanın `T`; `T`, `U`; `T1`, `T2`. + +Daha karmaşık durumlar için, sınıf adları için kuralları izleyin veya öneki ekleyin `T`. + +``` cpp +template +struct AggregatedStatElement +``` + +**6.** Şablon sabit argümanlarının adları: değişken adları için kurallara uyun veya kullanın `N` basit durumlarda. + +``` cpp +template +struct ExtractDomain +``` + +**7.** Soyut sınıflar (arayüzler) için şunları ekleyebilirsiniz `I` önek. + +``` cpp +class IBlockInputStream +``` + +**8.** Yerel olarak bir değişken kullanırsanız, kısa adı kullanabilirsiniz. + +Diğer tüm durumlarda, anlamı açıklayan bir isim kullanın. + +``` cpp +bool info_successfully_loaded = false; +``` + +**9.** İsimleri `define`s ve genel sabitler alt çizgi ile ALL\_CAPS kullanın. + +``` cpp +#define MAX_SRC_TABLE_NAMES_TO_STORE 1000 +``` + +**10.** Dosya adları, içerikleriyle aynı stili kullanmalıdır. + +Bir dosya tek bir sınıf içeriyorsa, dosyayı sınıfla aynı şekilde adlandırın (CamelCase). + +Dosya tek bir işlev içeriyorsa, dosyayı işlevle aynı şekilde adlandırın (camelCase). + +**11.** İsim bir kısaltma içeriyorsa, o zaman: + +- Değişken adları için kısaltma küçük harfler kullanmalıdır `mysql_connection` (değil `mySQL_connection`). +- Sınıfların ve işlevlerin adları için, büyük harfleri kısaltmada tutun`MySQLConnection` (değil `MySqlConnection`). + +**12.** Yalnızca sınıf üyelerini başlatmak için kullanılan yapıcı bağımsız değişkenleri, sınıf üyeleri ile aynı şekilde, ancak sonunda bir alt çizgi ile adlandırılmalıdır. + +``` cpp +FileQueueProcessor( + const std::string & path_, + const std::string & prefix_, + std::shared_ptr handler_) + : path(path_), + prefix(prefix_), + handler(handler_), + log(&Logger::get("FileQueueProcessor")) +{ +} +``` + +Bağımsız değişken yapıcı gövdesinde kullanılmazsa, alt çizgi soneki atlanabilir. + +**13.** Yerel değişkenlerin ve sınıf üyelerinin adlarında fark yoktur (önek gerekmez). + +``` cpp +timer (not m_timer) +``` + +**14.** Bir de SAB theitler için `enum`, büyük harfle CamelCase kullanın. ALL\_CAPS da kabul edilebilir. Eğer... `enum` yerel olmayan, bir `enum class`. + +``` cpp +enum class CompressionMethod +{ + QuickLZ = 0, + LZ4 = 1, +}; +``` + +**15.** Tüm isimler İngilizce olmalıdır. Rusça kelimelerin çevirisi izin verilmez. + + not Stroka + +**16.** Kısaltmalar iyi biliniyorsa kabul edilebilir (kısaltmanın anlamını Wikipedia'da veya bir arama motorunda kolayca bulabilirsiniz). + + `AST`, `SQL`. + + Not `NVDH` (some random letters) + +Kısaltılmış versiyon ortak kullanım ise eksik kelimeler kabul edilebilir. + +Yorumlarda tam ad yanında yer alıyorsa bir kısaltma da kullanabilirsiniz. + +**17.** C++ kaynak kodu ile dosya adları olmalıdır `.cpp` uzantı. Başlık dosyaları olmalıdır `.h` uzantı. + +## Kod nasıl yazılır {#how-to-write-code} + +**1.** Bellek yönetimi. + +El ile bellek ayırma (`delete`) sadece kütüphane kodunda kullanılabilir. + +Kütüphane kod inunda, `delete` operatör yalnızca yıkıcılarda kullanılabilir. + +Uygulama kodunda, bellek sahibi olan nesne tarafından serbest bırakılmalıdır. + +Örnekler: + +- En kolay yol, bir nesneyi yığına yerleştirmek veya onu başka bir sınıfın üyesi yapmaktır. +- Çok sayıda küçük nesne için kapları kullanın. +- Öbekte bulunan az sayıda nesnenin otomatik olarak ayrılması için şunları kullanın `shared_ptr/unique_ptr`. + +**2.** Kaynak yönetimi. + +Kullanmak `RAII` ve yukarıya bakın. + +**3.** Hata işleme. + +İstisnaları kullanın. Çoğu durumda, yalnızca bir istisna atmanız gerekir ve onu yakalamanız gerekmez (çünkü `RAII`). + +Çevrimdışı veri işleme uygulamalarında, istisnaları yakalamamak genellikle kabul edilebilir. + +Kullanıcı isteklerini işleyen sunucularda, bağlantı işleyicisinin en üst düzeyindeki istisnaları yakalamak genellikle yeterlidir. + +İş parçacığı işlevlerinde, bunları ana iş parçacığında yeniden taramak için tüm istisnaları yakalamalı ve tutmalısınız `join`. + +``` cpp +/// If there weren't any calculations yet, calculate the first block synchronously +if (!started) +{ + calculate(); + started = true; +} +else /// If calculations are already in progress, wait for the result + pool.wait(); + +if (exception) + exception->rethrow(); +``` + +İşleme olmadan istisnaları asla gizlemeyin. Sadece körü körüne log tüm istisnaları koymak asla. + +``` cpp +//Not correct +catch (...) {} +``` + +Eğer bazı özel durumlar göz ardı etmek gerekiyorsa, sadece özel olanlar için bunu yapmak ve diğerleri yeniden oluşturma. + +``` cpp +catch (const DB::Exception & e) +{ + if (e.code() == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION) + return nullptr; + else + throw; +} +``` + +Yanıt kodlarıyla işlevleri kullanırken veya `errno`, her zaman sonucu kontrol edin ve hata durumunda bir istisna atın. + +``` cpp +if (0 != close(fd)) + throwFromErrno("Cannot close file " + file_name, ErrorCodes::CANNOT_CLOSE_FILE); +``` + +`Do not use assert`. + +**4.** İstisna türleri. + +Uygulama kodunda karmaşık özel durum hiyerarşisini kullanmaya gerek yoktur. Özel durum metni bir sistem yöneticisi için anlaşılabilir olmalıdır. + +**5.** Yıkıcılardan istisnalar atmak. + +Bu tavsiye edilmez, ancak izin verilir. + +Aşağıdaki seçenekleri kullanın: + +- Bir işlev oluşturma (`done()` veya `finalize()`) bu, bir istisnaya yol açabilecek tüm işleri önceden yapacaktır. Bu işlev çağrıldıysa, daha sonra yıkıcıda istisna olmamalıdır. +- Çok karmaşık olan görevler (ağ üzerinden ileti gönderme gibi), sınıf kullanıcısının imha edilmeden önce çağırması gereken ayrı bir yöntemle yerleştirilebilir. +- Yıkıcıda bir istisna varsa, onu gizlemek yerine günlüğe kaydetmek daha iyidir (logger mevcutsa). +- Basit uygulamalarda, güvenmek kabul edilebilir `std::terminate` bu (vakaların `noexcept` varsayılan olarak C++11) istisnaları işlemek için. + +**6.** Anonim kod blokları. + +Belirli değişkenleri yerel hale getirmek için tek bir işlevin içinde ayrı bir kod bloğu oluşturabilirsiniz, böylece bloktan çıkarken yıkıcılar çağrılır. + +``` cpp +Block block = data.in->read(); + +{ + std::lock_guard lock(mutex); + data.ready = true; + data.block = block; +} + +ready_any.set(); +``` + +**7.** Multithreading. + +Çevrimdışı veri işleme programlarında: + +- Tek bir CPU çekirdeğinde mümkün olan en iyi performansı elde etmeye çalışın. Daha sonra gerekirse kodunuzu parallelize edebilirsiniz. + +Sunucu uygulamalarında: + +- İstekleri işlemek için iş parçacığı havuzunu kullanın. Bu noktada, userspace bağlam değiştirme gerektiren herhangi bir görevimiz olmadı. + +Çatal paralelleştirme için kullanılmaz. + +**8.** İş parçacıklarını senkronize etme. + +Genellikle farklı iş parçacıklarının farklı bellek hücreleri kullanmasını sağlamak mümkündür (daha da iyisi: farklı önbellek çizgileri) ve herhangi bir iş parçacığı senkronizasyonu kullanmamak (hariç `joinAll`). + +Senkronizasyon gerekiyorsa, çoğu durumda, mutex altında kullanmak yeterlidir `lock_guard`. + +Diğer durumlarda sistem senkronizasyonu ilkellerini kullanın. Meşgul bekleme kullanmayın. + +Atomik işlemler sadece en basit durumlarda kullanılmalıdır. + +Birincil uzmanlık alanınız olmadığı sürece kilitsiz veri yapılarını uygulamaya çalışmayın. + +**9.** İşaretçiler vs referanslar. + +Çoğu durumda, referansları tercih edin. + +**10.** const. + +Sabit referanslar, sabitler için işaretçiler kullanın, `const_iterator` ve const yöntemleri. + +Düşünmek `const` varsayılan olmak ve olmayan kullanmak-`const` sadece gerektiğinde. + +Değişkenleri değere göre geçirirken, `const` genellikle mantıklı değil. + +**11.** imzasız. + +Kullanmak `unsigned` gerekirse. + +**12.** Sayısal türleri. + +Türleri kullanın `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, ve `Int64` gibi `size_t`, `ssize_t`, ve `ptrdiff_t`. + +Bu türleri sayılar için kullanmayın: `signed/unsigned long`, `long long`, `short`, `signed/unsigned char`, `char`. + +**13.** Argümanları geçmek. + +Karmaşık değerleri referansla geçirin (dahil `std::string`). + +Bir işlev öbekte oluşturulan bir nesnenin sahipliğini yakalarsa, bağımsız değişken türünü yapın `shared_ptr` veya `unique_ptr`. + +**14.** Değerleri döndürür. + +Çoğu durumda, sadece kullanın `return`. Yaz domayın `[return std::move(res)]{.strike}`. + +İşlev öbek üzerinde bir nesne ayırır ve döndürürse, şunları kullanın `shared_ptr` veya `unique_ptr`. + +Nadir durumlarda, değeri bir argüman aracılığıyla döndürmeniz gerekebilir. Bu durumda, argüman bir referans olmalıdır. + +``` cpp +using AggregateFunctionPtr = std::shared_ptr; + +/** Allows creating an aggregate function by its name. + */ +class AggregateFunctionFactory +{ +public: + AggregateFunctionFactory(); + AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const; +``` + +**15.** ad. + +Ayrı bir kullanmaya gerek yoktur `namespace` uygulama kodu için. + +Küçük kütüphanelerin de buna ihtiyacı yok. + +Orta ve büyük kütüphaneler için her şeyi bir `namespace`. + +Kütüphan theede `.h` dosya, kullanabilirsiniz `namespace detail` uygulama kodu için gerekli olmayan uygulama ayrıntılarını gizlemek için. + +İn a `.cpp` dosya, bir kullanabilirsiniz `static` veya sembolleri gizlemek için anonim ad alanı. + +Ayrıca, bir `namespace` bir için kullanılabilir `enum` ilgili isimlerin harici bir yere düşmesini önlemek için `namespace` (ama kullanmak daha iyidir `enum class`). + +**16.** Ertelenmiş başlatma. + +Başlatma için bağımsız değişkenler gerekiyorsa, normalde varsayılan bir yapıcı yazmamalısınız. + +Daha sonra başlatmayı geciktirmeniz gerekiyorsa, geçersiz bir nesne oluşturacak varsayılan bir yapıcı ekleyebilirsiniz. Veya, az sayıda nesne için şunları kullanabilirsiniz `shared_ptr/unique_ptr`. + +``` cpp +Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); + +/// For deferred initialization +Loader() {} +``` + +**17.** Sanal fonksiyonlar. + +Sınıf polimorfik kullanım için tasarlanmamışsa, işlevleri sanal hale getirmeniz gerekmez. Bu aynı zamanda yıkıcı için de geçerlidir. + +**18.** Kodlamalar. + +Her yerde UTF-8 kullanın. Kullanmak `std::string`ve`char *`. Kullanmayın `std::wstring`ve`wchar_t`. + +**19.** Günlük. + +Koddaki her yerde örneklere bakın. + +Taahhütte bulunmadan önce, tüm anlamsız ve hata ayıklama günlüğünü ve diğer hata ayıklama çıktı türlerini silin. + +İzleme düzeyinde bile döngülerde oturum açmaktan kaçınılmalıdır. + +Günlükleri herhangi bir günlük düzeyinde okunabilir olmalıdır. + +Günlük kaydı yalnızca uygulama kodunda, çoğunlukla kullanılmalıdır. + +Günlük mesajları İngilizce olarak yazılmalıdır. + +Günlük, tercihen Sistem Yöneticisi için anlaşılabilir olmalıdır. + +Günlüğünde küfür kullanmayın. + +Günlüğünde UTF-8 kodlamasını kullanın. Nadir durumlarda, günlüğünde ASCII olmayan karakterler kullanabilirsiniz. + +**20.** Giriş-çıkış. + +Kullanmayın `iostreams` uygulama performansı için kritik olan iç döngülerde (ve asla kullanmayın `stringstream`). + +Kullan... `DB/IO` kütüphane yerine. + +**21.** Tarih ve zaman. + +Görmek `DateLUT` kitaplık. + +**22.** içermek. + +Her zaman kullanın `#pragma once` korumaları dahil etmek yerine. + +**23.** kullanım. + +`using namespace` kullanılmaz. Kullanabilirsiniz `using` özel bir şeyle. Ancak bir sınıf veya işlev içinde yerel yapın. + +**24.** Kullanmayın `trailing return type` gerekli olmadıkça fonksiyonlar için. + +``` cpp +[auto f() -> void;]{.strike} +``` + +**25.** Değişkenlerin bildirimi ve başlatılması. + +``` cpp +//right way +std::string s = "Hello"; +std::string s{"Hello"}; + +//wrong way +auto s = std::string{"Hello"}; +``` + +**26.** Sanal işlevler için yaz `virtual` temel sınıfta, ama yaz `override` yerine `virtual` soyundan gelen sınıflarda. + +## C++ ' ın kullanılmayan özellikleri {#unused-features-of-c} + +**1.** Sanal devralma kullanılmaz. + +**2.** C++03 özel durum belirteçleri kullanılmaz. + +## Platform {#platform} + +**1.** Belirli bir platform için kod yazıyoruz. + +Ama diğer şeyler eşit olmak, çapraz platform veya taşınabilir kod tercih edilir. + +**2.** Dil: C++17. + +**3.** Derleyici: `gcc`. Şu anda (Aralık 2017), kod sürüm 7.2 kullanılarak derlenmiştir. (Ayrıca kullanılarak derlenebilir `clang 4`.) + +Standart kütüphane kullanılır (`libstdc++` veya `libc++`). + +**4.**OS: Linux UB .untu, daha eski değil. + +**5.**Kod x86\_64 CPU mimarisi için yazılmıştır. + +CPU komut seti, sunucularımız arasında desteklenen minimum kümedir. Şu anda, sse 4.2. + +**6.** Kullanmak `-Wall -Wextra -Werror` derleme bayrakları. + +**7.** Statik olarak bağlanması zor olanlar hariç tüm kitaplıklarla statik bağlantı kullanın (bkz. `ldd` komut). + +**8.** Kod geliştirilmiş ve yayın ayarları ile ayıklanır. + +## Araçlar {#tools} + +**1.** KDevelop iyi bir IDE. + +**2.** Hata ayıklama için kullanın `gdb`, `valgrind` (`memcheck`), `strace`, `-fsanitize=...`, veya `tcmalloc_minimal_debug`. + +**3.** Profilleme için kullanın `Linux Perf`, `valgrind` (`callgrind`), veya `strace -cf`. + +**4.** Kaynaklar Git'te. + +**5.** Montaj kullanımları `CMake`. + +**6.** Programlar kullanılarak serbest bırakılır `deb` paketler. + +**7.** Ana taahhüt yapı kırmak gerekir. + +Sadece seçilen revizyonlar uygulanabilir olarak kabul edilir. + +**8.** Kod yalnızca kısmen hazır olsa bile, mümkün olduğunca sık taahhüt yapın. + +Bu amaçla dalları kullanın. + +Eğer kod inunuz `master` şube henüz imara değil, önce inşa onu hariç `push`. Bunu bitirmek veya birkaç gün içinde kaldırmak gerekir. + +**9.** Non-önemsiz değişiklik, kullanım şubeleri ve sunucu bunları yayımlamak. + +**10.** Kullanılmayan kod depodan kaldırılır. + +## Kitaplık {#libraries} + +**1.** C++14 standart Kütüphanesi kullanılır (deneysel uzantılara izin verilir) ve `boost` ve `Poco` çerçeveler. + +**2.** Gerekirse, OS paketinde bulunan iyi bilinen kütüphaneleri kullanabilirsiniz. + +Zaten mevcut olan iyi bir çözüm varsa, başka bir kütüphane yüklemeniz gerektiği anlamına gelse bile kullanın. + +(Ancak kötü kütüphaneleri koddan kaldırmaya hazır olun .) + +**3.** Paketlerde ihtiyacınız olan şey yoksa veya eski bir sürüme veya yanlış derleme türüne sahip değilseniz, paketlerde olmayan bir kitaplık yükleyebilirsiniz. + +**4.** Kütüphane küçükse ve kendi karmaşık yapı sistemine sahip değilse, kaynak dosyaları `contrib` klasör. + +**5.** Tercih her zaman zaten kullanımda olan kütüphanelere verilir. + +## Genel Öneriler {#general-recommendations-1} + +**1.** Mümkün olduğunca az kod yazın. + +**2.** En basit çözümü deneyin. + +**3.** Nasıl çalışacağını ve iç döngünün nasıl çalışacağını bilene kadar kod yazmayın. + +**4.** En basit durumlarda, kullanın `using` sınıflar veya yapılar yerine. + +**5.** Mümkünse, kopya oluşturucuları, atama işleçleri, yıkıcılar (sınıf en az bir sanal işlev içeriyorsa, sanal bir işlev dışında) yazmayın, oluşturucuları taşıyın veya atama işleçlerini taşıyın. Başka bir deyişle, derleyici tarafından oluşturulan işlevleri düzgün çalışması gerekir. Kullanabilirsiniz `default`. + +**6.** Kod sadeleştirme teşvik edilir. Mümkünse kodunuzun boyutunu azaltın. + +## Ek Öneriler {#additional-recommendations} + +**1.** Açıkça belirtme `std::` türleri için `stddef.h` + +tavsiye edilmez. Başka bir deyişle, yazmanızı öneririz `size_t` yerine `std::size_t` daha kısa olduğu için. + +Eklemek kabul edilebilir `std::`. + +**2.** Açıkça belirtme `std::` standart C kitap fromlığından fonksiyonlar için + +tavsiye edilmez. Başka bir deyişle, yazın `memcpy` yerine `std::memcpy`. + +Bunun nedeni, aşağıdaki gibi benzer standart dışı işlevlerin olmasıdır `memmem`. Bu işlevleri zaman zaman kullanıyoruz. Bu işlevler mevcut değil `namespace std`. + +Yazar yousan `std::memcpy` yerine `memcpy` her yerde, o zaman `memmem` olarak `std::` garip görünecek. + +Yine de, hala kullanabilirsiniz `std::` eğer tercih ederseniz edin. + +**3.** Aynı olanlar standart C++ kütüphanesinde mevcut olduğunda C'den işlevleri kullanma. + +Daha verimli ise bu kabul edilebilir. + +Örneğin, kullanın `memcpy` yerine `std::copy` büyük bellek parçalarını kopyalamak için. + +**4.** Çok satırlı fonksiyon argümanları. + +Aşağıdaki sarma stillerinden herhangi birine izin verilir: + +``` cpp +function( + T1 x1, + T2 x2) +``` + +``` cpp +function( + size_t left, size_t right, + const & RangesInDataParts ranges, + size_t limit) +``` + +``` cpp +function(size_t left, size_t right, + const & RangesInDataParts ranges, + size_t limit) +``` + +``` cpp +function(size_t left, size_t right, + const & RangesInDataParts ranges, + size_t limit) +``` + +``` cpp +function( + size_t left, + size_t right, + const & RangesInDataParts ranges, + size_t limit) +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/development/style/) diff --git a/docs/tr/development/tests.md b/docs/tr/development/tests.md new file mode 100644 index 00000000000..1d39c24da5f --- /dev/null +++ b/docs/tr/development/tests.md @@ -0,0 +1,252 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 69 +toc_title: "ClickHouse testleri nas\u0131l \xE7al\u0131\u015Ft\u0131r\u0131l\u0131\ + r" +--- + +# ClickHouse Testi {#clickhouse-testing} + +## Fonksiyonel Testler {#functional-tests} + +Fonksiyonel testler en basit ve kullanımı kolay olanlardır. ClickHouse özelliklerinin çoğu fonksiyonel testlerle test edilebilir ve bu şekilde test edilebilecek ClickHouse kodundaki her değişiklik için kullanılması zorunludur. + +Her işlevsel test, çalışan ClickHouse sunucusuna bir veya birden çok sorgu gönderir ve sonucu referansla karşılaştırır. + +Testler bulunur `queries` dizin. İki alt dizin var: `stateless` ve `stateful`. Durumsuz testler, önceden yüklenmiş test verileri olmadan sorguları çalıştırır - genellikle testin kendisinde anında küçük sentetik veri kümeleri oluştururlar. Durum bilgisi testleri, Yandex'ten önceden yüklenmiş test verileri gerektirir.Metrica ve halka açık değil. Biz sadece kullanmak eğilimindedir `stateless` testler ve yeni eklemekten kaçının `stateful` testler. + +Her test iki tipten biri olabilir: `.sql` ve `.sh`. `.sql` test için borulu basit SQL komut dosyasıdır `clickhouse-client --multiquery --testmode`. `.sh` test kendisi tarafından çalıştırılan bir komut dosyasıdır. + +Tüm testleri çalıştırmak için şunları kullanın `clickhouse-test` aracı. Bak `--help` Olası seçeneklerin listesi için. Sadece tüm testleri çalıştırmak veya test adı alt dize tarafından süzülmüş testlerin alt kümesini çalıştırabilirsiniz: `./clickhouse-test substring`. + +Fonksiyonel testleri çağırmanın en basit yolu kopyalamaktır `clickhouse-client` -e doğru `/usr/bin/`, çalıştırmak `clickhouse-server` ve sonra koş `./clickhouse-test` kendi dizininden. + +Yeni test eklemek için, bir `.sql` veya `.sh` dosya içinde `queries/0_stateless` dizin, elle kontrol edin ve sonra oluşturun `.reference` aşağıdaki şekilde dosya: `clickhouse-client -n --testmode < 00000_test.sql > 00000_test.reference` veya `./00000_test.sh > ./00000_test.reference`. + +Testler yalnızca tabloları (create, drop, vb.) kullanmalıdır `test` önceden oluşturulduğu varsayılır veritabanı; ayrıca testler geçici tablolar kullanabilirsiniz. + +İşlevsel testlerde dağıtılmış sorgular kullanmak istiyorsanız, kaldıraç `remote` tablo fonksiyonu ile `127.0.0.{1..2}` sunucunun kendisini sorgulaması için adresler; veya sunucu yapılandırma dosyasında önceden tanımlanmış test kümelerini kullanabilirsiniz `test_shard_localhost`. + +Bazı testler ile işaretlenir `zookeeper`, `shard` veya `long` kendi adlarına. +`zookeeper` ZooKeeper kullanan testler içindir. `shard` testler içindir +dinlemek için sunucu gerektirir `127.0.0.*`; `distributed` veya `global` aynı var +anlama. `long` bir saniye biraz daha uzun süren testler içindir. Yapabilirsin +kullanarak bu test gruplarını devre dışı bırakın `--no-zookeeper`, `--no-shard` ve +`--no-long` sırasıyla seçenekler. + +## Bilinen Hatalar {#known-bugs} + +Fonksiyonel testlerle kolayca çoğaltılabilen bazı hatalar biliyorsak, hazırlanmış fonksiyonel testleri `tests/queries/bugs` dizin. Bu testler taşınacaktır `tests/queries/0_stateless` hatalar düzeltildiğinde. + +## Entegrasyon Testleri {#integration-tests} + +Entegrasyon testleri, kümelenmiş konfigürasyonda Clickhouse'u ve MySQL, Postgres, MongoDB gibi diğer sunucularla ClickHouse etkileşimini test etmeyi sağlar. Ağ bölmelerini, paket damlalarını vb. taklit etmek için kullanışlıdırlar. Bu testler Docker altında çalıştırılır ve çeşitli yazılımlarla birden fazla konteyner oluşturur. + +Görmek `tests/integration/README.md` bu testlerin nasıl çalıştırılacağı hakkında. + +Clickhouse'un üçüncü taraf sürücülerle entegrasyonunun sınanmadığını unutmayın. Ayrıca şu anda JDBC ve ODBC sürücülerimizle entegrasyon testlerimiz yok. + +## Ünite Testleri {#unit-tests} + +Birim testleri, Clickhouse'u bir bütün olarak değil, tek bir yalıtılmış kitaplık veya sınıfı test etmek istediğinizde kullanışlıdır. Etkinleştirebilir veya devre dışı bırakma ile testlerin yapı `ENABLE_TESTS` Cmake seçeneği. Birim testleri (ve diğer test programları) bulunur `tests` kodun alt dizinleri. Birim testlerini çalıştırmak için şunları yazın `ninja test`. Bazı testler kullanın `gtest`, ancak bazıları test başarısızlığında sıfır olmayan çıkış kodunu döndüren programlardır. + +Kodun zaten işlevsel testler tarafından kapsanması durumunda birim testlerine sahip olmak zorunlu değildir (ve işlevsel testler genellikle kullanımı çok daha basittir). + +## Performans Testleri {#performance-tests} + +Performans testleri ölçmek ve sentetik sorguları ClickHouse bazı izole kısmının performansını karşılaştırmak için izin verir. Testler bulunur `tests/performance`. Her test ile temsil edilir `.xml` test durumunun açıklaması ile dosya. Testler ile çalıştırılır `clickhouse performance-test` Aracı (Bu gömülü `clickhouse` ikilik). Görmek `--help` çağırma için. + +Her test, durdurma için bazı koşullarla (örneğin, bir döngüde bir veya birden fazla sorgu (muhtemelen parametre kombinasyonlarıyla) çalıştırır “maximum execution speed is not changing in three seconds”) ve sorgu performansı ile ilgili bazı metrikleri ölçün (örneğin “maximum execution speed”). Bazı testler önceden yüklenmiş test veri kümesinde Önkoşullar içerebilir. + +Bazı senaryoda Clickhouse'un performansını artırmak istiyorsanız ve basit sorgularda iyileştirmeler gözlemlenebiliyorsa, bir performans testi yazmanız önerilir. Her zaman kullanmak mantıklı `perf top` testleriniz sırasında veya diğer perf araçları. + +## Test araçları ve komut dosyaları {#test-tools-and-scripts} + +Bazı programlar `tests` dizin testleri hazırlanmış değil, ancak test araçlarıdır. Örneğin, için `Lexer` bir araç var `src/Parsers/tests/lexer` bu sadece stdin'in tokenizasyonunu yapar ve renklendirilmiş sonucu stdout'a yazar. Bu tür araçları kod örnekleri olarak ve keşif ve manuel test için kullanabilirsiniz. + +Ayrıca Çift Dosya yerleştirebilirsiniz `.sh` ve `.reference` aracı ile birlikte bazı önceden tanımlanmış giriş üzerinde çalıştırmak için-daha sonra komut sonucu karşılaştırılabilir `.reference` Dosya. Bu tür testler otomatik değildir. + +## Çeşitli Testler {#miscellaneous-tests} + +Bulunan dış sözlükler için testler vardır `tests/external_dictionaries` ve makine öğrenilen modeller için `tests/external_models`. Bu testler güncelleştirilmez ve tümleştirme testlerine aktarılmalıdır. + +Çekirdek ekler için ayrı bir test var. Bu test, ayrı sunucularda ClickHouse kümesini çalıştırır ve çeşitli arıza durumlarını taklit eder: ağ bölünmesi, paket bırakma (ClickHouse düğümleri arasında, ClickHouse ve ZooKeeper arasında, ClickHouse sunucusu ve istemci arasında, vb.), `kill -9`, `kill -STOP` ve `kill -CONT` , istemek [Jepsen](https://aphyr.com/tags/Jepsen). Daha sonra test, kabul edilen tüm eklerin yazıldığını ve reddedilen tüm eklerin olmadığını kontrol eder. + +Clickhouse açık kaynaklı önce çekirdek testi ayrı ekip tarafından yazılmıştır. Bu takım artık ClickHouse ile çalışmıyor. Test yanlışlıkla Java ile yazılmıştır. Bu nedenlerden dolayı, çekirdek testi yeniden yazılmalı ve entegrasyon testlerine taşınmalıdır. + +## Manuel Test {#manual-testing} + +Yeni bir özellik geliştirdiğinizde, el ile de test etmek mantıklıdır. Bunu aşağıdaki adımlarla yapabilirsiniz: + +ClickHouse Oluşturun. Terminalden Clickhouse'u çalıştırın: dizini değiştir `programs/clickhouse-server` ve ile çalıştırın `./clickhouse-server`. Bu yapılandırma kullanacak (`config.xml`, `users.xml` ve içindeki dosyalar `config.d` ve `users.d` dizinler) geçerli dizinden varsayılan olarak. ClickHouse sunucusuna bağlanmak için, çalıştırın `programs/clickhouse-client/clickhouse-client`. + +Tüm clickhouse araçlarının (sunucu, istemci, vb.) sadece tek bir ikili için symlinks olduğunu unutmayın `clickhouse`. Bu ikili bulabilirsiniz `programs/clickhouse`. Tüm araçlar olarak da çağrılabilir `clickhouse tool` yerine `clickhouse-tool`. + +Alternatif olarak ClickHouse paketini yükleyebilirsiniz: Yandex deposundan kararlı sürüm veya kendiniz için paket oluşturabilirsiniz `./release` ClickHouse kaynakları kökünde. Ardından sunucuyu şu şekilde başlatın `sudo service clickhouse-server start` (veya sunucuyu durdurmak için durdurun). Günlükleri arayın `/etc/clickhouse-server/clickhouse-server.log`. + +ClickHouse sisteminizde zaten yüklü olduğunda, yeni bir `clickhouse` ikili ve mevcut ikili değiştirin: + +``` bash +$ sudo service clickhouse-server stop +$ sudo cp ./clickhouse /usr/bin/ +$ sudo service clickhouse-server start +``` + +Ayrıca sistem clickhouse-server durdurmak ve aynı yapılandırma ile ancak terminale günlüğü ile kendi çalıştırabilirsiniz: + +``` bash +$ sudo service clickhouse-server stop +$ sudo -u clickhouse /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml +``` + +Gdb ile örnek: + +``` bash +$ sudo -u clickhouse gdb --args /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml +``` + +Sistem clickhouse-sunucu zaten çalışıyorsa ve bunu durdurmak istemiyorsanız, sizin port numaralarını değiştirebilirsiniz `config.xml` (veya bunları bir dosyada geçersiz kılma `config.d` dizin), uygun veri yolu sağlayın ve çalıştırın. + +`clickhouse` ikili neredeyse hiçbir bağımlılıkları vardır ve Linux dağıtımları geniş genelinde çalışır. Hızlı ve kirli bir sunucuda değişikliklerinizi test etmek için, sadece yapabilirsiniz `scp` taze inşa `clickhouse` sunucunuza ikili ve daha sonra yukarıdaki örneklerde olduğu gibi çalıştırın. + +## Test Ortamı {#testing-environment} + +Kararlı olarak yayınlamadan önce test ortamında dağıtın. Test ortamı, 1/39 bölümünü işleyen bir kümedir [Üye.Metrica](https://metrica.yandex.com/) veriler. Test ortamımızı Yandex ile paylaşıyoruz.Metrica takımı. ClickHouse mevcut verilerin üstünde kesinti olmadan yükseltilir. İlk önce verilerin gerçek zamanlı olarak gecikmeden başarıyla işlendiğine bakıyoruz, çoğaltma çalışmaya devam ediyor ve Yandex tarafından görülebilen herhangi bir sorun yok.Metrica takımı. İlk kontrol aşağıdaki şekilde yapılabilir: + +``` sql +SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h; +``` + +Bazı durumlarda yandex'teki arkadaş ekiplerimizin test ortamına da dağıtım yapıyoruz: Pazar, Bulut, vb. Ayrıca geliştirme amacıyla kullanılan bazı donanım sunucularımız var. + +## Yük Testi {#load-testing} + +Test ortamına dağıtıldıktan sonra, üretim kümesinden gelen sorgularla yük testini çalıştırıyoruz. Bu elle yapılır. + +Etkinleştirdiğinizden emin olun `query_log` üretim kümenizde. + +Bir gün veya daha fazla sorgu günlüğü toplayın: + +``` bash +$ clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv +``` + +Bu şekilde karmaşık bir örnektir. `type = 2` başarıyla yürütülen sorguları süzer. `query LIKE '%ym:%'` yandex'ten ilgili sorguları seçmektir.Metrica. `is_initial_query` yalnızca istemci tarafından başlatılan sorguları seçmektir, Clickhouse'un kendisi tarafından değil (dağıtılmış sorgu işlemenin parçaları olarak). + +`scp` bu test kümenize günlük ve aşağıdaki gibi çalıştırın: + +``` bash +$ clickhouse benchmark --concurrency 16 < queries.tsv +``` + +(muhtemelen de belirtmek istiyorum `--user`) + +Sonra bir gece ya da hafta sonu için bırakın ve dinlenin. + +Kontrol etmelisiniz `clickhouse-server` çökmez, bellek ayak izi sınırlıdır ve performans zamanla aşağılayıcı değildir. + +Kesin sorgu yürütme zamanlamaları kaydedilmez ve sorguların ve ortamın yüksek değişkenliği nedeniyle karşılaştırılmaz. + +## Yapı Testleri {#build-tests} + +Yapı testleri, yapının çeşitli alternatif konfigürasyonlarda ve bazı yabancı sistemlerde bozulmadığını kontrol etmeyi sağlar. Testler bulunur `ci` dizin. Docker, Vagrant ve bazen de `qemu-user-static` Docker'ın içinde. Bu testler geliştirme aşamasındadır ve test çalıştırmaları otomatik değildir. + +Motivasyon: + +Normalde tüm testleri ClickHouse yapısının tek bir varyantında serbest bırakırız ve çalıştırırız. Ancak, iyice test edilmeyen alternatif yapı varyantları vardır. Örnekler: + +- FreeBSD üzerine inşa; +- sistem paketlerinden kütüphaneler ile Debian üzerine inşa; +- kütüphanelerin paylaşılan bağlantısı ile oluşturun; +- AArch64 platformunda oluşturun; +- PowerPc platformunda oluşturun. + +Örneğin, sistem paketleri ile oluştur kötü bir uygulamadır, çünkü bir sistemin hangi paketlerin tam sürümüne sahip olacağını garanti edemeyiz. Ancak bu gerçekten Debian bakıcılarına ihtiyaç duyuyor. Bu nedenle en azından bu yapı varyantını desteklemeliyiz. Başka bir örnek: paylaşılan bağlantı ortak bir sorun kaynağıdır, ancak bazı Meraklılar için gereklidir. + +Tüm yapı varyantlarında tüm testleri çalıştıramasak da, en azından çeşitli yapı varyantlarının bozulmadığını kontrol etmek istiyoruz. Bu amaçla yapı testlerini kullanıyoruz. + +## Protokol uyumluluğu testi {#testing-for-protocol-compatibility} + +ClickHouse ağ protokolünü genişlettiğimizde, eski clickhouse istemcisinin yeni clickhouse sunucusu ile çalıştığını ve yeni clickhouse istemcisinin eski clickhouse sunucusu ile çalıştığını (sadece ilgili paketlerden ikili dosyaları çalıştırarak) manuel olarak test ediyoruz. + +## Derleyiciden yardım {#help-from-the-compiler} + +Ana ClickHouse kodu (bu `dbms` dizin) ile inşa edilmiştir `-Wall -Wextra -Werror` ve bazı ek etkin uyarılar ile. Bu seçenekler üçüncü taraf kitaplıkları için etkin olmasa da. + +Clang daha yararlı uyarılar vardır-Sen ile onları arayabilirsiniz `-Weverything` ve varsayılan oluşturmak için bir şey seçin. + +Üretim yapıları için gcc kullanılır (hala clang'dan biraz daha verimli kod üretir). Geliştirme için, clang genellikle kullanımı daha uygundur. Hata ayıklama modu ile kendi makinenizde inşa edebilirsiniz (dizüstü bilgisayarınızın pilinden tasarruf etmek için), ancak derleyicinin daha fazla uyarı üretebileceğini lütfen unutmayın `-O3` daha iyi kontrol akışı ve prosedürler arası analiz nedeniyle. Clang ile inşa ederken, `libc++` yerine kullanılır `libstdc++` ve hata ayıklama modu ile oluştururken, hata ayıklama sürümü `libc++` çalışma zamanında daha fazla hata yakalamak için izin verir kullanılır. + +## Dezenfektanlar {#sanitizers} + +**Adres dezenfektanı**. +Biz başına taahhüt bazında ASan altında fonksiyonel ve entegrasyon testleri çalıştırın. + +**Valgrind (Memcheck)**. +Bir gecede valgrind altında fonksiyonel testler yapıyoruz. Birden fazla saat sürer. Şu anda bilinen bir yanlış pozitif var `re2` kütüphane, bkz [bu makale](https://research.swtch.com/sparse). + +**Tanımsız davranış dezenfektanı.** +Biz başına taahhüt bazında ASan altında fonksiyonel ve entegrasyon testleri çalıştırın. + +**İplik dezenfektanı**. +Biz başına taahhüt bazında tsan altında fonksiyonel testler çalıştırın. Tsan altında hala taahhüt bazında entegrasyon testleri yapmıyoruz. + +**Bellek temizleyici**. +Şu anda hala MSan kullanmıyoruz. + +**Hata ayıklama ayırıcısı.** +Hata ayıklama sürümü `jemalloc` hata ayıklama oluşturmak için kullanılır. + +## Fuzzing {#fuzzing} + +Rastgele SQL sorguları oluşturmak ve sunucunun ölmediğini kontrol etmek için basit fuzz testi kullanıyoruz. Fuzz testi Adres dezenfektanı ile yapılır. İçinde bulabilirsiniz `00746_sql_fuzzy.pl`. Bu test sürekli olarak (gece ve daha uzun) çalıştırılmalıdır. + +Aralık 2018 itibariyle, hala kütüphane kodunun izole fuzz testini kullanmıyoruz. + +## Güvenlik Denetimi {#security-audit} + +Yandex Bulut departmanından insanlar, güvenlik açısından ClickHouse yeteneklerine bazı temel genel bakışlar yaparlar. + +## Statik Analizörler {#static-analyzers} + +Koş weuyoruz `PVS-Studio` taahhüt bazında. Değerlendir havedik `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. Sen kullanım talimatları bulacaksınız `tests/instructions/` dizin. Ayrıca okuyabilirsiniz [Rusça makale](https://habr.com/company/yandex/blog/342018/). + +Kullanıyorsanız `CLion` bir IDE olarak, bazı kaldıraç `clang-tidy` kutudan kontrol eder. + +## Sertleşme {#hardening} + +`FORTIFY_SOURCE` varsayılan olarak kullanılır. Neredeyse işe yaramaz, ancak nadir durumlarda hala mantıklı ve bunu devre dışı bırakmıyoruz. + +## Kod Stili {#code-style} + +Kod stili kuralları açıklanmıştır [burada](https://clickhouse.tech/docs/en/development/style/). + +Bazı ortak stil ihlallerini kontrol etmek için şunları kullanabilirsiniz `utils/check-style` komut. + +Kodunuzun uygun stilini zorlamak için şunları kullanabilirsiniz `clang-format`. Dosya `.clang-format` kaynak rootlarında yer almaktadır. Çoğunlukla gerçek kod stilimizle karşılık gelir. Ancak uygulanması tavsiye edilmez `clang-format` varolan dosyalara biçimlendirmeyi daha da kötüleştirdiği için. Kullanabilirsiniz `clang-format-diff` eğer clang kaynak deposunda bulabilirsiniz aracı. + +Alternatif olarak deneyebilirsiniz `uncrustify` kodunuzu yeniden biçimlendirmek için bir araç. Yapılandırma içinde `uncrustify.cfg` kaynaklarda kök. Daha az test edilmiştir `clang-format`. + +`CLion` kod stilimiz için ayarlanması gereken kendi kod biçimlendiricisine sahiptir. + +## Metrica B2B testleri {#metrica-b2b-tests} + +Her ClickHouse sürümü Yandex Metrica ve AppMetrica motorları ile test edilir. Clickhouse'un Test ve kararlı sürümleri Vm'lerde dağıtılır ve Giriş verilerinin sabit örneğini işleyen Metrica motorunun küçük bir kopyasıyla çalışır. Daha sonra Metrica motorunun iki örneğinin sonuçları birlikte karşılaştırılır. + +Bu testler ayrı ekip tarafından otomatikleştirilir. Yüksek sayıda hareketli parça nedeniyle, testler çoğu zaman tamamen ilgisiz nedenlerle başarısız olur, bu da anlaşılması çok zordur. Büyük olasılıkla bu testlerin bizim için negatif değeri var. Bununla birlikte, bu testlerin yüzlerce kişiden yaklaşık bir veya iki kez yararlı olduğu kanıtlanmıştır. + +## Test Kapsamı {#test-coverage} + +Temmuz 2018 itibariyle test kapsamını takip etmiyoruz. + +## Test Otomasyonu {#test-automation} + +Yandex dahili CI ve iş otomasyon sistemi ile testler yapıyoruz “Sandbox”. + +Yapı işleri ve testler, taahhüt bazında sanal alanda çalıştırılır. Ortaya çıkan paketler ve test sonuçları Github'da yayınlanır ve doğrudan bağlantılar tarafından indirilebilir. Eserler sonsuza dek saklanır. Eğer GitHub bir çekme isteği gönderdiğinizde, biz olarak etiketlemek “can be tested” ve bizim CI sistemi sizin için ClickHouse paketleri (yayın, hata ayıklama, Adres dezenfektanı ile, vb) inşa edecek. + +Travis CI, zaman ve hesaplama gücü sınırı nedeniyle kullanmıyoruz. +Jenkins'i kullanmayız. Daha önce kullanıldı ve şimdi Jenkins kullanmadığımız için mutluyuz. + +[Orijinal makale](https://clickhouse.tech/docs/en/development/tests/) diff --git a/docs/tr/engines/database_engines/index.md b/docs/tr/engines/database_engines/index.md new file mode 100644 index 00000000000..c0e0bea8ab5 --- /dev/null +++ b/docs/tr/engines/database_engines/index.md @@ -0,0 +1,21 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Veritaban\u0131 Motorlar\u0131" +toc_priority: 27 +toc_title: "Giri\u015F" +--- + +# Veritabanı Motorları {#database-engines} + +Veritabanı motorları, tablolarla çalışmanıza izin verir. + +Varsayılan olarak, ClickHouse yapılandırılabilir sağlayan yerel veritabanı altyapısını kullanır [masa motorları](../../engines/table_engines/index.md) ve bir [SQL lehçesi](../../sql_reference/syntax.md). + +Aşağıdaki veritabanı altyapılarını da kullanabilirsiniz: + +- [MySQL](mysql.md) + +- [Tembel](lazy.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/database_engines/) diff --git a/docs/tr/engines/database_engines/lazy.md b/docs/tr/engines/database_engines/lazy.md new file mode 100644 index 00000000000..85436fdc164 --- /dev/null +++ b/docs/tr/engines/database_engines/lazy.md @@ -0,0 +1,18 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 31 +toc_title: Tembel +--- + +# Tembel {#lazy} + +Tabloları yalnızca RAM'de tutar `expiration_time_in_seconds` son erişimden saniyeler sonra. Sadece \* Log tabloları ile kullanılabilir. + +Erişimler arasında uzun bir zaman aralığı olan birçok küçük \* günlük tablosunu saklamak için optimize edilmiştir. + +## Veritabanı oluşturma {#creating-a-database} + + CREATE DATABASE testlazy ENGINE = Lazy(expiration_time_in_seconds); + +[Orijinal makale](https://clickhouse.tech/docs/en/database_engines/lazy/) diff --git a/docs/tr/engines/database_engines/mysql.md b/docs/tr/engines/database_engines/mysql.md new file mode 100644 index 00000000000..974515ab655 --- /dev/null +++ b/docs/tr/engines/database_engines/mysql.md @@ -0,0 +1,135 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 30 +toc_title: MySQL +--- + +# MySQL {#mysql} + +Uzak bir MySQL sunucusunda veritabanlarına bağlanmak ve gerçekleştirmek için izin verir `INSERT` ve `SELECT` ClickHouse ve MySQL arasında veri alışverişi için sorgular. + +Bu `MySQL` veritabanı motoru sorguları MySQL sunucusuna çevirir, böylece aşağıdaki gibi işlemleri gerçekleştirebilirsiniz `SHOW TABLES` veya `SHOW CREATE TABLE`. + +Aşağıdaki sorguları gerçekleştiremiyor: + +- `RENAME` +- `CREATE TABLE` +- `ALTER` + +## Veritabanı oluşturma {#creating-a-database} + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') +``` + +**Motor Parametreleri** + +- `host:port` — MySQL server address. +- `database` — Remote database name. +- `user` — MySQL user. +- `password` — User password. + +## Veri Türleri Desteği {#data_types-support} + +| MySQL | ClickHouse | +|----------------------------------|--------------------------------------------------------------| +| UNSIGNED TINYINT | [Uİnt8](../../sql_reference/data_types/int_uint.md) | +| TINYINT | [Int8](../../sql_reference/data_types/int_uint.md) | +| UNSIGNED SMALLINT | [Uınt16](../../sql_reference/data_types/int_uint.md) | +| SMALLINT | [Int16](../../sql_reference/data_types/int_uint.md) | +| UNSIGNED INT, UNSIGNED MEDIUMINT | [Uİnt32](../../sql_reference/data_types/int_uint.md) | +| INT, MEDIUMINT | [Int32](../../sql_reference/data_types/int_uint.md) | +| UNSIGNED BIGINT | [Uİnt64](../../sql_reference/data_types/int_uint.md) | +| BIGINT | [Int64](../../sql_reference/data_types/int_uint.md) | +| FLOAT | [Float32](../../sql_reference/data_types/float.md) | +| DOUBLE | [Float64](../../sql_reference/data_types/float.md) | +| DATE | [Tarihli](../../sql_reference/data_types/date.md) | +| DATETIME, TIMESTAMP | [DateTime](../../sql_reference/data_types/datetime.md) | +| BINARY | [FixedString](../../sql_reference/data_types/fixedstring.md) | + +Diğer tüm MySQL veri türleri dönüştürülür [Dize](../../sql_reference/data_types/string.md). + +[Nullable](../../sql_reference/data_types/nullable.md) desteklenir. + +## Kullanım Örnekleri {#examples-of-use} + +MySQL tablo: + +``` text +mysql> USE test; +Database changed + +mysql> CREATE TABLE `mysql_table` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `float` FLOAT NOT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from mysql_table; ++------+-----+ +| int_id | value | ++------+-----+ +| 1 | 2 | ++------+-----+ +1 row in set (0,00 sec) +``` + +Clickhouse'daki veritabanı, MySQL sunucusu ile veri alışverişi: + +``` sql +CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') +``` + +``` sql +SHOW DATABASES +``` + +``` text +┌─name─────┐ +│ default │ +│ mysql_db │ +│ system │ +└──────────┘ +``` + +``` sql +SHOW TABLES FROM mysql_db +``` + +``` text +┌─name─────────┐ +│ mysql_table │ +└──────────────┘ +``` + +``` sql +SELECT * FROM mysql_db.mysql_table +``` + +``` text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +└────────┴───────┘ +``` + +``` sql +INSERT INTO mysql_db.mysql_table VALUES (3,4) +``` + +``` sql +SELECT * FROM mysql_db.mysql_table +``` + +``` text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +│ 3 │ 4 │ +└────────┴───────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/database_engines/mysql/) diff --git a/docs/tr/engines/index.md b/docs/tr/engines/index.md new file mode 100644 index 00000000000..48004afa1c8 --- /dev/null +++ b/docs/tr/engines/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Motorlar +toc_priority: 25 +--- + + diff --git a/docs/tr/engines/table_engines/index.md b/docs/tr/engines/table_engines/index.md new file mode 100644 index 00000000000..fad3823453c --- /dev/null +++ b/docs/tr/engines/table_engines/index.md @@ -0,0 +1,85 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Masa Motorlar\u0131" +toc_priority: 26 +toc_title: "Giri\u015F" +--- + +# Masa Motorları {#table_engines} + +Tablo motoru (tablo türü) belirler: + +- Verilerin nasıl ve nerede depolandığı, nereye yazılacağı ve nereden okunacağı. +- Hangi sorgular desteklenir ve nasıl. +- Eşzamanlı veri erişimi. +- Varsa indeks uselerin kullanımı. +- Çok iş parçacıklı istek yürütme mümkün olup olmadığı. +- Veri çoğaltma parametreleri. + +## Motor Aileleri {#engine-families} + +### MergeTree {#mergetree} + +Yüksek yük görevleri için en evrensel ve fonksiyonel masa motorları. Bu motorlar tarafından paylaşılan özellik, sonraki arka plan veri işleme ile hızlı veri ekleme ' dir. `MergeTree` aile motorları destek veri çoğaltma (ile [Çoğaltıyordu\*](mergetree_family/replication.md) sürümleri), bölümleme ve diğer özellikler diğer motorlarda desteklenmez. + +Ailede motorlar: + +- [MergeTree](mergetree_family/mergetree.md) +- [ReplacingMergeTree](mergetree_family/replacingmergetree.md) +- [SummingMergeTree](mergetree_family/summingmergetree.md) +- [AggregatingMergeTree](mergetree_family/aggregatingmergetree.md) +- [CollapsingMergeTree](mergetree_family/collapsingmergetree.md) +- [VersionedCollapsingMergeTree](mergetree_family/versionedcollapsingmergetree.md) +- [Graphıtemergetree](mergetree_family/graphitemergetree.md) + +### Günlük {#log} + +Hafiflik [motorlar](log_family/index.md) minimum işlevsellik ile. Birçok küçük tabloyu (yaklaşık 1 milyon satıra kadar) hızlı bir şekilde yazmanız ve daha sonra bir bütün olarak okumanız gerektiğinde en etkili olanlardır. + +Ailede motorlar: + +- [TinyLog](log_family/tinylog.md) +- [StripeLog](log_family/stripelog.md) +- [Günlük](log_family/log.md) + +### Entegrasyon Motorları {#integration-engines} + +Diğer veri depolama ve işleme sistemleri ile iletişim kurmak için motorlar. + +Ailede motorlar: + +- [Kafka](integrations/kafka.md) +- [MySQL](integrations/mysql.md) +- [ODBC](integrations/odbc.md) +- [JDBC](integrations/jdbc.md) +- [HDFS](integrations/hdfs.md) + +### Özel Motorlar {#special-engines} + +Ailede motorlar: + +- [Dağılı](special/distributed.md) +- [MaterializedView](special/materializedview.md) +- [Sözlük](special/dictionary.md) +- [Birleştirmek](special/merge.md) +- [Dosya](special/file.md) +- [Boş](special/null.md) +- [Koymak](special/set.md) +- [Katmak](special/join.md) +- [URL](special/url.md) +- [Görünüm](special/view.md) +- [Bellek](special/memory.md) +- [Arabellek](special/buffer.md) + +## Sanal Sütunlar {#table_engines-virtual-columns} + +Sanal sütun, motor kaynak kodunda tanımlanan ayrılmaz bir tablo altyapısı özniteliğidir. + +Sanal sütunları belirtmemelisiniz `CREATE TABLE` sorgula ve onları göremezsin `SHOW CREATE TABLE` ve `DESCRIBE TABLE` sorgu sonuçları. Sanal sütunlar da salt okunur, bu nedenle sanal sütunlara veri ekleyemezsiniz. + +Sanal bir sütundan veri seçmek için, adını `SELECT` sorgu. `SELECT *` sanal sütunlardan değerler döndürmez. + +Tablo sanal sütunlarından biriyle aynı ada sahip bir sütuna sahip bir tablo oluşturursanız, sanal sütuna erişilemez hale gelir. Bunu yapmayı önermiyoruz. Çakışmaları önlemek için, sanal sütun adları genellikle bir alt çizgi ile öneki. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/) diff --git a/docs/tr/engines/table_engines/integrations/hdfs.md b/docs/tr/engines/table_engines/integrations/hdfs.md new file mode 100644 index 00000000000..42e1e31b80f --- /dev/null +++ b/docs/tr/engines/table_engines/integrations/hdfs.md @@ -0,0 +1,123 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 36 +toc_title: HDFS +--- + +# HDFS {#table_engines-hdfs} + +Bu motor ile entegrasyon sağlar [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) üzerinde veri Yönet allowingilmesine izin vererek ekosist dataem [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)ClickHouse aracılığıyla. Bu motor benzer +to the [Dosya](../special/file.md) ve [URL](../special/url.md) motorlar, ancak hadoop özgü özellikleri sağlar. + +## Kullanma {#usage} + +``` sql +ENGINE = HDFS(URI, format) +``` + +Bu `URI` parametre, HDFS'DEKİ tüm dosya URI'SIDIR. +Bu `format` parametre kullanılabilir dosya biçimlerinden birini belirtir. Gerçekleştirmek +`SELECT` sorgular, biçim giriş için desteklenmeli ve gerçekleştirmek için +`INSERT` queries – for output. The available formats are listed in the +[Biçimliler](../../../interfaces/formats.md#formats) bölme. +Yol kısmı `URI` globs içerebilir. Bu durumda tablo salt okunur olurdu. + +**Örnek:** + +**1.** Set up the `hdfs_engine_table` Tablo: + +``` sql +CREATE TABLE hdfs_engine_table (name String, value UInt32) ENGINE=HDFS('hdfs://hdfs1:9000/other_storage', 'TSV') +``` + +**2.** Fil filel file: + +``` sql +INSERT INTO hdfs_engine_table VALUES ('one', 1), ('two', 2), ('three', 3) +``` + +**3.** Verileri sorgula: + +``` sql +SELECT * FROM hdfs_engine_table LIMIT 2 +``` + +``` text +┌─name─┬─value─┐ +│ one │ 1 │ +│ two │ 2 │ +└──────┴───────┘ +``` + +## Uygulama Detayları {#implementation-details} + +- Okuma ve yazma paralel olabilir +- Desteklenmiyor: + - `ALTER` ve `SELECT...SAMPLE` harekat. + - Dizinler. + - Çoğalma. + +**Yolda Globs** + +Birden çok yol bileşenleri globs olabilir. İşlenmek için dosya var olmalı ve tüm yol deseniyle eşleşmelidir. Sırasında dosyaların listelen ofmesini belirler `SELECT` (not at `CREATE` an). + +- `*` — Substitutes any number of any characters except `/` boş dize dahil. +- `?` — Substitutes any single character. +- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. +- `{N..M}` — Substitutes any number in range from N to M including both borders. + +İle yapılar `{}` benzer olan [uzak](../../../sql_reference/table_functions/remote.md) tablo işlevi. + +**Örnek** + +1. HDFS'DE aşağıdaki Urı'lerle TSV formatında birkaç dosyamız olduğunu varsayalım: + +- ‘hdfs://hdfs1:9000/some\_dir/some\_file\_1’ +- ‘hdfs://hdfs1:9000/some\_dir/some\_file\_2’ +- ‘hdfs://hdfs1:9000/some\_dir/some\_file\_3’ +- ‘hdfs://hdfs1:9000/another\_dir/some\_file\_1’ +- ‘hdfs://hdfs1:9000/another\_dir/some\_file\_2’ +- ‘hdfs://hdfs1:9000/another\_dir/some\_file\_3’ + +1. Altı dosyadan oluşan bir tablo oluşturmanın birkaç yolu vardır: + + + +``` sql +CREATE TABLE table_with_range (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/some_file_{1..3}', 'TSV') +``` + +Başka bir yol: + +``` sql +CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/some_file_?', 'TSV') +``` + +Tablo, her iki dizindeki tüm dosyalardan oluşur (tüm dosyalar, sorguda açıklanan biçimi ve şemayı karşılamalıdır): + +``` sql +CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV') +``` + +!!! warning "Uyarıcı" + Dosyaların listelenmesi, önde gelen sıfırlarla sayı aralıkları içeriyorsa, her basamak için parantez içeren yapıyı ayrı ayrı kullanın veya kullanın `?`. + +**Örnek** + +Adlı dosyaları içeren tablo oluşturma `file000`, `file001`, … , `file999`: + +``` sql +CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV') +``` + +## Sanal Sütunlar {#virtual-columns} + +- `_path` — Path to the file. +- `_file` — Name of the file. + +**Ayrıca Bakınız** + +- [Sanal sütunlar](../index.md#table_engines-virtual_columns) + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/hdfs/) diff --git a/docs/tr/engines/table_engines/integrations/index.md b/docs/tr/engines/table_engines/integrations/index.md new file mode 100644 index 00000000000..608fc900e62 --- /dev/null +++ b/docs/tr/engines/table_engines/integrations/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Entegrasyonlar +toc_priority: 30 +--- + + diff --git a/docs/tr/engines/table_engines/integrations/jdbc.md b/docs/tr/engines/table_engines/integrations/jdbc.md new file mode 100644 index 00000000000..08196116cf8 --- /dev/null +++ b/docs/tr/engines/table_engines/integrations/jdbc.md @@ -0,0 +1,90 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 34 +toc_title: JDBC +--- + +# JDBC {#table-engine-jdbc} + +ClickHouse üzerinden harici veritabanlarına bağlanmak için izin verir [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity). + +JDBC bağlantısını uygulamak için ClickHouse ayrı programı kullanır [clickhouse-JDBC-köprü](https://github.com/alex-krash/clickhouse-jdbc-bridge) bu bir daemon olarak çalışmalıdır. + +Bu motor destekler [Nullable](../../../sql_reference/data_types/nullable.md) veri türü. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name +( + columns list... +) +ENGINE = JDBC(dbms_uri, external_database, external_table) +``` + +**Motor Parametreleri** + +- `dbms_uri` — URI of an external DBMS. + + Biçimli: `jdbc:://:/?user=&password=`. + MySQL örneği: `jdbc:mysql://localhost:3306/?user=root&password=root`. + +- `external_database` — Database in an external DBMS. + +- `external_table` — Name of the table in `external_database`. + +## Kullanım Örneği {#usage-example} + +Doğrudan konsol istemcisine bağlanarak MySQL sunucusunda bir tablo oluşturma: + +``` text +mysql> CREATE TABLE `test`.`test` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `int_nullable` INT NULL DEFAULT NULL, + -> `float` FLOAT NOT NULL, + -> `float_nullable` FLOAT NULL DEFAULT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into test (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from test; ++------+----------+-----+----------+ +| int_id | int_nullable | float | float_nullable | ++------+----------+-----+----------+ +| 1 | NULL | 2 | NULL | ++------+----------+-----+----------+ +1 row in set (0,00 sec) +``` + +ClickHouse Server'da bir tablo oluşturma ve ondan veri seçme: + +``` sql +CREATE TABLE jdbc_table +( + `int_id` Int32, + `int_nullable` Nullable(Int32), + `float` Float32, + `float_nullable` Nullable(Float32) +) +ENGINE JDBC('jdbc:mysql://localhost:3306/?user=root&password=root', 'test', 'test') +``` + +``` sql +SELECT * +FROM jdbc_table +``` + +``` text +┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ 2 │ ᴺᵁᴸᴸ │ +└────────┴──────────────┴───────┴────────────────┘ +``` + +## Ayrıca Bakınız {#see-also} + +- [JDBC tablo işlevi](../../../sql_reference/table_functions/jdbc.md). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/jdbc/) diff --git a/docs/tr/engines/table_engines/integrations/kafka.md b/docs/tr/engines/table_engines/integrations/kafka.md new file mode 100644 index 00000000000..baf0adcd080 --- /dev/null +++ b/docs/tr/engines/table_engines/integrations/kafka.md @@ -0,0 +1,176 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 32 +toc_title: Kafka +--- + +# Kafka {#kafka} + +Bu motor ile çalışır [Apache Kafka](http://kafka.apache.org/). + +Kafka sağlar: + +- Veri akışlarını yayınlayın veya abone olun. +- Hataya dayanıklı depolama düzenlemek. +- Kullanılabilir hale geldikçe akışları işleyin. + +## Tablo oluşturma {#table_engine-kafka-creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = Kafka() +SETTINGS + kafka_broker_list = 'host:port', + kafka_topic_list = 'topic1,topic2,...', + kafka_group_name = 'group_name', + kafka_format = 'data_format'[,] + [kafka_row_delimiter = 'delimiter_symbol',] + [kafka_schema = '',] + [kafka_num_consumers = N,] + [kafka_skip_broken_messages = N] +``` + +Gerekli parametreler: + +- `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). +- `kafka_topic_list` – A list of Kafka topics. +- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` fonksiyon gibi `JSONEachRow`. Daha fazla bilgi için, bkz: [Biçimliler](../../../interfaces/formats.md) bölme. + +İsteğe bağlı parametreler: + +- `kafka_row_delimiter` – Delimiter character, which ends the message. +- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) şema dosyasının yolunu ve kök adını gerektirir `schema.capnp:Message` nesne. +- `kafka_num_consumers` – The number of consumers per table. Default: `1`. Bir tüketicinin verimi yetersizse daha fazla tüketici belirtin. Bölüm başına yalnızca bir tüketici atanabileceğinden, toplam tüketici sayısı konudaki bölüm sayısını geçmemelidir. +- `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. Eğer `kafka_skip_broken_messages = N` sonra motor atlar *N* Ayrıştırılamayan Kafka iletileri (bir ileti bir veri satırına eşittir). + +Örnekler: + +``` sql + CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + SELECT * FROM queue LIMIT 5; + + CREATE TABLE queue2 ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka SETTINGS kafka_broker_list = 'localhost:9092', + kafka_topic_list = 'topic', + kafka_group_name = 'group1', + kafka_format = 'JSONEachRow', + kafka_num_consumers = 4; + + CREATE TABLE queue2 ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1') + SETTINGS kafka_format = 'JSONEachRow', + kafka_num_consumers = 4; +``` + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın. Mümkünse, eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format + [, kafka_row_delimiter, kafka_schema, kafka_num_consumers, kafka_skip_broken_messages]) +``` + +
+ +## Açıklama {#description} + +Teslim edilen mesajlar otomatik olarak izlenir, bu nedenle bir gruptaki her mesaj yalnızca bir kez sayılır. Verileri iki kez almak istiyorsanız, tablonun başka bir grup adıyla bir kopyasını oluşturun. + +Gruplar esnek ve kümede senkronize edilir. Örneğin, bir kümede 10 konu ve bir tablonun 5 kopyası varsa, her kopya 2 konu alır. Kopya sayısı değişirse, konular kopyalar arasında otomatik olarak yeniden dağıtılır. Bu konuda daha fazla bilgi edinin http://kafka.apache.org/intro. + +`SELECT` mesajları okumak için özellikle yararlı değildir (hata ayıklama hariç), çünkü her mesaj yalnızca bir kez okunabilir. Hayata görünümler kullanarak gerçek zamanlı iş parçacıkları oluşturmak daha pratiktir. Bunu yapmak için : + +1. Bir Kafka tüketici oluşturmak için motoru kullanın ve bir veri akışı düşünün. +2. İstenen yapıya sahip bir tablo oluşturun. +3. Verileri motordan dönüştüren ve daha önce oluşturulmuş bir tabloya koyan materyalleştirilmiş bir görünüm oluşturun. + +Ne zaman `MATERIALIZED VIEW` motora katılır, arka planda veri toplamaya başlar. Bu, kafka'dan sürekli olarak mesaj almanızı ve bunları kullanarak gerekli biçime dönüştürmenizi sağlar `SELECT`. +Bir kafka tablosu istediğiniz kadar materialized görüşe sahip olabilir, kafka tablosundan doğrudan veri okumazlar, ancak yeni kayıtlar (bloklar halinde) alırlar, bu şekilde farklı ayrıntı seviyesine sahip birkaç tabloya yazabilirsiniz (gruplama-toplama ve olmadan). + +Örnek: + +``` sql + CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + CREATE TABLE daily ( + day Date, + level String, + total UInt64 + ) ENGINE = SummingMergeTree(day, (day, level), 8192); + + CREATE MATERIALIZED VIEW consumer TO daily + AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total + FROM queue GROUP BY day, level; + + SELECT level, sum(total) FROM daily GROUP BY level; +``` + +Performansı artırmak için, alınan iletiler bloklar halinde gruplandırılır [max\_ınsert\_block\_size](../../../operations/server_configuration_parameters/settings.md#settings-max_insert_block_size). İçinde blok oluş ifma ifdıysa [stream\_flush\_interval\_ms](../../../operations/server_configuration_parameters/settings.md) milisaniye, veri blok bütünlüğü ne olursa olsun tabloya temizlendi. + +Konu verilerini almayı durdurmak veya dönüşüm mantığını değiştirmek için, hayata geçirilmiş görünümü ayırın: + +``` sql + DETACH TABLE consumer; + ATTACH MATERIALIZED VIEW consumer; +``` + +Kullanarak hedef tabloyu değiştirmek istiyorsanız `ALTER` hedef tablo ile görünümdeki veriler arasındaki tutarsızlıkları önlemek için malzeme görünümünü devre dışı bırakmanızı öneririz. + +## Yapılandırma {#configuration} + +GraphiteMergeTree benzer şekilde, Kafka motoru ClickHouse yapılandırma dosyasını kullanarak genişletilmiş yapılandırmayı destekler. Kullanabileceğiniz iki yapılandırma anahtarı vardır: global (`kafka`) ve konu düzeyinde (`kafka_*`). Genel yapılandırma önce uygulanır ve sonra konu düzeyinde yapılandırma uygulanır (varsa). + +``` xml + + + cgrp + smallest + + + + + 250 + 100000 + +``` + +Olası yapılandırma seçeneklerinin listesi için bkz. [librdkafka yapılandırma referansı](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). Alt çizgiyi kullan (`_`) ClickHouse yapılandırmasında bir nokta yerine. Mesela, `check.crcs=true` olacak `true`. + +## Sanal Sütunlar {#virtual-columns} + +- `_topic` — Kafka topic. +- `_key` — Key of the message. +- `_offset` — Offset of the message. +- `_timestamp` — Timestamp of the message. +- `_partition` — Partition of Kafka topic. + +**Ayrıca Bakınız** + +- [Sanal sütunlar](../index.md#table_engines-virtual_columns) + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) diff --git a/docs/tr/engines/table_engines/integrations/mysql.md b/docs/tr/engines/table_engines/integrations/mysql.md new file mode 100644 index 00000000000..f6c811465e6 --- /dev/null +++ b/docs/tr/engines/table_engines/integrations/mysql.md @@ -0,0 +1,105 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 33 +toc_title: MySQL +--- + +# Mysql {#mysql} + +MySQL motoru gerçekleştirmek için izin verir `SELECT` uzak bir MySQL sunucusunda depolanan veriler üzerinde sorgular. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], + ... +) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); +``` + +Ayrıntılı bir açıklamasını görmek [CREATE TABLE](../../../sql_reference/statements/create.md#create-table-query) sorgu. + +Tablo yapısı orijinal MySQL tablo yapısından farklı olabilir: + +- Sütun adları orijinal MySQL tablosundaki ile aynı olmalıdır, ancak bu sütunların sadece bazılarını ve herhangi bir sırada kullanabilirsiniz. +- Sütun türleri orijinal MySQL tablosundakilerden farklı olabilir. ClickHouse çalışır [döküm](../../../sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) ClickHouse veri türleri için değerler. + +**Motor Parametreleri** + +- `host:port` — MySQL server address. + +- `database` — Remote database name. + +- `table` — Remote table name. + +- `user` — MySQL user. + +- `password` — User password. + +- `replace_query` — Flag that converts `INSERT INTO` için sorgular `REPLACE INTO`. Eğer `replace_query=1`, sorgu değiştirilir. + +- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` eklenen ifade `INSERT` sorgu. + + Örnek: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, nere `on_duplicate_clause` oluyor `UPDATE c2 = c2 + 1`. Görmek [MySQL dökü documentationmanları](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html) bulmak için hangi `on_duplicate_clause` ile kullanabilirsiniz `ON DUPLICATE KEY` yan. + + Belirtmek `on_duplicate_clause` sen geçmek gerekir `0` to the `replace_query` parametre. Aynı anda geçerseniz `replace_query = 1` ve `on_duplicate_clause`, ClickHouse bir özel durum oluşturur. + +Basit `WHERE` gibi maddeler `=, !=, >, >=, <, <=` MySQL sunucusunda yürütülür. + +Geri kalan şartlar ve `LIMIT` örnekleme kısıtlaması, yalnızca MySQL sorgusu bittikten sonra Clickhouse'da yürütülür. + +## Kullanım Örneği {#usage-example} + +MySQL tablo: + +``` text +mysql> CREATE TABLE `test`.`test` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `int_nullable` INT NULL DEFAULT NULL, + -> `float` FLOAT NOT NULL, + -> `float_nullable` FLOAT NULL DEFAULT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into test (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from test; ++------+----------+-----+----------+ +| int_id | int_nullable | float | float_nullable | ++------+----------+-----+----------+ +| 1 | NULL | 2 | NULL | ++------+----------+-----+----------+ +1 row in set (0,00 sec) +``` + +Clickhouse'daki tablo, yukarıda oluşturulan MySQL tablosundan veri alma: + +``` sql +CREATE TABLE mysql_table +( + `float_nullable` Nullable(Float32), + `int_id` Int32 +) +ENGINE = MySQL('localhost:3306', 'test', 'test', 'bayonet', '123') +``` + +``` sql +SELECT * FROM mysql_table +``` + +``` text +┌─float_nullable─┬─int_id─┐ +│ ᴺᵁᴸᴸ │ 1 │ +└────────────────┴────────┘ +``` + +## Ayrıca Bakınız {#see-also} + +- [Bu ‘mysql’ tablo fonksiyonu](../../../sql_reference/table_functions/mysql.md) +- [Harici sözlük kaynağı olarak MySQL kullanma](../../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql) + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/mysql/) diff --git a/docs/tr/engines/table_engines/integrations/odbc.md b/docs/tr/engines/table_engines/integrations/odbc.md new file mode 100644 index 00000000000..86ded26587a --- /dev/null +++ b/docs/tr/engines/table_engines/integrations/odbc.md @@ -0,0 +1,132 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 35 +toc_title: ODBC +--- + +# ODBC {#table-engine-odbc} + +ClickHouse üzerinden harici veritabanlarına bağlanmak için izin verir [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). + +ODBC bağlantılarını güvenli bir şekilde uygulamak için ClickHouse ayrı bir program kullanır `clickhouse-odbc-bridge`. ODBC sürücüsü doğrudan yüklenmişse `clickhouse-server`, sürücü sorunları ClickHouse sunucu çökmesine neden olabilir. ClickHouse otomatik olarak başlar `clickhouse-odbc-bridge` gerekli olduğunda. ODBC Köprüsü programı aynı paketten yüklenir `clickhouse-server`. + +Bu motor destekler [Nullable](../../../sql_reference/data_types/nullable.md) veri türü. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1], + name2 [type2], + ... +) +ENGINE = ODBC(connection_settings, external_database, external_table) +``` + +Ayrıntılı bir açıklamasını görmek [CREATE TABLE](../../../sql_reference/statements/create.md#create-table-query) sorgu. + +Tablo yapısı kaynak tablo yapısından farklı olabilir: + +- Sütun adları kaynak tablodaki ile aynı olmalıdır, ancak yalnızca bu sütunlardan bazılarını ve herhangi bir sırada kullanabilirsiniz. +- Sütun türleri kaynak tablodakilerden farklı olabilir. ClickHouse çalışır [döküm](../../../sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) ClickHouse veri türleri için değerler. + +**Motor Parametreleri** + +- `connection_settings` — Name of the section with connection settings in the `odbc.ini` Dosya. +- `external_database` — Name of a database in an external DBMS. +- `external_table` — Name of a table in the `external_database`. + +## Kullanım Örneği {#usage-example} + +**ODBC üzerinden yerel MySQL kurulumundan veri alma** + +Bu örnek Ubuntu Linux 18.04 ve MySQL server 5.7 için kontrol edilir. + +UnixODBC ve MySQL Connector yüklü olduğundan emin olun. + +Varsayılan olarak (paketlerden yüklüyse), ClickHouse kullanıcı olarak başlar `clickhouse`. Bu nedenle, bu kullanıcıyı MySQL sunucusunda oluşturmanız ve yapılandırmanız gerekir. + +``` bash +$ sudo mysql +``` + +``` sql +mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse'; +mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION; +``` + +Sonra bağlantıyı yapılandırın `/etc/odbc.ini`. + +``` bash +$ cat /etc/odbc.ini +[mysqlconn] +DRIVER = /usr/local/lib/libmyodbc5w.so +SERVER = 127.0.0.1 +PORT = 3306 +DATABASE = test +USERNAME = clickhouse +PASSWORD = clickhouse +``` + +Kullanarak bağlantıyı kontrol edebilirsiniz `isql` unixodbc yüklemesinden yardımcı program. + +``` bash +$ isql -v mysqlconn ++-------------------------+ +| Connected! | +| | +... +``` + +MySQL tablo: + +``` text +mysql> CREATE TABLE `test`.`test` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `int_nullable` INT NULL DEFAULT NULL, + -> `float` FLOAT NOT NULL, + -> `float_nullable` FLOAT NULL DEFAULT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into test (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from test; ++------+----------+-----+----------+ +| int_id | int_nullable | float | float_nullable | ++------+----------+-----+----------+ +| 1 | NULL | 2 | NULL | ++------+----------+-----+----------+ +1 row in set (0,00 sec) +``` + +Clickhouse'daki tablo, MySQL tablosundan veri alma: + +``` sql +CREATE TABLE odbc_t +( + `int_id` Int32, + `float_nullable` Nullable(Float32) +) +ENGINE = ODBC('DSN=mysqlconn', 'test', 'test') +``` + +``` sql +SELECT * FROM odbc_t +``` + +``` text +┌─int_id─┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ +└────────┴────────────────┘ +``` + +## Ayrıca Bakınız {#see-also} + +- [ODBC harici sözlükler](../../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc) +- [ODBC tablo işlevi](../../../sql_reference/table_functions/odbc.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/odbc/) diff --git a/docs/tr/engines/table_engines/log_family/index.md b/docs/tr/engines/table_engines/log_family/index.md new file mode 100644 index 00000000000..062087a5874 --- /dev/null +++ b/docs/tr/engines/table_engines/log_family/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "G\xFCnl\xFCk Aile" +toc_priority: 29 +--- + + diff --git a/docs/tr/engines/table_engines/log_family/log.md b/docs/tr/engines/table_engines/log_family/log.md new file mode 100644 index 00000000000..ca1f8c4c1f4 --- /dev/null +++ b/docs/tr/engines/table_engines/log_family/log.md @@ -0,0 +1,16 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 33 +toc_title: "G\xFCnl\xFCk" +--- + +# Günlük {#log} + +Motor günlük motorları ailesine aittir. Günlük motorlarının ortak özelliklerini ve farklılıklarını görün [Log Engine Ailesi](log_family.md) makale. + +Log differsar differsit fromma [TinyLog](tinylog.md) bu küçük bir dosyada “marks” sütun dosyaları ile bulunur. Bu işaretler her veri bloğuna yazılır ve belirtilen satır sayısını atlamak için dosyayı okumaya nereden başlayacağınızı gösteren uzaklıklar içerir. Bu, tablo verilerini birden çok iş parçacığında okumayı mümkün kılar. +Eşzamanlı veri erişimi için, okuma işlemleri aynı anda gerçekleştirilebilirken, yazma işlemleri okur ve birbirlerini engeller. +Günlük altyapısı dizinleri desteklemez. Benzer şekilde, bir tabloya yazma başarısız olursa, tablo bozulur ve Okuma bir hata döndürür. Günlük altyapısı, geçici veriler, bir kez yazma tabloları ve sınama veya gösteri amaçları için uygundur. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/log/) diff --git a/docs/tr/engines/table_engines/log_family/log_family.md b/docs/tr/engines/table_engines/log_family/log_family.md new file mode 100644 index 00000000000..2a954c60def --- /dev/null +++ b/docs/tr/engines/table_engines/log_family/log_family.md @@ -0,0 +1,46 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 31 +toc_title: "Giri\u015F" +--- + +# Log Engine Ailesi {#log-engine-family} + +Bu motorlar, birçok küçük tabloyu (yaklaşık 1 milyon satıra kadar) hızlı bir şekilde yazmanız ve daha sonra bir bütün olarak okumanız gerektiğinde senaryolar için geliştirilmiştir. + +Ailenin motorları: + +- [StripeLog](stripelog.md) +- [Günlük](log.md) +- [TinyLog](tinylog.md) + +## Ortak Özellikler {#common-properties} + +Motorlar: + +- Verileri bir diskte saklayın. + +- Yazarken dosyanın sonuna veri ekleyin. + +- Eşzamanlı veri erişimi için destek kilitleri. + + Sırasında `INSERT` sorgular, tablo kilitlenir ve veri okumak ve yazmak için diğer sorgular hem tablonun kilidini açmak için bekler. Veri yazma sorguları varsa, herhangi bir sayıda veri okuma sorguları aynı anda gerçekleştirilebilir. + +- Destek yok [mutasyon](../../../sql_reference/statements/alter.md#alter-mutations) harekat. + +- Dizinleri desteklemez. + + Bu demektir ki `SELECT` veri aralıkları için sorgular verimli değildir. + +- Atomik veri yazmayın. + + Bir şey yazma işlemini bozarsa, örneğin anormal sunucu kapatma gibi bozuk verilerle bir tablo alabilirsiniz. + +## Farklılıklar {#differences} + +Bu `TinyLog` motor, ailenin en basitidir ve en fakir işlevselliği ve en düşük verimliliği sağlar. Bu `TinyLog` motor, birkaç iş parçacığı tarafından paralel veri okumayı desteklemez. Paralel okumayı destekleyen ailedeki diğer motorlardan daha yavaş veri okur ve neredeyse birçok tanımlayıcı kullanır `Log` motor, her sütunu ayrı bir dosyada sakladığı için. Basit düşük yük senaryolarında kullanın. + +Bu `Log` ve `StripeLog` motorlar paralel veri okumayı destekler. Veri okurken, ClickHouse birden çok iş parçacığı kullanır. Her iş parçacığı ayrı bir veri bloğu işler. Bu `Log` engine, tablonun her sütunu için ayrı bir dosya kullanır. `StripeLog` tüm verileri tek bir dosyada saklar. Sonuç olarak, `StripeLog` motor işletim sisteminde daha az tanımlayıcı kullanır, ancak `Log` motor veri okurken daha yüksek verimlilik sağlar. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/log_family/) diff --git a/docs/tr/engines/table_engines/log_family/stripelog.md b/docs/tr/engines/table_engines/log_family/stripelog.md new file mode 100644 index 00000000000..1ff251c3ee0 --- /dev/null +++ b/docs/tr/engines/table_engines/log_family/stripelog.md @@ -0,0 +1,95 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 32 +toc_title: StripeLog +--- + +# Stripelog {#stripelog} + +Bu motor günlük motor ailesine aittir. Günlük motorlarının ortak özelliklerini ve farklılıklarını görün [Log Engine Ailesi](log_family.md) makale. + +Az miktarda veri içeren (1 milyondan az satır) birçok tablo yazmanız gerektiğinde, bu altyapıyı senaryolarda kullanın. + +## Tablo oluşturma {#table_engines-stripelog-creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + column1_name [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + column2_name [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = StripeLog +``` + +Ayrıntılı açıklamasına bakın [CREATE TABLE](../../../sql_reference/statements/create.md#create-table-query) sorgu. + +## Veri yazma {#table_engines-stripelog-writing-the-data} + +Bu `StripeLog` motor tüm sütunları tek bir dosyada saklar. Her biri için `INSERT` sorgu, ClickHouse veri bloğunu bir tablo dosyasının sonuna ekler, sütunları tek tek yazar. + +Her tablo için ClickHouse dosyaları yazar: + +- `data.bin` — Data file. +- `index.mrk` — File with marks. Marks contain offsets for each column of each data block inserted. + +Bu `StripeLog` motor desteklemiyor `ALTER UPDATE` ve `ALTER DELETE` harekat. + +## Verileri okuma {#table_engines-stripelog-reading-the-data} + +İşaretli dosya, Clickhouse'un verilerin okunmasını paralelleştirmesine izin verir. Bu demektir `SELECT` sorgu satırları öngörülemeyen bir sırayla döndürür. Kullan... `ORDER BY` satırları sıralamak için yan tümce. + +## Kullanım Örneği {#table_engines-stripelog-example-of-use} + +Tablo oluşturma: + +``` sql +CREATE TABLE stripe_log_table +( + timestamp DateTime, + message_type String, + message String +) +ENGINE = StripeLog +``` + +Veri ekleme: + +``` sql +INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The first regular message') +INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The second regular message'),(now(),'WARNING','The first warning message') +``` + +İki kullandık `INSERT` içinde iki veri bloğu oluşturmak için sorgular `data.bin` Dosya. + +ClickHouse veri seçerken birden çok iş parçacığı kullanır. Her iş parçacığı ayrı bir veri bloğu okur ve sonuç olarak satırları bağımsız olarak döndürür. Sonuç olarak, çıktıdaki satır bloklarının sırası, çoğu durumda girişteki aynı blokların sırasına uymuyor. Mesela: + +``` sql +SELECT * FROM stripe_log_table +``` + +``` text +┌───────────timestamp─┬─message_type─┬─message────────────────────┐ +│ 2019-01-18 14:27:32 │ REGULAR │ The second regular message │ +│ 2019-01-18 14:34:53 │ WARNING │ The first warning message │ +└─────────────────────┴──────────────┴────────────────────────────┘ +┌───────────timestamp─┬─message_type─┬─message───────────────────┐ +│ 2019-01-18 14:23:43 │ REGULAR │ The first regular message │ +└─────────────────────┴──────────────┴───────────────────────────┘ +``` + +Sonuçları sıralama (varsayılan olarak artan sipariş): + +``` sql +SELECT * FROM stripe_log_table ORDER BY timestamp +``` + +``` text +┌───────────timestamp─┬─message_type─┬─message────────────────────┐ +│ 2019-01-18 14:23:43 │ REGULAR │ The first regular message │ +│ 2019-01-18 14:27:32 │ REGULAR │ The second regular message │ +│ 2019-01-18 14:34:53 │ WARNING │ The first warning message │ +└─────────────────────┴──────────────┴────────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/stripelog/) diff --git a/docs/tr/engines/table_engines/log_family/tinylog.md b/docs/tr/engines/table_engines/log_family/tinylog.md new file mode 100644 index 00000000000..e4eccf220b1 --- /dev/null +++ b/docs/tr/engines/table_engines/log_family/tinylog.md @@ -0,0 +1,16 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 34 +toc_title: TinyLog +--- + +# TinyLog {#tinylog} + +Motor log engine ailesine aittir. Görmek [Log Engine Ailesi](log_family.md) günlük motorlarının ortak özellikleri ve farklılıkları için. + +Bu tablo motoru genellikle write-once yöntemi ile kullanılır: verileri bir kez yazın, ardından gerektiği kadar okuyun. Örneğin, kullanabilirsiniz `TinyLog`- küçük gruplar halinde işlenen Ara veriler için tablolar yazın. Çok sayıda küçük tabloda veri depolamanın verimsiz olduğunu unutmayın. + +Sorgular tek bir akışta yürütülür. Başka bir deyişle, bu motor nispeten küçük tablolar için tasarlanmıştır (yaklaşık 1.000.000 satıra kadar). Çok sayıda küçük tablonuz varsa, bu tablo motorunu kullanmak mantıklıdır, çünkü [Günlük](log.md) motor (daha az dosya açılması gerekir). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/tinylog/) diff --git a/docs/tr/engines/table_engines/mergetree_family/aggregatingmergetree.md b/docs/tr/engines/table_engines/mergetree_family/aggregatingmergetree.md new file mode 100644 index 00000000000..8034be02969 --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/aggregatingmergetree.md @@ -0,0 +1,102 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 35 +toc_title: AggregatingMergeTree +--- + +# Aggregatingmergetree {#aggregatingmergetree} + +Motor devralır [MergeTree](mergetree.md#table_engines-mergetree), veri parçaları birleştirme mantığı değiştirme. ClickHouse, tüm satırları aynı birincil anahtarla değiştirir (veya daha doğru olarak, aynı [sıralama anahtarı](mergetree.md)) tek bir satırla (bir veri parçası içinde), toplama işlevlerinin durumlarının bir kombinasyonunu saklar. + +Kullanabilirsiniz `AggregatingMergeTree` artımlı veri toplama, toplanan materialized görünümleri de dahil olmak üzere tablolar. + +Motor tüm sütunları ile işler [AggregateFunction](../../../sql_reference/data_types/aggregatefunction.md) tür. + +Kullanmak uygundur `AggregatingMergeTree` siparişlere göre satır sayısını azaltırsa. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = AggregatingMergeTree() +[PARTITION BY expr] +[ORDER BY expr] +[SAMPLE BY expr] +[TTL expr] +[SETTINGS name=value, ...] +``` + +İstek parametrelerinin açıklaması için bkz. [istek açıklaması](../../../sql_reference/statements/create.md). + +**Sorgu yan tümceleri** + +Oluştururken bir `AggregatingMergeTree` tablo aynı [yanlar](mergetree.md) oluşturul ,urken olduğu gibi gerekli `MergeTree` Tablo. + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın ve mümkünse eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE [=] AggregatingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity) +``` + +Tüm parametreler, aşağıdaki gibi aynı anlama sahiptir `MergeTree`. +
+ +## Seç ve Ekle {#select-and-insert} + +Veri eklemek için şunları kullanın [INSERT SELECT](../../../sql_reference/statements/insert_into.md) agrega-Devlet-fonksiyonları ile sorgu. +Veri seçerken `AggregatingMergeTree` tablo kullanın `GROUP BY` yan tümce ve veri eklerken aynı toplama işlevleri, ancak kullanarak `-Merge` sonek. + +Sonuç inlarında `SELECT` sorgu, değerleri `AggregateFunction` türü, Tüm ClickHouse çıktı biçimleri için uygulamaya özgü ikili gösterime sahiptir. Örneğin, veri dökümü, `TabSeparated` ile format `SELECT` sorgu daha sonra bu dökümü kullanarak geri yüklenebilir `INSERT` sorgu. + +## Toplu bir Somutlaştırılmış Görünüm örneği {#example-of-an-aggregated-materialized-view} + +`AggregatingMergeTree` saatler hayata görünüm `test.visits` Tablo: + +``` sql +CREATE MATERIALIZED VIEW test.basic +ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate) +AS SELECT + CounterID, + StartDate, + sumState(Sign) AS Visits, + uniqState(UserID) AS Users +FROM test.visits +GROUP BY CounterID, StartDate; +``` + +Veri ekleme `test.visits` Tablo. + +``` sql +INSERT INTO test.visits ... +``` + +Veriler hem tablo hem de görünümde eklenir `test.basic` toplama işlemini gerçekleştir .ecektir. + +Toplanan verileri almak için, aşağıdaki gibi bir sorgu yürütmemiz gerekir `SELECT ... GROUP BY ...` görünüm fromden `test.basic`: + +``` sql +SELECT + StartDate, + sumMerge(Visits) AS Visits, + uniqMerge(Users) AS Users +FROM test.basic +GROUP BY StartDate +ORDER BY StartDate; +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/aggregatingmergetree/) diff --git a/docs/tr/engines/table_engines/mergetree_family/collapsingmergetree.md b/docs/tr/engines/table_engines/mergetree_family/collapsingmergetree.md new file mode 100644 index 00000000000..3ef53846f32 --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/collapsingmergetree.md @@ -0,0 +1,309 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 36 +toc_title: CollapsingMergeTree +--- + +# CollapsingMergeTree {#table_engine-collapsingmergetree} + +Motor devralır [MergeTree](mergetree.md) ve veri parçaları birleştirme algoritmasına çöken satırların mantığını ekler. + +`CollapsingMergeTree` sıralama anahtarındaki tüm alanlar zaman uyumsuz olarak siler (daraltır) satır çiftleri (`ORDER BY`) belirli alan hariç eşdeğerdir `Sign` hangi olabilir `1` ve `-1` değerler. Çift olmayan satırlar tutulur. Daha fazla bilgi için bkz: [Çökme](#table_engine-collapsingmergetree-collapsing) belgenin bölümü. + +Motor depolama hacmini önemli ölçüde azaltabilir ve `SELECT` sonuç olarak sorgu. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = CollapsingMergeTree(sign) +[PARTITION BY expr] +[ORDER BY expr] +[SAMPLE BY expr] +[SETTINGS name=value, ...] +``` + +Sorgu parametrelerinin açıklaması için bkz. [sorgu açıklaması](../../../sql_reference/statements/create.md). + +**CollapsingMergeTree Parametreleri** + +- `sign` — Name of the column with the type of row: `1` is a “state” satır, `-1` is a “cancel” satır. + + Column data type — `Int8`. + +**Sorgu yan tümceleri** + +Oluştururken bir `CollapsingMergeTree` tablo, aynı [sorgu yan tümceleri](mergetree.md#table_engine-mergetree-creating-a-table) oluşturul ,urken olduğu gibi gerekli `MergeTree` Tablo. + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın ve mümkünse eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE [=] CollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign) +``` + +Hariç tüm parametreler `sign` içinde olduğu gibi aynı anlama sahip `MergeTree`. + +- `sign` — Name of the column with the type of row: `1` — “state” satır, `-1` — “cancel” satır. + + Column Data Type — `Int8`. + +
+ +## Çökme {#table_engine-collapsingmergetree-collapsing} + +### Veriler {#data} + +Bazı nesneler için sürekli değişen verileri kaydetmeniz gereken durumu düşünün. Bir nesne için bir satıra sahip olmak ve herhangi bir değişiklikte güncellemek mantıklı geliyor, ancak güncelleme işlemi dbms için pahalı ve yavaş çünkü depolama alanındaki verilerin yeniden yazılmasını gerektiriyor. Verileri hızlı bir şekilde yazmanız gerekiyorsa, güncelleme kabul edilemez, ancak bir nesnenin değişikliklerini sırayla aşağıdaki gibi yazabilirsiniz. + +Belirli sütunu kullanın `Sign`. Eğer `Sign = 1` bu, satırın bir nesnenin durumu olduğu anlamına gelir, diyelim ki “state” satır. Eğer `Sign = -1` aynı özelliklere sahip bir nesnenin durumunun iptali anlamına gelir, diyelim ki “cancel” satır. + +Örneğin, kullanıcıların bazı sitelerde ne kadar sayfa kontrol ettiğini ve ne kadar süre orada olduklarını hesaplamak istiyoruz. Bir anda kullanıcı etkinliği durumu ile aşağıdaki satırı yazıyoruz: + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +Bir an sonra kullanıcı aktivitesinin değişikliğini kaydedip aşağıdaki iki satırla yazıyoruz. + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +İlk satır, nesnenin (kullanıcı) önceki durumunu iptal eder. İptal edilen durumun sıralama anahtar alanlarını kopyalamalıdır `Sign`. + +İkinci satır geçerli durumu içerir. + +Sadece kullanıcı etkinliğinin son durumuna ihtiyacımız olduğu için, satırlar + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +bir nesnenin geçersiz (eski) durumunu daraltarak silinebilir. `CollapsingMergeTree` veri parçalarının birleştirilmesi sırasında bunu yapar. + +Neden her değişiklik için 2 satıra ihtiyacımız var [Algoritma](#table_engine-collapsingmergetree-collapsing-algorithm) paragraf. + +**Bu yaklaşımın kendine özgü özellikleri** + +1. Verileri yazan program, iptal edebilmek için bir nesnenin durumunu hatırlamalıdır. “Cancel” dize, sıralama anahtar alanlarının kopyalarını içermelidir. “state” dize ve tersi `Sign`. Bu depolama başlangıç boyutunu artırır ama hızlı bir şekilde veri yazmak için izin verir. +2. Sütunlardaki uzun büyüyen diziler, yazma yükü nedeniyle motorun verimliliğini azaltır. Daha basit veriler, verimlilik o kadar yüksek olur. +3. Bu `SELECT` sonuçlara itiraz değişiklikler tarihin tutarlılık bağlıdır. Ekleme için veri hazırlarken doğru olun. Tutarsız verilerde öngörülemeyen sonuçlar elde edebilirsiniz, örneğin, oturum derinliği gibi negatif olmayan metrikler için negatif değerler. + +### Algoritma {#table_engine-collapsingmergetree-collapsing-algorithm} + +ClickHouse veri parçalarını birleştirdiğinde, her ardışık satır grubu aynı sıralama anahtarıyla (`ORDER BY`) en fazla iki satır reduceda indir isgen ,ir, biri `Sign = 1` (“state” satır) ve başka bir `Sign = -1` (“cancel” satır). Başka bir deyişle, girişler çöker. + +Elde edilen her veri parçası için ClickHouse kaydeder: + +1. Birincilik “cancel” ve son “state” satır sayısı ise “state” ve “cancel” satırlar eşleşir ve son satır bir “state” satır. + +2. Son “state” satır, daha varsa “state” satırlar daha “cancel” satırlar. + +3. Birincilik “cancel” satır, daha varsa “cancel” satırlar daha “state” satırlar. + +4. Diğer tüm durumlarda satırların hiçbiri. + +Ayrıca en az 2 tane daha olduğunda “state” satırlar daha “cancel” satırlar veya en az 2 tane daha “cancel” r rowsows th thenen “state” satırlar, birleştirme devam eder, ancak ClickHouse bu durumu mantıksal bir hata olarak değerlendirir ve sunucu günlüğüne kaydeder. Aynı veriler birden çok kez eklendiğinde, bu hata oluşabilir. + +Bu nedenle, çöken istatistik hesaplama sonuçlarını değiştirmemelidir. +Değişiklikler yavaş yavaş çöktü, böylece sonunda hemen hemen her nesnenin sadece son durumu kaldı. + +Bu `Sign` birleştirme algoritması, aynı sıralama anahtarına sahip tüm satırların aynı sonuçtaki veri bölümünde ve hatta aynı fiziksel sunucuda olacağını garanti etmediğinden gereklidir. ClickHouse süreci `SELECT` birden çok iş parçacığına sahip sorgular ve sonuçtaki satırların sırasını tahmin edemez. Tamamen almak için bir ihtiyaç varsa toplama gereklidir “collapsed” veri `CollapsingMergeTree` Tablo. + +Daraltmayı sonuçlandırmak için bir sorgu yazın `GROUP BY` yan tümce ve işareti için hesap toplama işlevleri. Örneğin, miktarı hesaplamak için kullanın `sum(Sign)` yerine `count()`. Bir şeyin toplamını hesaplamak için şunları kullanın `sum(Sign * x)` yerine `sum(x)`, ve böylece, ve ayrıca ekleyin `HAVING sum(Sign) > 0`. + +Toplanan `count`, `sum` ve `avg` bu şekilde hesaplanmış olabilir. Toplanan `uniq` bir nesnenin en az bir durumu çökmüş değilse hesaplanabilir. Toplanan `min` ve `max` hesaplan becauseamadı çünkü `CollapsingMergeTree` daraltılmış durumların değerleri geçmişini kaydetmez. + +Toplama olmadan veri ayıklamanız gerekiyorsa (örneğin, en yeni değerleri belirli koşullarla eşleşen satırların mevcut olup olmadığını kontrol etmek için) `FINAL` değiştirici için `FROM` yan. Bu yaklaşım önemli ölçüde daha az etkilidir. + +## Kullanım Örneği {#example-of-use} + +Örnek veriler: + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +Tablonun oluşturulması: + +``` sql +CREATE TABLE UAct +( + UserID UInt64, + PageViews UInt8, + Duration UInt8, + Sign Int8 +) +ENGINE = CollapsingMergeTree(Sign) +ORDER BY UserID +``` + +Veri ekleme: + +``` sql +INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1) +``` + +``` sql +INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1),(4324182021466249494, 6, 185, 1) +``` + +Biz iki kullanın `INSERT` iki farklı veri parçası oluşturmak için sorgular. Verileri bir sorgu ile eklersek ClickHouse bir veri parçası oluşturur ve hiç bir birleştirme gerçekleştirmez. + +Veri alma: + +``` sql +SELECT * FROM UAct +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +Ne görüyoruz ve nerede çöküyor? + +İki ile `INSERT` sorgular, 2 Veri parçası oluşturduk. Bu `SELECT` sorgu 2 iş parçacığında yapıldı ve rastgele bir satır sırası aldık. Veri parçalarının henüz birleştirilmediği için çökme gerçekleşmedi. ClickHouse biz tahmin edemez bilinmeyen bir anda veri kısmını birleştirir. + +Böylece toplama ihtiyacımız var: + +``` sql +SELECT + UserID, + sum(PageViews * Sign) AS PageViews, + sum(Duration * Sign) AS Duration +FROM UAct +GROUP BY UserID +HAVING sum(Sign) > 0 +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┐ +│ 4324182021466249494 │ 6 │ 185 │ +└─────────────────────┴───────────┴──────────┘ +``` + +Toplamaya ihtiyacımız yoksa ve çökmeyi zorlamak istiyorsak, şunları kullanabiliriz `FINAL` değiştirici için `FROM` yan. + +``` sql +SELECT * FROM UAct FINAL +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +Verileri seçmenin bu yolu çok verimsizdir. Büyük masalar için kullanmayın. + +## Başka Bir Yaklaşım Örneği {#example-of-another-approach} + +Örnek veriler: + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ +│ 4324182021466249494 │ -5 │ -146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +Fikir, birleştirmelerin yalnızca anahtar alanları hesaba katmasıdır. Ve içinde “Cancel” satır işareti sütununu kullanmadan toplanırken satırın önceki sürümünü eşitleyen negatif değerleri belirtebiliriz. Bu yaklaşım için veri türünü değiştirmek gerekir `PageViews`,`Duration` uint8 -\> Int16 negatif değerlerini saklamak için. + +``` sql +CREATE TABLE UAct +( + UserID UInt64, + PageViews Int16, + Duration Int16, + Sign Int8 +) +ENGINE = CollapsingMergeTree(Sign) +ORDER BY UserID +``` + +Yaklaşımı test edelim: + +``` sql +insert into UAct values(4324182021466249494, 5, 146, 1); +insert into UAct values(4324182021466249494, -5, -146, -1); +insert into UAct values(4324182021466249494, 6, 185, 1); + +select * from UAct final; // avoid using final in production (just for a test or small tables) +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +``` sql +SELECT + UserID, + sum(PageViews) AS PageViews, + sum(Duration) AS Duration +FROM UAct +GROUP BY UserID +```text +┌──────────────UserID─┬─PageViews─┬─Duration─┐ +│ 4324182021466249494 │ 6 │ 185 │ +└─────────────────────┴───────────┴──────────┘ +``` + +``` sqk +select count() FROM UAct +``` + +``` text +┌─count()─┐ +│ 3 │ +└─────────┘ +``` + +``` sql +optimize table UAct final; + +select * FROM UAct +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/collapsingmergetree/) diff --git a/docs/tr/engines/table_engines/mergetree_family/custom_partitioning_key.md b/docs/tr/engines/table_engines/mergetree_family/custom_partitioning_key.md new file mode 100644 index 00000000000..ba09aa08cbb --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/custom_partitioning_key.md @@ -0,0 +1,127 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 32 +toc_title: "\xD6zel B\xF6l\xFCmleme Anahtar\u0131" +--- + +# Özel Bölümleme Anahtarı {#custom-partitioning-key} + +Bölümleme için kullanılabilir [MergeTree](mergetree.md) aile tabloları (dahil [çoğaltıyordu](replication.md) Tablolar). [Hayata görünümler](../special/materializedview.md) MergeTree tablolarına dayanarak bölümlemeyi de destekler. + +Bir bölüm, bir tablodaki kayıtların belirtilen bir kritere göre mantıksal bir birleşimidir. Bir bölümü, ay, gün veya olay türü gibi rasgele bir ölçütle ayarlayabilirsiniz. Bu verilerin manipülasyonlarını basitleştirmek için her bölüm ayrı ayrı saklanır. Verilere erişirken, ClickHouse mümkün olan en küçük bölüm alt kümesini kullanır. + +Bölüm belirtilen `PARTITION BY expr` fık whenra ne zaman [tablo oluşturma](mergetree.md#table_engine-mergetree-creating-a-table). Bölüm anahtarı tablo sütunlarından herhangi bir ifade olabilir. Örneğin, aya göre bölümleme belirtmek için ifadeyi kullanın `toYYYYMM(date_column)`: + +``` sql +CREATE TABLE visits +( + VisitDate Date, + Hour UInt8, + ClientID UUID +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(VisitDate) +ORDER BY Hour; +``` + +Bölüm anahtarı ayrıca bir ifade kümesi olabilir ( [birincil anahtar](mergetree.md#primary-keys-and-indexes-in-queries)). Mesela: + +``` sql +ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign) +PARTITION BY (toMonday(StartDate), EventType) +ORDER BY (CounterID, StartDate, intHash32(UserID)); +``` + +Bu örnekte, bölümlemeyi geçerli hafta boyunca meydana gelen olay türlerine göre ayarladık. + +Bir tabloya yeni veri eklerken, bu veriler birincil anahtara göre sıralanmış ayrı bir parça (yığın) olarak depolanır. Taktıktan 10-15 dakika sonra, aynı bölümün parçaları tüm parçaya birleştirilir. + +!!! info "Bilgin" + Birleştirme yalnızca bölümleme ifadesi için aynı değere sahip veri parçaları için çalışır. Bu demektir **aşırı granüler bölümler yapmamalısınız** (yaklaşık binden fazla bölüm). Aksi takdirde, `SELECT` sorgu, dosya sistemindeki ve açık dosya tanımlayıcılarındaki makul olmayan sayıda dosya nedeniyle yetersiz performans gösterir. + +Kullan... [sistem.parçalar](../../../operations/system_tables.md#system_tables-parts) tablo tablo parçaları ve bölümleri görüntülemek için. Örneğin, bir var varsayalım `visits` aya göre bölümleme ile tablo. Hadi gerçekleştirelim `SELECT` sorgu için `system.parts` Tablo: + +``` sql +SELECT + partition, + name, + active +FROM system.parts +WHERE table = 'visits' +``` + +``` text +┌─partition─┬─name───────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1 │ 1 │ +│ 201902 │ 201902_10_10_0 │ 1 │ +│ 201902 │ 201902_11_11_0 │ 1 │ +└───────────┴────────────────┴────────┘ +``` + +Bu `partition` sütun bölümlerin adlarını içerir. Bu örnekte iki bölüm vardır: `201901` ve `201902`. Bölüm adını belirtmek için bu sütun değerini kullanabilirsiniz [ALTER … PARTITION](#alter_manipulations-with-partitions) sorgular. + +Bu `name` sütun, bölüm veri parçalarının adlarını içerir. Bölümün adını belirtmek için bu sütunu kullanabilirsiniz. [ALTER ATTACH PART](#alter_attach-partition) sorgu. + +İlk bölümün adını kıralım: `201901_1_3_1`: + +- `201901` bölüm adıdır. +- `1` en az veri bloğu sayısıdır. +- `3` veri bloğunun maksimum sayısıdır. +- `1` yığın düzeyidir (oluşturduğu birleştirme ağacının derinliği). + +!!! info "Bilgin" + Eski tip tabloların parçaları adı vardır: `20190117_20190123_2_2_0` (minimum tarih - maksimum tarih - minimum blok numarası - maksimum blok numarası - seviye). + +Bu `active` sütun, parçanın durumunu gösterir. `1` aktif istir; `0` etkin değil. Etkin olmayan parçalar, örneğin, daha büyük bir parçaya birleştirildikten sonra kalan kaynak parçalarıdır. Bozuk veri parçaları da etkin olarak gösterilir. + +Örnekte gördüğünüz gibi, aynı bölümün birkaç ayrı parçası vardır (örneğin, `201901_1_3_1` ve `201901_1_9_2`). Bu, bu parçaların henüz birleştirilmediği anlamına gelir. ClickHouse, eklendikten yaklaşık 15 dakika sonra eklenen veri parçalarını periyodik olarak birleştirir. Buna ek olarak, kullanarak zamanlanmış olmayan birleştirme gerçekleştirebilirsiniz [OPTIMIZE](../../../sql_reference/statements/misc.md#misc_operations-optimize) sorgu. Örnek: + +``` sql +OPTIMIZE TABLE visits PARTITION 201902; +``` + +``` text +┌─partition─┬─name───────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1 │ 0 │ +│ 201902 │ 201902_4_11_2 │ 1 │ +│ 201902 │ 201902_10_10_0 │ 0 │ +│ 201902 │ 201902_11_11_0 │ 0 │ +└───────────┴────────────────┴────────┘ +``` + +Etkin olmayan parçalar birleştirildikten yaklaşık 10 dakika sonra silinecektir. + +Bir parça ve bölüm kümesini görüntülemenin başka bir yolu da tablonun dizinine gitmektir: `/var/lib/clickhouse/data///`. Mesela: + +``` bash +/var/lib/clickhouse/data/default/visits$ ls -l +total 40 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 201901_1_3_1 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201901_1_9_2 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_8_8_0 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_9_9_0 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_10_10_0 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_11_11_0 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:19 201902_4_11_2 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 12:09 201902_4_6_1 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached +``` + +Klasör ‘201901\_1\_1\_0’, ‘201901\_1\_7\_1’ ve böylece parçaların dizinleri vardır. Her bölüm karşılık gelen bir bölümle ilgilidir ve yalnızca belirli bir ay için veri içerir (Bu örnekteki tabloda aylara göre bölümleme vardır). + +Bu `detached` dizin kullanarak tablodan ayrılmış parçaları içerir [DETACH](#alter_detach-partition) sorgu. Bozuk parçalar da silinmek yerine bu dizine taşınır. Sunucu parçaları kullanmaz `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql_reference/statements/alter.md#alter_attach-partition) sorgu. + +İşletim sunucusunda, sunucu bunu bilmediğinden, dosya sistemindeki parça kümesini veya verilerini el ile değiştiremeyeceğinizi unutmayın. Çoğaltılmamış tablolar için, sunucu durdurulduğunda bunu yapabilirsiniz, ancak önerilmez. Çoğaltılmış tablolar için, parça kümesi her durumda değiştirilemez. + +ClickHouse, bölümlerle işlemleri gerçekleştirmenize izin verir: bunları silin, bir tablodan diğerine kopyalayın veya bir yedek oluşturun. Bölümdeki tüm işlemlerin listesine bakın [Bölümler ve parçalar ile manipülasyonlar](../../../sql_reference/statements/alter.md#alter_manipulations-with-partitions). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) diff --git a/docs/tr/engines/table_engines/mergetree_family/graphitemergetree.md b/docs/tr/engines/table_engines/mergetree_family/graphitemergetree.md new file mode 100644 index 00000000000..5d4a349631b --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/graphitemergetree.md @@ -0,0 +1,174 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 38 +toc_title: "Graph\u0131temergetree" +--- + +# Graphıtemergetree {#graphitemergetree} + +Bu motor inceltme ve toplama/ortalama (toplaması) için tasarlanmıştır) [Grafit](http://graphite.readthedocs.io/en/latest/index.html) veriler. Clickhouse'u Grafit için bir veri deposu olarak kullanmak isteyen geliştiriciler için yararlı olabilir. + +Toplamaya ihtiyacınız yoksa Grafit verilerini depolamak için herhangi bir ClickHouse tablo motorunu kullanabilirsiniz, ancak bir toplamaya ihtiyacınız varsa `GraphiteMergeTree`. Motor, depolama hacmini azaltır ve grafitten gelen sorguların verimliliğini arttırır. + +Motor özellikleri devralır [MergeTree](mergetree.md). + +## Tablo oluşturma {#creating-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + Path String, + Time DateTime, + Value , + Version + ... +) ENGINE = GraphiteMergeTree(config_section) +[PARTITION BY expr] +[ORDER BY expr] +[SAMPLE BY expr] +[SETTINGS name=value, ...] +``` + +Ayrıntılı bir açıklamasını görmek [CREATE TABLE](../../../sql_reference/statements/create.md#create-table-query) sorgu. + +Grafit verileri için bir tablo aşağıdaki veriler için aşağıdaki sütunlara sahip olmalıdır: + +- Metrik adı (Grafit sensörü). Veri türü: `String`. + +- Metrik ölçme zamanı. Veri türü: `DateTime`. + +- Metrik değeri. Veri türü: herhangi bir sayısal. + +- Metrik sürümü. Veri türü: herhangi bir sayısal. + + ClickHouse en yüksek sürümü veya sürümleri aynı ise son yazılan satırları kaydeder. Veri parçalarının birleştirilmesi sırasında diğer satırlar silinir. + +Bu sütunların adları toplaması yapılandırmasında ayarlanmalıdır. + +**Graphıtemergetree parametreleri** + +- `config_section` — Name of the section in the configuration file, where are the rules of rollup set. + +**Sorgu yan tümceleri** + +Oluştururken bir `GraphiteMergeTree` tablo, aynı [yanlar](mergetree.md#table_engine-mergetree-creating-a-table) oluşturul ,urken olduğu gibi gerekli `MergeTree` Tablo. + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın ve mümkünse eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + EventDate Date, + Path String, + Time DateTime, + Value , + Version + ... +) ENGINE [=] GraphiteMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, config_section) +``` + +Hariç tüm parametreler `config_section` içinde olduğu gibi aynı anlama sahip `MergeTree`. + +- `config_section` — Name of the section in the configuration file, where are the rules of rollup set. + +
+ +## Toplaması Yapılandırması {#rollup-configuration} + +Toplaması için ayarları tarafından tanımlanan [graphite\_rollup](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-graphite_rollup) sunucu yapılandırmasında parametre. Parametrenin adı herhangi biri olabilir. Birkaç yapılandırma oluşturabilir ve bunları farklı tablolar için kullanabilirsiniz. + +Toplaması yapılandırma yapısı: + + required-columns + patterns + +### Gerekli Sütunlar {#required-columns} + +- `path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`. +- `time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`. +- `value_column_name` — The name of the column storing the value of the metric at the time set in `time_column_name`. Varsayılan değer: `Value`. +- `version_column_name` — The name of the column storing the version of the metric. Default value: `Timestamp`. + +### Desenler {#patterns} + +Bu yapı `patterns` bölme: + +``` text +pattern + regexp + function +pattern + regexp + age + precision + ... +pattern + regexp + function + age + precision + ... +pattern + ... +default + function + age + precision + ... +``` + +!!! warning "Dikkat" + Desenler kesinlikle sipariş edilmelidir: + + 1. Patterns without `function` or `retention`. + 1. Patterns with both `function` and `retention`. + 1. Pattern `default`. + +Bir satır işlerken, ClickHouse kuralları denetler `pattern` bölmeler. Tüm `pattern` (içeren `default`) bölümler içerebilir `function` toplama için parametre, `retention` parametreler veya her ikisi. Metrik adı eşleşirse `regexp` gelen kuralları `pattern` bölüm (veya bölümler) uygulanır; aksi takdirde, kurallar `default` bölüm kullanılır. + +Alanlar için `pattern` ve `default` bölmeler: + +- `regexp`– A pattern for the metric name. +- `age` – The minimum age of the data in seconds. +- `precision`– How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day). +- `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. + +### Yapılandırma Örneği {#configuration-example} + +``` xml + + Version + + click_cost + any + + 0 + 5 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/graphitemergetree/) diff --git a/docs/tr/engines/table_engines/mergetree_family/index.md b/docs/tr/engines/table_engines/mergetree_family/index.md new file mode 100644 index 00000000000..e722564f4dd --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: MergeTree Ailesi +toc_priority: 28 +--- + + diff --git a/docs/tr/engines/table_engines/mergetree_family/mergetree.md b/docs/tr/engines/table_engines/mergetree_family/mergetree.md new file mode 100644 index 00000000000..5adb6c2b038 --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/mergetree.md @@ -0,0 +1,654 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 30 +toc_title: MergeTree +--- + +# MergeTree {#table_engines-mergetree} + +Bu `MergeTree` bu ailenin motoru ve diğer motorları (`*MergeTree`) en sağlam ClickHouse masa motorlarıdır. + +Motor inlarda `MergeTree` aile, bir tabloya çok büyük miktarda veri eklemek için tasarlanmıştır. Veriler hızlı bir şekilde tabloya kısmen yazılır, daha sonra parçaları arka planda birleştirmek için kurallar uygulanır. Bu yöntem, ınsert sırasında depolama alanındaki verileri sürekli olarak yeniden yazmaktan çok daha etkilidir. + +Ana özellikler: + +- Birincil anahtara göre sıralanmış verileri saklar. + + Bu, verileri daha hızlı bulmanıza yardımcı olan küçük bir seyrek dizin oluşturmanıza olanak sağlar. + +- Bölümler eğer kullanılabilir [bölümleme anahtarı](custom_partitioning_key.md) belirt .ilmektedir. + + ClickHouse, aynı sonuçla aynı veriler üzerindeki genel işlemlerden daha etkili olan bölümlerle belirli işlemleri destekler. ClickHouse, bölümleme anahtarının sorguda belirtildiği bölüm verilerini de otomatik olarak keser. Bu da sorgu performansını artırır. + +- Veri çoğaltma desteği. + + The family of `ReplicatedMergeTree` tablolar veri çoğaltma sağlar. Daha fazla bilgi için, bkz. [Veri çoğaltma](replication.md). + +- Veri örnekleme desteği. + + Gerekirse, tabloda veri örnekleme yöntemini ayarlayabilirsiniz. + +!!! info "Bilgin" + Bu [Birleştirmek](../special/merge.md) motor ait değil `*MergeTree` aile. + +## Tablo oluşturma {#table_engine-mergetree-creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], + ... + INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1, + INDEX index_name2 expr2 TYPE type2(...) GRANULARITY value2 +) ENGINE = MergeTree() +[PARTITION BY expr] +[ORDER BY expr] +[PRIMARY KEY expr] +[SAMPLE BY expr] +[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...] +[SETTINGS name=value, ...] +``` + +Parametrelerin açıklaması için bkz. [Sorgu açıklaması oluştur](../../../sql_reference/statements/create.md). + +!!! note "Not" + `INDEX` deneysel bir özelliktir, bkz [Veri Atlama Dizinleri](#table_engine-mergetree-data_skipping-indexes). + +### Sorgu Yan Tümceleri {#mergetree-query-clauses} + +- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. Bu `MergeTree` motor parametreleri yok. + +- `PARTITION BY` — The [bölümleme anahtarı](custom_partitioning_key.md). + + Aylara göre bölümleme için `toYYYYMM(date_column)` ifade, nerede `date_column` türün tarihi olan bir sütun mu [Tarihli](../../../sql_reference/data_types/date.md). Burada bölüm isimleri var `"YYYYMM"` biçimli. + +- `ORDER BY` — The sorting key. + + Sütun veya keyfi ifadeler bir tuple. Örnek: `ORDER BY (CounterID, EventDate)`. + +- `PRIMARY KEY` — The primary key if it [sıralama anahtarından farklıdır](mergetree.md). + + Varsayılan olarak, birincil anahtar sıralama anahtarıyla aynıdır (bu anahtar tarafından belirtilir). `ORDER BY` yan). Bu nedenle çoğu durumda ayrı bir belirtmek gereksizdir `PRIMARY KEY` yan. + +- `SAMPLE BY` — An expression for sampling. + + Bir örnekleme ifadesi kullanılırsa, birincil anahtar onu içermelidir. Örnek: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. + +- `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [diskler ve birimler arasında](#table_engine-mergetree-multiple-volumes). + + İfade bir olmalıdır `Date` veya `DateTime` sonuç olarak sütun. Örnek: + `TTL date + INTERVAL 1 DAY` + + Kuralın türü `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` ifade tatmin edildiyse (geçerli zamana ulaşırsa) parça ile yapılacak bir eylemi belirtir: süresi dolmuş satırların kaldırılması, bir parçanın (bir parçadaki tüm satırlar için ifade tatmin edildiyse) belirtilen diske taşınması (`TO DISK 'xxx'`) veya hacim (`TO VOLUME 'xxx'`). Kuralın varsayılan türü kaldırma (`DELETE`). Birden fazla kural listesi belirtilebilir, ancak birden fazla olmamalıdır `DELETE` kural. + + Daha fazla ayrıntı için bkz. [Sütunlar ve tablolar için TTL](#table_engine-mergetree-ttl) + +- `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: + + - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Veri Depolama](#mergetree-data-storage). + - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Veri Depolama](#mergetree-data-storage). + - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` ayar. 19.11 sürümünden önce, sadece `index_granularity` granül boyutunu kısıtlamak için ayar. Bu `index_granularity_bytes` büyük satırlar (onlarca ve megabayt yüzlerce) ile tablolardan veri seçerken ayarı ClickHouse performansını artırır. Büyük satırlara sahip tablolarınız varsa, tabloların verimliliğini artırmak için bu ayarı etkinleştirebilirsiniz. `SELECT` sorgular. + - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, daha sonra ZooKeeper daha az veri depolar. Daha fazla bilgi için, bkz: [ayar açıklaması](../../../operations/server_configuration_parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) içinde “Server configuration parameters”. + - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bayt, ClickHouse okur ve doğrudan I/O arabirimi kullanarak depolama diskine veri yazar (`O_DIRECT` seçenek). Eğer `min_merge_bytes_to_use_direct_io = 0`, sonra doğrudan g / Ç devre dışı bırakılır. Varsayılan değer: `10 * 1024 * 1024 * 1024` baytlar. + + - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with TTL. Default value: 86400 (1 day). + - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don't turn it off. + - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. + - `storage_policy` — Storage policy. See [Veri depolama için birden fazla blok cihazı kullanma](#table_engine-mergetree-multiple-volumes). + +**Bölüm ayarı örneği** + +``` sql +ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192 +``` + +Örnekte, aylara göre bölümleme ayarladık. + +Biz de kullanıcı kimliği ile karma olarak örnekleme için bir ifade ayarlayın. Bu, her biri için tablodaki verileri pseudorandomize etmenizi sağlar `CounterID` ve `EventDate`. Tanım yoularsanız bir [SAMPLE](../../../sql_reference/statements/select.md#select-sample-clause) yan tümcesi verileri seçerken, ClickHouse kullanıcıların bir alt kümesi için eşit pseudorandom veri örneği döndürür. + +Bu `index_granularity` 8192 varsayılan değer olduğundan ayarı atlanabilir. + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın. Mümkünse, eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE [=] MergeTree(date-column [, sampling_expression], (primary, key), index_granularity) +``` + +**MergeTree () Parametreleri** + +- `date-column` — The name of a column of the [Tarihli](../../../sql_reference/data_types/date.md) tür. ClickHouse otomatik olarak bu sütuna göre ay bölümleri oluşturur. Bölüm adları `"YYYYMM"` biçimli. +- `sampling_expression` — An expression for sampling. +- `(primary, key)` — Primary key. Type: [Demet()](../../../sql_reference/data_types/tuple.md) +- `index_granularity` — The granularity of an index. The number of data rows between the “marks” bir dizinin. 8192 değeri çoğu görev için uygundur. + +**Örnek** + +``` sql +MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) +``` + +Bu `MergeTree` motor, Ana motor yapılandırma yöntemi için yukarıdaki örnekte olduğu gibi yapılandırılır. +
+ +## Veri Depolama {#mergetree-data-storage} + +Bir tabloda birincil anahtar tarafından sıralanmış verileri bölümden oluşmaktadır. + +Veri bir tabloya eklendiğinde, ayrı veri parçaları oluşturulur ve bunların her biri birincil anahtara göre lexicographically sıralanır. Örneğin, birincil anahtar `(CounterID, Date)`, parçadaki veriler şu şekilde sıralanır `CounterID` ve içinde her `CounterID` tarafından sipariş edilir `Date`. + +Farklı bölümlere ait veriler farklı parçalara ayrılır. Arka planda, ClickHouse daha verimli depolama için veri parçalarını birleştirir. Farklı bölümlere ait parçalar birleştirilmez. Birleştirme mekanizması, aynı birincil anahtara sahip tüm satırların aynı veri bölümünde olacağını garanti etmez. + +Her veri parçası mantıksal olarak granüllere ayrılmıştır. Bir granül, Clickhouse'un veri seçerken okuduğu en küçük bölünmez veri kümesidir. ClickHouse satırları veya değerleri bölmez, bu nedenle her granül her zaman bir tamsayı satır içerir. Bir granülün ilk satırı, satır için birincil anahtarın değeri ile işaretlenir. Her veri bölümü için ClickHouse işaretleri depolayan bir dizin dosyası oluşturur. Her sütun için, birincil anahtarda olsun ya da olmasın, ClickHouse aynı işaretleri de saklar. Bu işaretler, verileri doğrudan sütun dosyalarında bulmanızı sağlar. + +Granül boyutu ile sınırlıdır `index_granularity` ve `index_granularity_bytes` tablo motorunun ayarları. Bir granüldeki satır sayısı `[1, index_granularity]` Aralık, satırların boyutuna bağlı olarak. Bir granülün boyutu aşabilir `index_granularity_bytes` tek bir satırın boyutu ayarın değerinden büyükse. Bu durumda, granülün boyutu satırın boyutuna eşittir. + +## Sorgularda birincil anahtarlar ve dizinler {#primary-keys-and-indexes-in-queries} + +Tak thee the `(CounterID, Date)` örnek olarak birincil anahtar. Bu durumda, sıralama ve dizin aşağıdaki gibi gösterilebilir: + + Whole data: [---------------------------------------------] + CounterID: [aaaaaaaaaaaaaaaaaabbbbcdeeeeeeeeeeeeefgggggggghhhhhhhhhiiiiiiiiikllllllll] + Date: [1111111222222233331233211111222222333211111112122222223111112223311122333] + Marks: | | | | | | | | | | | + a,1 a,2 a,3 b,3 e,2 e,3 g,1 h,2 i,1 i,3 l,3 + Marks numbers: 0 1 2 3 4 5 6 7 8 9 10 + +Veri sorgusu belirtirse: + +- `CounterID in ('a', 'h')`, sunucu işaretleri aralıklarında verileri okur `[0, 3)` ve `[6, 8)`. +- `CounterID IN ('a', 'h') AND Date = 3`, sunucu işaretleri aralıklarında verileri okur `[1, 3)` ve `[7, 8)`. +- `Date = 3`, sunucu işaretleri aralığında veri okur `[1, 10]`. + +Yukarıdaki örnekler, her zaman bir dizin tam taramadan daha etkili olduğunu göstermektedir. + +Seyrek bir dizin, ekstra verilerin okunmasına izin verir. Birincil anahtarın tek bir aralığını okurken, `index_granularity * 2` her veri bloğundaki ekstra satırlar okunabilir. + +Seyrek dizinler, çok sayıda tablo satırı ile çalışmanıza izin verir, çünkü çoğu durumda, bu tür dizinler bilgisayarın RAM'İNE sığar. + +ClickHouse benzersiz bir birincil anahtar gerektirmez. Aynı birincil anahtar ile birden çok satır ekleyebilirsiniz. + +### Birincil anahtar seçme {#selecting-the-primary-key} + +Birincil anahtardaki sütun sayısı açıkça sınırlı değildir. Veri yapısına bağlı olarak, birincil anahtara daha fazla veya daha az sütun ekleyebilirsiniz. Bu Mayıs: + +- Bir dizin performansını artırın. + + Birincil anahtar ise `(a, b)`, sonra başka bir sütun ekleyerek `c` aşağıdaki koşullar yerine getirilirse performansı artıracaktır: + + - Sütun üzerinde bir koşulu olan sorgular var `c`. + - Uzun veri aralıkları (birkaç kat daha uzun `index_granularity`) için aynı değer withlerle `(a, b)` yaygındır. Başka bir deyişle, başka bir sütun eklerken oldukça uzun veri aralıklarını atlamanıza izin verir. + +- Veri sıkıştırmasını geliştirin. + + ClickHouse verileri birincil anahtarla sıralar, bu nedenle tutarlılık ne kadar yüksek olursa sıkıştırma o kadar iyi olur. + +- Veri parçalarını birleştirirken ek mantık sağlayın [CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) ve [SummingMergeTree](summingmergetree.md) motorlar. + + Bu durumda belirtmek mantıklı *sıralama anahtarı* bu birincil anahtardan farklıdır. + +Uzun bir birincil anahtar, ekleme performansını ve bellek tüketimini olumsuz yönde etkiler, ancak birincil anahtardaki ek sütunlar, ClickHouse performansını etkilemez `SELECT` sorgular. + +### Sıralama anahtarından farklı bir birincil anahtar seçme {#choosing-a-primary-key-that-differs-from-the-sorting-key} + +Sıralama anahtarından (veri bölümlerindeki satırları sıralamak için bir ifade) farklı bir birincil anahtar (her işaret için dizin dosyasında yazılan değerlere sahip bir ifade) belirtmek mümkündür. Bu durumda, birincil anahtar ifadesi tuple, sıralama anahtarı ifadesi tuple'ın bir öneki olmalıdır. + +Bu özellik kullanırken yararlıdır [SummingMergeTree](summingmergetree.md) ve +[AggregatingMergeTree](aggregatingmergetree.md) masa motorları. Bu motorları kullanırken yaygın bir durumda, tablonun iki tür sütunu vardır: *boyutlular* ve *ölçümler*. Tipik sorgular, rasgele ölçü sütunlarının değerlerini toplar `GROUP BY` ve boyutlara göre filtreleme. Çünkü SummingMergeTree ve AggregatingMergeTree sıralama anahtarının aynı değere sahip satırları toplamak, tüm boyutları eklemek doğaldır. Sonuç olarak, anahtar ifadesi uzun bir sütun listesinden oluşur ve bu liste yeni eklenen boyutlarla sık sık güncelleştirilmelidir. + +Bu durumda, birincil anahtarda verimli Aralık taramaları sağlayacak ve kalan boyut sütunlarını sıralama anahtarı kümesine ekleyecek yalnızca birkaç sütun bırakmak mantıklıdır. + +[ALTER](../../../sql_reference/statements/alter.md) yeni bir sütun aynı anda tabloya ve sıralama anahtarı eklendiğinde, varolan veri parçaları değiştirilmesi gerekmez, çünkü sıralama anahtarının hafif bir işlemdir. Eski sıralama anahtarı yeni sıralama anahtarının bir öneki olduğundan ve yeni eklenen sütunda veri olmadığından, veriler tablo değişikliği anında hem eski hem de yeni sıralama anahtarlarına göre sıralanır. + +### Sorgularda dizin ve bölümlerin kullanımı {#use-of-indexes-and-partitions-in-queries} + +İçin `SELECT` sorgular, ClickHouse bir dizin kullanılabilir olup olmadığını analiz eder. Eğer bir dizin kullanılabilir `WHERE/PREWHERE` yan tümce, bir eşitlik veya eşitsizlik karşılaştırma işlemini temsil eden bir ifadeye (bağlantı öğelerinden biri olarak veya tamamen) sahiptir veya varsa `IN` veya `LIKE` sütun veya birincil anahtar veya bölümleme anahtar veya bu sütunların belirli kısmen tekrarlayan işlevleri veya bu ifadelerin mantıksal ilişkileri olan ifadeler üzerinde sabit bir önek ile. + +Bu nedenle, birincil anahtarın bir veya daha fazla aralığındaki sorguları hızlı bir şekilde çalıştırmak mümkündür. Bu örnekte, belirli bir izleme etiketi, belirli bir etiket ve tarih aralığı, belirli bir etiket ve tarih için, tarih aralığına sahip birden çok etiket için vb. çalıştırıldığında sorgular hızlı olacaktır. + +Aşağıdaki gibi yapılandırılmış motora bakalım: + + ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate) SETTINGS index_granularity=8192 + +Bu durumda, sorgularda: + +``` sql +SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 +SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) +SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) +``` + +ClickHouse, uygun olmayan verileri kırpmak için birincil anahtar dizinini ve uygun olmayan tarih aralıklarındaki bölümleri kırpmak için aylık bölümleme anahtarını kullanır. + +Yukarıdaki sorgular, dizinin karmaşık ifadeler için bile kullanıldığını göstermektedir. Tablodan okuma, dizini kullanarak tam taramadan daha yavaş olamayacak şekilde düzenlenmiştir. + +Aşağıdaki örnekte, dizin kullanılamaz. + +``` sql +SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' +``` + +Clickhouse'un bir sorgu çalıştırırken dizini kullanıp kullanamayacağını kontrol etmek için ayarları kullanın [force\_index\_by\_date](../../../operations/settings/settings.md#settings-force_index_by_date) ve [force\_primary\_key](../../../operations/settings/settings.md). + +Aylara göre bölümleme anahtarı, yalnızca uygun aralıktaki tarihleri içeren veri bloklarını okumanıza izin verir. Bu durumda, veri bloğu birçok tarih için veri içerebilir (bir aya kadar). Bir blok içinde veriler, ilk sütun olarak tarihi içermeyen birincil anahtara göre sıralanır. Bu nedenle, birincil anahtar önekini belirtmeyen yalnızca bir tarih koşulu ile bir sorgu kullanarak tek bir tarih için okunacak daha fazla veri neden olur. + +### Kısmen monotonik birincil anahtarlar için Endeks kullanımı {#use-of-index-for-partially-monotonic-primary-keys} + +Örneğin, Ayın günlerini düşünün. Onlar formu bir [monotonik dizisi](https://en.wikipedia.org/wiki/Monotonic_function) bir ay boyunca, ancak daha uzun süreler için monotonik değil. Bu kısmen monotonik bir dizidir. Bir kullanıcı kısmen monoton birincil anahtar ile tablo oluşturursa, ClickHouse her zamanki gibi seyrek bir dizin oluşturur. Bir kullanıcı bu tür bir tablodan veri seçtiğinde, ClickHouse sorgu koşullarını analiz eder. Kullanıcı, dizinin iki işareti arasında veri almak isterse ve bu işaretlerin her ikisi de bir ay içinde düşerse, ClickHouse bu özel durumda dizini kullanabilir, çünkü sorgu parametreleri ile dizin işaretleri arasındaki mesafeyi hesaplayabilir. + +Sorgu parametresi aralığındaki birincil anahtarın değerleri monotonik bir sırayı temsil etmiyorsa, ClickHouse bir dizin kullanamaz. Bu durumda, ClickHouse Tam Tarama yöntemini kullanır. + +ClickHouse bu mantığı yalnızca ay dizilerinin günleri için değil, kısmen monotonik bir diziyi temsil eden herhangi bir birincil anahtar için kullanır. + +### Veri atlama indeksleri (deneysel) {#table_engine-mergetree-data_skipping-indexes} + +Dizin bildirimi sütunlar bölümünde `CREATE` sorgu. + +``` sql +INDEX index_name expr TYPE type(...) GRANULARITY granularity_value +``` + +Tablolar için `*MergeTree` aile, veri atlama endeksleri belirtilebilir. + +Bu endeksler, bloklarda belirtilen ifade hakkında bazı bilgileri toplar ve bunlardan oluşur `granularity_value` granüller (granül boyutu kullanılarak belirtilir `index_granularity` tablo motoru ayarı). Daha sonra bu agregalar `SELECT` büyük veri bloklarını atlayarak diskten okunacak veri miktarını azaltmak için sorgular `where` sorgu tatmin edilemez. + +**Örnek** + +``` sql +CREATE TABLE table_name +( + u64 UInt64, + i32 Int32, + s String, + ... + INDEX a (u64 * i32, s) TYPE minmax GRANULARITY 3, + INDEX b (u64 * length(s)) TYPE set(1000) GRANULARITY 4 +) ENGINE = MergeTree() +... +``` + +Örneğin endeksleri aşağıdaki sorgularda diskten okunacak veri miktarını azaltmak için ClickHouse tarafından kullanılabilir: + +``` sql +SELECT count() FROM table WHERE s < 'z' +SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 +``` + +#### Mevcut Endeks Türleri {#available-types-of-indices} + +- `minmax` + + Belirtilen ifad (eyi saklar (ifad (enin `tuple`, sonra her eleman için aşırı depolar `tuple`), birincil anahtar gibi veri bloklarını atlamak için saklanan bilgileri kullanır. + +- `set(max_rows)` + + Belirtilen ifadenin benzersiz değerlerini depolar (en fazla `max_rows` satırlar, `max_rows=0` anlama “no limits”). Kontrol etmek için değerleri kullanır `WHERE` ifade, bir veri bloğu üzerinde tatmin edilemez değildir. + +- `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` + + Mağazalar a [Blo filterom filtre](https://en.wikipedia.org/wiki/Bloom_filter) bu, bir veri bloğundaki tüm ngramları içerir. Sadece dizeleri ile çalışır. Optimizasyonu için kullanılabilir `equals`, `like` ve `in` ifadeler. + + - `n` — ngram size, + - `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). + - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. + - `random_seed` — The seed for Bloom filter hash functions. + +- `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` + + Olarak aynı `ngrambf_v1`, ancak ngrams yerine simgeleri saklar. Belirteçler alfasayısal olmayan karakterlerle ayrılmış dizilerdir. + +- `bloom_filter([false_positive])` — Stores a [Blo filterom filtre](https://en.wikipedia.org/wiki/Bloom_filter) belirtilen sütunlar için. + + Opsiyonel `false_positive` parametre, filtreden yanlış pozitif yanıt alma olasılığıdır. Olası değerler: (0, 1). Varsayılan değer: 0.025. + + Desteklenen veri türleri: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`. + + Aşağıdaki işlevleri kullanabilirsiniz: [eşitlikler](../../../sql_reference/functions/comparison_functions.md), [notEquals](../../../sql_reference/functions/comparison_functions.md), [içinde](../../../sql_reference/functions/in_functions.md), [notİn](../../../sql_reference/functions/in_functions.md), [var](../../../sql_reference/functions/array_functions.md). + + + +``` sql +INDEX sample_index (u64 * length(s)) TYPE minmax GRANULARITY 4 +INDEX sample_index2 (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARITY 4 +INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4 +``` + +#### Fonksiyonları Destek {#functions-support} + +Koşulları `WHERE` yan tümcesi, sütunlarla çalışan işlevlerin çağrılarını içerir. Sütun bir dizinin bir parçasıysa, ClickHouse işlevleri gerçekleştirirken bu dizini kullanmaya çalışır. ClickHouse, dizinleri kullanmak için farklı işlev alt kümelerini destekler. + +Bu `set` dizin tüm fonksiyonları ile kullanılabilir. Diğer dizinler için işlev alt kümeleri aşağıdaki tabloda gösterilmiştir. + +| Fonksiyon (operatör) / dizin | birincil anahtar | minmax | ngrambf\_v1 | tokenbf\_v1 | bloom\_filter | +|------------------------------------------------------------------------------------------------------------|------------------|--------|-------------|-------------|---------------| +| [eşitlikler (=, ==)](../../../sql_reference/functions/comparison_functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, \<\>)](../../../sql_reference/functions/comparison_functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [hoşlanmak](../../../sql_reference/functions/string_search_functions.md#function-like) | ✔ | ✔ | ✔ | ✗ | ✗ | +| [notLike](../../../sql_reference/functions/string_search_functions.md#function-notlike) | ✔ | ✔ | ✔ | ✗ | ✗ | +| [startsWith](../../../sql_reference/functions/string_functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [endsWith](../../../sql_reference/functions/string_functions.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | +| [multiSearchAny](../../../sql_reference/functions/string_search_functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | +| [içinde](../../../sql_reference/functions/in_functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notİn](../../../sql_reference/functions/in_functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [daha az (\<)](../../../sql_reference/functions/comparison_functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [büyük (\>)](../../../sql_reference/functions/comparison_functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [lessOrEquals (\<=)](../../../sql_reference/functions/comparison_functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greaterOrEquals (\>=)](../../../sql_reference/functions/comparison_functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [boş](../../../sql_reference/functions/array_functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [notEmpty](../../../sql_reference/functions/array_functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | + +Ngram boyutundan daha az olan sabit bir argümana sahip işlevler tarafından kullanılamaz `ngrambf_v1` sorgu optimizasyonu için. + +Bloom filtreleri yanlış pozitif eşleşmelere sahip olabilir, bu yüzden `ngrambf_v1`, `tokenbf_v1`, ve `bloom_filter` dizinler, örneğin bir işlevin sonucunun false olması beklenen sorguları en iyi duruma getirmek için kullanılamaz: + +- Optimize edilebilir: + - `s LIKE '%test%'` + - `NOT s NOT LIKE '%test%'` + - `s = 1` + - `NOT s != 1` + - `startsWith(s, 'test')` +- Optimize edilemez: + - `NOT s LIKE '%test%'` + - `s NOT LIKE '%test%'` + - `NOT s = 1` + - `s != 1` + - `NOT startsWith(s, 'test')` + +## Eşzamanlı Veri Erişimi {#concurrent-data-access} + +Eşzamanlı tablo erişimi için çoklu sürüm kullanıyoruz. Başka bir deyişle, bir tablo aynı anda okunup güncelleştirildiğinde, sorgu sırasında geçerli olan bir parça kümesinden veri okunur. Uzun kilitler yok. Ekler okuma işlemlerinin yoluna girmez. + +Bir tablodan okuma otomatik olarak paralelleştirilir. + +## Sütunlar ve tablolar için TTL {#table_engine-mergetree-ttl} + +Değerlerin ömrünü belirler. + +Bu `TTL` yan tümcesi tüm tablo ve her sütun için ayarlanabilir. Tablo düzeyinde TTL ayrıca diskler ve birimler arasında otomatik veri taşıma mantığını belirtebilirsiniz. + +İfadeleri değerlendirmek gerekir [Tarihli](../../../sql_reference/data_types/date.md) veya [DateTime](../../../sql_reference/data_types/datetime.md) veri türü. + +Örnek: + +``` sql +TTL time_column +TTL time_column + interval +``` + +Tanımlamak `interval`, kullanma [zaman aralığı](../../../sql_reference/operators.md#operators-datetime) operatörler. + +``` sql +TTL date_time + INTERVAL 1 MONTH +TTL date_time + INTERVAL 15 HOUR +``` + +### Sütun TTL {#mergetree-column-ttl} + +Sütundaki değerler sona erdiğinde, ClickHouse bunları sütun veri türü için varsayılan değerlerle değiştirir. Veri bölümündeki tüm sütun değerleri sona ererse, ClickHouse bu sütunu bir dosya sistemindeki veri bölümünden siler. + +Bu `TTL` yan tümcesi anahtar sütunlar için kullanılamaz. + +Örnekler: + +TTL ile tablo oluşturma + +``` sql +CREATE TABLE example_table +( + d DateTime, + a Int TTL d + INTERVAL 1 MONTH, + b Int TTL d + INTERVAL 1 MONTH, + c String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d; +``` + +Varolan bir tablonun sütununa TTL ekleme + +``` sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 DAY; +``` + +Sütun TTL değiştirme + +``` sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 MONTH; +``` + +### Tablo TTL {#mergetree-table-ttl} + +Tablo, süresi dolmuş satırların kaldırılması için bir ifadeye ve parçaların arasında otomatik olarak taşınması için birden fazla ifadeye sahip olabilir [diskler veya birimler](#table_engine-mergetree-multiple-volumes). Tablodaki satırların süresi dolduğunda, ClickHouse ilgili tüm satırları siler. Parça taşıma özelliği için, bir parçanın tüm satırları hareket ifadesi ölçütlerini karşılaması gerekir. + +``` sql +TTL expr [DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'], ... +``` + +TTL kuralı türü her TTL ifadesini takip edebilir. İfade tatmin edildikten sonra yapılacak bir eylemi etkiler (şimdiki zamana ulaşır): + +- `DELETE` - süresi dolmuş satırları sil (varsayılan eylem); +- `TO DISK 'aaa'` - parçayı diske taşı `aaa`; +- `TO VOLUME 'bbb'` - parçayı diske taşı `bbb`. + +Örnekler: + +TTL ile tablo oluşturma + +``` sql +CREATE TABLE example_table +( + d DateTime, + a Int +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d +TTL d + INTERVAL 1 MONTH [DELETE], + d + INTERVAL 1 WEEK TO VOLUME 'aaa', + d + INTERVAL 2 WEEK TO DISK 'bbb'; +``` + +Tablonun TTL değiştirme + +``` sql +ALTER TABLE example_table + MODIFY TTL d + INTERVAL 1 DAY; +``` + +**Verileri Kaldırma** + +ClickHouse veri parçalarını birleştirdiğinde süresi dolmuş bir TTL ile veri kaldırılır. + +ClickHouse, verilerin süresi dolduğunu gördüğünde, zamanlama dışı bir birleştirme gerçekleştirir. Bu tür birleştirmelerin sıklığını kontrol etmek için şunları ayarlayabilirsiniz [merge\_with\_ttl\_timeout](#mergetree_setting-merge_with_ttl_timeout). Değer çok düşükse, çok fazla kaynak tüketebilecek birçok zamanlama dışı birleştirme gerçekleştirir. + +Gerçekleştir theirseniz `SELECT` birleştirme arasında sorgu, süresi dolmuş veri alabilirsiniz. Bunu önlemek için, [OPTIMIZE](../../../sql_reference/statements/misc.md#misc_operations-optimize) önce sorgu `SELECT`. + +## Veri Depolama İçin Birden Fazla Blok Cihazı Kullanma {#table_engine-mergetree-multiple-volumes} + +### Giriş {#introduction} + +`MergeTree` aile tablo motorları birden fazla blok cihazlarda veri saklayabilirsiniz. Örneğin, belirli bir tablonun verileri örtük olarak bölündüğünde yararlı olabilir “hot” ve “cold”. En son veriler düzenli olarak talep edilir, ancak yalnızca az miktarda alan gerektirir. Aksine, yağ kuyruklu tarihsel veriler nadiren talep edilir. Birkaç disk varsa, “hot” veriler hızlı disklerde (örneğin, NVMe SSD'ler veya bellekte) bulunabilir; “cold” veri-nispeten yavaş olanlar (örneğin, HDD). + +Veri kısmı için minimum hareketli birimdir `MergeTree`- motor masaları. Bir parçaya ait veriler bir diskte saklanır. Veri parçaları arka planda diskler arasında (kullanıcı ayarlarına göre) ve aynı zamanda [ALTER](../../../sql_reference/statements/alter.md#alter_move-partition) sorgular. + +### Şartlar {#terms} + +- Disk — Block device mounted to the filesystem. +- Default disk — Disk that stores the path specified in the [yol](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-path) sunucu ayarı. +- Volume — Ordered set of equal disks (similar to [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)). +- Storage policy — Set of volumes and the rules for moving data between them. + +Açıklanan varlıklara verilen isimler sistem tablolarında bulunabilir, [sistem.storage\_policies](../../../operations/system_tables.md#system_tables-storage_policies) ve [sistem.diskler](../../../operations/system_tables.md#system_tables-disks). Bir tablo için yapılandırılmış depolama ilkelerinden birini uygulamak için `storage_policy` ayarı `MergeTree`- motor aile tabloları. + +### Yapılandırma {#table_engine-mergetree-multiple-volumes-configure} + +Diskler, birimler ve depolama politikaları içinde bildirilmelidir `` ana dosyada ya etiket `config.xml` veya farklı bir dosyada `config.d` dizin. + +Yapılandırma yapısı: + +``` xml + + + + /mnt/fast_ssd/clickhouse/ + + + /mnt/hdd1/clickhouse/ + 10485760 + + + /mnt/hdd2/clickhouse/ + 10485760 + + + ... + + + ... + +``` + +Etiketler: + +- `` — Disk name. Names must be different for all disks. +- `path` — path under which a server will store data (`data` ve `shadow` klasörler) ile Sonlandır shouldılmalıdır ‘/’. +- `keep_free_space_bytes` — the amount of free disk space to be reserved. + +Disk tanımının sırası önemli değildir. + +Depolama ilkeleri yapılandırma biçimlendirme: + +``` xml + + ... + + + + + disk_name_from_disks_configuration + 1073741824 + + + + + + + 0.2 + + + + + + + + ... + +``` + +Etiketler: + +- `policy_name_N` — Policy name. Policy names must be unique. +- `volume_name_N` — Volume name. Volume names must be unique. +- `disk` — a disk within a volume. +- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. +- `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). + +Cofiguration örnekleri: + +``` xml + + ... + + + + + disk1 + disk2 + + + + + + + + fast_ssd + 1073741824 + + + disk1 + + + 0.2 + + + ... + +``` + +Verilen örnekte, `hdd_in_order` politika uygular [Ro -und-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) yaklaşma. Böylece bu politika yalnızca bir birim tanımlar (`single`), veri parçaları tüm disklerinde dairesel sırayla saklanır. Bu tür bir politika, sisteme birkaç benzer disk takılıysa, ancak RAID yapılandırılmamışsa oldukça yararlı olabilir. Her bir disk sürücüsünün güvenilir olmadığını ve bunu 3 veya daha fazla çoğaltma faktörü ile telafi etmek isteyebileceğinizi unutmayın. + +Sistemde farklı türde diskler varsa, `moving_from_ssd_to_hdd` politika yerine kullanılabilir. Birim `hot` bir SSD disk oluşur (`fast_ssd`) ve bu birimde saklanabilecek bir parçanın maksimum boyutu 1GB. Tüm parçaları ile boyutu daha büyük 1 GB üzerinde doğrudan saklanır `cold` bir HDD diski içeren birim `disk1`. +Ayrıca, bir kez disk `fast_ssd` 80'den fazla % tarafından doldurulur, veri transfer edilecektir `disk1` bir arka plan işlemi ile. + +Depolama ilkesi içindeki birim numaralandırma sırası önemlidir. Bir birim aşırı doldurulduktan sonra, veriler bir sonrakine taşınır. Disk numaralandırma sırası da önemlidir, çünkü veriler sırayla depolanır. + +Bir tablo oluştururken, yapılandırılmış depolama ilkelerinden birini ona uygulayabilirsiniz: + +``` sql +CREATE TABLE table_with_non_default_policy ( + EventDate Date, + OrderID UInt64, + BannerID UInt64, + SearchPhrase String +) ENGINE = MergeTree +ORDER BY (OrderID, BannerID) +PARTITION BY toYYYYMM(EventDate) +SETTINGS storage_policy = 'moving_from_ssd_to_hdd' +``` + +Bu `default` depolama ilkesi, Yalnızca verilen bir diskten oluşan yalnızca bir birim kullanmayı ima eder ``. Bir tablo oluşturulduktan sonra, depolama ilkesi değiştirilemez. + +### Ayrıntı {#details} + +Bu durumda `MergeTree` tablolar, veriler diske farklı şekillerde giriyor: + +- Bir ekleme sonucunda (`INSERT` sorgu). +- Arka plan birleştirmeleri sırasında ve [mutasyonlar](../../../sql_reference/statements/alter.md#alter-mutations). +- Başka bir kopyadan indirirken. +- Bölüm Don ofması sonucu [ALTER TABLE … FREEZE PARTITION](../../../sql_reference/statements/alter.md#alter_freeze-partition). + +Mutasyonlar ve bölüm dondurma hariç tüm bu durumlarda, bir parça verilen depolama politikasına göre bir birim ve bir diskte saklanır: + +1. Bir parçayı depolamak için yeterli disk alanına sahip olan ilk birim (tanım sırasına göre) (`unreserved_space > current_part_size`) ve belirli bir boyuttaki parçaların saklanmasına izin verir (`max_data_part_size_bytes > current_part_size`) seçilir. +2. Bu birimde, önceki veri yığınını depolamak için kullanılan ve parça boyutundan daha fazla boş alana sahip olan diski izleyen disk seçilir (`unreserved_space - keep_free_space_bytes > current_part_size`). + +Kap hoodut underun altında, [sabit linkler](https://en.wikipedia.org/wiki/Hard_link). Farklı diskler arasındaki sabit bağlantılar desteklenmez, bu nedenle bu gibi durumlarda ortaya çıkan parçalar ilk disklerle aynı disklerde saklanır. + +Arka planda, parçalar boş alan miktarına göre hacimler arasında taşınır (`move_factor` parametre) sırasına göre birimler yapılandırma dosyasında beyan edilir. +Veriler asla sonuncudan ve birincisine aktarılmaz. Bir sistem tabloları kullanabilirsiniz [sistem.part\_log](../../../operations/system_tables.md#system_tables-part-log) (alan `type = MOVE_PART`) ve [sistem.parçalar](../../../operations/system_tables.md#system_tables-parts) (alanlar `path` ve `disk`) arka plan hareketlerini izlemek için. Ayrıca, ayrıntılı bilgi sunucu günlüklerinde bulunabilir. + +Kullanıcı, sorguyu kullanarak bir bölümü veya bölümü bir birimden diğerine taşımaya zorlayabilir [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql_reference/statements/alter.md#alter_move-partition), arka plan işlemleri için tüm kısıtlamalar dikkate alınır. Sorgu, kendi başına bir hareket başlatır ve tamamlanması için arka plan işlemleri beklemez. Yeterli boş alan yoksa veya gerekli koşullardan herhangi biri karşılanmazsa kullanıcı bir hata mesajı alır. + +Veri taşıma veri çoğaltma ile müdahale etmez. Bu nedenle, farklı depolama ilkeleri aynı tablo için farklı yinelemeler üzerinde belirtilebilir. + +Arka plan birleşimlerinin ve mutasyonlarının tamamlanmasından sonra, eski parçalar yalnızca belirli bir süre sonra çıkarılır (`old_parts_lifetime`). +Bu süre zarfında, diğer birimlere veya disklere taşınmazlar. Bu nedenle, parçalar nihayet çıkarılıncaya kadar, işgal edilen disk alanının değerlendirilmesi için hala dikkate alınır. + +[Orijinal makale](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/tr/engines/table_engines/mergetree_family/replacingmergetree.md b/docs/tr/engines/table_engines/mergetree_family/replacingmergetree.md new file mode 100644 index 00000000000..baf2fc98b07 --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/replacingmergetree.md @@ -0,0 +1,69 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 33 +toc_title: ReplacingMergeTree +--- + +# ReplacingMergeTree {#replacingmergetree} + +Motor farklıdır [MergeTree](mergetree.md#table_engines-mergetree) aynı birincil anahtar değerine sahip yinelenen girdileri kaldırır (veya daha doğru bir şekilde, aynı [sıralama anahtarı](mergetree.md) değer). + +Veri tekilleştirme yalnızca birleştirme sırasında oluşur. Birleştirme, arka planda bilinmeyen bir zamanda gerçekleşir, bu nedenle bunu planlayamazsınız. Bazı veriler işlenmemiş kalabilir. Kullanarak programsız bir birleştirme çalıştırabilirsiniz, ancak `OPTIMIZE` sorgu, kullanmaya güvenmeyin, çünkü `OPTIMIZE` sorgu büyük miktarda veri okuyacak ve yazacaktır. + +Böyle, `ReplacingMergeTree` yerden tasarruf etmek için arka planda yinelenen verileri temizlemek için uygundur, ancak kopyaların yokluğunu garanti etmez. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = ReplacingMergeTree([ver]) +[PARTITION BY expr] +[ORDER BY expr] +[PRIMARY KEY expr] +[SAMPLE BY expr] +[SETTINGS name=value, ...] +``` + +İstek parametrelerinin açıklaması için bkz. [istek açıklaması](../../../sql_reference/statements/create.md). + +**ReplacingMergeTree Parametreleri** + +- `ver` — column with version. Type `UInt*`, `Date` veya `DateTime`. İsteğe bağlı parametre. + + Birleş whenirken, `ReplacingMergeTree` aynı birincil anahtara sahip tüm satırlardan sadece bir tane bırakır: + + - Seç inimde son, eğer `ver` set değil. + - Maksimum sürümü ile, eğer `ver` belirtilen. + +**Sorgu yan tümceleri** + +Oluştururken bir `ReplacingMergeTree` tablo aynı [yanlar](mergetree.md) oluşturul ,urken olduğu gibi gerekli `MergeTree` Tablo. + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın ve mümkünse eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE [=] ReplacingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [ver]) +``` + +Hariç tüm parametreler `ver` içinde olduğu gibi aynı anlama sahip `MergeTree`. + +- `ver` - sürümü ile sütun. İsteğe bağlı parametre. Bir açıklama için yukarıdaki metne bakın. + +
+ +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/replacingmergetree/) diff --git a/docs/tr/engines/table_engines/mergetree_family/replication.md b/docs/tr/engines/table_engines/mergetree_family/replication.md new file mode 100644 index 00000000000..e9890c2652f --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/replication.md @@ -0,0 +1,218 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 31 +toc_title: "Veri \xC7o\u011Faltma" +--- + +# Veri Çoğaltma {#table_engines-replication} + +Çoğaltma yalnızca mergetree ailesindeki tablolar için desteklenir: + +- ReplicatedMergeTree +- ReplicatedSummingMergeTree +- ReplicatedReplacingMergeTree +- ReplicatedAggregatingMergeTree +- ReplicatedCollapsingMergeTree +- ReplicatedVersionedCollapsingMergetree +- ReplicatedGraphiteMergeTree + +Çoğaltma, tüm sunucu değil, tek bir tablo düzeyinde çalışır. Bir sunucu hem çoğaltılmış hem de çoğaltılmamış tabloları aynı anda depolayabilir. + +Çoğaltma, parçaya bağlı değildir. Her parçanın kendi bağımsız çoğaltması vardır. + +İçin sıkıştırılmış veri `INSERT` ve `ALTER` sorgular çoğaltılır (daha fazla bilgi için bkz. [ALTER](../../../sql_reference/statements/alter.md#query_language_queries_alter)). + +`CREATE`, `DROP`, `ATTACH`, `DETACH` ve `RENAME` sorgular tek bir sunucuda yürütülür ve çoğaltılmaz: + +- Bu `CREATE TABLE` sorgu sorgu çalıştırıldığı sunucuda yeni bir replicatable tablo oluşturur. Bu tablo diğer sunucularda zaten varsa, yeni bir yineleme ekler. +- Bu `DROP TABLE` sorgu, sorgunun çalıştırıldığı sunucuda bulunan yinelemeyi siler. +- Bu `RENAME` sorgu yinelemeler birinde tabloyu yeniden adlandırır. Başka bir deyişle, çoğaltılmış tablolar farklı yinelemeler üzerinde farklı adlara sahip olabilir. + +ClickHouse kullanır [Apache ZooKeeper](https://zookeeper.apache.org) kopyaları meta bilgilerini saklamak için. ZooKeeper sürüm 3.4.5 veya daha yeni kullanın. + +Çoğaltma kullanmak için, parametreleri [zookeeper](../../../operations/server_configuration_parameters/settings.md#server-settings_zookeeper) sunucu yapılandırma bölümü. + +!!! attention "Dikkat" + Güvenlik ayarını ihmal etmeyin. ClickHouse destekler `digest` [ACLL şeması](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) ZooKeeper Güvenlik alt sisteminin. + +ZooKeeper kümesinin adreslerini ayarlama örneği: + +``` xml + + + example1 + 2181 + + + example2 + 2181 + + + example3 + 2181 + + +``` + +Varolan herhangi bir ZooKeeper kümesini belirtebilirsiniz ve sistem kendi verileri için bir dizin kullanır (replicatable tablo oluştururken dizin belirtilir). + +Zookeeper yapılandırma dosyasında ayarlanmamışsa, çoğaltılmış tablolar oluşturamazsınız ve varolan çoğaltılmış tablolar salt okunur olacaktır. + +ZooKeeper kullanılmaz `SELECT` çoğaltma performansını etkilemez çünkü sorgular `SELECT` ve sorgular, çoğaltılmamış tablolar için yaptıkları kadar hızlı çalışır. Dağıtılmış çoğaltılmış tabloları sorgularken, ClickHouse davranışı ayarlar tarafından denetlenir [max\_replica\_delay\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) ve [fallback\_to\_stale\_replicas\_for\_distributed\_queries](../../../operations/settings/settings.md#settings-fallback_to_stale_replicas_for_distributed_queries). + +Her biri için `INSERT` sorgu, yaklaşık on girişleri zookeeper birkaç işlemler aracılığıyla eklenir. (Daha kesin olmak gerekirse, bu eklenen her veri bloğu içindir; bir ekleme sorgusu her bir blok veya bir blok içerir `max_insert_block_size = 1048576` satırlar.) Bu, biraz daha uzun gecikmelere yol açar `INSERT` çoğaltılmamış tablolarla karşılaştırıldığında. Ancak, birden fazla olmayan gruplar halinde veri eklemek için önerileri izlerseniz `INSERT` saniyede, herhangi bir sorun yaratmaz. Bir ZooKeeper kümesini koordine etmek için kullanılan tüm ClickHouse kümesinin toplam birkaç yüzü vardır `INSERTs` saniyede. Veri eklerindeki verim (saniyede satır sayısı), çoğaltılmamış veriler için olduğu kadar yüksektir. + +Çok büyük kümeler için, farklı kırıklar için farklı ZooKeeper kümelerini kullanabilirsiniz. Ancak, bu Yandex'de gerekli değildir.Metrica küme (yaklaşık 300 sunucu). + +Çoğaltma zaman uyumsuz ve çok ana. `INSERT` sorgular (yanı sıra `ALTER`) mevcut herhangi bir sunucuya gönderilebilir. Veri sorgu çalıştırıldığı sunucuda eklenir ve sonra diğer sunuculara kopyalanır. Zaman uyumsuz olduğundan, son eklenen veriler bazı gecikme ile diğer yinelemeler görünür. Yinelemelerin bir kısmı mevcut değilse, veriler kullanılabilir olduklarında yazılır. Bir çoğaltma varsa, gecikme, sıkıştırılmış veri bloğunu ağ üzerinden aktarmak için gereken süredir. + +Varsayılan olarak, bir INSERT sorgusu yalnızca bir yinelemeden veri yazma onayı bekler. Verileri başarıyla yalnızca bir yineleme için yazılmıştır ve bu yineleme ile sunucu varolmaya sona erer, depolanan veriler kaybolur. Birden çok yinelemeden veri yazma onayını almayı etkinleştirmek için `insert_quorum` seçenek. + +Her veri bloğu atomik olarak yazılır. Ekle sorgusu kadar bloklara ayrılmıştır `max_insert_block_size = 1048576` satırlar. Diğer bir deyişle, `INSERT` sorgu 1048576 satırdan daha az, atomik olarak yapılır. + +Veri blokları tekilleştirilmiştir. Aynı veri bloğunun (aynı sırayla aynı satırları içeren aynı boyuttaki veri blokları) birden fazla yazımı için, blok yalnızca bir kez yazılır. Bunun nedeni, istemci uygulaması verilerin DB'YE yazılıp yazılmadığını bilmediğinde ağ arızaları durumunda, `INSERT` sorgu sadece tekrar edilebilir. Hangi çoğaltma eklerinin aynı verilerle gönderildiği önemli değildir. `INSERTs` idempotent vardır. Tekilleştirme parametreleri tarafından kontrol edilir [merge\_tree](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-merge_tree) sunucu ayarları. + +Çoğaltma sırasında, yalnızca eklenecek kaynak veriler ağ üzerinden aktarılır. Daha fazla veri dönüşümü (birleştirme), tüm kopyalarda aynı şekilde koordine edilir ve gerçekleştirilir. Bu, ağ kullanımını en aza indirir; bu, çoğaltmaların farklı veri merkezlerinde bulunduğu zaman çoğaltmanın iyi çalıştığı anlamına gelir. (Farklı veri merkezlerinde çoğaltmanın çoğaltmanın ana hedefi olduğunu unutmayın .) + +Aynı verilerin çoğaltmaları herhangi bir sayıda olabilir. Üye.Metrica üretimde çift çoğaltma kullanır. Her sunucu, bazı durumlarda RAID-5 veya RAID-6 ve RAID-10 kullanır. Bu nispeten güvenilir ve kullanışlı bir çözümdür. + +Sistem, yinelemelerdeki veri senkronizasyonunu izler ve bir hatadan sonra kurtarabilir. Yük devretme otomatik (verilerde küçük farklılıklar için) veya yarı otomatik (veriler çok fazla farklılık gösterdiğinde, bu da bir yapılandırma hatasını gösterebilir). + +## Çoğaltılmış Tablolar Oluşturma {#creating-replicated-tables} + +Bu `Replicated` önek tablo motoru adına eklenir. Mesela:`ReplicatedMergeTree`. + +**Çoğaltılan \* MergeTree parametreleri** + +- `zoo_path` — The path to the table in ZooKeeper. +- `replica_name` — The replica name in ZooKeeper. + +Örnek: + +``` sql +CREATE TABLE table_name +( + EventDate DateTime, + CounterID UInt32, + UserID UInt32 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}') +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate, intHash32(UserID)) +SAMPLE BY intHash32(UserID) +``` + +
+ +Kullanımdan kaldırılmış sözdizimi örneği + +``` sql +CREATE TABLE table_name +( + EventDate DateTime, + CounterID UInt32, + UserID UInt32 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) +``` + +
+ +Örnekte gösterildiği gibi, bu parametreler kıvırcık köşeli ayraçlarda ikameler içerebilir. İkame edilen değerler ‘macros’ yapılandırma dosyasının bölümü. Örnek: + +``` xml + + 05 + 02 + example05-02-1.yandex.ru + +``` + +Zookeeper tablonun yolunu her çoğaltılmış tablo için benzersiz olmalıdır. Farklı parçalardaki tabloların farklı yolları olmalıdır. +Bu durumda, yol aşağıdaki parçalardan oluşur: + +`/clickhouse/tables/` ortak önek. Tam olarak bunu kullanmanızı öneririz. + +`{layer}-{shard}` shard tanımlayıcısıdır. Bu örnekte Yandex'den beri iki bölümden oluşmaktadır.Metrica küme iki seviyeli sharding kullanır. Çoğu görev için, yalnızca shard tanımlayıcısına genişletilecek olan {shard} ikamesini bırakabilirsiniz. + +`table_name` ZooKeeper tablo için düğüm adıdır. Tablo adı ile aynı yapmak için iyi bir fikirdir. Açıkça tanımlanır, çünkü tablo adının aksine, bir yeniden adlandırma sorgusundan sonra değişmez. +*HINT*: önüne bir veritabanı adı ekleyebilirsiniz `table_name` yanında. E. g. `db_name.table_name` + +Çoğaltma adı, aynı tablonun farklı yinelemelerini tanımlar. Örnekte olduğu gibi bunun için sunucu adını kullanabilirsiniz. Adın sadece her parça içinde benzersiz olması gerekir. + +Değiştirmeleri kullanmak yerine parametreleri açıkça tanımlayabilirsiniz. Bu, test etmek ve küçük kümeleri yapılandırmak için uygun olabilir. Ancak, dağıtılmış DDL sorguları kullanamazsınız (`ON CLUSTER` bu durumda). + +Büyük kümelerle çalışırken, hata olasılığını azalttıkları için değiştirmeleri kullanmanızı öneririz. + +Run the `CREATE TABLE` her yineleme üzerinde sorgu. Bu sorgu, yeni bir çoğaltılmış tablo oluşturur veya varolan bir yeni bir yineleme ekler. + +Tablo zaten diğer yinelemeler üzerinde bazı veriler içerdikten sonra yeni bir yineleme eklerseniz, verileri diğer yinelemeler için yeni bir sorgu çalıştırdıktan sonra kopyalanır. Başka bir deyişle, yeni çoğaltma kendisini diğerleriyle eşitler. + +Bir yineleme silmek için çalıştırın `DROP TABLE`. However, only one replica is deleted – the one that resides on the server where you run the query. + +## Arızalardan Sonra Kurtarma {#recovery-after-failures} + +Bir sunucu başlatıldığında ZooKeeper kullanılamıyorsa, çoğaltılmış tablolar salt okunur moda geçer. Sistem periyodik olarak ZooKeeper bağlanmaya çalışır. + +ZooKeeper sırasında kullanılamıyorsa bir `INSERT`, veya ZooKeeper ile etkileşimde bulunurken bir hata oluşur, bir istisna atılır. + +ZooKeeper bağlandıktan sonra, sistem yerel dosya sistemindeki veri kümesinin beklenen veri kümesiyle eşleşip eşleşmediğini kontrol eder (ZooKeeper bu bilgileri saklar). Küçük tutarsızlıklar varsa, sistem verileri kopyalarla senkronize ederek bunları çözer. + +Sistem bozuk veri parçalarını (yanlış dosya boyutu ile) veya tanınmayan parçaları (dosya sistemine yazılmış ancak Zookeeper'da kaydedilmemiş parçalar) tespit ederse, bunları `detached` alt dizin (silinmez). Eksik parçalar kopyalardan kopyalanır. + +Clickhouse'un büyük miktarda veriyi otomatik olarak silme gibi yıkıcı eylemler gerçekleştirmediğini unutmayın. + +Sunucu başlatıldığında (veya ZooKeeper ile yeni bir oturum kurduğunda), yalnızca tüm dosyaların miktarını ve boyutlarını kontrol eder. Dosya boyutları eşleşirse, ancak bayt ortasında bir yerde değiştirilmişse, bu hemen algılanmaz, ancak yalnızca bir dosya için verileri okumaya çalışırken algılanmaz. `SELECT` sorgu. Sorgu, eşleşen olmayan bir sağlama toplamı veya sıkıştırılmış bir bloğun boyutu hakkında bir özel durum atar. Bu durumda, veri parçaları doğrulama kuyruğuna eklenir ve gerekirse kopyalardan kopyalanır. + +Yerel veri kümesi beklenenden çok fazla farklıysa, bir güvenlik mekanizması tetiklenir. Sunucu bunu günlüğe girer ve başlatmayı reddeder. Bunun nedeni, bu durumda, bir parçadaki bir kopya yanlışlıkla farklı bir parçadaki bir kopya gibi yapılandırılmışsa gibi bir yapılandırma hatası gösterebilir. Ancak, bu mekanizma için eşikleri oldukça düşük ayarlanır ve bu durum normal hata kurtarma sırasında ortaya çıkabilir. Bu durumda, veriler yarı otomatik olarak geri yüklenir “pushing a button”. + +Kurtarma işlemini başlatmak için düğümü oluşturun `/path_to_table/replica_name/flags/force_restore_data` herhangi bir içerik ile ZooKeeper veya tüm çoğaltılmış tabloları geri yüklemek için komutu çalıştırın: + +``` bash +sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data +``` + +Sunucuyu yeniden başlatın. Başlangıçta, sunucu bu bayrakları siler ve kurtarma işlemini başlatır. + +## Tam Veri Kaybından Sonra Kurtarma {#recovery-after-complete-data-loss} + +Tüm veriler ve meta veriler sunuculardan birinden kaybolduysa, kurtarma için şu adımları izleyin: + +1. Clickhouse'u sunucuya yükleyin. Bunları kullanırsanız, shard tanımlayıcısı ve yinelemeleri içeren yapılandırma dosyasında doğru değiştirmelerin tanımlayın. +2. Sunucularda el ile çoğaltılması gereken yinelenmemiş tablolar varsa, verilerini bir kopyadan kopyalayın (dizinde `/var/lib/clickhouse/data/db_name/table_name/`). +3. Bulunan tablo tanım copylarını kopyala `/var/lib/clickhouse/metadata/` bir kopyadan. Tablo tanımlarında bir parça veya çoğaltma tanımlayıcısı açıkça tanımlanmışsa, bu kopyaya karşılık gelecek şekilde düzeltin. (Alternatif olarak, sunucuyu başlatın ve tüm `ATTACH TABLE` içinde olması gereken sorgular .sql dosyaları `/var/lib/clickhouse/metadata/`.) +4. Kurtarma işlemini başlatmak için ZooKeeper düğümünü oluşturun `/path_to_table/replica_name/flags/force_restore_data` herhangi bir içerikle veya tüm çoğaltılmış tabloları geri yüklemek için komutu çalıştırın: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` + +Ardından sunucuyu başlatın (zaten çalışıyorsa yeniden başlatın). Veriler kopyalardan indirilecektir. + +Alternatif bir kurtarma seçeneği zookeeper kayıp yineleme hakkında bilgi silmektir (`/path_to_table/replica_name`), daha sonra açıklandığı gibi yinelemeyi tekrar oluşturun “[Çoğaltılmış tablolar oluşturma](#creating-replicated-tables)”. + +Kurtarma sırasında ağ bant genişliği üzerinde herhangi bir kısıtlama yoktur. Aynı anda birçok yinelemeyi geri yüklüyorsanız bunu aklınızda bulundurun. + +## Mergetree'den Replicatedmergetree'ye Dönüştürme {#converting-from-mergetree-to-replicatedmergetree} + +Terimi kullanıyoruz `MergeTree` tüm tablo motorlarına başvurmak için `MergeTree family` için aynı `ReplicatedMergeTree`. + +Eğer olsaydı bir `MergeTree` el ile çoğaltılmış tablo, çoğaltılmış bir tabloya dönüştürebilirsiniz. Zaten büyük miktarda veri topladıysanız bunu yapmanız gerekebilir. `MergeTree` tablo ve şimdi çoğaltmayı etkinleştirmek istiyorsunuz. + +Veriler çeşitli yinelemelerde farklılık gösteriyorsa, önce onu eşitleyin veya bu verileri biri dışındaki tüm yinelemelerde silin. + +Varolan MergeTree tablosunu yeniden adlandırın, sonra bir `ReplicatedMergeTree` eski adı olan tablo. +Eski tablodan veri taşıma `detached` yeni tablo verileri ile dizin içindeki alt dizin (`/var/lib/clickhouse/data/db_name/table_name/`). +Sonra koş `ALTER TABLE ATTACH PARTITION` bu veri parçalarını çalışma kümesine eklemek için yinelemelerden birinde. + +## Replicatedmergetree'den Mergetree'ye Dönüştürme {#converting-from-replicatedmergetree-to-mergetree} + +Farklı bir adla bir MergeTree tablosu oluşturun. İle dizinden tüm verileri taşıyın `ReplicatedMergeTree` yeni tablonun veri dizinine tablo verileri. Sonra Sil `ReplicatedMergeTree` tablo ve sunucuyu yeniden başlatın. + +Eğer bir kurtulmak istiyorsanız `ReplicatedMergeTree` sunucu başlatmadan tablo: + +- İlgili sil `.sql` meta veri dizinindeki dosya (`/var/lib/clickhouse/metadata/`). +- ZooKeeper ilgili yolu silin (`/path_to_table/replica_name`). + +Bundan sonra, sunucuyu başlatabilir, bir `MergeTree` tablo, verileri kendi dizinine taşıyın ve sonra sunucuyu yeniden başlatın. + +## Zookeeper kümesindeki meta veriler kaybolduğunda veya zarar gördüğünde kurtarma {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged} + +ZooKeeper içindeki veriler kaybolduysa veya hasar gördüyse, verileri yukarıda açıklandığı gibi yinelenmemiş bir tabloya taşıyarak kaydedebilirsiniz. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/tr/engines/table_engines/mergetree_family/summingmergetree.md b/docs/tr/engines/table_engines/mergetree_family/summingmergetree.md new file mode 100644 index 00000000000..15a58064ab4 --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/summingmergetree.md @@ -0,0 +1,141 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 34 +toc_title: SummingMergeTree +--- + +# SummingMergeTree {#summingmergetree} + +Motor devralır [MergeTree](mergetree.md#table_engines-mergetree). Fark, veri parçalarını birleştirirken `SummingMergeTree` tablolar ClickHouse tüm satırları aynı birincil anahtarla değiştirir (veya daha doğru olarak, aynı [sıralama anahtarı](mergetree.md)) sayısal veri türüne sahip sütunlar için özetlenen değerleri içeren bir satır ile. Sıralama anahtarı, tek bir anahtar değeri çok sayıda satıra karşılık gelecek şekilde oluşturulursa, bu, depolama birimini önemli ölçüde azaltır ve veri seçimini hızlandırır. + +Motoru birlikte kullanmanızı öneririz `MergeTree`. Mağaza tam veri `MergeTree` tablo ve kullanım `SummingMergeTree` örneğin, rapor hazırlarken toplu veri depolamak için. Böyle bir yaklaşım, yanlış oluşturulmuş bir birincil anahtar nedeniyle değerli verileri kaybetmenizi önleyecektir. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = SummingMergeTree([columns]) +[PARTITION BY expr] +[ORDER BY expr] +[SAMPLE BY expr] +[SETTINGS name=value, ...] +``` + +İstek parametrelerinin açıklaması için bkz. [istek açıklaması](../../../sql_reference/statements/create.md). + +**SummingMergeTree parametreleri** + +- `columns` - değerlerin özetleneceği sütunların adlarına sahip bir tuple. İsteğe bağlı parametre. + Sütunlar sayısal tipte olmalı ve birincil anahtarda olmamalıdır. + + Eğer `columns` belirtilmemiş, ClickHouse birincil anahtarda olmayan bir sayısal veri türü ile tüm sütunlardaki değerleri özetler. + +**Sorgu yan tümceleri** + +Oluştururken bir `SummingMergeTree` tablo aynı [yanlar](mergetree.md) oluşturul ,urken olduğu gibi gerekli `MergeTree` Tablo. + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın ve mümkünse eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE [=] SummingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [columns]) +``` + +Hariç tüm parametreler `columns` içinde olduğu gibi aynı anlama sahip `MergeTree`. + +- `columns` — tuple with names of columns values of which will be summarized. Optional parameter. For a description, see the text above. + +
+ +## Kullanım Örneği {#usage-example} + +Aşağıdaki tabloyu düşünün: + +``` sql +CREATE TABLE summtt +( + key UInt32, + value UInt32 +) +ENGINE = SummingMergeTree() +ORDER BY key +``` + +Veri Ekle: + +``` sql +INSERT INTO summtt Values(1,1),(1,2),(2,1) +``` + +ClickHouse tüm satırları tamamen toplayabilir ([aşağıya bakın](#data-processing)), bu yüzden bir toplama işlevi kullanıyoruz `sum` ve `GROUP BY` sorguda yan tümcesi. + +``` sql +SELECT key, sum(value) FROM summtt GROUP BY key +``` + +``` text +┌─key─┬─sum(value)─┐ +│ 2 │ 1 │ +│ 1 │ 3 │ +└─────┴────────────┘ +``` + +## Veri İşleme {#data-processing} + +Veriler bir tabloya eklendiğinde, bunlar olduğu gibi kaydedilir. ClickHouse, verilerin eklenen bölümlerini periyodik olarak birleştirir ve bu, aynı birincil anahtara sahip satırların toplandığı ve sonuçta elde edilen her veri parçası için bir tane ile değiştirildiği zamandır. + +ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) bir toplama fonksiyonu [toplam()](../../../sql_reference/aggregate_functions/reference.md#agg_function-sum) ve `GROUP BY` yukarıdaki örnekte açıklandığı gibi yan tümcesi bir sorguda kullanılmalıdır. + +### Toplama İçin Ortak Kurallar {#common-rules-for-summation} + +Sayısal veri türüne sahip sütunlardaki değerler özetlenir. Sütun kümesi parametre tarafından tanımlanır `columns`. + +Değerler toplamı için tüm sütunlarda 0 ise, satır silinir. + +Sütun birincil anahtarda değilse ve özetlenmezse, mevcut olanlardan rasgele bir değer seçilir. + +Değerler, birincil anahtardaki sütunlar için özetlenmez. + +### Aggregatefunction Sütunlarındaki toplama {#the-summation-in-the-aggregatefunction-columns} + +Sütunlar için [AggregateFunction türü](../../../sql_reference/data_types/aggregatefunction.md) ClickHouse olarak davranır [AggregatingMergeTree](aggregatingmergetree.md) işleve göre motor toplama. + +### İç İçe Yapılar {#nested-structures} + +Tablo, özel bir şekilde işlenen iç içe geçmiş veri yapılarına sahip olabilir. + +İç içe geçmiş bir tablonun adı ile bitiyorsa `Map` ve aşağıdaki kriterleri karşılayan en az iki sütun içerir: + +- ilk sütun sayısal `(*Int*, Date, DateTime)` veya bir dize `(String, FixedString)` hadi diyelim `key`, +- diğer sütunlar aritmetik `(*Int*, Float32/64)` hadi diyelim `(values...)`, + +sonra bu iç içe geçmiş tablo bir eşleme olarak yorumlanır `key => (values...)` ve satırlarını birleştirirken, iki veri kümesinin öğeleri şu şekilde birleştirilir `key` karşılık gelen bir toplamı ile `(values...)`. + +Örnekler: + +``` text +[(1, 100)] + [(2, 150)] -> [(1, 100), (2, 150)] +[(1, 100)] + [(1, 150)] -> [(1, 250)] +[(1, 100)] + [(1, 150), (2, 150)] -> [(1, 250), (2, 150)] +[(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] +``` + +Veri isterken, [sumMap (anahtar, değer)](../../../sql_reference/aggregate_functions/reference.md) toplama fonksiyonu `Map`. + +İç içe geçmiş veri yapısı için, sütunlarının toplamı için sütun kümesinde belirtmeniz gerekmez. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/summingmergetree/) diff --git a/docs/tr/engines/table_engines/mergetree_family/versionedcollapsingmergetree.md b/docs/tr/engines/table_engines/mergetree_family/versionedcollapsingmergetree.md new file mode 100644 index 00000000000..b9adb381783 --- /dev/null +++ b/docs/tr/engines/table_engines/mergetree_family/versionedcollapsingmergetree.md @@ -0,0 +1,238 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 37 +toc_title: VersionedCollapsingMergeTree +--- + +# VersionedCollapsingMergeTree {#versionedcollapsingmergetree} + +Bu motor: + +- Sürekli değişen nesne durumlarının hızlı yazılmasını sağlar. +- Arka planda eski nesne durumlarını siler. Bu, depolama hacmini önemli ölçüde azaltır. + +Bölümüne bakınız [Çökme](#table_engines_versionedcollapsingmergetree) ayrıntılar için. + +Motor devralır [MergeTree](mergetree.md#table_engines-mergetree) ve veri parçalarını birleştirmek için algoritmaya satırları daraltmak için mantığı ekler. `VersionedCollapsingMergeTree` aynı amaca hizmet eder [CollapsingMergeTree](collapsingmergetree.md) ancak, verilerin birden çok iş parçacığıyla herhangi bir sıraya yerleştirilmesine izin veren farklı bir çökme algoritması kullanır. Özellikle, `Version` sütun, yanlış sıraya yerleştirilmiş olsalar bile satırları düzgün bir şekilde daraltmaya yardımcı olur. Tersine, `CollapsingMergeTree` sadece kesinlikle ardışık ekleme sağlar. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = VersionedCollapsingMergeTree(sign, version) +[PARTITION BY expr] +[ORDER BY expr] +[SAMPLE BY expr] +[SETTINGS name=value, ...] +``` + +Sorgu parametrelerinin açıklaması için bkz: [sorgu açıklaması](../../../sql_reference/statements/create.md). + +**Motor Parametreleri** + +``` sql +VersionedCollapsingMergeTree(sign, version) +``` + +- `sign` — Name of the column with the type of row: `1` is a “state” satır, `-1` is a “cancel” satır. + + Sütun veri türü olmalıdır `Int8`. + +- `version` — Name of the column with the version of the object state. + + Sütun veri türü olmalıdır `UInt*`. + +**Sorgu Yan Tümceleri** + +Oluştururken bir `VersionedCollapsingMergeTree` tablo, aynı [yanlar](mergetree.md) oluşturul aurken gerekli `MergeTree` Tablo. + +
+ +Bir tablo oluşturmak için kullanımdan kaldırılan yöntem + +!!! attention "Dikkat" + Bu yöntemi yeni projelerde kullanmayın. Mümkünse, eski projeleri yukarıda açıklanan yönteme geçin. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE [=] VersionedCollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign, version) +``` + +Dışındaki tüm parametreler `sign` ve `version` içinde olduğu gibi aynı anlama sahip `MergeTree`. + +- `sign` — Name of the column with the type of row: `1` is a “state” satır, `-1` is a “cancel” satır. + + Column Data Type — `Int8`. + +- `version` — Name of the column with the version of the object state. + + Sütun veri türü olmalıdır `UInt*`. + +
+ +## Çökme {#table_engines-versionedcollapsingmergetree} + +### Veriler {#data} + +Bazı nesneler için sürekli değişen verileri kaydetmeniz gereken bir durumu düşünün. Bir nesne için bir satıra sahip olmak ve değişiklikler olduğunda satırı güncellemek mantıklıdır. Ancak, depolama alanındaki verileri yeniden yazmayı gerektirdiğinden, güncelleştirme işlemi bir DBMS için pahalı ve yavaştır. Verileri hızlı bir şekilde yazmanız gerekiyorsa güncelleştirme kabul edilemez, ancak değişiklikleri bir nesneye sırayla aşağıdaki gibi yazabilirsiniz. + +Kullan... `Sign` satır yazarken sütun. Eğer `Sign = 1` bu, satırın bir nesnenin durumu olduğu anlamına gelir (diyelim “state” satır). Eğer `Sign = -1` aynı özelliklere sahip bir nesnenin durumunun iptal edildiğini gösterir (buna “cancel” satır). Ayrıca kullanın `Version` bir nesnenin her durumunu ayrı bir sayı ile tanımlaması gereken sütun. + +Örneğin, kullanıcıların bazı sitede kaç sayfa ziyaret ettiğini ve ne kadar süre orada olduklarını hesaplamak istiyoruz. Bir noktada, kullanıcı etkinliği durumu ile aşağıdaki satırı yazıyoruz: + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 | +└─────────────────────┴───────────┴──────────┴──────┴─────────┘ +``` + +Bir noktada daha sonra kullanıcı aktivitesinin değişikliğini kaydediyoruz ve aşağıdaki iki satırla yazıyoruz. + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 | +│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 | +└─────────────────────┴───────────┴──────────┴──────┴─────────┘ +``` + +İlk satır, nesnenin (kullanıcı) önceki durumunu iptal eder. Dışında iptal edilen Devletin tüm alanlarını kopya shouldlama shouldlıdır `Sign`. + +İkinci satır geçerli durumu içerir. + +Sadece kullanıcı etkinliğinin son durumuna ihtiyacımız olduğundan, satırlar + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 | +│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 | +└─────────────────────┴───────────┴──────────┴──────┴─────────┘ +``` + +nesnenin geçersiz (eski) durumunu daraltarak silinebilir. `VersionedCollapsingMergeTree` veri parçalarını birleştirirken bunu yapar. + +Her değişiklik için neden iki satıra ihtiyacımız olduğunu bulmak için bkz. [Algoritma](#table_engines-versionedcollapsingmergetree-algorithm). + +**Kullanımı ile ilgili notlar** + +1. Verileri yazan program, iptal etmek için bir nesnenin durumunu hatırlamalıdır. Bu “cancel” dize bir kopyası olmalıdır “state” tersi ile dize `Sign`. Bu, ilk depolama boyutunu arttırır, ancak verileri hızlı bir şekilde yazmanıza izin verir. +2. Sütunlardaki uzun büyüyen diziler, yazma yükü nedeniyle motorun verimliliğini azaltır. Daha basit veri, daha iyi verim. +3. `SELECT` sonuçlara itiraz değişiklikleri tarihinin tutarlılık bağlıdır. Ekleme için veri hazırlarken doğru olun. Oturum derinliği gibi negatif olmayan metrikler için negatif değerler gibi tutarsız verilerle öngörülemeyen sonuçlar alabilirsiniz. + +### Algoritma {#table_engines-versionedcollapsingmergetree-algorithm} + +ClickHouse veri parçalarını birleştirdiğinde, aynı birincil anahtar ve sürüm ve farklı olan her satır çiftini siler `Sign`. Satırların sırası önemli değil. + +ClickHouse veri eklediğinde, satırları birincil anahtarla sipariş eder. Eğer... `Version` sütun birincil anahtarda değil, ClickHouse onu birincil anahtara örtük olarak son alan olarak ekler ve sipariş vermek için kullanır. + +## Veri Seçme {#selecting-data} + +ClickHouse, aynı birincil anahtara sahip tüm satırların aynı sonuçtaki veri bölümünde veya hatta aynı fiziksel sunucuda olacağını garanti etmez. Bu, hem verileri yazmak hem de veri parçalarının daha sonra birleştirilmesi için geçerlidir. Ayrıca, ClickHouse süreçleri `SELECT` birden çok iş parçacıklarıyla sorgular ve sonuçtaki satırların sırasını tahmin edemez. Bu tamamen almak için bir ihtiyaç varsa toplama gerekli olduğu anlamına gelir “collapsed” bir veri `VersionedCollapsingMergeTree` Tablo. + +Daraltmayı sonuçlandırmak için, bir sorgu ile bir sorgu yazın `GROUP BY` yan tümce ve işareti için hesap toplama işlevleri. Örneğin, miktarı hesaplamak için kullanın `sum(Sign)` yerine `count()`. Bir şeyin toplamını hesaplamak için şunları kullanın `sum(Sign * x)` yerine `sum(x)` ve Ekle `HAVING sum(Sign) > 0`. + +Toplanan `count`, `sum` ve `avg` bu şekilde hesaplanabilir. Toplanan `uniq` bir nesnenin en az bir daraltılmamış durumu varsa hesaplanabilir. Toplanan `min` ve `max` hesaplan becauseamaz çünkü `VersionedCollapsingMergeTree` çökmüş durumların değerlerinin geçmişini kaydetmez. + +İle verileri ayıklamak gerekiyorsa “collapsing” ancak toplama olmadan (örneğin, en yeni değerleri belirli koşullarla eşleşen satırların mevcut olup olmadığını kontrol etmek için) `FINAL` değiştirici için `FROM` yan. Bu yaklaşım verimsizdir ve büyük tablolarla kullanılmamalıdır. + +## Kullanım Örneği {#example-of-use} + +Örnek veriler: + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 | +│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 | +│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 | +└─────────────────────┴───────────┴──────────┴──────┴─────────┘ +``` + +Tablo oluşturma: + +``` sql +CREATE TABLE UAct +( + UserID UInt64, + PageViews UInt8, + Duration UInt8, + Sign Int8, + Version UInt8 +) +ENGINE = VersionedCollapsingMergeTree(Sign, Version) +ORDER BY UserID +``` + +Veri ekleme: + +``` sql +INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1, 1) +``` + +``` sql +INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1, 1),(4324182021466249494, 6, 185, 1, 2) +``` + +Biz iki kullanın `INSERT` iki farklı veri parçası oluşturmak için sorgular. Verileri tek bir sorgu ile eklersek, ClickHouse bir veri parçası oluşturur ve hiçbir zaman birleştirme gerçekleştirmez. + +Veri alma: + +``` sql +SELECT * FROM UAct +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┴─────────┘ +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ 1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 │ +└─────────────────────┴───────────┴──────────┴──────┴─────────┘ +``` + +Burada ne görüyoruz ve çökmüş parçalar nerede? +İki veri parçasını iki kullanarak oluşturduk `INSERT` sorgular. Bu `SELECT` sorgu iki iş parçacığında gerçekleştirildi ve sonuç rastgele bir satır sırasıdır. +Veri bölümleri henüz birleştirilmediği için çökme gerçekleşmedi. ClickHouse biz tahmin edemez zaman içinde bilinmeyen bir noktada veri parçalarını birleştirir. + +Bu yüzden toplamaya ihtiyacımız var: + +``` sql +SELECT + UserID, + sum(PageViews * Sign) AS PageViews, + sum(Duration * Sign) AS Duration, + Version +FROM UAct +GROUP BY UserID, Version +HAVING sum(Sign) > 0 +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Version─┐ +│ 4324182021466249494 │ 6 │ 185 │ 2 │ +└─────────────────────┴───────────┴──────────┴─────────┘ +``` + +Toplamaya ihtiyacımız yoksa ve çökmeyi zorlamak istiyorsak, `FINAL` değiştirici için `FROM` yan. + +``` sql +SELECT * FROM UAct FINAL +``` + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ 2 │ +└─────────────────────┴───────────┴──────────┴──────┴─────────┘ +``` + +Bu, verileri seçmek için çok verimsiz bir yoldur. Büyük tablolar için kullanmayın. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/versionedcollapsingmergetree/) diff --git a/docs/tr/engines/table_engines/special/buffer.md b/docs/tr/engines/table_engines/special/buffer.md new file mode 100644 index 00000000000..fa53822ab2b --- /dev/null +++ b/docs/tr/engines/table_engines/special/buffer.md @@ -0,0 +1,71 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 45 +toc_title: Arabellek +--- + +# Arabellek {#buffer} + +RAM'de yazmak için verileri tamponlar, periyodik olarak başka bir tabloya temizler. Okuma işlemi sırasında veri arabellekten ve diğer tablodan aynı anda okunur. + +``` sql +Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) +``` + +Motor parametreleri: + +- `database` – Database name. Instead of the database name, you can use a constant expression that returns a string. +- `table` – Table to flush data to. +- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` bağımsız tamponların. Önerilen değer: 16. +- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, ve `max_bytes` – Conditions for flushing data from the buffer. + +Veri arabellekten temizlendi ve hedef tabloya yazılır eğer tüm `min*` koşulları veya en az bir `max*` koşul karşı arelanır. + +- `min_time`, `max_time` – Condition for the time in seconds from the moment of the first write to the buffer. +- `min_rows`, `max_rows` – Condition for the number of rows in the buffer. +- `min_bytes`, `max_bytes` – Condition for the number of bytes in the buffer. + +Yazma işlemi sırasında veri bir `num_layers` rastgele tampon sayısı. Veya, eklenecek veri kısmı yeterince büyükse (daha büyük `max_rows` veya `max_bytes`), arabelleği atlayarak doğrudan hedef tabloya yazılır. + +Verilerin yıkanması için koşullar, her biri için ayrı ayrı hesaplanır. `num_layers` arabellekler. Örneğin, `num_layers = 16` ve `max_bytes = 100000000`, maksimum RAM tüketimi 1,6 GB'DİR. + +Örnek: + +``` sql +CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000) +``` + +Oluşturma Bir ‘merge.hits\_buffer’ ile aynı yapıya sahip tablo ‘merge.hits’ ve Tampon motorunu kullanarak. Bu tabloya yazarken, veriler RAM'de arabelleğe alınır ve daha sonra ‘merge.hits’ Tablo. 16 tamponlar oluşturulur. 100 saniye geçti veya bir milyon satır yazılmış veya 100 MB veri yazılmıştır; ya da aynı anda 10 saniye geçti ve 10.000 satır ve 10 MB veri yazılmıştır, bunların her veri temizlendi. Örneğin, sadece bir satır yazılmışsa, 100 saniye sonra ne olursa olsun, yıkanacaktır. Ancak, birçok satır yazılmışsa, veriler daha erken temizlenecektir. + +Sunucu DROP TABLE veya DETACH TABLE ile durdurulduğunda, arabellek verileri de hedef tabloya temizlendi. + +Veritabanı ve tablo adı için tek tırnak içinde boş dizeleri ayarlayabilirsiniz. Bu, bir hedef tablonun yokluğunu gösterir. Bu durumda, Veri Temizleme koşullarına ulaşıldığında, arabellek basitçe temizlenir. Bu, bir veri penceresini bellekte tutmak için yararlı olabilir. + +Bir arabellek tablosundan okurken, veriler hem arabellekten hem de hedef tablodan (varsa) işlenir. +Arabellek tabloları bir dizin desteklemediğini unutmayın. Başka bir deyişle, arabellekteki veriler tamamen taranır, bu da büyük arabellekler için yavaş olabilir. (Alt tablodaki veriler için, desteklediği dizin kullanılacaktır.) + +Arabellek tablosundaki sütun kümesi, alt tablodaki sütun kümesiyle eşleşmiyorsa, her iki tabloda da bulunan sütunların bir alt kümesi eklenir. + +Türleri arabellek tablo ve alt tablo sütunlarından biri için eşleşmiyorsa, sunucu günlüğüne bir hata iletisi girilir ve arabellek temizlenir. +Arabellek temizlendiğinde alt tablo yoksa aynı şey olur. + +Eğer bağımlı bir tablo ve Tampon tablo için ALTER çalıştırmak gerekiyorsa, ilk Tampon tablo silme, alt tablo için ALTER çalışan, sonra tekrar Tampon tablo oluşturma öneririz. + +Sunucu anormal şekilde yeniden başlatılırsa, arabellekteki veriler kaybolur. + +Son ve örnek arabellek tabloları için düzgün çalışmıyor. Bu koşullar hedef tabloya geçirilir, ancak arabellekte veri işlemek için kullanılmaz. Bu özellikler gerekiyorsa, hedef tablodan okurken yalnızca yazma için arabellek tablosunu kullanmanızı öneririz. + +Bir arabelleğe veri eklerken, arabelleklerden biri kilitlenir. Bir okuma işlemi aynı anda tablodan gerçekleştiriliyor, bu gecikmelere neden olur. + +Bir arabellek tablosuna eklenen veriler, alt tabloda farklı bir sırada ve farklı bloklarda sonuçlanabilir. Bu nedenle, bir arabellek tablo CollapsingMergeTree doğru yazmak için kullanmak zordur. Sorunları önlemek için şunları ayarlayabilirsiniz ‘num\_layers’ 1'e. + +Hedef tablo yinelenirse, bir arabellek tablosuna yazarken yinelenmiş tabloların bazı beklenen özellikleri kaybolur. Satır ve veri parçaları boyutlarda sipariş için rasgele değişiklikler veri çoğaltma güvenilir olması mümkün olmadığını ifade eden çalışma, kapanmasına neden ‘exactly once’ çoğaltılan tablolara yazın. + +Bu dezavantajlardan dolayı, nadir durumlarda yalnızca bir arabellek tablosu kullanmanızı önerebiliriz. + +Bir arabellek tablosu, bir zaman birimi üzerinden çok sayıda sunucudan çok fazla ekleme alındığında kullanılır ve ekleme işleminden önce veri arabelleğe alınamaz, bu da eklerin yeterince hızlı çalışamayacağı anlamına gelir. + +Arabellek tabloları için bile, her seferinde bir satır veri eklemek mantıklı olmadığını unutmayın. Bu, yalnızca saniyede birkaç bin satırlık bir hız üretirken, daha büyük veri blokları eklemek saniyede bir milyondan fazla satır üretebilir (bölüme bakın “Performance”). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/buffer/) diff --git a/docs/tr/engines/table_engines/special/dictionary.md b/docs/tr/engines/table_engines/special/dictionary.md new file mode 100644 index 00000000000..ce02429a31f --- /dev/null +++ b/docs/tr/engines/table_engines/special/dictionary.md @@ -0,0 +1,97 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 35 +toc_title: "S\xF6zl\xFCk" +--- + +# Sözlük {#dictionary} + +Bu `Dictionary` motor görüntüler [sözlük](../../../sql_reference/dictionaries/external_dictionaries/external_dicts.md) bir ClickHouse tablo olarak veri. + +Örnek olarak, bir sözlük düşünün `products` aşağıdaki yapılandırma ile: + +``` xml + + + products + + +
products
+ DSN=some-db-server + + + + 300 + 360 + + + + + + + product_id + + + title + String + + + + + +``` + +Sözlük verilerini sorgula: + +``` sql +SELECT + name, + type, + key, + attribute.names, + attribute.types, + bytes_allocated, + element_count, + source +FROM system.dictionaries +WHERE name = 'products' +``` + +``` text +┌─name─────┬─type─┬─key────┬─attribute.names─┬─attribute.types─┬─bytes_allocated─┬─element_count─┬─source──────────┐ +│ products │ Flat │ UInt64 │ ['title'] │ ['String'] │ 23065376 │ 175032 │ ODBC: .products │ +└──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ +``` + +Kullanabilirsiniz [dictGet\*](../../../sql_reference/functions/ext_dict_functions.md#ext_dict_functions) sözlük verilerini bu formatta almak için işlev. + +Bu görünüm, ham veri almanız gerektiğinde veya bir `JOIN` işleyiş. Bu durumlar için şunları kullanabilirsiniz `Dictionary` bir tabloda sözlük verilerini görüntüleyen motor. + +Sözdizimi: + +``` sql +CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)` +``` + +Kullanım örneği: + +``` sql +create table products (product_id UInt64, title String) Engine = Dictionary(products); +``` + + Ok + +Masada ne olduğuna bir bak. + +``` sql +select * from products limit 1; +``` + +``` text +┌────product_id─┬─title───────────┐ +│ 152689 │ Some item │ +└───────────────┴─────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/dictionary/) diff --git a/docs/tr/engines/table_engines/special/distributed.md b/docs/tr/engines/table_engines/special/distributed.md new file mode 100644 index 00000000000..28cb0e0855d --- /dev/null +++ b/docs/tr/engines/table_engines/special/distributed.md @@ -0,0 +1,152 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 33 +toc_title: "Da\u011F\u0131l\u0131" +--- + +# Dağılı {#distributed} + +**Dağıtılmış altyapısı olan tablolar kendileri tarafından herhangi bir veri depolamaz**, ancak birden çok sunucuda dağıtılmış sorgu işleme izin verir. +Okuma otomatik olarak paralelleştirilir. Bir okuma sırasında, varsa uzak sunucularda tablo dizinleri kullanılır. + +Dağıtılmış motor parametreleri kabul eder: + +- sunucunun yapılandırma dosyasındaki küme adı + +- uzak veritabanı adı + +- uzak bir tablonun adı + +- (isteğe bağlı olarak) sharding anahtarı + +- (isteğe bağlı olarak) ilke adı, zaman uyumsuz göndermek için geçici dosyaları depolamak için kullanılacaktır + + Ayrıca bakınız: + + - `insert_distributed_sync` ayar + - [MergeTree](../mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes) örnekler için + +Örnek: + +``` sql +Distributed(logs, default, hits[, sharding_key[, policy_name]]) +``` + +Veri tüm sunuculardan okunacak ‘logs’ küme, varsayılan değerden.kümedeki her sunucuda bulunan hits tablosu. +Veriler yalnızca okunmakla kalmaz, aynı zamanda uzak sunucularda kısmen işlenir (bunun mümkün olduğu ölçüde). +Örneğin, GROUP BY ile bir sorgu için uzak sunucularda veri toplanır ve toplama işlevlerinin Ara durumları istek sahibi sunucuya gönderilir. Daha sonra veriler daha fazla toplanacaktır. + +Veritabanı adı yerine, bir dize döndüren sabit bir ifade kullanabilirsiniz. Örneğin: currentDatabase (). + +logs – The cluster name in the server's config file. + +Kümeler şöyle ayarlanır: + +``` xml + + + + + 1 + + false + + example01-01-1 + 9000 + + + example01-01-2 + 9000 + + + + 2 + false + + example01-02-1 + 9000 + + + example01-02-2 + 1 + 9440 + + + + +``` + +Burada bir küme adı ile tanımlanır ‘logs’ bu, her biri iki kopya içeren iki parçadan oluşur. +Kırıklar, verilerin farklı bölümlerini içeren sunuculara başvurur (tüm verileri okumak için tüm kırıklara erişmeniz gerekir). +Yinelemeler sunucuları çoğaltılıyor (tüm verileri okumak için, yinelemelerden herhangi birinde verilere erişebilirsiniz). + +Küme adları nokta içermemelidir. + +Parametre `host`, `port` ve isteğe bağlı olarak `user`, `password`, `secure`, `compression` her sunucu için belirtilir: +- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. +- `port` – The TCP port for messenger activity (‘tcp\_port’ yapılandırmada, genellikle 9000 olarak ayarlanır). Http\_port ile karıştırmayın. +- `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Erişim hakları](../../../operations/access_rights.md). +- `password` – The password for connecting to a remote server (not masked). Default value: empty string. +- `secure` - Bağlantı için ssl kullanın, genellikle de tanımlamanız gerekir `port` = 9440. Sunucu dinlem shouldeli 9440 ve doğru sertifikalara sahip. +- `compression` - Kullanım veri sıkıştırma. Varsayılan değer: true. + +When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) – see the [dengeleme](../../../operations/settings/settings.md#settings-load_balancing) ayar. +Sunucu ile bağlantı kurulmamışsa, kısa bir zaman aşımı ile bağlanma girişimi olacaktır. Bağlantı başarısız olursa, sonraki yineleme seçilir ve benzeri tüm yinelemeler için. Bağlantı girişimi tüm yinelemeler için başarısız olursa, girişimi aynı şekilde, birkaç kez tekrarlanır. +Bu esneklik lehine çalışır, ancak tam hataya dayanıklılık sağlamaz: uzak bir sunucu bağlantıyı kabul edebilir, ancak çalışmayabilir veya kötü çalışabilir. + +Parçalardan yalnızca birini belirtebilirsiniz (bu durumda, sorgu işleme dağıtılmış yerine uzak olarak adlandırılmalıdır) veya herhangi bir sayıda parçaya kadar. Her parçada, bir ila herhangi bir sayıda yinelemeyi belirtebilirsiniz. Her parça için farklı sayıda çoğaltma belirtebilirsiniz. + +Yapılandırmada istediğiniz kadar küme belirtebilirsiniz. + +Kümelerinizi görüntülemek için ‘system.clusters’ Tablo. + +Dağıtılmış motor, yerel bir sunucu gibi bir küme ile çalışmaya izin verir. Ancak, küme uzatılamaz: yapılandırmasını sunucu yapılandırma dosyasına yazmanız gerekir (tüm kümenin sunucuları için daha da iyisi). + +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the ‘remote’ bunun yerine tablo işlevi. Bölümüne bakınız [Tablo fonksiyonları](../../../sql_reference/table_functions/index.md). + +Bir kümeye veri yazmak için iki yöntem vardır: + +İlk olarak, hangi sunucuların hangi verileri yazacağını ve her bir parçaya doğrudan yazmayı gerçekleştireceğini tanımlayabilirsiniz. Başka bir deyişle, dağıtılmış tablo içinde INSERT gerçekleştirmek “looks at”. Bu, konu alanının gereklilikleri nedeniyle önemsiz olmayan herhangi bir sharding şemasını kullanabileceğiniz için en esnek çözümdür. Bu aynı zamanda en uygun çözümdür, çünkü veriler farklı parçalara tamamen bağımsız olarak yazılabilir. + +İkinci olarak, dağıtılmış bir tabloda ekleme gerçekleştirebilirsiniz. Bu durumda, tablo eklenen verileri sunucuların kendisine dağıtacaktır. Dağıtılmış bir tabloya yazmak için, bir sharding anahtar kümesi (son parametre) olmalıdır. Ek olarak, yalnızca bir parça varsa, yazma işlemi sharding anahtarını belirtmeden çalışır, çünkü bu durumda hiçbir şey ifade etmez. + +Her parça yapılandırma dosyasında tanımlanan bir ağırlığa sahip olabilir. Varsayılan olarak, ağırlık bir eşittir. Veriler, parça ağırlığı ile orantılı miktarda parçalara dağıtılır. Örneğin, iki parça varsa ve birincisi 9'luk bir ağırlığa sahipse, ikincisi 10'luk bir ağırlığa sahipse, ilk satırların 9 / 19 parçası gönderilir ve ikincisi 10 / 19 gönderilir. + +Her shard olabilir ‘internal\_replication’ yapılandırma dosyasında tanımlanan parametre. + +Bu parametre şu şekilde ayarlanırsa ‘true’, yazma işlemi ilk sağlıklı yinelemeyi seçer ve ona veri yazar. Dağıtılmış tablo ise bu alternatifi kullanın “looks at” çoğaltılan tablolar. Başka bir deyişle, verilerin yazılacağı tablo kendilerini çoğaltacaktır. + +Olarak ayarlan ifmışsa ‘false’ (varsayılan), veriler tüm kopyalara yazılır. Özünde, bu, dağıtılmış tablonun verilerin kendisini çoğalttığı anlamına gelir. Bu, çoğaltılmış tabloları kullanmaktan daha kötüdür, çünkü kopyaların tutarlılığı denetlenmez ve zamanla biraz farklı veriler içerirler. + +Bir veri satırının gönderildiği parçayı seçmek için, parçalama ifadesi analiz edilir ve kalan kısmı, parçaların toplam ağırlığına bölünmesinden alınır. Satır, kalanların yarı aralığına karşılık gelen parçaya gönderilir. ‘prev\_weight’ -e doğru ‘prev\_weights + weight’, nere ‘prev\_weights’ en küçük sayıya sahip parçaların toplam ağırlığı ve ‘weight’ bu parçanın ağırlığı. Örneğin, iki parça varsa ve birincisi 9'luk bir ağırlığa sahipse, ikincisi 10'luk bir ağırlığa sahipse, satır \[0, 9) aralığından kalanlar için ilk parçaya ve ikincisine \[9, 19) aralığından kalanlar için gönderilecektir. + +Sharding ifadesi, bir tamsayı döndüren sabitler ve tablo sütunlarından herhangi bir ifade olabilir. Örneğin, ifadeyi kullanabilirsiniz ‘rand()’ verilerin rastgele dağılımı için veya ‘UserID’ kullanıcının kimliğinin bölünmesinden kalanın dağıtımı için (daha sonra tek bir kullanıcının verileri, kullanıcılar tarafından çalışmayı ve katılmayı basitleştiren tek bir parçada bulunur). Sütunlardan biri yeterince eşit olarak dağıtılmazsa, onu bir karma işleve sarabilirsiniz: ınthash64(Userıd). + +Bölüm'den basit bir hatırlatma, sharding için sınırlı bir çözümdür ve her zaman uygun değildir. Orta ve büyük hacimlerde veri (düzinelerce sunucu) için çalışır, ancak çok büyük hacimlerde veri (yüzlerce sunucu veya daha fazla) için değildir. İkinci durumda, dağıtılmış tablolarda girdileri kullanmak yerine konu alanı tarafından gerekli olan sharding şemasını kullanın. + +SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. + +Aşağıdaki durumlarda sharding şeması hakkında endişelenmelisiniz: + +- Belirli bir anahtar tarafından veri (veya birleştirme) birleştirme gerektiren sorgular kullanılır. Veriler bu anahtar tarafından parçalanırsa, GLOBAL IN veya GLOBAL JOİN yerine local IN veya JOİN kullanabilirsiniz, bu da çok daha etkilidir. +- Çok sayıda küçük Sorgu ile çok sayıda sunucu (yüzlerce veya daha fazla) kullanılır (bireysel müşterilerin sorguları - web siteleri, reklamverenler veya ortaklar). Küçük sorguların tüm kümeyi etkilememesi için, tek bir istemci için tek bir parça üzerinde veri bulmak mantıklıdır. Alternatif olarak, Yandex'te yaptığımız gibi.Metrica, iki seviyeli sharding kurabilirsiniz: tüm kümeyi bölün “layers”, bir katmanın birden fazla parçadan oluşabileceği yer. Tek bir istemci için veriler tek bir katmanda bulunur, ancak kırıklar gerektiğinde bir katmana eklenebilir ve veriler rastgele dağıtılır. Her katman için dağıtılmış tablolar oluşturulur ve genel sorgular için tek bir paylaşılan dağıtılmış tablo oluşturulur. + +Veriler zaman uyumsuz olarak yazılır. Tabloya eklendiğinde, veri bloğu sadece yerel dosya sistemine yazılır. Veriler en kısa sürede arka planda uzak sunuculara gönderilir. Veri gönderme süresi tarafından yönetilir [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) ve [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) ayarlar. Bu `Distributed` motor ayrı ayrı eklenen verilerle her dosyayı gönderir, ancak toplu dosya gönderme etkinleştirebilirsiniz [distributed\_directory\_monitor\_batch\_ınserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) ayar. Bu ayar, yerel sunucu ve ağ kaynaklarını daha iyi kullanarak küme performansını artırır. Tablo dizinindeki dosyaların listesini (gönderilmeyi bekleyen veriler) kontrol ederek verilerin başarıyla gönderilip gönderilmediğini kontrol etmelisiniz: `/var/lib/clickhouse/data/database/table/`. + +Sunucu varlığını durdurdu veya (örneğin, bir aygıt arızasından sonra) dağıtılmış bir tabloya bir ekleme sonra kaba bir yeniden başlatma vardı, eklenen veriler kaybolabilir. Tablo dizininde bozuk bir veri parçası tespit edilirse, ‘broken’ alt dizin ve artık kullanılmıyor. + +Max\_parallel\_replicas seçeneği etkinleştirildiğinde, sorgu işleme tek bir parça içindeki tüm yinelemeler arasında paralelleştirilir. Daha fazla bilgi için bölüme bakın [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). + +## Sanal Sütunlar {#virtual-columns} + +- `_shard_num` — Contains the `shard_num` (itibaren `system.clusters`). Tür: [Uİnt32](../../../sql_reference/data_types/int_uint.md). + +!!! note "Not" + Beri [`remote`](../../../sql_reference/table_functions/remote.md)/`cluster` tablo işlevleri DAHİLİ olarak aynı dağıtılmış altyapının geçici örneğini oluşturur, `_shard_num` de kullanılabilir. + +**Ayrıca Bakınız** + +- [Sanal sütunlar](index.md#table_engines-virtual_columns) + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/distributed/) diff --git a/docs/tr/engines/table_engines/special/external_data.md b/docs/tr/engines/table_engines/special/external_data.md new file mode 100644 index 00000000000..123b2dbdf0e --- /dev/null +++ b/docs/tr/engines/table_engines/special/external_data.md @@ -0,0 +1,68 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 34 +toc_title: "D\u0131\u015F veri" +--- + +# Sorgu işleme için harici veriler {#external-data-for-query-processing} + +ClickHouse bir sunucu bir SELECT sorgusu ile birlikte bir sorgu işlemek için gerekli olan verileri gönderme sağlar. Bu veriler geçici bir tabloya konur (bölüme bakın “Temporary tables”) ve sorguda kullanılabilir (örneğin, işleçlerde). + +Örneğin, önemli kullanıcı tanımlayıcılarına sahip bir metin dosyanız varsa, bu listeyi süzme kullanan bir sorgu ile birlikte sunucuya yükleyebilirsiniz. + +Büyük hacimli dış verilerle birden fazla sorgu çalıştırmanız gerekiyorsa, bu özelliği kullanmayın. Verileri vaktinden önce DB'YE yüklemek daha iyidir. + +Harici veriler komut satırı istemcisi (etkileşimli olmayan modda) veya HTTP arabirimi kullanılarak yüklenebilir. + +Komut satırı istemcisinde, formatta bir parametreler bölümü belirtebilirsiniz + +``` bash +--external --file=... [--name=...] [--format=...] [--types=...|--structure=...] +``` + +İletilen tablo sayısı için bunun gibi birden çok bölümünüz olabilir. + +**–external** – Marks the beginning of a clause. +**–file** – Path to the file with the table dump, or -, which refers to stdin. +Stdın'den yalnızca tek bir tablo alınabilir. + +Aşağıdaki parametreler isteğe bağlıdır: **–name**– Name of the table. If omitted, \_data is used. +**–format** – Data format in the file. If omitted, TabSeparated is used. + +Aşağıdaki parametrelerden biri gereklidir:**–types** – A list of comma-separated column types. For example: `UInt64,String`. The columns will be named \_1, \_2, … +**–structure**– The table structure in the format`UserID UInt64`, `URL String`. Sütun adlarını ve türlerini tanımlar. + +Belirtilen dosyalar ‘file’ belirtilen biçimde ayrıştırılır ‘format’, belirtilen veri türlerini kullanarak ‘types’ veya ‘structure’. Tablo sunucuya yüklenecek ve orada adı ile geçici bir tablo olarak erişilebilir ‘name’. + +Örnekler: + +``` bash +$ echo -ne "1\n2\n3\n" | clickhouse-client --query="SELECT count() FROM test.visits WHERE TraficSourceID IN _data" --external --file=- --types=Int8 +849897 +$ cat /etc/passwd | sed 's/:/\t/g' | clickhouse-client --query="SELECT shell, count() AS c FROM passwd GROUP BY shell ORDER BY c DESC" --external --file=- --name=passwd --structure='login String, unused String, uid UInt16, gid UInt16, comment String, home String, shell String' +/bin/sh 20 +/bin/false 5 +/bin/bash 4 +/usr/sbin/nologin 1 +/bin/sync 1 +``` + +HTTP arabirimini kullanırken, dış veriler çok parçalı/form veri biçiminde geçirilir. Her tablo ayrı bir dosya olarak iletilir. Tablo adı dosya adından alınır. Bu ‘query\_string’ parametreleri geçirilir ‘name\_format’, ‘name\_types’, ve ‘name\_structure’, nere ‘name’ bu parametreler karşılık gelen tablonun adıdır. Parametrelerin anlamı, komut satırı istemcisini kullanırken olduğu gibi aynıdır. + +Örnek: + +``` bash +$ cat /etc/passwd | sed 's/:/\t/g' > passwd.tsv + +$ curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+count()+AS+c+FROM+passwd+GROUP+BY+shell+ORDER+BY+c+DESC&passwd_structure=login+String,+unused+String,+uid+UInt16,+gid+UInt16,+comment+String,+home+String,+shell+String' +/bin/sh 20 +/bin/false 5 +/bin/bash 4 +/usr/sbin/nologin 1 +/bin/sync 1 +``` + +Dağıtılmış sorgu işleme için geçici tablolar tüm uzak sunuculara gönderilir. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/external_data/) diff --git a/docs/tr/engines/table_engines/special/file.md b/docs/tr/engines/table_engines/special/file.md new file mode 100644 index 00000000000..c132d1dd5a4 --- /dev/null +++ b/docs/tr/engines/table_engines/special/file.md @@ -0,0 +1,90 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 37 +toc_title: Dosya +--- + +# Dosya {#table_engines-file} + +Dosya tablosu altyapısı, verileri desteklenen dosyalardan birinde tutar [Dosya +biçimliler](../../../interfaces/formats.md#formats) (TabSeparated, yerli, vb.). + +Kullanım örnekleri: + +- Clickhouse'dan dosyaya veri aktarımı. +- Verileri bir biçimden diğerine dönüştürün. +- Bir diskte bir dosya düzenleme yoluyla ClickHouse veri güncelleme. + +## ClickHouse Sunucusunda Kullanım {#usage-in-clickhouse-server} + +``` sql +File(Format) +``` + +Bu `Format` parametre kullanılabilir dosya biçimlerinden birini belirtir. Gerçekleştirmek +`SELECT` sorgular, biçim giriş için desteklenmeli ve gerçekleştirmek için +`INSERT` queries – for output. The available formats are listed in the +[Biçimliler](../../../interfaces/formats.md#formats) bölme. + +ClickHouse dosya sistemi yolunu belirtmek için izin vermiyor`File`. Tarafından tanımlanan klasörü kullan willacaktır [yol](../../../operations/server_configuration_parameters/settings.md) sunucu yapılandırmasında ayarlama. + +Kullanarak tablo oluştururken `File(Format)` bu klasörde boş bir alt dizin oluşturur. Veri o tabloya yazıldığında, içine konur `data.Format` bu alt dizinde dosya. + +Bu alt klasörü ve dosyayı sunucu dosya sisteminde el ile oluşturabilir ve sonra [ATTACH](../../../sql_reference/statements/misc.md) eşleşen ada sahip tablo bilgilerine, böylece bu dosyadan veri sorgulayabilirsiniz. + +!!! warning "Uyarıcı" + Bu işlevselliğe dikkat edin, çünkü ClickHouse bu tür dosyalarda harici değişiklikleri izlemez. ClickHouse ve ClickHouse dışında eşzamanlı yazma sonucu tanımsızdır. + +**Örnek:** + +**1.** Set up the `file_engine_table` Tablo: + +``` sql +CREATE TABLE file_engine_table (name String, value UInt32) ENGINE=File(TabSeparated) +``` + +Varsayılan olarak ClickHouse klasör oluşturur `/var/lib/clickhouse/data/default/file_engine_table`. + +**2.** El ile oluştur `/var/lib/clickhouse/data/default/file_engine_table/data.TabSeparated` içerme: + +``` bash +$ cat data.TabSeparated +one 1 +two 2 +``` + +**3.** Verileri sorgula: + +``` sql +SELECT * FROM file_engine_table +``` + +``` text +┌─name─┬─value─┐ +│ one │ 1 │ +│ two │ 2 │ +└──────┴───────┘ +``` + +## Clickhouse'da kullanım-yerel {#usage-in-clickhouse-local} + +İçinde [clickhouse-yerel](../../../operations/utilities/clickhouse-local.md) Dosya motoru ek olarak dosya yolunu kabul eder `Format`. Varsayılan giriş / çıkış akışları gibi sayısal veya insan tarafından okunabilir isimler kullanılarak belirtilebilir `0` veya `stdin`, `1` veya `stdout`. +**Örnek:** + +``` bash +$ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" +``` + +## Uygulama Detayları {#details-of-implementation} + +- Çoklu `SELECT` sorgular aynı anda yapılabilir, ancak `INSERT` sorgular birbirini bekler. +- Tarafından yeni dosya oluşturma desteklenen `INSERT` sorgu. +- Dosya varsa, `INSERT` içinde yeni değerler ekler. +- Desteklenmiyor: + - `ALTER` + - `SELECT ... SAMPLE` + - Dizinler + - Çoğalma + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/file/) diff --git a/docs/tr/engines/table_engines/special/generate.md b/docs/tr/engines/table_engines/special/generate.md new file mode 100644 index 00000000000..01d2534441e --- /dev/null +++ b/docs/tr/engines/table_engines/special/generate.md @@ -0,0 +1,61 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 46 +toc_title: GenerateRandom +--- + +# Generaterandom {#table_engines-generate} + +GenerateRandom tablo motoru, verilen tablo şeması için rasgele veri üretir. + +Kullanım örnekleri: + +- Tekrarlanabilir büyük tabloyu doldurmak için testte kullanın. +- Fuzzing testleri için rastgele girdi oluşturun. + +## ClickHouse Sunucusunda Kullanım {#usage-in-clickhouse-server} + +``` sql +ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) +``` + +Bu `max_array_length` ve `max_string_length` parametreler tüm maksimum uzunluğu belirtin +oluşturulan verilerde dizi sütunları ve dizeleri. + +Tablo motoru oluşturmak yalnızca destekler `SELECT` sorgular. + +Tüm destekler [Veri türleri](../../../sql_reference/data_types/index.md) dışında bir tabloda saklanabilir `LowCardinality` ve `AggregateFunction`. + +**Örnek:** + +**1.** Set up the `generate_engine_table` Tablo: + +``` sql +CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE = GenerateRandom(1, 5, 3) +``` + +**2.** Verileri sorgula: + +``` sql +SELECT * FROM generate_engine_table LIMIT 3 +``` + +``` text +┌─name─┬──────value─┐ +│ c4xJ │ 1412771199 │ +│ r │ 1791099446 │ +│ 7#$ │ 124312908 │ +└──────┴────────────┘ +``` + +## Uygulama Detayları {#details-of-implementation} + +- Desteklenmiyor: + - `ALTER` + - `SELECT ... SAMPLE` + - `INSERT` + - Dizinler + - Çoğalma + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/generate/) diff --git a/docs/tr/engines/table_engines/special/index.md b/docs/tr/engines/table_engines/special/index.md new file mode 100644 index 00000000000..2e754a86bc8 --- /dev/null +++ b/docs/tr/engines/table_engines/special/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "\xD6zellikli" +toc_priority: 31 +--- + + diff --git a/docs/tr/engines/table_engines/special/join.md b/docs/tr/engines/table_engines/special/join.md new file mode 100644 index 00000000000..2fa786148e0 --- /dev/null +++ b/docs/tr/engines/table_engines/special/join.md @@ -0,0 +1,111 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 40 +toc_title: Katmak +--- + +# Katmak {#join} + +Kullanılmak üzere hazırlanmış veri yapısı [JOIN](../../../sql_reference/statements/select.md#select-join) harekat. + +## Tablo oluşturma {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], +) ENGINE = Join(join_strictness, join_type, k1[, k2, ...]) +``` + +Ayrıntılı açıklamasına bakın [CREATE TABLE](../../../sql_reference/statements/create.md#create-table-query) sorgu. + +**Motor Parametreleri** + +- `join_strictness` – [Katılık katılın](../../../sql_reference/statements/select.md#select-join-strictness). +- `join_type` – [Birleştirme türü](../../../sql_reference/statements/select.md#select-join-types). +- `k1[, k2, ...]` – Key columns from the `USING` fık thera: `JOIN` işlemi yapılmamaktadır. + +Girmek `join_strictness` ve `join_type` tırnak işaretleri olmadan parametreler, örneğin, `Join(ANY, LEFT, col1)`. Onlar eşleşmelidir `JOIN` tablo için kullanılacak işlem. Parametreler eşleşmezse, ClickHouse bir istisna atmaz ve yanlış veri döndürebilir. + +## Tablo Kullanımı {#table-usage} + +### Örnek {#example} + +Sol taraftaki tablo oluşturma: + +``` sql +CREATE TABLE id_val(`id` UInt32, `val` UInt32) ENGINE = TinyLog +``` + +``` sql +INSERT INTO id_val VALUES (1,11)(2,12)(3,13) +``` + +Sağ tarafı oluşturma `Join` Tablo: + +``` sql +CREATE TABLE id_val_join(`id` UInt32, `val` UInt8) ENGINE = Join(ANY, LEFT, id) +``` + +``` sql +INSERT INTO id_val_join VALUES (1,21)(1,22)(3,23) +``` + +Tabloları birleştirme: + +``` sql +SELECT * FROM id_val ANY LEFT JOIN id_val_join USING (id) SETTINGS join_use_nulls = 1 +``` + +``` text +┌─id─┬─val─┬─id_val_join.val─┐ +│ 1 │ 11 │ 21 │ +│ 2 │ 12 │ ᴺᵁᴸᴸ │ +│ 3 │ 13 │ 23 │ +└────┴─────┴─────────────────┘ +``` + +Alternatif olarak, veri alabilirsiniz `Join` tablo, birleştirme anahtarı değerini belirterek: + +``` sql +SELECT joinGet('id_val_join', 'val', toUInt32(1)) +``` + +``` text +┌─joinGet('id_val_join', 'val', toUInt32(1))─┐ +│ 21 │ +└────────────────────────────────────────────┘ +``` + +### Veri seçme ve ekleme {#selecting-and-inserting-data} + +Kullanabilirsiniz `INSERT` veri eklemek için sorgular `Join`- motor masaları. Tablo ile oluşturulmuş ise `ANY` katılık, yinelenen anahtarlar için veriler göz ardı edilir. İle... `ALL` katılık, tüm satırlar eklenir. + +Gerçekleştir aemezsiniz `SELECT` doğrudan tablodan sorgulayın. Bunun yerine, aşağıdaki yöntemlerden birini kullanın: + +- Tabloyu sağ tarafa yerleştirin. `JOIN` yan. +- Call the [joinGet](../../../sql_reference/functions/other_functions.md#joinget) tablodan bir sözlükten aynı şekilde veri ayıklamanızı sağlayan işlev. + +### Sınırlamalar ve Ayarlar {#join-limitations-and-settings} + +Bir tablo oluştururken aşağıdaki ayarlar uygulanır: + +- [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) +- [max\_rows\_in\_join](../../../operations/settings/query_complexity.md#settings-max_rows_in_join) +- [max\_bytes\_in\_join](../../../operations/settings/query_complexity.md#settings-max_bytes_in_join) +- [join\_overflow\_mode](../../../operations/settings/query_complexity.md#settings-join_overflow_mode) +- [join\_any\_take\_last\_row](../../../operations/settings/settings.md#settings-join_any_take_last_row) + +Bu `Join`- motor tabloları kullanılamaz `GLOBAL JOIN` harekat. + +Bu `Join`- motor kullanımına izin verir [join\_use\_nulls](../../../operations/settings/settings.md#join_use_nulls) ayarı `CREATE TABLE` deyim. Ve [SELECT](../../../sql_reference/statements/select.md) sorgu kullanımına izin verir `join_use_nulls` çok. Eğer farklı varsa `join_use_nulls` ayarlar, tablo birleştirme bir hata alabilirsiniz. Bu katılmak türüne bağlıdır. Kullandığınızda [joinGet](../../../sql_reference/functions/other_functions.md#joinget) fonksiyonu, aynı kullanmak zorunda `join_use_nulls` ayarı `CRATE TABLE` ve `SELECT` deyimler. + +## Veri Depolama {#data-storage} + +`Join` tablo verileri her zaman RAM'de bulunur. Bir tabloya satır eklerken, sunucu yeniden başlatıldığında geri yüklenebilir, böylece ClickHouse disk üzerindeki dizine veri bloklarını yazar. + +Sunucu yanlış yeniden başlatılırsa, diskteki veri bloğu kaybolabilir veya zarar görebilir. Bu durumda, dosyayı hasarlı verilerle el ile silmeniz gerekebilir. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/join/) diff --git a/docs/tr/engines/table_engines/special/materializedview.md b/docs/tr/engines/table_engines/special/materializedview.md new file mode 100644 index 00000000000..23e7d3122eb --- /dev/null +++ b/docs/tr/engines/table_engines/special/materializedview.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 43 +toc_title: MaterializedView +--- + +# Materializedview {#materializedview} + +Somut görünümler uygulamak için kullanılır (Daha fazla bilgi için bkz . [CREATE TABLE](../../../sql_reference/statements/create.md)). Verileri depolamak için, görünümü oluştururken belirtilen farklı bir motor kullanır. Bir tablodan okurken, sadece bu motoru kullanır. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/materializedview/) diff --git a/docs/tr/engines/table_engines/special/memory.md b/docs/tr/engines/table_engines/special/memory.md new file mode 100644 index 00000000000..a5f985ff5e2 --- /dev/null +++ b/docs/tr/engines/table_engines/special/memory.md @@ -0,0 +1,19 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 44 +toc_title: Bellek +--- + +# Bellek {#memory} + +Bellek altyapısı verileri RAM, sıkıştırılmamış biçimde depolar. Veri okunduğunda alınan tam olarak aynı biçimde saklanır. Başka bir deyişle, bu tablodan okuma tamamen ücretsizdir. +Eşzamanlı veri erişimi senkronize edilir. Kilitler kısa: okuma ve yazma işlemleri birbirini engellemez. +Dizinler desteklenmiyor. Okuma paralelleştirilmiştir. +Basit sorgularda maksimum üretkenliğe (10 GB/sn'den fazla) ulaşılır, çünkü diskten okuma, açma veya veri serisini kaldırma yoktur. (Birçok durumda MergeTree motorunun verimliliğinin neredeyse yüksek olduğunu unutmamalıyız.) +Bir sunucu yeniden başlatılırken, veri tablodan kaybolur ve tablo boş olur. +Normalde, bu tablo motorunu kullanmak haklı değildir. Bununla birlikte, testler ve nispeten az sayıda satırda (yaklaşık 100.000.000'a kadar) maksimum hızın gerekli olduğu görevler için kullanılabilir. + +Bellek motoru, harici sorgu verilerine sahip geçici tablolar için sistem tarafından kullanılır (bkz. “External data for processing a query”) ve GLOBAL In uygulanması için (bkz. “IN operators”). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/memory/) diff --git a/docs/tr/engines/table_engines/special/merge.md b/docs/tr/engines/table_engines/special/merge.md new file mode 100644 index 00000000000..bd4150e1afb --- /dev/null +++ b/docs/tr/engines/table_engines/special/merge.md @@ -0,0 +1,70 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 36 +toc_title: "Birle\u015Ftirmek" +--- + +# Birleştirmek {#merge} + +Bu `Merge` motor (ile karıştırılmamalıdır `MergeTree`) verileri kendisi saklamaz, ancak aynı anda herhangi bir sayıda başka tablodan okumaya izin verir. +Okuma otomatik olarak paralelleştirilir. Bir tabloya yazma desteklenmiyor. Okurken, gerçekten okunmakta olan tabloların dizinleri varsa kullanılır. +Bu `Merge` motor parametreleri kabul eder: veritabanı adı ve tablolar için düzenli ifade. + +Örnek: + +``` sql +Merge(hits, '^WatchLog') +``` + +Veri tablolardan okunacak `hits` düzenli ifadeyle eşleşen adlara sahip veritabanı ‘`^WatchLog`’. + +Veritabanı adı yerine, bir dize döndüren sabit bir ifade kullanabilirsiniz. Mesela, `currentDatabase()`. + +Regular expressions — [re2](https://github.com/google/re2) (pcre bir alt kümesini destekler), büyük / küçük harf duyarlı. +Düzenli ifadelerde kaçan sembollerle ilgili notlara bakın “match” bölme. + +Okumak için tabloları seçerken, `Merge` regex ile eşleşse bile tablonun kendisi seçilmeyecektir. Bu döngülerden kaçınmaktır. +İki tane oluşturmak mümkündür `Merge` sonsuza kadar birbirlerinin verilerini okumaya çalışacak tablolar, ancak bu iyi bir fikir değil. + +Kullanmak için tipik bir yol `Merge` motor çok sayıda çalışma içindir `TinyLog` tablolar tek bir tablo ile sanki. + +Örnek 2: + +Diyelim ki eski bir tablonuz (WatchLog\_old) var ve verileri yeni bir tabloya (WatchLog\_new) taşımadan bölümlemeyi değiştirmeye karar verdiniz ve her iki tablodaki verileri görmeniz gerekiyor. + +``` sql +CREATE TABLE WatchLog_old(date Date, UserId Int64, EventType String, Cnt UInt64) +ENGINE=MergeTree(date, (UserId, EventType), 8192); +INSERT INTO WatchLog_old VALUES ('2018-01-01', 1, 'hit', 3); + +CREATE TABLE WatchLog_new(date Date, UserId Int64, EventType String, Cnt UInt64) +ENGINE=MergeTree PARTITION BY date ORDER BY (UserId, EventType) SETTINGS index_granularity=8192; +INSERT INTO WatchLog_new VALUES ('2018-01-02', 2, 'hit', 3); + +CREATE TABLE WatchLog as WatchLog_old ENGINE=Merge(currentDatabase(), '^WatchLog'); + +SELECT * +FROM WatchLog +``` + +``` text +┌───────date─┬─UserId─┬─EventType─┬─Cnt─┐ +│ 2018-01-01 │ 1 │ hit │ 3 │ +└────────────┴────────┴───────────┴─────┘ +┌───────date─┬─UserId─┬─EventType─┬─Cnt─┐ +│ 2018-01-02 │ 2 │ hit │ 3 │ +└────────────┴────────┴───────────┴─────┘ +``` + +## Sanal Sütunlar {#virtual-columns} + +- `_table` — Contains the name of the table from which data was read. Type: [Dize](../../../sql_reference/data_types/string.md). + + Sabit koşulları ayarlayabilirsiniz `_table` in the `WHERE/PREWHERE` fıkra (sı (örneğin, `WHERE _table='xyz'`). Bu durumda, okuma işlemi yalnızca koşulun açık olduğu tablolar için gerçekleştirilir `_table` memnun olduğunu, bu yüzden `_table` sütun bir dizin görevi görür. + +**Ayrıca Bakınız** + +- [Sanal sütunlar](index.md#table_engines-virtual_columns) + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/merge/) diff --git a/docs/tr/engines/table_engines/special/null.md b/docs/tr/engines/table_engines/special/null.md new file mode 100644 index 00000000000..19d518f415f --- /dev/null +++ b/docs/tr/engines/table_engines/special/null.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 38 +toc_title: "Bo\u015F" +--- + +# Boş {#null} + +Boş bir tabloya yazarken, veri yoksayılır. Boş bir tablodan okurken, yanıt boştur. + +Ancak, boş bir tablo üzerinde materialized bir görünüm oluşturabilirsiniz. Böylece tabloya yazılan veriler görünümde sona erecek. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/null/) diff --git a/docs/tr/engines/table_engines/special/set.md b/docs/tr/engines/table_engines/special/set.md new file mode 100644 index 00000000000..af94ada6e12 --- /dev/null +++ b/docs/tr/engines/table_engines/special/set.md @@ -0,0 +1,19 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 39 +toc_title: Koymak +--- + +# Koymak {#set} + +Her zaman RAM olan bir veri kümesi. In operatörünün sağ tarafında kullanılmak üzere tasarlanmıştır (bölüme bakın “IN operators”). + +Tabloya veri eklemek için INSERT kullanabilirsiniz. Veri kümesine yeni öğeler eklenirken, yinelenenler göz ardı edilir. +Ancak tablodan seçim yapamazsınız. Verileri almak için tek yol, IN operatörünün sağ yarısında kullanmaktır. + +Veri her zaman RAM yer almaktadır. INSERT için, eklenen veri blokları da diskteki tabloların dizinine yazılır. Sunucuyu başlatırken, bu veriler RAM'e yüklenir. Başka bir deyişle, yeniden başlattıktan sonra veriler yerinde kalır. + +Kaba bir sunucu yeniden başlatma için diskteki veri bloğu kaybolabilir veya zarar görebilir. İkinci durumda, dosyayı hasarlı verilerle el ile silmeniz gerekebilir. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/set/) diff --git a/docs/tr/engines/table_engines/special/url.md b/docs/tr/engines/table_engines/special/url.md new file mode 100644 index 00000000000..51657248e75 --- /dev/null +++ b/docs/tr/engines/table_engines/special/url.md @@ -0,0 +1,82 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 41 +toc_title: URL +--- + +# URL (URL, Biçim) {#table_engines-url} + +Uzak bir HTTP/HTTPS sunucusundaki verileri yönetir. Bu motor benzer +to the [Dosya](file.md) motor. + +## ClickHouse sunucusunda motoru kullanma {#using-the-engine-in-the-clickhouse-server} + +Bu `format` Clickhouse'un kullanabileceği bir tane olmalı +`SELECT` sorgular ve gerekirse `INSERTs`. Desteklenen formatların tam listesi için bkz. +[Biçimliler](../../../interfaces/formats.md#formats). + +Bu `URL` tekdüzen bir kaynak Bulucu yapısına uygun olmalıdır. Belirtilen URL bir sunucuya işaret etmelidir +bu HTTP veya HTTPS kullanır. Bu herhangi bir gerektirmez +sunucudan yanıt almak için ek başlıklar. + +`INSERT` ve `SELECT` sorgular dönüştürülür `POST` ve `GET` istemler, +sırasıyla. İşleme için `POST` istekleri, uzak sunucu desteklemesi gerekir +[Yığınlı aktarım kodlaması](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). + +Kullanarak HTTP get yönlendirme şerbetçiotu sayısını sınırlayabilirsiniz [max\_http\_get\_redirects](../../../operations/settings/settings.md#setting-max_http_get_redirects) ayar. + +**Örnek:** + +**1.** Create a `url_engine_table` sunucuda tablo : + +``` sql +CREATE TABLE url_engine_table (word String, value UInt64) +ENGINE=URL('http://127.0.0.1:12345/', CSV) +``` + +**2.** Standart Python 3 araçlarını kullanarak temel bir HTTP Sunucusu oluşturun ve +Başlat: + +``` python3 +from http.server import BaseHTTPRequestHandler, HTTPServer + +class CSVHTTPServer(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-type', 'text/csv') + self.end_headers() + + self.wfile.write(bytes('Hello,1\nWorld,2\n', "utf-8")) + +if __name__ == "__main__": + server_address = ('127.0.0.1', 12345) + HTTPServer(server_address, CSVHTTPServer).serve_forever() +``` + +``` bash +$ python3 server.py +``` + +**3.** Veri iste: + +``` sql +SELECT * FROM url_engine_table +``` + +``` text +┌─word──┬─value─┐ +│ Hello │ 1 │ +│ World │ 2 │ +└───────┴───────┘ +``` + +## Uygulama Detayları {#details-of-implementation} + +- Okuma ve yazma paralel olabilir +- Desteklenmiyor: + - `ALTER` ve `SELECT...SAMPLE` harekat. + - Dizinler. + - Çoğalma. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/url/) diff --git a/docs/tr/engines/table_engines/special/view.md b/docs/tr/engines/table_engines/special/view.md new file mode 100644 index 00000000000..e3b46a7b926 --- /dev/null +++ b/docs/tr/engines/table_engines/special/view.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 42 +toc_title: "G\xF6r\xFCn\xFCm" +--- + +# Görünüm {#table_engines-view} + +Görünümler uygulamak için kullanılır (Daha fazla bilgi için bkz. `CREATE VIEW query`). Verileri saklamaz, ancak yalnızca belirtilen `SELECT` sorgu. Bir tablodan okurken, bu sorguyu çalıştırır (ve gereksiz tüm sütunları sorgudan siler). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/table_engines/view/) diff --git a/docs/tr/faq/general.md b/docs/tr/faq/general.md new file mode 100644 index 00000000000..05779e132a9 --- /dev/null +++ b/docs/tr/faq/general.md @@ -0,0 +1,60 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 78 +toc_title: Genel Sorular +--- + +# Genel Sorular {#general-questions} + +## Neden MapReduce Gibi Bir Şey Kullanmıyorsun? {#why-not-use-something-like-mapreduce} + +Mapreduce gibi sistemlere, azaltma işleminin dağıtılmış sıralamaya dayandığı dağıtılmış bilgi işlem sistemleri olarak başvurabiliriz. Bu sınıftaki en yaygın açık kaynak çözümü [Apache Hadoop](http://hadoop.apache.org). Yandex, şirket içi çözümünü, YT'Yİ kullanıyor. + +Bu sistemler, yüksek gecikme süreleri nedeniyle çevrimiçi sorgular için uygun değildir. Başka bir deyişle, bir web arayüzü için arka uç olarak kullanılamazlar. Bu tür sistemler gerçek zamanlı veri güncellemeleri için yararlı değildir. Dağıtılmış sıralama, işlemin sonucu ve tüm ara sonuçlar (varsa) tek bir sunucunun RAM'İNDE bulunuyorsa, genellikle çevrimiçi sorgular için geçerli olan işlemleri azaltmanın en iyi yolu değildir. Böyle bir durumda, bir karma tablo azaltma işlemlerini gerçekleştirmek için en uygun yoldur. Harita azaltma görevlerini optimize etmek için ortak bir yaklaşım, RAM'de bir karma tablo kullanarak ön toplama (kısmi azaltma) ' dir. Kullanıcı bu optimizasyonu manuel olarak gerçekleştirir. Dağıtılmış sıralama, basit harita azaltma görevlerini çalıştırırken düşük performansın ana nedenlerinden biridir. + +Çoğu MapReduce uygulaması, bir kümede rasgele kod çalıştırmanıza izin verir. Ancak bildirimsel bir sorgu dili, deneyleri hızlı bir şekilde çalıştırmak için OLAP için daha uygundur. Örneğin, Hadoop kovanı ve domuz vardır. Ayrıca Spark için Cloudera Impala veya Shark'ı (modası geçmiş) ve Spark SQL, Presto ve Apache Drill'i de düşünün. Bu tür görevleri çalıştırırken performans, özel sistemlere kıyasla oldukça düşük bir seviyededir, ancak nispeten yüksek gecikme, bu sistemleri bir web arayüzü için arka uç olarak kullanmayı gerçekçi kılmaktadır. + +## Oracle aracılığıyla ODBC kullanırken Kodlamalarla ilgili bir sorunum varsa ne olur? {#oracle-odbc-encodings} + +Oracle ODBC sürücüsü aracılığıyla dış sözlükler kaynağı olarak kullanırsanız, doğru değeri ayarlamanız gerekir. `NLS_LANG` ortam değişkeni `/etc/default/clickhouse`. Daha fazla bilgi için, bkz: [Oracle NLS\_LANG SSS](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). + +**Örnek** + +``` sql +NLS_LANG=RUSSIAN_RUSSIA.UTF8 +``` + +## Clickhouse'dan bir dosyaya verileri nasıl dışa aktarırım? {#how-to-export-to-file} + +### INTO OUTFİLE yan tümcesini kullanma {#using-into-outfile-clause} + +Add an [INTO OUTFILE](../sql_reference/statements/select.md#into-outfile-clause) sorgunuza yan tümce. + +Mesela: + +``` sql +SELECT * FROM table INTO OUTFILE 'file' +``` + +Varsayılan olarak, ClickHouse kullanır [TabSeparated](../interfaces/formats.md#tabseparated) çıktı verileri için Biçim. Seçmek için [Veri formatı](../interfaces/formats.md), use the [FORMAT CLA clauseuse](../sql_reference/statements/select.md#format-clause). + +Mesela: + +``` sql +SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV +``` + +### Dosya altyapısı tablosu kullanma {#using-a-file-engine-table} + +Görmek [Dosya](../engines/table_engines/special/file.md). + +### Komut Satırı Yeniden Yönlendirmesini Kullanma {#using-command-line-redirection} + +``` sql +$ clickhouse-client --query "SELECT * from table" --format FormatName > result.txt +``` + +Görmek [clickhouse-müşteri](../interfaces/cli.md). + +{## [Orijinal makale](https://clickhouse.tech/docs/en/faq/general/) ##} diff --git a/docs/tr/faq/index.md b/docs/tr/faq/index.md new file mode 100644 index 00000000000..591011fb66d --- /dev/null +++ b/docs/tr/faq/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: F.A.Q. +toc_priority: 76 +--- + + diff --git a/docs/tr/getting_started/example_datasets/amplab_benchmark.md b/docs/tr/getting_started/example_datasets/amplab_benchmark.md new file mode 100644 index 00000000000..e6f95df68b8 --- /dev/null +++ b/docs/tr/getting_started/example_datasets/amplab_benchmark.md @@ -0,0 +1,129 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 17 +toc_title: "AMPLab B\xFCy\xFCk Veri Benchmark" +--- + +# AMPLab Büyük Veri Benchmark {#amplab-big-data-benchmark} + +Bkz. https://amplab.cs.berkeley.edu/benchmark/ + +Ücretsiz bir hesap için kaydolun https://aws.amazon.com. bir kredi kartı, e-posta ve telefon numarası gerektirir. Yeni bir erişim anahtarı alın https://console.aws.amazon.com/iam/home?nc2=h\_m\_sc\#security\_credential + +Konsolda aşağıdakileri çalıştırın: + +``` bash +$ sudo apt-get install s3cmd +$ mkdir tiny; cd tiny; +$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/tiny/ . +$ cd .. +$ mkdir 1node; cd 1node; +$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/1node/ . +$ cd .. +$ mkdir 5nodes; cd 5nodes; +$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/5nodes/ . +$ cd .. +``` + +Aşağıdaki ClickHouse sorguları çalıştırın: + +``` sql +CREATE TABLE rankings_tiny +( + pageURL String, + pageRank UInt32, + avgDuration UInt32 +) ENGINE = Log; + +CREATE TABLE uservisits_tiny +( + sourceIP String, + destinationURL String, + visitDate Date, + adRevenue Float32, + UserAgent String, + cCode FixedString(3), + lCode FixedString(6), + searchWord String, + duration UInt32 +) ENGINE = MergeTree(visitDate, visitDate, 8192); + +CREATE TABLE rankings_1node +( + pageURL String, + pageRank UInt32, + avgDuration UInt32 +) ENGINE = Log; + +CREATE TABLE uservisits_1node +( + sourceIP String, + destinationURL String, + visitDate Date, + adRevenue Float32, + UserAgent String, + cCode FixedString(3), + lCode FixedString(6), + searchWord String, + duration UInt32 +) ENGINE = MergeTree(visitDate, visitDate, 8192); + +CREATE TABLE rankings_5nodes_on_single +( + pageURL String, + pageRank UInt32, + avgDuration UInt32 +) ENGINE = Log; + +CREATE TABLE uservisits_5nodes_on_single +( + sourceIP String, + destinationURL String, + visitDate Date, + adRevenue Float32, + UserAgent String, + cCode FixedString(3), + lCode FixedString(6), + searchWord String, + duration UInt32 +) ENGINE = MergeTree(visitDate, visitDate, 8192); +``` + +Konsola geri dön: + +``` bash +$ for i in tiny/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_tiny FORMAT CSV"; done +$ for i in tiny/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_tiny FORMAT CSV"; done +$ for i in 1node/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_1node FORMAT CSV"; done +$ for i in 1node/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_1node FORMAT CSV"; done +$ for i in 5nodes/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_5nodes_on_single FORMAT CSV"; done +$ for i in 5nodes/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_5nodes_on_single FORMAT CSV"; done +``` + +Veri örnekleri almak için sorgular: + +``` sql +SELECT pageURL, pageRank FROM rankings_1node WHERE pageRank > 1000 + +SELECT substring(sourceIP, 1, 8), sum(adRevenue) FROM uservisits_1node GROUP BY substring(sourceIP, 1, 8) + +SELECT + sourceIP, + sum(adRevenue) AS totalRevenue, + avg(pageRank) AS pageRank +FROM rankings_1node ALL INNER JOIN +( + SELECT + sourceIP, + destinationURL AS pageURL, + adRevenue + FROM uservisits_1node + WHERE (visitDate > '1980-01-01') AND (visitDate < '1980-04-01') +) USING pageURL +GROUP BY sourceIP +ORDER BY totalRevenue DESC +LIMIT 1 +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/example_datasets/amplab_benchmark/) diff --git a/docs/tr/getting_started/example_datasets/criteo.md b/docs/tr/getting_started/example_datasets/criteo.md new file mode 100644 index 00000000000..37cb2d2b0e6 --- /dev/null +++ b/docs/tr/getting_started/example_datasets/criteo.md @@ -0,0 +1,81 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 19 +toc_title: "Criteo'dan Terabyte t\u0131klama g\xFCnl\xFCkleri" +--- + +# Criteo'dan tıklama günlüklerinin terabayt {#terabyte-of-click-logs-from-criteo} + +Verileri indirin http://labs.criteo.com/downloads/download-terabyte-click-logs/ + +Günlük almak için bir tablo oluşturun: + +``` sql +CREATE TABLE criteo_log (date Date, clicked UInt8, int1 Int32, int2 Int32, int3 Int32, int4 Int32, int5 Int32, int6 Int32, int7 Int32, int8 Int32, int9 Int32, int10 Int32, int11 Int32, int12 Int32, int13 Int32, cat1 String, cat2 String, cat3 String, cat4 String, cat5 String, cat6 String, cat7 String, cat8 String, cat9 String, cat10 String, cat11 String, cat12 String, cat13 String, cat14 String, cat15 String, cat16 String, cat17 String, cat18 String, cat19 String, cat20 String, cat21 String, cat22 String, cat23 String, cat24 String, cat25 String, cat26 String) ENGINE = Log +``` + +Verileri indirin: + +``` bash +$ for i in {00..23}; do echo $i; zcat datasets/criteo/day_${i#0}.gz | sed -r 's/^/2000-01-'${i/00/24}'\t/' | clickhouse-client --host=example-perftest01j --query="INSERT INTO criteo_log FORMAT TabSeparated"; done +``` + +Dönüştürülen veriler için bir tablo oluşturma: + +``` sql +CREATE TABLE criteo +( + date Date, + clicked UInt8, + int1 Int32, + int2 Int32, + int3 Int32, + int4 Int32, + int5 Int32, + int6 Int32, + int7 Int32, + int8 Int32, + int9 Int32, + int10 Int32, + int11 Int32, + int12 Int32, + int13 Int32, + icat1 UInt32, + icat2 UInt32, + icat3 UInt32, + icat4 UInt32, + icat5 UInt32, + icat6 UInt32, + icat7 UInt32, + icat8 UInt32, + icat9 UInt32, + icat10 UInt32, + icat11 UInt32, + icat12 UInt32, + icat13 UInt32, + icat14 UInt32, + icat15 UInt32, + icat16 UInt32, + icat17 UInt32, + icat18 UInt32, + icat19 UInt32, + icat20 UInt32, + icat21 UInt32, + icat22 UInt32, + icat23 UInt32, + icat24 UInt32, + icat25 UInt32, + icat26 UInt32 +) ENGINE = MergeTree(date, intHash32(icat1), (date, intHash32(icat1)), 8192) +``` + +Verileri ham günlüğünden dönüştürün ve ikinci tabloya koyun: + +``` sql +INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log; + +DROP TABLE criteo_log; +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/example_datasets/criteo/) diff --git a/docs/tr/getting_started/example_datasets/index.md b/docs/tr/getting_started/example_datasets/index.md new file mode 100644 index 00000000000..fecb39f219f --- /dev/null +++ b/docs/tr/getting_started/example_datasets/index.md @@ -0,0 +1,22 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "\xD6rnek Veri K\xFCmeleri" +toc_priority: 12 +toc_title: "Giri\u015F" +--- + +# Örnek Veri Kümeleri {#example-datasets} + +Bu bölümde, örnek veri kümelerinin nasıl elde edileceği ve bunları Clickhouse'a nasıl içe aktarılacağı açıklanmaktadır. +Bazı veri kümeleri için örnek sorgular da mevcuttur. + +- [Anonim Yandex.Metrica Veri Kümesi](metrica.md) +- [Yıldız Şema Ben Benchmarkch Benchmarkmark](star_schema.md) +- [WikiStat](wikistat.md) +- [Criteo'dan tıklama günlüklerinin terabayt](criteo.md) +- [AMPLab Büyük Veri Benchmark](amplab_benchmark.md) +- [New York Taksi Verileri](nyc_taxi.md) +- [OnTime](ontime.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/example_datasets) diff --git a/docs/tr/getting_started/example_datasets/metrica.md b/docs/tr/getting_started/example_datasets/metrica.md new file mode 100644 index 00000000000..a9d3d6743ee --- /dev/null +++ b/docs/tr/getting_started/example_datasets/metrica.md @@ -0,0 +1,70 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 21 +toc_title: "\xDCye.Metrica Verileri" +--- + +# Anonim Yandex.Metrica Verileri {#anonymized-yandex-metrica-data} + +Veri kümesi, isabetlerle ilgili anonimleştirilmiş verileri içeren iki tablodan oluşur (`hits_v1`) ve ziyaret visitsler (`visits_v1`(kayıt olmak için).Metrica. Yandex hakkında daha fazla bilgi edinebilirsiniz.Metrica içinde [ClickHouse geçmişi](../../introduction/history.md) bölme. + +Veri kümesi iki tablodan oluşur, bunlardan biri sıkıştırılmış olarak indirilebilir `tsv.xz` dosya veya hazırlanmış bölümler olarak. Buna ek olarak, genişletilmiş bir sürümü `hits` 100 milyon satır içeren tablo TSV olarak mevcuttur https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits\_100m\_obfuscated\_v1.tsv.xz ve hazırlanan bölümler olarak https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits\_100m\_obfuscated\_v1.tar.xz. + +## Hazırlanan bölümlerden tablolar elde etme {#obtaining-tables-from-prepared-partitions} + +İndirme ve ithalat tablo hits: + +``` bash +curl -O https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar +tar xvf hits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory +# check permissions on unpacked data, fix if required +sudo service clickhouse-server restart +clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" +``` + +İndirme ve ithalat ziyaretleri: + +``` bash +curl -O https://clickhouse-datasets.s3.yandex.net/visits/partitions/visits_v1.tar +tar xvf visits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory +# check permissions on unpacked data, fix if required +sudo service clickhouse-server restart +clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" +``` + +## Sıkıştırılmış TSV dosyasından Tablo alma {#obtaining-tables-from-compressed-tsv-file} + +Sıkıştırılmış TSV dosyasından indir ve İçe Aktar: + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +# now create table +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" +clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" +# import data +cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 +# optionally you can optimize table +clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" +clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" +``` + +Sıkıştırılmış tsv dosyasından ziyaretleri indirin ve içe aktarın: + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +# now create table +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" +clickhouse-client --query "CREATE TABLE datasets.visits_v1 ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" +# import data +cat visits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.visits_v1 FORMAT TSV" --max_insert_block_size=100000 +# optionally you can optimize table +clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" +clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" +``` + +## Örnek Sorgular {#example-queries} + +[ClickHouse eğitimi](../../getting_started/tutorial.md) Yandex dayanmaktadır.Metrica veri kümesi ve bu veri kümesine başlamak için önerilen yol sadece öğreticiden geçmektir. + +Bu tablolara ek sorgu örnekleri arasında bulunabilir [durum bilgisi testleri](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) ClickHouse of (onlar adlandırılır `test.hists` ve `test.visits` oralarda). diff --git a/docs/tr/getting_started/example_datasets/nyc_taxi.md b/docs/tr/getting_started/example_datasets/nyc_taxi.md new file mode 100644 index 00000000000..218a7f06f7a --- /dev/null +++ b/docs/tr/getting_started/example_datasets/nyc_taxi.md @@ -0,0 +1,390 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 16 +toc_title: New York Taksi Verileri +--- + +# New York Taksi Verileri {#new-york-taxi-data} + +Bu veri kümesi iki şekilde elde edilebilir: + +- ham verilerden içe aktarma +- hazırlanan bölüm downloadlerin indir downloadilmesi + +## Ham veri nasıl alınır {#how-to-import-the-raw-data} + +Bkz. https://github.com/toddwschneider/nyc-taxi-data ve http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html bir veri kümesinin açıklaması ve indirme talimatları için. + +İndirme, CSV dosyalarında yaklaşık 227 GB sıkıştırılmamış veri ile sonuçlanacaktır. İndirme, 1 Gbit bağlantısı üzerinden yaklaşık bir saat sürer (paralel indirme s3.amazonaws.com 1 Gbit kanalın en az yarısını kurtarır). +Bazı dosyalar tam olarak indirilmeyebilir. Dosya boyutlarını kontrol edin ve şüpheli görünen herhangi birini yeniden indirin. + +Bazı dosyalar geçersiz satırlar içerebilir. Bunları aşağıdaki gibi düzeltebilirsiniz: + +``` bash +sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-02.csv > data/yellow_tripdata_2010-02.csv_ +sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-03.csv > data/yellow_tripdata_2010-03.csv_ +mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv +mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv +``` + +Daha sonra veriler Postgresql'de önceden işlenmelidir. Bu, çokgenlerdeki noktaların seçimlerini oluşturacaktır (Haritadaki noktaları New York şehrinin ilçeleriyle eşleştirmek için) ve tüm verileri bir birleştirme kullanarak tek bir denormalize düz tabloda birleştirecektir. Bunu yapmak için Postgresql'i Postgıs desteği ile yüklemeniz gerekir. + +Çalışırken dikkatli olun `initialize_database.sh` ve tüm tabloların doğru şekilde oluşturulduğunu manuel olarak yeniden kontrol edin. + +Postgresql'deki her ayın verilerini işlemek yaklaşık 20-30 dakika sürer, toplam yaklaşık 48 saat sürer. + +İndirilen satır sayısını aşağıdaki gibi kontrol edebilirsiniz: + +``` bash +$ time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" +## Count + 1298979494 +(1 row) + +real 7m9.164s +``` + +(Bu, Mark Litwintschik tarafından bir dizi blog gönderisinde bildirilen 1.1 milyar satırdan biraz daha fazladır .) + +Postgresql'deki veriler 370 GB alan kullanıyor. + +PostgreSQL veri verme: + +``` sql +COPY +( + SELECT trips.id, + trips.vendor_id, + trips.pickup_datetime, + trips.dropoff_datetime, + trips.store_and_fwd_flag, + trips.rate_code_id, + trips.pickup_longitude, + trips.pickup_latitude, + trips.dropoff_longitude, + trips.dropoff_latitude, + trips.passenger_count, + trips.trip_distance, + trips.fare_amount, + trips.extra, + trips.mta_tax, + trips.tip_amount, + trips.tolls_amount, + trips.ehail_fee, + trips.improvement_surcharge, + trips.total_amount, + trips.payment_type, + trips.trip_type, + trips.pickup, + trips.dropoff, + + cab_types.type cab_type, + + weather.precipitation_tenths_of_mm rain, + weather.snow_depth_mm, + weather.snowfall_mm, + weather.max_temperature_tenths_degrees_celsius max_temp, + weather.min_temperature_tenths_degrees_celsius min_temp, + weather.average_wind_speed_tenths_of_meters_per_second wind, + + pick_up.gid pickup_nyct2010_gid, + pick_up.ctlabel pickup_ctlabel, + pick_up.borocode pickup_borocode, + pick_up.boroname pickup_boroname, + pick_up.ct2010 pickup_ct2010, + pick_up.boroct2010 pickup_boroct2010, + pick_up.cdeligibil pickup_cdeligibil, + pick_up.ntacode pickup_ntacode, + pick_up.ntaname pickup_ntaname, + pick_up.puma pickup_puma, + + drop_off.gid dropoff_nyct2010_gid, + drop_off.ctlabel dropoff_ctlabel, + drop_off.borocode dropoff_borocode, + drop_off.boroname dropoff_boroname, + drop_off.ct2010 dropoff_ct2010, + drop_off.boroct2010 dropoff_boroct2010, + drop_off.cdeligibil dropoff_cdeligibil, + drop_off.ntacode dropoff_ntacode, + drop_off.ntaname dropoff_ntaname, + drop_off.puma dropoff_puma + FROM trips + LEFT JOIN cab_types + ON trips.cab_type_id = cab_types.id + LEFT JOIN central_park_weather_observations_raw weather + ON weather.date = trips.pickup_datetime::date + LEFT JOIN nyct2010 pick_up + ON pick_up.gid = trips.pickup_nyct2010_gid + LEFT JOIN nyct2010 drop_off + ON drop_off.gid = trips.dropoff_nyct2010_gid +) TO '/opt/milovidov/nyc-taxi-data/trips.tsv'; +``` + +Veri anlık görüntüsü saniyede yaklaşık 50 MB hızında oluşturulur. Anlık görüntü oluştururken, PostgreSQL diskten saniyede yaklaşık 28 MB hızında okur. +Bu yaklaşık 5 saat sürer. Elde edilen TSV dosyası 590612904969 bayttır. + +Clickhouse'da geçici bir tablo oluşturma: + +``` sql +CREATE TABLE trips +( +trip_id UInt32, +vendor_id String, +pickup_datetime DateTime, +dropoff_datetime Nullable(DateTime), +store_and_fwd_flag Nullable(FixedString(1)), +rate_code_id Nullable(UInt8), +pickup_longitude Nullable(Float64), +pickup_latitude Nullable(Float64), +dropoff_longitude Nullable(Float64), +dropoff_latitude Nullable(Float64), +passenger_count Nullable(UInt8), +trip_distance Nullable(Float64), +fare_amount Nullable(Float32), +extra Nullable(Float32), +mta_tax Nullable(Float32), +tip_amount Nullable(Float32), +tolls_amount Nullable(Float32), +ehail_fee Nullable(Float32), +improvement_surcharge Nullable(Float32), +total_amount Nullable(Float32), +payment_type Nullable(String), +trip_type Nullable(UInt8), +pickup Nullable(String), +dropoff Nullable(String), +cab_type Nullable(String), +precipitation Nullable(UInt8), +snow_depth Nullable(UInt8), +snowfall Nullable(UInt8), +max_temperature Nullable(UInt8), +min_temperature Nullable(UInt8), +average_wind_speed Nullable(UInt8), +pickup_nyct2010_gid Nullable(UInt8), +pickup_ctlabel Nullable(String), +pickup_borocode Nullable(UInt8), +pickup_boroname Nullable(String), +pickup_ct2010 Nullable(String), +pickup_boroct2010 Nullable(String), +pickup_cdeligibil Nullable(FixedString(1)), +pickup_ntacode Nullable(String), +pickup_ntaname Nullable(String), +pickup_puma Nullable(String), +dropoff_nyct2010_gid Nullable(UInt8), +dropoff_ctlabel Nullable(String), +dropoff_borocode Nullable(UInt8), +dropoff_boroname Nullable(String), +dropoff_ct2010 Nullable(String), +dropoff_boroct2010 Nullable(String), +dropoff_cdeligibil Nullable(String), +dropoff_ntacode Nullable(String), +dropoff_ntaname Nullable(String), +dropoff_puma Nullable(String) +) ENGINE = Log; +``` + +Alanları daha doğru veri türlerine dönüştürmek ve mümkünse Boşları ortadan kaldırmak için gereklidir. + +``` bash +$ time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips.tsv + +real 75m56.214s +``` + +Veri 112-140 Mb/saniye hızında okunur. +Bir akışta bir günlük türü tablosuna veri yükleme 76 dakika sürdü. +Bu tablodaki veriler 142 GB kullanır. + +(Verileri doğrudan Postgres'ten içe aktarmak da mümkündür `COPY ... TO PROGRAM`.) + +Unfortunately, all the fields associated with the weather (precipitation…average\_wind\_speed) were filled with NULL. Because of this, we will remove them from the final data set. + +Başlamak için, tek bir sunucuda bir tablo oluşturacağız. Daha sonra tabloyu dağıtacağız. + +Özet Tablo oluşturma ve doldurma: + +``` sql +CREATE TABLE trips_mergetree +ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) +AS SELECT + +trip_id, +CAST(vendor_id AS Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14)) AS vendor_id, +toDate(pickup_datetime) AS pickup_date, +ifNull(pickup_datetime, toDateTime(0)) AS pickup_datetime, +toDate(dropoff_datetime) AS dropoff_date, +ifNull(dropoff_datetime, toDateTime(0)) AS dropoff_datetime, +assumeNotNull(store_and_fwd_flag) IN ('Y', '1', '2') AS store_and_fwd_flag, +assumeNotNull(rate_code_id) AS rate_code_id, +assumeNotNull(pickup_longitude) AS pickup_longitude, +assumeNotNull(pickup_latitude) AS pickup_latitude, +assumeNotNull(dropoff_longitude) AS dropoff_longitude, +assumeNotNull(dropoff_latitude) AS dropoff_latitude, +assumeNotNull(passenger_count) AS passenger_count, +assumeNotNull(trip_distance) AS trip_distance, +assumeNotNull(fare_amount) AS fare_amount, +assumeNotNull(extra) AS extra, +assumeNotNull(mta_tax) AS mta_tax, +assumeNotNull(tip_amount) AS tip_amount, +assumeNotNull(tolls_amount) AS tolls_amount, +assumeNotNull(ehail_fee) AS ehail_fee, +assumeNotNull(improvement_surcharge) AS improvement_surcharge, +assumeNotNull(total_amount) AS total_amount, +CAST((assumeNotNull(payment_type) AS pt) IN ('CSH', 'CASH', 'Cash', 'CAS', 'Cas', '1') ? 'CSH' : (pt IN ('CRD', 'Credit', 'Cre', 'CRE', 'CREDIT', '2') ? 'CRE' : (pt IN ('NOC', 'No Charge', 'No', '3') ? 'NOC' : (pt IN ('DIS', 'Dispute', 'Dis', '4') ? 'DIS' : 'UNK'))) AS Enum8('CSH' = 1, 'CRE' = 2, 'UNK' = 0, 'NOC' = 3, 'DIS' = 4)) AS payment_type_, +assumeNotNull(trip_type) AS trip_type, +ifNull(toFixedString(unhex(pickup), 25), toFixedString('', 25)) AS pickup, +ifNull(toFixedString(unhex(dropoff), 25), toFixedString('', 25)) AS dropoff, +CAST(assumeNotNull(cab_type) AS Enum8('yellow' = 1, 'green' = 2, 'uber' = 3)) AS cab_type, + +assumeNotNull(pickup_nyct2010_gid) AS pickup_nyct2010_gid, +toFloat32(ifNull(pickup_ctlabel, '0')) AS pickup_ctlabel, +assumeNotNull(pickup_borocode) AS pickup_borocode, +CAST(assumeNotNull(pickup_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS pickup_boroname, +toFixedString(ifNull(pickup_ct2010, '000000'), 6) AS pickup_ct2010, +toFixedString(ifNull(pickup_boroct2010, '0000000'), 7) AS pickup_boroct2010, +CAST(assumeNotNull(ifNull(pickup_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS pickup_cdeligibil, +toFixedString(ifNull(pickup_ntacode, '0000'), 4) AS pickup_ntacode, + +CAST(assumeNotNull(pickup_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS pickup_ntaname, + +toUInt16(ifNull(pickup_puma, '0')) AS pickup_puma, + +assumeNotNull(dropoff_nyct2010_gid) AS dropoff_nyct2010_gid, +toFloat32(ifNull(dropoff_ctlabel, '0')) AS dropoff_ctlabel, +assumeNotNull(dropoff_borocode) AS dropoff_borocode, +CAST(assumeNotNull(dropoff_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS dropoff_boroname, +toFixedString(ifNull(dropoff_ct2010, '000000'), 6) AS dropoff_ct2010, +toFixedString(ifNull(dropoff_boroct2010, '0000000'), 7) AS dropoff_boroct2010, +CAST(assumeNotNull(ifNull(dropoff_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS dropoff_cdeligibil, +toFixedString(ifNull(dropoff_ntacode, '0000'), 4) AS dropoff_ntacode, + +CAST(assumeNotNull(dropoff_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS dropoff_ntaname, + +toUInt16(ifNull(dropoff_puma, '0')) AS dropoff_puma + +FROM trips +``` + +Bu, saniyede yaklaşık 428.000 satırlık bir hızda 3030 saniye sürer. +Daha hızlı yüklemek için, tablo ile oluşturabilirsiniz `Log` motor yerine `MergeTree`. Bu durumda, indirme 200 saniyeden daha hızlı çalışır. + +Tablo 126 GB disk alanı kullanır. + +``` sql +SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active +``` + +``` text +┌─formatReadableSize(sum(bytes))─┐ +│ 126.18 GiB │ +└────────────────────────────────┘ +``` + +Diğer şeylerin yanı sıra, MERGETREE üzerinde en iyi duruma getirme sorgusunu çalıştırabilirsiniz. Ama her şey onsuz iyi olacak çünkü gerekli değildir. + +## Hazırlanan Bölüm downloadlerin indir downloadilmesi {#download-of-prepared-partitions} + +``` bash +$ curl -O https://clickhouse-datasets.s3.yandex.net/trips_mergetree/partitions/trips_mergetree.tar +$ tar xvf trips_mergetree.tar -C /var/lib/clickhouse # path to ClickHouse data directory +$ # check permissions of unpacked data, fix if required +$ sudo service clickhouse-server restart +$ clickhouse-client --query "select count(*) from datasets.trips_mergetree" +``` + +!!! info "Bilgin" + Aşağıda açıklanan sorguları çalıştıracaksanız, tam tablo adını kullanmanız gerekir, `datasets.trips_mergetree`. + +## Tek Server ile ilgili sonuçlar {#results-on-single-server} + +Q1: + +``` sql +SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type +``` + +0.490 saniye. + +Q2: + +``` sql +SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenger_count +``` + +1.224 saniye. + +Q3: + +``` sql +SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year +``` + +2.104 saniye. + +Q4: + +``` sql +SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) +FROM trips_mergetree +GROUP BY passenger_count, year, distance +ORDER BY year, count(*) DESC +``` + +3.593 saniye. + +Aşağıdaki sunucu kullanıldı: + +İki Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60 GHz, 16 fiziksel çekirdekler toplam,128 GiB RAM,8x6 TB HD donanım RAID-5 + +Yürütme süresi üç koşunun en iyisidir. Ancak ikinci çalıştırmadan başlayarak, sorgular dosya sistemi önbelleğinden verileri okur. Başka önbelleğe alma oluşur: veri okundu ve her vadede işlenir. + +Üç sunucuda tablo oluşturma: + +Her sunucuda: + +``` sql +CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) +``` + +Kaynak sunucuda: + +``` sql +CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()) +``` + +Aşağıdaki sorgu verileri yeniden dağıtır: + +``` sql +INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree +``` + +Bu 2454 saniye sürer. + +Üç sunucuda: + +Q1: 0.212 saniye. +Q2: 0.438 saniye. +Q3: 0.733 saniye. +Q4: 1.241 saniye. + +Sorgular doğrusal olarak ölçeklendiğinden, burada sürpriz yok. + +Ayrıca 140 sunucu kümesinden elde edilen sonuçlara sahibiz: + +Q1: 0.028 sn. +Q2: 0.043 sn. +Q3: 0.051 sn. +Q4: 0.072 sn. + +Bu durumda, sorgu işleme süresi her şeyden önce ağ gecikmesi ile belirlenir. +Finlandiya'daki bir Yandex veri merkezinde bulunan ve Rusya'daki bir kümede bulunan ve yaklaşık 20 ms gecikme süresi ekleyen bir istemci kullanarak sorgular çalıştırdık. + +## Özet {#summary} + +| hizmetçiler | Q1 | Q2 | Q3 | Q4 | +|-------------|-------|-------|-------|-------| +| 1 | 0.490 | 1.224 | 2.104 | 3.593 | +| 3 | 0.212 | 0.438 | 0.733 | 1.241 | +| 140 | 0.028 | 0.043 | 0.051 | 0.072 | + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/example_datasets/nyc_taxi/) diff --git a/docs/tr/getting_started/example_datasets/ontime.md b/docs/tr/getting_started/example_datasets/ontime.md new file mode 100644 index 00000000000..dc06b86189f --- /dev/null +++ b/docs/tr/getting_started/example_datasets/ontime.md @@ -0,0 +1,412 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 15 +toc_title: OnTime +--- + +# OnTime {#ontime} + +Bu veri kümesi iki şekilde elde edilebilir: + +- ham verilerden içe aktarma +- hazırlanan bölüm downloadlerin indir downloadilmesi + +## Ham Verilerden İçe Aktarma {#import-from-raw-data} + +Veri indirme: + +``` bash +for s in `seq 1987 2018` +do +for m in `seq 1 12` +do +wget https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_${s}_${m}.zip +done +done +``` + +(from https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh ) + +Tablo oluşturma: + +``` sql +CREATE TABLE `ontime` ( + `Year` UInt16, + `Quarter` UInt8, + `Month` UInt8, + `DayofMonth` UInt8, + `DayOfWeek` UInt8, + `FlightDate` Date, + `UniqueCarrier` FixedString(7), + `AirlineID` Int32, + `Carrier` FixedString(2), + `TailNum` String, + `FlightNum` String, + `OriginAirportID` Int32, + `OriginAirportSeqID` Int32, + `OriginCityMarketID` Int32, + `Origin` FixedString(5), + `OriginCityName` String, + `OriginState` FixedString(2), + `OriginStateFips` String, + `OriginStateName` String, + `OriginWac` Int32, + `DestAirportID` Int32, + `DestAirportSeqID` Int32, + `DestCityMarketID` Int32, + `Dest` FixedString(5), + `DestCityName` String, + `DestState` FixedString(2), + `DestStateFips` String, + `DestStateName` String, + `DestWac` Int32, + `CRSDepTime` Int32, + `DepTime` Int32, + `DepDelay` Int32, + `DepDelayMinutes` Int32, + `DepDel15` Int32, + `DepartureDelayGroups` String, + `DepTimeBlk` String, + `TaxiOut` Int32, + `WheelsOff` Int32, + `WheelsOn` Int32, + `TaxiIn` Int32, + `CRSArrTime` Int32, + `ArrTime` Int32, + `ArrDelay` Int32, + `ArrDelayMinutes` Int32, + `ArrDel15` Int32, + `ArrivalDelayGroups` Int32, + `ArrTimeBlk` String, + `Cancelled` UInt8, + `CancellationCode` FixedString(1), + `Diverted` UInt8, + `CRSElapsedTime` Int32, + `ActualElapsedTime` Int32, + `AirTime` Int32, + `Flights` Int32, + `Distance` Int32, + `DistanceGroup` UInt8, + `CarrierDelay` Int32, + `WeatherDelay` Int32, + `NASDelay` Int32, + `SecurityDelay` Int32, + `LateAircraftDelay` Int32, + `FirstDepTime` String, + `TotalAddGTime` String, + `LongestAddGTime` String, + `DivAirportLandings` String, + `DivReachedDest` String, + `DivActualElapsedTime` String, + `DivArrDelay` String, + `DivDistance` String, + `Div1Airport` String, + `Div1AirportID` Int32, + `Div1AirportSeqID` Int32, + `Div1WheelsOn` String, + `Div1TotalGTime` String, + `Div1LongestGTime` String, + `Div1WheelsOff` String, + `Div1TailNum` String, + `Div2Airport` String, + `Div2AirportID` Int32, + `Div2AirportSeqID` Int32, + `Div2WheelsOn` String, + `Div2TotalGTime` String, + `Div2LongestGTime` String, + `Div2WheelsOff` String, + `Div2TailNum` String, + `Div3Airport` String, + `Div3AirportID` Int32, + `Div3AirportSeqID` Int32, + `Div3WheelsOn` String, + `Div3TotalGTime` String, + `Div3LongestGTime` String, + `Div3WheelsOff` String, + `Div3TailNum` String, + `Div4Airport` String, + `Div4AirportID` Int32, + `Div4AirportSeqID` Int32, + `Div4WheelsOn` String, + `Div4TotalGTime` String, + `Div4LongestGTime` String, + `Div4WheelsOff` String, + `Div4TailNum` String, + `Div5Airport` String, + `Div5AirportID` Int32, + `Div5AirportSeqID` Int32, + `Div5WheelsOn` String, + `Div5TotalGTime` String, + `Div5LongestGTime` String, + `Div5WheelsOff` String, + `Div5TailNum` String +) ENGINE = MergeTree +PARTITION BY Year +ORDER BY (Carrier, FlightDate) +SETTINGS index_granularity = 8192; +``` + +Veri yükleme: + +``` bash +$ for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done +``` + +## Hazırlanan Bölüm downloadlerin indir downloadilmesi {#download-of-prepared-partitions} + +``` bash +$ curl -O https://clickhouse-datasets.s3.yandex.net/ontime/partitions/ontime.tar +$ tar xvf ontime.tar -C /var/lib/clickhouse # path to ClickHouse data directory +$ # check permissions of unpacked data, fix if required +$ sudo service clickhouse-server restart +$ clickhouse-client --query "select count(*) from datasets.ontime" +``` + +!!! info "Bilgin" + Aşağıda açıklanan sorguları çalıştıracaksanız, tam tablo adını kullanmanız gerekir, `datasets.ontime`. + +## Sorgular {#queries} + +Q0. + +``` sql +SELECT avg(c1) +FROM +( + SELECT Year, Month, count(*) AS c1 + FROM ontime + GROUP BY Year, Month +); +``` + +Q1. 2000 yılından 2008 yılına kadar günlük uçuş sayısı + +``` sql +SELECT DayOfWeek, count(*) AS c +FROM ontime +WHERE Year>=2000 AND Year<=2008 +GROUP BY DayOfWeek +ORDER BY c DESC; +``` + +S2. 2000-2008 için haftanın gününe göre gruplandırılmış 10 dakikadan fazla geciken uçuş sayısı + +``` sql +SELECT DayOfWeek, count(*) AS c +FROM ontime +WHERE DepDelay>10 AND Year>=2000 AND Year<=2008 +GROUP BY DayOfWeek +ORDER BY c DESC; +``` + +Q3. 2000-2008 için havaalanı tarafından gecikme sayısı + +``` sql +SELECT Origin, count(*) AS c +FROM ontime +WHERE DepDelay>10 AND Year>=2000 AND Year<=2008 +GROUP BY Origin +ORDER BY c DESC +LIMIT 10; +``` + +S4. 2007 yılı için taşıyıcı tarafından gecikme sayısı + +``` sql +SELECT Carrier, count(*) +FROM ontime +WHERE DepDelay>10 AND Year=2007 +GROUP BY Carrier +ORDER BY count(*) DESC; +``` + +Q5. 2007 yılı için taşıyıcı tarafından gecikme yüzdesi + +``` sql +SELECT Carrier, c, c2, c*100/c2 as c3 +FROM +( + SELECT + Carrier, + count(*) AS c + FROM ontime + WHERE DepDelay>10 + AND Year=2007 + GROUP BY Carrier +) +JOIN +( + SELECT + Carrier, + count(*) AS c2 + FROM ontime + WHERE Year=2007 + GROUP BY Carrier +) USING Carrier +ORDER BY c3 DESC; +``` + +Aynı sorgunun daha iyi sürümü: + +``` sql +SELECT Carrier, avg(DepDelay>10)*100 AS c3 +FROM ontime +WHERE Year=2007 +GROUP BY Carrier +ORDER BY c3 DESC +``` + +S6. Daha geniş bir yıl aralığı için önceki talep, 2000-2008 + +``` sql +SELECT Carrier, c, c2, c*100/c2 as c3 +FROM +( + SELECT + Carrier, + count(*) AS c + FROM ontime + WHERE DepDelay>10 + AND Year>=2000 AND Year<=2008 + GROUP BY Carrier +) +JOIN +( + SELECT + Carrier, + count(*) AS c2 + FROM ontime + WHERE Year>=2000 AND Year<=2008 + GROUP BY Carrier +) USING Carrier +ORDER BY c3 DESC; +``` + +Aynı sorgunun daha iyi sürümü: + +``` sql +SELECT Carrier, avg(DepDelay>10)*100 AS c3 +FROM ontime +WHERE Year>=2000 AND Year<=2008 +GROUP BY Carrier +ORDER BY c3 DESC; +``` + +Q7. Yıla göre 10 dakikadan fazla gecikmeli uçuş yüzdesi + +``` sql +SELECT Year, c1/c2 +FROM +( + select + Year, + count(*)*100 as c1 + from ontime + WHERE DepDelay>10 + GROUP BY Year +) +JOIN +( + select + Year, + count(*) as c2 + from ontime + GROUP BY Year +) USING (Year) +ORDER BY Year; +``` + +Aynı sorgunun daha iyi sürümü: + +``` sql +SELECT Year, avg(DepDelay>10)*100 +FROM ontime +GROUP BY Year +ORDER BY Year; +``` + +S8. Çeşitli yıl aralıkları için doğrudan bağlı şehirlerin sayısına göre en popüler yerler + +``` sql +SELECT DestCityName, uniqExact(OriginCityName) AS u +FROM ontime +WHERE Year >= 2000 and Year <= 2010 +GROUP BY DestCityName +ORDER BY u DESC LIMIT 10; +``` + +Q9. + +``` sql +SELECT Year, count(*) AS c1 +FROM ontime +GROUP BY Year; +``` + +Q10. + +``` sql +SELECT + min(Year), max(Year), Carrier, count(*) AS cnt, + sum(ArrDelayMinutes>30) AS flights_delayed, + round(sum(ArrDelayMinutes>30)/count(*),2) AS rate +FROM ontime +WHERE + DayOfWeek NOT IN (6,7) AND OriginState NOT IN ('AK', 'HI', 'PR', 'VI') + AND DestState NOT IN ('AK', 'HI', 'PR', 'VI') + AND FlightDate < '2010-01-01' +GROUP by Carrier +HAVING cnt>100000 and max(Year)>1990 +ORDER by rate DESC +LIMIT 1000; +``` + +Bonus: + +``` sql +SELECT avg(cnt) +FROM +( + SELECT Year,Month,count(*) AS cnt + FROM ontime + WHERE DepDel15=1 + GROUP BY Year,Month +); + +SELECT avg(c1) FROM +( + SELECT Year,Month,count(*) AS c1 + FROM ontime + GROUP BY Year,Month +); + +SELECT DestCityName, uniqExact(OriginCityName) AS u +FROM ontime +GROUP BY DestCityName +ORDER BY u DESC +LIMIT 10; + +SELECT OriginCityName, DestCityName, count() AS c +FROM ontime +GROUP BY OriginCityName, DestCityName +ORDER BY c DESC +LIMIT 10; + +SELECT OriginCityName, count() AS c +FROM ontime +GROUP BY OriginCityName +ORDER BY c DESC +LIMIT 10; +``` + +Bu performans testi Vadim Tkachenko tarafından oluşturuldu. Görmek: + +- https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/ +- https://www.percona.com/blog/2009/10/26/air-traffic-queries-in-luciddb/ +- https://www.percona.com/blog/2009/11/02/air-traffic-queries-in-infinidb-early-alpha/ +- https://www.percona.com/blog/2014/04/21/using-apache-hadoop-and-impala-together-with-mysql-for-data-analysis/ +- https://www.percona.com/blog/2016/01/07/apache-spark-with-air-ontime-performance-data/ +- http://nickmakos.blogspot.ru/2012/08/analyzing-air-traffic-performance-with.html + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/example_datasets/ontime/) diff --git a/docs/tr/getting_started/example_datasets/star_schema.md b/docs/tr/getting_started/example_datasets/star_schema.md new file mode 100644 index 00000000000..1326aab7ab2 --- /dev/null +++ b/docs/tr/getting_started/example_datasets/star_schema.md @@ -0,0 +1,370 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 20 +toc_title: "Y\u0131ld\u0131z \u015Eema Ben Benchmarkch Benchmarkmark" +--- + +# Yıldız Şema Ben Benchmarkch Benchmarkmark {#star-schema-benchmark} + +Dbgen derleme: + +``` bash +$ git clone git@github.com:vadimtk/ssb-dbgen.git +$ cd ssb-dbgen +$ make +``` + +Veri oluşturma: + +!!! warning "Dikkat" + İle `-s 100` dbgen, 600 milyon satır (67 GB) üretir `-s 1000` 6 milyar satır üretir (bu çok zaman alır) + +``` bash +$ ./dbgen -s 1000 -T c +$ ./dbgen -s 1000 -T l +$ ./dbgen -s 1000 -T p +$ ./dbgen -s 1000 -T s +$ ./dbgen -s 1000 -T d +``` + +Clickhouse'da tablolar oluşturma: + +``` sql +CREATE TABLE customer +( + C_CUSTKEY UInt32, + C_NAME String, + C_ADDRESS String, + C_CITY LowCardinality(String), + C_NATION LowCardinality(String), + C_REGION LowCardinality(String), + C_PHONE String, + C_MKTSEGMENT LowCardinality(String) +) +ENGINE = MergeTree ORDER BY (C_CUSTKEY); + +CREATE TABLE lineorder +( + LO_ORDERKEY UInt32, + LO_LINENUMBER UInt8, + LO_CUSTKEY UInt32, + LO_PARTKEY UInt32, + LO_SUPPKEY UInt32, + LO_ORDERDATE Date, + LO_ORDERPRIORITY LowCardinality(String), + LO_SHIPPRIORITY UInt8, + LO_QUANTITY UInt8, + LO_EXTENDEDPRICE UInt32, + LO_ORDTOTALPRICE UInt32, + LO_DISCOUNT UInt8, + LO_REVENUE UInt32, + LO_SUPPLYCOST UInt32, + LO_TAX UInt8, + LO_COMMITDATE Date, + LO_SHIPMODE LowCardinality(String) +) +ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY); + +CREATE TABLE part +( + P_PARTKEY UInt32, + P_NAME String, + P_MFGR LowCardinality(String), + P_CATEGORY LowCardinality(String), + P_BRAND LowCardinality(String), + P_COLOR LowCardinality(String), + P_TYPE LowCardinality(String), + P_SIZE UInt8, + P_CONTAINER LowCardinality(String) +) +ENGINE = MergeTree ORDER BY P_PARTKEY; + +CREATE TABLE supplier +( + S_SUPPKEY UInt32, + S_NAME String, + S_ADDRESS String, + S_CITY LowCardinality(String), + S_NATION LowCardinality(String), + S_REGION LowCardinality(String), + S_PHONE String +) +ENGINE = MergeTree ORDER BY S_SUPPKEY; +``` + +Veri ekleme: + +``` bash +$ clickhouse-client --query "INSERT INTO customer FORMAT CSV" < customer.tbl +$ clickhouse-client --query "INSERT INTO part FORMAT CSV" < part.tbl +$ clickhouse-client --query "INSERT INTO supplier FORMAT CSV" < supplier.tbl +$ clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl +``` + +Dönüşüm “star schema” denormalized için “flat schema”: + +``` sql +SET max_memory_usage = 20000000000; + +CREATE TABLE lineorder_flat +ENGINE = MergeTree +PARTITION BY toYear(LO_ORDERDATE) +ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS +SELECT + l.LO_ORDERKEY AS LO_ORDERKEY, + l.LO_LINENUMBER AS LO_LINENUMBER, + l.LO_CUSTKEY AS LO_CUSTKEY, + l.LO_PARTKEY AS LO_PARTKEY, + l.LO_SUPPKEY AS LO_SUPPKEY, + l.LO_ORDERDATE AS LO_ORDERDATE, + l.LO_ORDERPRIORITY AS LO_ORDERPRIORITY, + l.LO_SHIPPRIORITY AS LO_SHIPPRIORITY, + l.LO_QUANTITY AS LO_QUANTITY, + l.LO_EXTENDEDPRICE AS LO_EXTENDEDPRICE, + l.LO_ORDTOTALPRICE AS LO_ORDTOTALPRICE, + l.LO_DISCOUNT AS LO_DISCOUNT, + l.LO_REVENUE AS LO_REVENUE, + l.LO_SUPPLYCOST AS LO_SUPPLYCOST, + l.LO_TAX AS LO_TAX, + l.LO_COMMITDATE AS LO_COMMITDATE, + l.LO_SHIPMODE AS LO_SHIPMODE, + c.C_NAME AS C_NAME, + c.C_ADDRESS AS C_ADDRESS, + c.C_CITY AS C_CITY, + c.C_NATION AS C_NATION, + c.C_REGION AS C_REGION, + c.C_PHONE AS C_PHONE, + c.C_MKTSEGMENT AS C_MKTSEGMENT, + s.S_NAME AS S_NAME, + s.S_ADDRESS AS S_ADDRESS, + s.S_CITY AS S_CITY, + s.S_NATION AS S_NATION, + s.S_REGION AS S_REGION, + s.S_PHONE AS S_PHONE, + p.P_NAME AS P_NAME, + p.P_MFGR AS P_MFGR, + p.P_CATEGORY AS P_CATEGORY, + p.P_BRAND AS P_BRAND, + p.P_COLOR AS P_COLOR, + p.P_TYPE AS P_TYPE, + p.P_SIZE AS P_SIZE, + p.P_CONTAINER AS P_CONTAINER +FROM lineorder AS l +INNER JOIN customer AS c ON c.C_CUSTKEY = l.LO_CUSTKEY +INNER JOIN supplier AS s ON s.S_SUPPKEY = l.LO_SUPPKEY +INNER JOIN part AS p ON p.P_PARTKEY = l.LO_PARTKEY; +``` + +Sorguları çalıştırma: + +Q1.1 + +``` sql +SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue +FROM lineorder_flat +WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25; +``` + +Q1.2 + +``` sql +SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue +FROM lineorder_flat +WHERE toYYYYMM(LO_ORDERDATE) = 199401 AND LO_DISCOUNT BETWEEN 4 AND 6 AND LO_QUANTITY BETWEEN 26 AND 35; +``` + +Q1.3 + +``` sql +SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue +FROM lineorder_flat +WHERE toISOWeek(LO_ORDERDATE) = 6 AND toYear(LO_ORDERDATE) = 1994 + AND LO_DISCOUNT BETWEEN 5 AND 7 AND LO_QUANTITY BETWEEN 26 AND 35; +``` + +Q2.1 + +``` sql +SELECT + sum(LO_REVENUE), + toYear(LO_ORDERDATE) AS year, + P_BRAND +FROM lineorder_flat +WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' +GROUP BY + year, + P_BRAND +ORDER BY + year, + P_BRAND; +``` + +Q2.2 + +``` sql +SELECT + sum(LO_REVENUE), + toYear(LO_ORDERDATE) AS year, + P_BRAND +FROM lineorder_flat +WHERE P_BRAND >= 'MFGR#2221' AND P_BRAND <= 'MFGR#2228' AND S_REGION = 'ASIA' +GROUP BY + year, + P_BRAND +ORDER BY + year, + P_BRAND; +``` + +Q2.3 + +``` sql +SELECT + sum(LO_REVENUE), + toYear(LO_ORDERDATE) AS year, + P_BRAND +FROM lineorder_flat +WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE' +GROUP BY + year, + P_BRAND +ORDER BY + year, + P_BRAND; +``` + +Q3.1 + +``` sql +SELECT + C_NATION, + S_NATION, + toYear(LO_ORDERDATE) AS year, + sum(LO_REVENUE) AS revenue +FROM lineorder_flat +WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 +GROUP BY + C_NATION, + S_NATION, + year +ORDER BY + year ASC, + revenue DESC; +``` + +Q3.2 + +``` sql +SELECT + C_CITY, + S_CITY, + toYear(LO_ORDERDATE) AS year, + sum(LO_REVENUE) AS revenue +FROM lineorder_flat +WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997 +GROUP BY + C_CITY, + S_CITY, + year +ORDER BY + year ASC, + revenue DESC; +``` + +Q3.3 + +``` sql +SELECT + C_CITY, + S_CITY, + toYear(LO_ORDERDATE) AS year, + sum(LO_REVENUE) AS revenue +FROM lineorder_flat +WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997 +GROUP BY + C_CITY, + S_CITY, + year +ORDER BY + year ASC, + revenue DESC; +``` + +Q3.4 + +``` sql +SELECT + C_CITY, + S_CITY, + toYear(LO_ORDERDATE) AS year, + sum(LO_REVENUE) AS revenue +FROM lineorder_flat +WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND toYYYYMM(LO_ORDERDATE) = 199712 +GROUP BY + C_CITY, + S_CITY, + year +ORDER BY + year ASC, + revenue DESC; +``` + +Q4.1 + +``` sql +SELECT + toYear(LO_ORDERDATE) AS year, + C_NATION, + sum(LO_REVENUE - LO_SUPPLYCOST) AS profit +FROM lineorder_flat +WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') +GROUP BY + year, + C_NATION +ORDER BY + year ASC, + C_NATION ASC; +``` + +Q4.2 + +``` sql +SELECT + toYear(LO_ORDERDATE) AS year, + S_NATION, + P_CATEGORY, + sum(LO_REVENUE - LO_SUPPLYCOST) AS profit +FROM lineorder_flat +WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') +GROUP BY + year, + S_NATION, + P_CATEGORY +ORDER BY + year ASC, + S_NATION ASC, + P_CATEGORY ASC; +``` + +Q4.3 + +``` sql +SELECT + toYear(LO_ORDERDATE) AS year, + S_CITY, + P_BRAND, + sum(LO_REVENUE - LO_SUPPLYCOST) AS profit +FROM lineorder_flat +WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' +GROUP BY + year, + S_CITY, + P_BRAND +ORDER BY + year ASC, + S_CITY ASC, + P_BRAND ASC; +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/example_datasets/star_schema/) diff --git a/docs/tr/getting_started/example_datasets/wikistat.md b/docs/tr/getting_started/example_datasets/wikistat.md new file mode 100644 index 00000000000..0fc24dd5bb1 --- /dev/null +++ b/docs/tr/getting_started/example_datasets/wikistat.md @@ -0,0 +1,35 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 18 +toc_title: WikiStat +--- + +# WikiStat {#wikistat} + +Bakınız: http://dumps.wikimedia.org/other/pagecounts-raw/ + +Tablo oluşturma: + +``` sql +CREATE TABLE wikistat +( + date Date, + time DateTime, + project String, + subproject String, + path String, + hits UInt64, + size UInt64 +) ENGINE = MergeTree(date, (path, time), 8192); +``` + +Veri yükleme: + +``` bash +$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt +$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done +$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/example_datasets/wikistat/) diff --git a/docs/tr/getting_started/index.md b/docs/tr/getting_started/index.md new file mode 100644 index 00000000000..e97f9add1f0 --- /dev/null +++ b/docs/tr/getting_started/index.md @@ -0,0 +1,17 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Ba\u015Flarken" +toc_hidden: true +toc_priority: 8 +toc_title: "gizlenmi\u015F" +--- + +# Başlarken {#getting-started} + +Eğer ClickHouse için yeni ve performans bir hands-on hissi almak istiyorsanız, her şeyden önce, sen [yükleme işlemi](install.md). Bundan sonra şunları yapabilirsiniz: + +- [Ayrıntılı öğretici geçmesi](tutorial.md) +- [Örnek veri kümeleri ile deneme](example_datasets/ontime.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/) diff --git a/docs/tr/getting_started/install.md b/docs/tr/getting_started/install.md new file mode 100644 index 00000000000..8597a378fb4 --- /dev/null +++ b/docs/tr/getting_started/install.md @@ -0,0 +1,191 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 11 +toc_title: Kurulum +--- + +# Kurulum {#installation} + +## Sistem Gereksinimleri {#system-requirements} + +ClickHouse, x86\_64, AArch64 veya PowerPC64LE CPU mimarisine sahip herhangi bir Linux, FreeBSD veya Mac OS X üzerinde çalışabilir. + +Resmi önceden oluşturulmuş ikili dosyalar genellikle x86\_64 ve kaldıraç sse 4.2 komut seti için derlenir, bu nedenle destekleyen CPU'nun aksi belirtilmedikçe ek bir sistem gereksinimi haline gelir. Geçerli CPU'nun sse 4.2 desteği olup olmadığını kontrol etmek için komut: + +``` bash +$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +SSE 4.2'yi desteklemeyen veya AArch64 veya PowerPC64LE mimarisine sahip işlemcilerde Clickhouse'u çalıştırmak için şunları yapmalısınız [kaynaklardan ClickHouse oluşturun](#from-sources) uygun yapılandırma ayarlamaları ile. + +## Mevcut Kurulum Seçenekleri {#available-installation-options} + +### DEB paket fromlerinden {#install-from-deb-packages} + +Resmi önceden derlenmiş kullanılması tavsiye edilir `deb` Debian veya Ubuntu için paketler. + +Resmi paketleri yüklemek için Yandex deposunu ekleyin `/etc/apt/sources.list` veya ayrı bir `/etc/apt/sources.list.d/clickhouse.list` Dosya: + + deb http://repo.clickhouse.tech/deb/stable/ main/ + +En son sürümü kullanmak istiyorsanız, değiştirin `stable` ile `testing` (bu, test ortamlarınız için önerilir). + +Sonra paketleri yüklemek için bu komutları çalıştırın: + +``` bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +Paketleri buradan manuel olarak indirebilir ve kurabilirsiniz: https://repo.yandex.ru/clickhouse/deb/stable/main/. + +#### Paketler {#packages} + +- `clickhouse-common-static` — Installs ClickHouse compiled binary files. +- `clickhouse-server` — Creates a symbolic link for `clickhouse-server` ve varsayılan sunucu yapılandırmasını yükler. +- `clickhouse-client` — Creates a symbolic link for `clickhouse-client` ve diğer istemci ile ilgili araçlar. ve istemci yapılandırma dosyalarını yükler. +- `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. + +### RPM paket fromlerinden {#from-rpm-packages} + +Resmi önceden derlenmiş kullanılması tavsiye edilir `rpm` CentOS, RedHat ve diğer tüm rpm tabanlı Linux dağıtımları için paketler. + +İlk olarak, resmi depoyu eklemeniz gerekir: + +``` bash +sudo yum install yum-utils +sudo rpm --import https://repo.clickhouse.tech/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.clickhouse.tech/rpm/stable/x86_64 +``` + +En son sürümü kullanmak istiyorsanız, değiştirin `stable` ile `testing` (bu, test ortamlarınız için önerilir). Bu `prestable` etiket de bazen kullanılabilir. + +Sonra paketleri yüklemek için bu komutları çalıştırın: + +``` bash +sudo yum install clickhouse-server clickhouse-client +``` + +Paketleri buradan manuel olarak indirebilir ve kurabilirsiniz: https://repo.clickhouse.teknoloji / rpm / kararlı / x86\_64. + +### Tgz Arşivlerinden {#from-tgz-archives} + +Resmi önceden derlenmiş kullanılması tavsiye edilir `tgz` Arch ,iv ,es for tüm Linux dağıtım installationları, kurulumu `deb` veya `rpm` paketler mümkün değildir. + +Gerekli sürümü ile indirilebilir `curl` veya `wget` depo fromdan https://repo.yandex.ru/clickhouse/tgz/. +Bundan sonra indirilen arşivler açılmalı ve kurulum komut dosyaları ile kurulmalıdır. En son sürüm için örnek: + +``` bash +export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` +curl -O https://repo.clickhouse.tech/tgz/clickhouse-common-static-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.tech/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.tech/tgz/clickhouse-server-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.tech/tgz/clickhouse-client-$LATEST_VERSION.tgz + +tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz +sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh + +tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz +sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh + +tar -xzvf clickhouse-server-$LATEST_VERSION.tgz +sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh +sudo /etc/init.d/clickhouse-server start + +tar -xzvf clickhouse-client-$LATEST_VERSION.tgz +sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh +``` + +Üretim ortamları için en son teknolojiyi kullanmanız önerilir `stable`-sürüm. Numarasını GitHub sayfasında bulabilirsiniz https://github.com/ClickHouse/ClickHouse/tags postfix ile `-stable`. + +### Docker Görüntüden {#from-docker-image} + +Docker içinde ClickHouse çalıştırmak için kılavuzu izleyin [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Bu görüntüler resmi `deb` paketler içinde. + +### Kaynaklardan {#from-sources} + +Clickhouse'u el ile derlemek için aşağıdaki talimatları izleyin [Linux](../development/build.md) veya [Mac OS X](../development/build_osx.md). + +Paketleri derleyebilir ve yükleyebilir veya paketleri yüklemeden programları kullanabilirsiniz. Ayrıca elle inşa ederek SSE 4.2 gereksinimini devre dışı bırakabilir veya AArch64 CPU'lar için oluşturabilirsiniz. + + Client: programs/clickhouse-client + Server: programs/clickhouse-server + +Bir veri ve meta veri klasörleri oluşturmanız gerekir ve `chown` onları istenen kullanıcı için. Yolları sunucu yapılandırmasında değiştirilebilir (src / programlar / sunucu / config.xml), varsayılan olarak: + + /opt/clickhouse/data/default/ + /opt/clickhouse/metadata/default/ + +Gentoo üzerinde, sadece kullanabilirsiniz `emerge clickhouse` Clickhouse'u kaynaklardan yüklemek için. + +## Başlamak {#launch} + +Sunucuyu bir daemon olarak başlatmak için çalıştırın: + +``` bash +$ sudo service clickhouse-server start +``` + +Yok eğer `service` command, run as + +``` bash +$ sudo /etc/init.d/clickhouse-server start +``` + +Günlükleri görmek `/var/log/clickhouse-server/` dizin. + +Sunucu başlatılmazsa, dosyadaki yapılandırmaları kontrol edin `/etc/clickhouse-server/config.xml`. + +Ayrıca sunucuyu konsoldan manuel olarak başlatabilirsiniz: + +``` bash +$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +Bu durumda, günlük geliştirme sırasında uygun olan konsola yazdırılacaktır. +Yapılandırma dosyası geçerli dizinde ise, `--config-file` parametre. Varsayılan olarak, kullanır `./config.xml`. + +ClickHouse erişim kısıtlama ayarlarını destekler. Bulun theurlar. `users.xml` dosya (yanındaki `config.xml`). +Varsayılan olarak, erişim için herhangi bir yerden izin verilir `default` Kullanıcı, şifre olmadan. Görmek `user/default/networks`. +Daha fazla bilgi için bölüme bakın [“Configuration Files”](../operations/configuration_files.md). + +Sunucuyu başlattıktan sonra, ona bağlanmak için komut satırı istemcisini kullanabilirsiniz: + +``` bash +$ clickhouse-client +``` + +Varsayılan olarak, bağlanır `localhost:9000` kullanıcı adına `default` şifre olmadan. Kullanarak uzak bir sunucuya bağlanmak için de kullanılabilir `--host` değişken. + +Terminal UTF-8 kodlamasını kullanmalıdır. +Daha fazla bilgi için bölüme bakın [“Command-line client”](../interfaces/cli.md). + +Örnek: + +``` bash +$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +**Tebrikler, sistem çalışıyor!** + +Denemeye devam etmek için, test veri kümelerinden birini indirebilir veya şunları yapabilirsiniz [öğretici](https://clickhouse.tech/tutorial.html). + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/install/) diff --git a/docs/tr/getting_started/playground.md b/docs/tr/getting_started/playground.md new file mode 100644 index 00000000000..7c0f3641544 --- /dev/null +++ b/docs/tr/getting_started/playground.md @@ -0,0 +1,48 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 14 +toc_title: "Bah\xE7e" +--- + +# ClickHouse Oyun Alanı {#clickhouse-playground} + +[ClickHouse Oyun Alanı](https://play.clickhouse.tech?file=welcome) kullanıcıların kendi sunucu veya küme kurmadan, anında sorguları çalıştırarak ClickHouse ile deneme sağlar. +Oyun alanında çeşitli örnek veri kümelerinin yanı sıra ClickHouse özelliklerini gösteren örnek sorgular da mevcuttur. + +Sorgular salt okunur bir kullanıcı olarak yürütülür. Bazı sınırlamaları ima eder: + +- DDL sorgularına İzin Verilmiyor +- Sorgu Ekle izin verilmez + +Aşağıdaki ayarlar da uygulanır: +- [`max_result_bytes=10485760`](../operations/settings/query_complexity/#max-result-bytes) +- [`max_result_rows=2000`](../operations/settings/query_complexity/#setting-max_result_rows) +- [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode) +- [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) + +ClickHouse oyun alanı m2 deneyimini sunar.küçükler +[ClickHouse için yönetilen hizmet](https://cloud.yandex.com/services/managed-clickhouse) +örnek host hosteded in [Üye.Bulut](https://cloud.yandex.com/). +Hakkında daha fazla bilgi [bulut sağlayıcıları](../commercial/cloud.md). + +ClickHouse Playground web arayüzü clickhouse üzerinden istekleri yapar [HTTP API](../interfaces/http.md). +Bahçesi arka uç herhangi bir ek sunucu tarafı uygulaması olmadan sadece bir ClickHouse kümesidir. +ClickHouse HTTPS bitiş noktası da oyun alanının bir parçası olarak kullanılabilir. + +Herhangi bir HTTP istemcisi kullanarak oyun alanına sorgu yapabilirsiniz, örneğin [kıvrılma](https://curl.haxx.se) veya [wget](https://www.gnu.org/software/wget/), veya kullanarak bir bağlantı kurmak [JDBC](../interfaces/jdbc.md) veya [ODBC](../interfaces/odbc.md) sürücüler. +Clickhouse'u destekleyen yazılım ürünleri hakkında daha fazla bilgi mevcuttur [burada](../interfaces/index.md). + +| Parametre | Değer | +|:----------|:----------------------------------------| +| Nokta | https://play-api.clickhouse.teknik:8443 | +| Kullanan | `playground` | +| Şifre | `clickhouse` | + +Bu bitiş noktasının güvenli bir bağlantı gerektirdiğini unutmayın. + +Örnek: + +``` bash +curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets" +``` diff --git a/docs/tr/getting_started/tutorial.md b/docs/tr/getting_started/tutorial.md new file mode 100644 index 00000000000..9d3d31ba898 --- /dev/null +++ b/docs/tr/getting_started/tutorial.md @@ -0,0 +1,671 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 12 +toc_title: "\xD6\u011Fretici" +--- + +# ClickHouse Eğitimi {#clickhouse-tutorial} + +## Bu Öğreticiden ne beklenir? {#what-to-expect-from-this-tutorial} + +Bu öğreticiden geçerek, basit bir ClickHouse kümesinin nasıl kurulacağını öğreneceksiniz. Küçük ama hataya dayanıklı ve ölçeklenebilir olacak. Ardından, verilerle doldurmak ve bazı demo sorguları yürütmek için örnek veri kümelerinden birini kullanacağız. + +## Tek Düğüm Kurulumu {#single-node-setup} + +Dağıtılmış bir ortamın karmaşıklığını ertelemek için, Clickhouse'u tek bir sunucu veya sanal makinede dağıtmaya başlayacağız. ClickHouse genellikle [deb](index.md#install-from-deb-packages) veya [rpm](index.md#from-rpm-packages) paketler, ama var [alternatifler](index.md#from-docker-image) onları desteklemeyen işletim sistemleri için. + +Örneğin, seçtiğiniz `deb` paketler ve yürütülen: + +``` bash +sudo apt-get install dirmngr +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 + +echo "deb http://repo.clickhouse.tech/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list +sudo apt-get update + +sudo apt-get install -y clickhouse-server clickhouse-client +``` + +Yüklü olan paketlerde ne var: + +- `clickhouse-client` paket içerir [clickhouse-müşteri](../interfaces/cli.md) uygulama, interaktif ClickHouse konsol istemcisi. +- `clickhouse-common` paket clickhouse yürütülebilir dosya içerir. +- `clickhouse-server` paket bir sunucu olarak ClickHouse çalıştırmak için yapılandırma dosyalarını içerir. + +Sunucu yapılandırma dosyaları bulunur `/etc/clickhouse-server/`. Daha fazla gitmeden önce, fark lütfen `` element in `config.xml`. Path, veri depolama için konumu belirler, bu nedenle büyük disk kapasitesine sahip birimde bulunmalıdır; varsayılan değer `/var/lib/clickhouse/`. Yapılandırmayı ayarlamak istiyorsanız, doğrudan düzenlemek kullanışlı değildir `config.xml` dosya, gelecekteki paket güncellemelerinde yeniden yazılabileceğini düşünüyor. Yapılandırma öğelerini geçersiz kılmak için önerilen yol oluşturmaktır [config dosyaları.d dizin](../operations/configuration_files.md) hizmet olarak “patches” config için.xml. + +Fark etmiş olabileceğiniz gibi, `clickhouse-server` paket kurulumdan sonra otomatik olarak başlatılmaz. Güncellemelerden sonra otomatik olarak yeniden başlatılmaz. Sunucuyu başlatma şekliniz init sisteminize bağlıdır, genellikle: + +``` bash +sudo service clickhouse-server start +``` + +veya + +``` bash +sudo /etc/init.d/clickhouse-server start +``` + +Sunucu günlükleri için varsayılan konum `/var/log/clickhouse-server/`. Sunucu, oturum açtıktan sonra istemci bağlantılarını işlemeye hazırdır. `Ready for connections` ileti. + +Bir kez `clickhouse-server` yukarı ve çalışıyor, biz kullanabilirsiniz `clickhouse-client` sunucuya bağlanmak ve aşağıdaki gibi bazı test sorguları çalıştırmak için `SELECT "Hello, world!";`. + +
+ +Clickhouse-client için hızlı ipuçları +İnteraktif mod: + +``` bash +clickhouse-client +clickhouse-client --host=... --port=... --user=... --password=... +``` + +Çok satırlı sorguları etkinleştirme: + +``` bash +clickhouse-client -m +clickhouse-client --multiline +``` + +Toplu iş modunda sorguları çalıştırma: + +``` bash +clickhouse-client --query='SELECT 1' +echo 'SELECT 1' | clickhouse-client +clickhouse-client <<< 'SELECT 1' +``` + +Belirtilen biçimde bir dosyadan veri ekleme: + +``` bash +clickhouse-client --query='INSERT INTO table VALUES' < data.txt +clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv +``` + +
+ +## Örnek Veri Kümesini İçe Aktar {#import-sample-dataset} + +Şimdi ClickHouse sunucumuzu bazı örnek verilerle doldurmanın zamanı geldi. Bu eğitimde, yandex'in anonim verilerini kullanacağız.Metrica, açık kaynak olmadan önce Clickhouse'u üretim yolunda çalıştıran ilk hizmet (daha fazlası [tarih bölümü](../introduction/history.md)). Var [Yandex'i içe aktarmanın birden fazla yolu.Metrica veri kümesi](example_datasets/metrica.md) ve öğretici uğruna, en gerçekçi olanı ile gideceğiz. + +### Tablo verilerini indirin ve ayıklayın {#download-and-extract-table-data} + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +``` + +Çıkarılan dosyalar yaklaşık 10GB boyutundadır. + +### Tablo Oluşturma {#create-tables} + +Çoğu veritabanı yönetim sisteminde olduğu gibi, ClickHouse tabloları mantıksal olarak gruplar “databases”. Bir `default` veritabanı, ancak adında yeni bir tane oluşturacağız `tutorial`: + +``` bash +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" +``` + +Tablolar oluşturmak için sözdizimi veritabanlarına kıyasla çok daha karmaşıktır (bkz. [başvurma](../sql_reference/statements/create.md). Genelde `CREATE TABLE` deyimi üç önemli şeyi belirtmek zorundadır: + +1. Oluşturulacak tablonun adı. +2. Table schema, i.e. list of columns and their [veri türleri](../sql_reference/data_types/index.md). +3. [Masa motoru](../engines/table_engines/index.md) ve bu tabloya yapılan sorguların fiziksel olarak nasıl yürütüleceği ile ilgili tüm ayrıntıları belirleyen ayarları. + +Üye.Metrica bir web analitiği hizmetidir ve örnek veri kümesi tam işlevselliğini kapsamaz, bu nedenle oluşturmak için sadece iki tablo vardır: + +- `hits` hizmet kapsamındaki tüm web sitelerinde tüm kullanıcılar tarafından yapılan her eylem ile bir tablodur. +- `visits` bireysel eylemler yerine önceden oluşturulmuş oturumlar içeren bir tablodur. + +Bu tablolar için gerçek create table sorgularını görelim ve yürütelim: + +``` sql +CREATE TABLE tutorial.hits_v1 +( + `WatchID` UInt64, + `JavaEnable` UInt8, + `Title` String, + `GoodEvent` Int16, + `EventTime` DateTime, + `EventDate` Date, + `CounterID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RegionID` UInt32, + `UserID` UInt64, + `CounterClass` Int8, + `OS` UInt8, + `UserAgent` UInt8, + `URL` String, + `Referer` String, + `URLDomain` String, + `RefererDomain` String, + `Refresh` UInt8, + `IsRobot` UInt8, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `FlashMinor2` String, + `NetMajor` UInt8, + `NetMinor` UInt8, + `UserAgentMajor` UInt16, + `UserAgentMinor` FixedString(2), + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `MobilePhone` UInt8, + `MobilePhoneModel` String, + `Params` String, + `IPNetworkID` UInt32, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `IsArtifical` UInt8, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `ClientTimeZone` Int16, + `ClientEventTime` DateTime, + `SilverlightVersion1` UInt8, + `SilverlightVersion2` UInt8, + `SilverlightVersion3` UInt32, + `SilverlightVersion4` UInt16, + `PageCharset` String, + `CodeVersion` UInt32, + `IsLink` UInt8, + `IsDownload` UInt8, + `IsNotBounce` UInt8, + `FUniqID` UInt64, + `HID` UInt32, + `IsOldCounter` UInt8, + `IsEvent` UInt8, + `IsParameter` UInt8, + `DontCountHits` UInt8, + `WithHash` UInt8, + `HitColor` FixedString(1), + `UTCEventTime` DateTime, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `WindowName` Int32, + `OpenerName` Int32, + `HistoryLength` Int16, + `BrowserLanguage` FixedString(2), + `BrowserCountry` FixedString(2), + `SocialNetwork` String, + `SocialAction` String, + `HTTPError` UInt16, + `SendTiming` Int32, + `DNSTiming` Int32, + `ConnectTiming` Int32, + `ResponseStartTiming` Int32, + `ResponseEndTiming` Int32, + `FetchTiming` Int32, + `RedirectTiming` Int32, + `DOMInteractiveTiming` Int32, + `DOMContentLoadedTiming` Int32, + `DOMCompleteTiming` Int32, + `LoadEventStartTiming` Int32, + `LoadEventEndTiming` Int32, + `NSToDOMContentLoadedTiming` Int32, + `FirstPaintTiming` Int32, + `RedirectCount` Int8, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `ParamPrice` Int64, + `ParamOrderID` String, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `GoalsReached` Array(UInt32), + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `RefererHash` UInt64, + `URLHash` UInt64, + `CLID` UInt32, + `YCLID` UInt64, + `ShareService` String, + `ShareURL` String, + `ShareTitle` String, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `IslandID` FixedString(16), + `RequestNum` UInt32, + `RequestTry` UInt8 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate, intHash32(UserID)) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +``` sql +CREATE TABLE tutorial.visits_v1 +( + `CounterID` UInt32, + `StartDate` Date, + `Sign` Int8, + `IsNew` UInt8, + `VisitID` UInt64, + `UserID` UInt64, + `StartTime` DateTime, + `Duration` UInt32, + `UTCStartTime` DateTime, + `PageViews` Int32, + `Hits` Int32, + `IsBounce` UInt8, + `Referer` String, + `StartURL` String, + `RefererDomain` String, + `StartURLDomain` String, + `EndURL` String, + `LinkURL` String, + `IsDownload` UInt8, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `PlaceID` Int32, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `IsYandex` UInt8, + `GoalReachesDepth` Int32, + `GoalReachesURL` Int32, + `GoalReachesAny` Int32, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `MobilePhoneModel` String, + `ClientEventTime` DateTime, + `RegionID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `IPNetworkID` UInt32, + `SilverlightVersion3` UInt32, + `CodeVersion` UInt32, + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `UserAgentMajor` UInt16, + `UserAgentMinor` UInt16, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `SilverlightVersion2` UInt8, + `SilverlightVersion4` UInt16, + `FlashVersion3` UInt16, + `FlashVersion4` UInt16, + `ClientTimeZone` Int16, + `OS` UInt8, + `UserAgent` UInt8, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `NetMajor` UInt8, + `NetMinor` UInt8, + `MobilePhone` UInt8, + `SilverlightVersion1` UInt8, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `JavaEnable` UInt8, + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `BrowserLanguage` UInt16, + `BrowserCountry` UInt16, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `Params` Array(String), + `Goals` Nested( + ID UInt32, + Serial UInt32, + EventTime DateTime, + Price Int64, + OrderID String, + CurrencyID UInt32), + `WatchIDs` Array(UInt64), + `ParamSumPrice` Int64, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `ClickLogID` UInt64, + `ClickEventID` Int32, + `ClickGoodEvent` Int32, + `ClickEventTime` DateTime, + `ClickPriorityID` Int32, + `ClickPhraseID` Int32, + `ClickPageID` Int32, + `ClickPlaceID` Int32, + `ClickTypeID` Int32, + `ClickResourceID` Int32, + `ClickCost` UInt32, + `ClickClientIP` UInt32, + `ClickDomainID` UInt32, + `ClickURL` String, + `ClickAttempt` UInt8, + `ClickOrderID` UInt32, + `ClickBannerID` UInt32, + `ClickMarketCategoryID` UInt32, + `ClickMarketPP` UInt32, + `ClickMarketCategoryName` String, + `ClickMarketPPName` String, + `ClickAWAPSCampaignName` String, + `ClickPageName` String, + `ClickTargetType` UInt16, + `ClickTargetPhraseID` UInt64, + `ClickContextType` UInt8, + `ClickSelectType` Int8, + `ClickOptions` String, + `ClickGroupBannerID` Int32, + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `FirstVisit` DateTime, + `PredLastVisit` Date, + `LastVisit` Date, + `TotalVisits` UInt32, + `TraficSource` Nested( + ID Int8, + SearchEngineID UInt16, + AdvEngineID UInt8, + PlaceID UInt16, + SocialSourceNetworkID UInt8, + Domain String, + SearchPhrase String, + SocialSourcePage String), + `Attendance` FixedString(16), + `CLID` UInt32, + `YCLID` UInt64, + `NormalizedRefererHash` UInt64, + `SearchPhraseHash` UInt64, + `RefererDomainHash` UInt64, + `NormalizedStartURLHash` UInt64, + `StartURLDomainHash` UInt64, + `NormalizedEndURLHash` UInt64, + `TopLevelDomain` UInt64, + `URLScheme` UInt64, + `OpenstatServiceNameHash` UInt64, + `OpenstatCampaignIDHash` UInt64, + `OpenstatAdIDHash` UInt64, + `OpenstatSourceIDHash` UInt64, + `UTMSourceHash` UInt64, + `UTMMediumHash` UInt64, + `UTMCampaignHash` UInt64, + `UTMContentHash` UInt64, + `UTMTermHash` UInt64, + `FromHash` UInt64, + `WebVisorEnabled` UInt8, + `WebVisorActivity` UInt32, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `Market` Nested( + Type UInt8, + GoalID UInt32, + OrderID String, + OrderPrice Int64, + PP UInt32, + DirectPlaceID UInt32, + DirectOrderID UInt32, + DirectBannerID UInt32, + GoodID String, + GoodName String, + GoodQuantity Int32, + GoodPrice Int64), + `IslandID` FixedString(16) +) +ENGINE = CollapsingMergeTree(Sign) +PARTITION BY toYYYYMM(StartDate) +ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +Etkileşimli modunu kullanarak bu sorguları yürütebilirsiniz `clickhouse-client` (sadece önceden bir sorgu belirtmeden bir terminalde başlatın) veya bazı deneyin [alternatif arayüz](../interfaces/index.md) Eğer isterseniz. + +Gördüğümüz gibi, `hits_v1` kullanır [temel MergeTree motoru](../engines/table_engines/mergetree_family/mergetree.md), WH whileile the `visits_v1` kullanır [Çökme](../engines/table_engines/mergetree_family/collapsingmergetree.md) varyant. + +### Verileri İçe Aktar {#import-data} + +Clickhouse'a veri aktarımı yapılır [INSERT INTO](../sql_reference/statements/insert_into.md) diğer birçok SQL veritabanlarında olduğu gibi sorgu. Bununla birlikte, veriler genellikle [desteklenen seri hale getirme biçimleri](../interfaces/formats.md) yerine `VALUES` fıkra clausesı (ayrıca desteklenmektedir). + +Onları almak için ne kadar daha önce indirdiğimiz dosyaları sekme ayrılmış biçimde, yani burada konsol istemci ile : + +``` bash +clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv +clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv +``` + +ClickHouse bir yeri vardır [ayarlan settingsacak ayarlar](../operations/settings/index.md) ve bunları konsol istemcisinde belirtmenin bir yolu, görebildiğimiz gibi argümanlar aracılığıyla `--max_insert_block_size`. Hangi ayarların mevcut olduğunu, ne anlama geldiğini ve varsayılanların ne olduğunu anlamanın en kolay yolu `system.settings` Tablo: + +``` sql +SELECT name, value, changed, description +FROM system.settings +WHERE name LIKE '%max_insert_b%' +FORMAT TSV + +max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." +``` + +İsteğe bağlı olarak şunları yapabilirsiniz [OPTIMIZE](../sql_reference/statements/misc.md#misc_operations-optimize) ithalattan sonra tablolar. MergeTree-family'den bir motorla yapılandırılmış tablolar, veri depolamayı en iyi duruma getirmek (veya en azından mantıklı olup olmadığını kontrol etmek) için her zaman arka planda veri parçalarının birleştirilmesini sağlar. Bu sorgular, tablo motorunu bir süre sonra yerine şu anda depolama optimizasyonu yapmaya zorlar: + +``` bash +clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" +clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" +``` + +Bu sorgular bir G/Ç ve CPU yoğun işlem başlatır, bu nedenle tablo sürekli olarak yeni veriler alırsa, onu yalnız bırakmak ve birleştirmelerin arka planda çalışmasına izin vermek daha iyidir. + +Şimdi tablo ithalatının başarılı olup olmadığını kontrol edebiliriz: + +``` bash +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" +``` + +## Örnek Sorgular {#example-queries} + +``` sql +SELECT + StartURL AS URL, + AVG(Duration) AS AvgDuration +FROM tutorial.visits_v1 +WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' +GROUP BY URL +ORDER BY AvgDuration DESC +LIMIT 10 +``` + +``` sql +SELECT + sum(Sign) AS visits, + sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, + (100. * goal_visits) / visits AS goal_percent +FROM tutorial.visits_v1 +WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') +``` + +## Küme Dağıtımı {#cluster-deployment} + +ClickHouse kümesi homojen bir kümedir. Kurulum adımları: + +1. Kümenin tüm makinelerine ClickHouse Server'ı yükleyin +2. Yapılandırma dosyalarında küme yapılandırmalarını ayarlama +3. Her örnekte yerel tablolar oluşturun +4. Create a [Dağıtılmış tablo](../engines/table_engines/special/distributed.md) + +[Dağıtılmış tablo](../engines/table_engines/special/distributed.md) aslında bir tür “view” ClickHouse kümesinin yerel tablolarına. Dağıtılmış bir tablodan sorgu seçin, tüm kümenin parçalarının kaynaklarını kullanarak yürütür. Birden çok küme için yapılandırmalar belirtebilir ve farklı kümelere görünümler sağlayan birden çok dağıtılmış tablo oluşturabilirsiniz. + +Her biri bir kopya olan üç parçalı bir küme için örnek yapılandırma: + +``` xml + + + + + example-perftest01j.yandex.ru + 9000 + + + + + example-perftest02j.yandex.ru + 9000 + + + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +Daha fazla gösteri için, aynı ile yeni bir yerel tablo oluşturalım `CREATE TABLE` için kullandığımız sorgu `hits_v1` ama farklı bir tablo adı: + +``` sql +CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... +``` + +Kümenin yerel tablolarına bir görünüm sağlayan dağıtılmış bir tablo oluşturma: + +``` sql +CREATE TABLE tutorial.hits_all AS tutorial.hits_local +ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); +``` + +Yaygın bir uygulama, kümenin tüm makinelerinde benzer dağıtılmış tablolar oluşturmaktır. Kümenin herhangi bir makinesinde dağıtılmış sorguları çalıştırmaya izin verir. Ayrıca, belirli bir SELECT sorgusu için geçici dağıtılmış tablo oluşturmak için alternatif bir seçenek vardır [uzak](../sql_reference/table_functions/remote.md) tablo işlevi. + +Hadi koşalım [INSERT SELECT](../sql_reference/statements/insert_into.md) tabloyu birden çok sunucuya yaymak için dağıtılmış tabloya. + +``` sql +INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; +``` + +!!! warning "Bildirim" + Bu yaklaşım büyük tabloların parçalanması için uygun değildir. Ayrı bir araç var [clickhouse-fotokopi makinesi](../operations/utilities/clickhouse-copier.md) bu keyfi büyük tabloları yeniden parçalayabilir. + +Tahmin edebileceğiniz gibi, hesaplamalı olarak ağır sorgular, bir yerine 3 sunucu kullanıyorsa n kat daha hızlı çalışır. + +Bu durumda, 3 parçaya sahip bir küme kullandık ve her biri tek bir kopya içeriyor. + +Bir üretim ortamında esneklik sağlamak için, her bir parçanın birden çok kullanılabilirlik bölgesi veya veri merkezleri (veya en azından raflar) arasında yayılmış 2-3 kopya içermesi önerilir. ClickHouse yinelemeler sınırsız sayıda desteklediğini unutmayın. + +Üç yineleme içeren bir parça kümesi için örnek yapılandırma: + +``` xml + + ... + + + + example-perftest01j.yandex.ru + 9000 + + + example-perftest02j.yandex.ru + 9000 + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +Yerel çoğaltmayı etkinleştirmek için [ZooKeeper](http://zookeeper.apache.org/) gereklidir. ClickHouse tüm yinelemeler üzerinde veri tutarlılığı ilgilenir ve otomatik olarak hatadan sonra prosedürü geri çalışır. ZooKeeper kümesinin ayrı sunuculara dağıtılması önerilir(ClickHouse dahil başka hiçbir işlem çalışmaz). + +!!! note "Not" + ZooKeeper sıkı bir gereklilik değildir: bazı basit durumlarda, verileri uygulama kodunuzdan tüm kopyalara yazarak çoğaltabilirsiniz. Bu yaklaşım **değil** önerilen, bu durumda, ClickHouse tüm yinelemelerde veri tutarlılığını garanti edemez. Böylece başvurunuzun sorumluluğu haline gelir. + +Zookeeper konumları yapılandırma dosyasında belirtilir: + +``` xml + + + zoo01.yandex.ru + 2181 + + + zoo02.yandex.ru + 2181 + + + zoo03.yandex.ru + 2181 + + +``` + +Ayrıca, tablo oluşturulmasında kullanılan her bir parça ve kopyayı tanımlamak için makrolar ayarlamamız gerekir: + +``` xml + + 01 + 01 + +``` + +Yinelenmiş tablo oluşturma şu anda hiçbir yinelemeler varsa, yeni bir ilk yineleme örneği. Zaten canlı yinelemeler varsa, yeni yineleme varolan verileri klonlar. Önce tüm çoğaltılmış tablolar oluşturmak ve sonra veri eklemek için bir seçeneğiniz vardır. Başka bir seçenek, bazı yinelemeler oluşturmak ve veri ekleme sırasında veya sonrasında diğerlerini eklemektir. + +``` sql +CREATE TABLE tutorial.hits_replica (...) +ENGINE = ReplcatedMergeTree( + '/clickhouse_perftest/tables/{shard}/hits', + '{replica}' +) +... +``` + +Burada kullanıyoruz [ReplicatedMergeTree](../engines/table_engines/mergetree_family/replication.md) masa motoru. Parametrelerde, Shard ve çoğaltma tanımlayıcılarını içeren ZooKeeper yolunu belirtiyoruz. + +``` sql +INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; +``` + +Çoğaltma çok ana modda çalışır. Veriler herhangi bir kopyaya yüklenebilir ve sistem daha sonra otomatik olarak diğer örneklerle eşitler. Çoğaltma zaman uyumsuz olduğundan, belirli bir anda, tüm yinelemeler son eklenen verileri içerebilir. Veri alımına izin vermek için en az bir kopya olmalıdır. Diğerleri verileri senkronize eder ve tekrar aktif hale geldiklerinde tutarlılığı onarır. Bu yaklaşımın, yakın zamanda eklenen veri kaybı olasılığının düşük olmasına izin verdiğini unutmayın. + +[Orijinal makale](https://clickhouse.tech/docs/en/getting_started/tutorial/) diff --git a/docs/tr/guides/apply_catboost_model.md b/docs/tr/guides/apply_catboost_model.md new file mode 100644 index 00000000000..92bfac226f2 --- /dev/null +++ b/docs/tr/guides/apply_catboost_model.md @@ -0,0 +1,239 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 41 +toc_title: CatBoost Modellerini Uygulamak +--- + +# Clickhouse'da bir Catboost modeli uygulamak {#applying-catboost-model-in-clickhouse} + +[CatBoost](https://catboost.ai) geliştirilen ücretsiz ve açık kaynak kodlu bir GRA anddi libraryent kütüphan aesidir. [Yandex](https://yandex.com/company/) makine öğrenimi için. + +Bu Talimatla, Sql'den model çıkarımı çalıştırarak Clickhouse'da önceden eğitilmiş modelleri uygulamayı öğreneceksiniz. + +Clickhouse'da bir CatBoost modeli uygulamak için: + +1. [Tablo oluşturma](#create-table). +2. [Verileri tabloya ekleme](#insert-data-to-table). +3. [Catboost'u Clickhouse'a entegre edin](#integrate-catboost-into-clickhouse) (İsteğe bağlı adım). +4. [SQL'DEN Model Çıkarımını çalıştırın](#run-model-inference). + +Eğitim CatBoost modelleri hakkında daha fazla bilgi için bkz [Eğitim ve uygulama modelleri](https://catboost.ai/docs/features/training.html#training). + +## Önkoşullar {#prerequisites} + +Eğer yoksa [Docker](https://docs.docker.com/install/) yine de yükleyin. + +!!! note "Not" + [Docker](https://www.docker.com) sistemin geri kalanından bir CatBoost ve ClickHouse kurulumunu izole eden kaplar oluşturmanıza olanak sağlayan bir yazılım platformudur. + +Bir CatBoost modeli uygulamadan önce: + +**1.** P pullull the [Docker görüntü](https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) kayıt defter theinden: + +``` bash +$ docker pull yandex/tutorial-catboost-clickhouse +``` + +Kod, çalışma zamanı, kütüphaneler, ortam değişkenleri ve Yapılandırma Dosyaları: bu Docker görüntü catboost ve ClickHouse çalıştırmak için gereken her şeyi içerir. + +**2.** Docker görüntüsünün başarıyla çekildiğinden emin olun: + +``` bash +$ docker image ls +REPOSITORY TAG IMAGE ID CREATED SIZE +yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB +``` + +**3.** Bu görüntüye dayalı bir Docker kabı başlatın: + +``` bash +$ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse +``` + +## 1. Tablo oluşturma {#create-table} + +Eğitim örneği için bir ClickHouse tablosu oluşturmak için: + +**1.** Etkileşimli modda ClickHouse konsol istemcisini başlatın: + +``` bash +$ clickhouse client +``` + +!!! note "Not" + Clickhouse sunucusu Docker kapsayıcısı içinde zaten çalışıyor. + +**2.** Komutu kullanarak tablo oluşturun: + +``` sql +:) CREATE TABLE amazon_train +( + date Date MATERIALIZED today(), + ACTION UInt8, + RESOURCE UInt32, + MGR_ID UInt32, + ROLE_ROLLUP_1 UInt32, + ROLE_ROLLUP_2 UInt32, + ROLE_DEPTNAME UInt32, + ROLE_TITLE UInt32, + ROLE_FAMILY_DESC UInt32, + ROLE_FAMILY UInt32, + ROLE_CODE UInt32 +) +ENGINE = MergeTree ORDER BY date +``` + +**3.** ClickHouse konsol istemcisinden çıkış: + +``` sql +:) exit +``` + +## 2. Verileri tabloya ekleme {#insert-data-to-table} + +Verileri eklemek için: + +**1.** Aşağıdaki komutu çalıştırın: + +``` bash +$ clickhouse client --host 127.0.0.1 --query 'INSERT INTO amazon_train FORMAT CSVWithNames' < ~/amazon/train.csv +``` + +**2.** Etkileşimli modda ClickHouse konsol istemcisini başlatın: + +``` bash +$ clickhouse client +``` + +**3.** Verilerin yüklendiğinden emin olun: + +``` sql +:) SELECT count() FROM amazon_train + +SELECT count() +FROM amazon_train + ++-count()-+ +| 65538 | ++-------+ +``` + +## 3. Catboost'u Clickhouse'a entegre edin {#integrate-catboost-into-clickhouse} + +!!! note "Not" + **İsteğe bağlı adım.** Docker görüntü catboost ve ClickHouse çalıştırmak için gereken her şeyi içerir. + +Catboost'u Clickhouse'a entegre etmek için: + +**1.** Değerlendirme kitaplığı oluşturun. + +Bir CatBoost modelini değerlendirmenin en hızlı yolu derlemedir `libcatboostmodel.` kitaplık. Kitaplığın nasıl oluşturulacağı hakkında daha fazla bilgi için bkz. [CatBoost belgeleri](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). + +**2.** Herhangi bir yerde ve herhangi bir adla yeni bir dizin oluşturun, örneğin, `data` ve oluşturulan kütüphaneyi içine koyun. Docker görüntüsü zaten kütüphaneyi içeriyor `data/libcatboostmodel.so`. + +**3.** Yapılandırma modeli için herhangi bir yerde ve herhangi bir adla yeni bir dizin oluşturun, örneğin, `models`. + +**4.** Örneğin, herhangi bir ada sahip bir model yapılandırma dosyası oluşturun, `models/amazon_model.xml`. + +**5.** Model yapılandırmasını açıklayın: + +``` xml + + + + catboost + + amazon + + /home/catboost/tutorial/catboost_model.bin + + 0 + + +``` + +**6.** Catboost yolunu ve model yapılandırmasını ClickHouse yapılandırmasına ekleyin: + +``` xml + +/home/catboost/data/libcatboostmodel.so +/home/catboost/models/*_model.xml +``` + +## 4. SQL'DEN Model Çıkarımını çalıştırın {#run-model-inference} + +Test modeli için ClickHouse istemcisini çalıştırın `$ clickhouse client`. + +Modelin çalıştığından emin olalım: + +``` sql +:) SELECT + modelEvaluate('amazon', + RESOURCE, + MGR_ID, + ROLE_ROLLUP_1, + ROLE_ROLLUP_2, + ROLE_DEPTNAME, + ROLE_TITLE, + ROLE_FAMILY_DESC, + ROLE_FAMILY, + ROLE_CODE) > 0 AS prediction, + ACTION AS target +FROM amazon_train +LIMIT 10 +``` + +!!! note "Not" + İşlev [modelEvaluate](../sql_reference/functions/other_functions.md#function-modelevaluate) multiclass modelleri için sınıf başına ham tahminleri ile tuple döndürür. + +Olasılığı tahmin edelim: + +``` sql +:) SELECT + modelEvaluate('amazon', + RESOURCE, + MGR_ID, + ROLE_ROLLUP_1, + ROLE_ROLLUP_2, + ROLE_DEPTNAME, + ROLE_TITLE, + ROLE_FAMILY_DESC, + ROLE_FAMILY, + ROLE_CODE) AS prediction, + 1. / (1 + exp(-prediction)) AS probability, + ACTION AS target +FROM amazon_train +LIMIT 10 +``` + +!!! note "Not" + Hakkında daha fazla bilgi [exp()](../sql_reference/functions/math_functions.md) işlev. + +Örnek üzerinde LogLoss hesaplayalım: + +``` sql +:) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss +FROM +( + SELECT + modelEvaluate('amazon', + RESOURCE, + MGR_ID, + ROLE_ROLLUP_1, + ROLE_ROLLUP_2, + ROLE_DEPTNAME, + ROLE_TITLE, + ROLE_FAMILY_DESC, + ROLE_FAMILY, + ROLE_CODE) AS prediction, + 1. / (1. + exp(-prediction)) AS prob, + ACTION AS tg + FROM amazon_train +) +``` + +!!! note "Not" + Hakkında daha fazla bilgi [avg()](../sql_reference/aggregate_functions/reference.md#agg_function-avg) ve [günlük()](../sql_reference/functions/math_functions.md) işlevler. + +[Orijinal makale](https://clickhouse.tech/docs/en/guides/apply_catboost_model/) diff --git a/docs/tr/guides/index.md b/docs/tr/guides/index.md new file mode 100644 index 00000000000..95ad65443b0 --- /dev/null +++ b/docs/tr/guides/index.md @@ -0,0 +1,16 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "K\u0131lavuzlar" +toc_priority: 38 +toc_title: "Genel bak\u0131\u015F" +--- + +# ClickHouse Kılavuzları {#clickhouse-guides} + +ClickHouse kullanarak çeşitli görevleri çözmeye yardımcı olan ayrıntılı adım adım talimatların listesi: + +- [Basit küme kurulumu eğitimi](../getting_started/tutorial.md) +- [Clickhouse'da bir CatBoost modeli uygulamak](apply_catboost_model.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/guides/) diff --git a/docs/tr/images b/docs/tr/images new file mode 120000 index 00000000000..73937c941ec --- /dev/null +++ b/docs/tr/images @@ -0,0 +1 @@ +../en/images \ No newline at end of file diff --git a/docs/tr/index.md b/docs/tr/index.md new file mode 100644 index 00000000000..5cbf9330750 --- /dev/null +++ b/docs/tr/index.md @@ -0,0 +1,95 @@ +--- +toc_priority: 0 +toc_title: "Genel bak\u0131\u015F" +--- + +# ClickHouse nedir? {#what-is-clickhouse} + +ClickHouse, sorguların çevrimiçi analitik işlenmesi (*Online Analytical Processing* - OLAP) için sütun odaklı bir Veritabanı Yönetim Sistemidir (*DataBase Management System* - DBMS). + +“Normal” bir satır odaklı DBMS içinde veriler şu şekilde saklanır: + +| Satır | WatchId | JavaEnable | Başlık | İyiOlay | OlayZamanı | +|-------|-------------|------------|----------------------|-----------|---------------------| +| \#0 | 89354350662 | 1 | Yatırımcı İlişkileri | 1 | 2016-05-18 05:19:20 | +| \#1 | 90329509958 | 0 | Bize ulaşın | 1 | 2016-05-18 08:10:20 | +| \#2 | 89953706054 | 1 | Görev | 1 | 2016-05-18 07:38:00 | +| \#N | … | … | … | … | … | + +Başka bir deyişle, bir satırla ilgili tüm değerler fiziksel olarak yan yana depolanır. + +MySQL, Postgres ve MS SQL Server gibi veritabanları satır odaklı DBMS örnekleridir. + +Sütun odaklı bir DBMS'de ise veriler şu şekilde saklanır: + +| Satır: | \#0 | \#1 | \#2 | \#N | +|-------------|----------------------|---------------------|---------------------|-----| +| WatchId: | 89354350662 | 90329509958 | 89953706054 | … | +| JavaEnable: | 1 | 0 | 1 | … | +| Başlık: | Yatırımcı İlişkileri | Bize ulaşın | Görev | … | +| İyiOlay: | 1 | 1 | 1 | … | +| OlayZamanı: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | … | + +Bu örnekler yalnızca verilerin düzenlendiği sırayı gösterir. Farklı sütunlardaki değerler ayrı olarak depolanır ve aynı sütundaki veriler birlikte depolanır. + +Sütun odaklı DBMS örnekleri: Vertica, Paraccel (Actian Matrix ve Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise ve Actian vektör), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid ve kdb+. + +Verinin farklı bir şekilde sıralanarak depolanması, bazı veri erişim senaryoları için daha uygundur. Veri erişim senaryosu, hangi sorguların ne kadar sıklıkla yapıldığını, ne kadar verinin okunduğu, bunların hangi tiplerde hangi kolonlardan, satırlardan ve hangi miktarda(bayt olarak) okunacağını; verinin okunması ile güncellenmesi arasındaki ilişkiyi; verinin işlenen boyutu ve ne kadar yerel olduğunu; veri değiş-tokuşunun(transaction) olup olmayacağını, olacaksa diğer işlemlerden ne kadat yalıtılacağını; verilerin kopyalanması ve mantıksal bütünlük intiyaçlarını; her sorgu türünün gecikme ve iletim debisi ihtiyaçlarını gösterir. + +Sistem üzerindeki yük ne kadar fazlaysa, sistem ayarlarının kullanım senaryolarına uyarlanması ve bu ayarların ne kadar hassas olduğu da o kadar önemli hale gelir. Birbirinden büyük ölçüde farklı olan veri erişim senaryolarına tam uyum sağlayan, yani her işe ve yüke gelen bir sistem yoktur. Eğer bir sistem yük altında her türlü veri erişim senaryosuna adapte olabiliyorsa, o halde böyle bir sistem ya tüm senaryolara ya da senaryoların bir veya birkaçına karşı zayıp bir performans gösterir. + +## OLAP senaryosunun temel özellikleri {#key-properties-of-olap-scenario} + +- İsteklerin büyük çoğunluğu, okuma erişimi içindir. +- Veriler, tek satırlarla değil, oldukça büyük gruplar halinde (\> 1000 satır) güncellenir; veya hiç güncellenmez. +- Veri, veritabanına eklenir, ancak değiştirilmez. +- Bazı sorgular için veritabanından den oldukça fazla sayıda satır çekilir, ancak sonuç sadece birkaç satır ve sütunludur. +- Tablolar "geniştir", yani bir tabloda çok sayıda kolon vardır(onlarca). +- Sorgular sıkılığı diğer senaryolara göre daha azdır (genellikle sunucu başına saniyede 100 veya daha az sorgu gelir). +- Basit sorgular için, 50 ms civarında gecikmelere izin verilir. +- Saklanan veriler oldukça küçüktür: genelde sadece sayılar ve kısa metinler içerir(örneğin, URL başına 60 bayt). +- Tek bir sorguyu işlemek yüksek miktarda veri okunmasını gerektirir(sunucu başına saniyede milyarlarca satıra kadar). +- Veri değiş-tokuşu(transaction) gerekli değildir. +- Veri tutarlılığı o kadar da önemli değildir. +- Genelde bir tane çok büyük tablo vardır, gerisi küçük tablolardan oluşur +- Bir sorgu sonucu elde edilen veri, okuanan veri miktarından oldukça küçüktür. Başka bir deyişle, milyarlarca satır içinden veriler süzgeçlenerek veya birleştirilerek elde edilen verilerin tek bir sunucunun RAM'ine sığar. + +OLAP senaryosunun diğer popüler senaryolardan (*Online Transactional Processing* - OLTP veya *Key-Value* veritabanı) çok farklı olduğu açıkça görülebilir. Bu nedenle, iyi bir performans elde etmek istiyorsanız, analitik sorguları işlemek için OLTP veya *Key-Value* veritabanlarını kullanmak pek mantıklı olmaz. Örneğin, analitik için MongoDB veya Redis kullanmaya çalışırsanız, OLAP veritabanlarına kıyasla çok düşük performans elde edersiniz. + +## Sütun yönelimli veritabanları OLAP senaryosunda neden daha iyi çalışır {#why-column-oriented-databases-work-better-in-the-olap-scenario} + +Sütun yönelimli veritabanları OLAP senaryolarına daha uygundur: hatta o kadar ki, çoğu sorgunun işlenmesi en az 100 kat daha hızlıdır. Her ne kadar OLAP veritabanlarının neden bu kadar hızlı olduğuna dair nedenler aşağıda ayrıntılı verilmiş olsa da görseller üzerinden anlatmak daha kolay olacakttır: + +**Satır yönelimli DBMS** + +![Row-oriented](images/row_oriented.gif#) + +**Sütun yönelimli DBMS** + +![Column-oriented](images/column_oriented.gif#) + +Farkı görüyor musunuz? + +### Giriş/çıkış {#inputoutput} + +1. Analitik bir sorgu için, yalnızca az sayıda tablo sütununun okunması gerekir. Sütun yönelimli bir veritabanında, yalnızca ihtiyacınız olan verileri okuyabilirsiniz. Örneğin, 100 üzerinden 5 sütun gerekiyorsa, g/Ç'de 20 kat azalma bekleyebilirsiniz. +2. Veri paketler halinde okunduğundan, sıkıştırılması daha kolaydır. Sütunlardaki verilerin sıkıştırılması da daha kolaydır. Bu, G/Ç hacmini daha da azaltır. +3. Azaltılmış G/Ç nedeniyle, sistem önbelleğine daha fazla veri sığar. + +Örneğin, sorgu “count the number of records for each advertising platform” bir okuma gerektirir “advertising platform ID” 1 bayt sıkıştırılmamış kadar alır sütun. Trafiğin çoğu reklam platformlarından değilse, bu sütunun en az 10 kat sıkıştırılmasını bekleyebilirsiniz. Hızlı bir sıkıştırma algoritması kullanırken, saniyede en az birkaç gigabayt sıkıştırılmamış veri hızında veri dekompresyonu mümkündür. Başka bir deyişle, bu sorgu, tek bir sunucuda saniyede yaklaşık birkaç milyar satır hızında işlenebilir. Bu hız aslında pratikte elde edilir. + +### CPU {#cpu} + +Bir sorguyu yürütmek çok sayıda satırı işlemeyi gerektirdiğinden, ayrı satırlar yerine tüm vektörler için tüm işlemlerin gönderilmesine veya sorgu motorunun neredeyse hiç gönderim maliyeti olmaması için uygulanmasına yardımcı olur. Bunu yapmazsanız, yarı iyi bir disk alt sistemi ile, sorgu yorumlayıcısı kaçınılmaz olarak CPU'yu durdurur. Hem verileri sütunlarda depolamak hem de mümkün olduğunda sütunlarla işlemek mantıklıdır. + +Bunu yapmanın iki yolu vardır: + +1. Bir vektör motoru. Tüm işlemler ayrı değerler yerine vektörler için yazılır. Bu, işlemleri çok sık aramanıza gerek olmadığı ve sevkiyatın maliyetlerinin ihmal edilebilir olduğu anlamına gelir. İşlem kodu optimize edilmiş bir iç döngü içerir. + +2. Kod üretimi. Sorgu için oluşturulan kod, içindeki tüm dolaylı çağrılara sahiptir. + +Bu yapılmaz “normal” veritabanları, çünkü basit sorguları çalıştırırken mantıklı değil. Ancak, istisnalar vardır. Örneğin, MemSQL SQL sorgularını işlerken gecikmeyi azaltmak için kod oluşturma kullanır. (Karşılaştırma için, analitik Dbms'ler gecikme değil, verim optimizasyonunu gerektirir .) + +CPU verimliliği için sorgu dilinin bildirimsel (SQL veya MDX) veya en az bir vektör (J, K) olması gerektiğini unutmayın. Sorgu yalnızca en iyi duruma getirme için izin veren örtük döngüler içermelidir. + +{## [Orijinal makale](https://clickhouse.tech/docs/en/) ##} diff --git a/docs/tr/interfaces/cli.md b/docs/tr/interfaces/cli.md new file mode 100644 index 00000000000..c526105dfe7 --- /dev/null +++ b/docs/tr/interfaces/cli.md @@ -0,0 +1,149 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 17 +toc_title: "Komut Sat\u0131r\u0131 \u0130stemcisi" +--- + +# Komut satırı istemcisi {#command-line-client} + +ClickHouse yerel bir komut satırı istemcisi sağlar: `clickhouse-client`. İstemci komut satırı seçeneklerini ve yapılandırma dosyalarını destekler. Daha fazla bilgi için, bkz. [Yapılandırma](#interfaces_cli_configuration). + +[Yüklemek](../getting_started/index.md) ıt from the `clickhouse-client` paketleyin ve komutla çalıştırın `clickhouse-client`. + +``` bash +$ clickhouse-client +ClickHouse client version 19.17.1.1579 (official build). +Connecting to localhost:9000 as user default. +Connected to ClickHouse server version 19.17.1 revision 54428. + +:) +``` + +Farklı istemci ve sunucu sürümleri birbiriyle uyumludur, ancak bazı özellikler eski istemcilerde kullanılamayabilir. Biz sunucu uygulaması olarak istemci aynı sürümünü kullanmanızı öneririz. Eski sürümün bir istemcisini kullanmaya çalıştığınızda, daha sonra sunucu, `clickhouse-client` mesajı görüntüler: + + ClickHouse client version is older than ClickHouse server. It may lack support for new features. + +## Kullanma {#cli_usage} + +İstemci etkileşimli ve etkileşimli olmayan (toplu iş) modunda kullanılabilir. Toplu iş modunu kullanmak için ‘query’ parametre veya veri göndermek ‘stdin’ (bunu doğrular ‘stdin’ bir terminal değildir) veya her ikisi de. HTTP arayüzüne benzer, kullanırken ‘query’ parametre ve veri gönderme ‘stdin’, istek bir birleştirme olduğunu ‘query’ parametre, bir satır besleme ve veri ‘stdin’. Bu, büyük ekleme sorguları için uygundur. + +Veri eklemek için istemci kullanma örneği: + +``` bash +$ echo -ne "1, 'some text', '2016-08-14 00:00:00'\n2, 'some more text', '2016-08-14 00:00:01'" | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; + +$ cat <<_EOF | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; +3, 'some text', '2016-08-14 00:00:00' +4, 'some more text', '2016-08-14 00:00:01' +_EOF + +$ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; +``` + +Toplu iş modunda, varsayılan veri biçimi TabSeparated. Sorgunun biçim yan tümcesinde biçimi ayarlayabilirsiniz. + +Varsayılan olarak, yalnızca tek bir sorguyu toplu iş modunda işleyebilirsiniz. Birden çok sorgu yapmak için bir “script,” kullan... `--multiquery` parametre. Bu, INSERT dışındaki tüm sorgular için çalışır. Sorgu sonuçları, ek ayırıcılar olmadan ardışık olarak çıktılanır. Benzer şekilde, çok sayıda sorgu işlemek için, çalıştırabilirsiniz ‘clickhouse-client’ her sorgu için. Başlatmak için onlarca milisaniye sürebilir unutmayın ‘clickhouse-client’ program. + +Etkileşimli modda, sorguları girebileceğiniz bir komut satırı alırsınız. + +Eğer ‘multiline’ belirtilmemiş (varsayılan): sorguyu çalıştırmak için Enter tuşuna basın. Noktalı virgül, sorgunun sonunda gerekli değildir. Çok satırlı bir sorgu girmek için ters eğik çizgi girin `\` hat beslemeden önce. Enter tuşuna bastıktan sonra, sorgunun sonraki satırını girmeniz istenecektir. + +Çok satırlı belirtilirse: bir sorguyu çalıştırmak için, noktalı virgülle sonlandırın ve Enter tuşuna basın. Noktalı virgül, girilen satırın sonunda atlandıysa, sorgunun bir sonraki satırını girmeniz istenecektir. + +Yalnızca tek bir sorgu çalıştırılır, bu nedenle noktalı virgülden sonra her şey göz ardı edilir. + +Belirtebilirsiniz `\G` noktalı virgül yerine veya sonra. Bu dikey biçimi gösterir. Bu formatta, her değer geniş tablolar için uygun olan ayrı bir satıra yazdırılır. Bu sıradışı özellik MySQL CLI ile uyumluluk için eklendi. + +Komut satırı dayanmaktadır ‘replxx’ (benzer ‘readline’). Başka bir deyişle, tanıdık klavye kısayollarını kullanır ve bir geçmişi tutar. Tarih yazılır `~/.clickhouse-client-history`. + +Varsayılan olarak, kullanılan biçim PrettyCompact. Sorgunun biçim yan tümcesinde veya belirterek biçimi değiştirebilirsiniz `\G` sorgunun sonunda, `--format` veya `--vertical` komut satırında veya istemci yapılandırma dosyasını kullanarak bağımsız değişken. + +İstemciden çıkmak için Ctrl+D (veya Ctrl+C) tuşlarına basın veya bir sorgu yerine aşağıdakilerden birini girin: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q” + +Bir sorguyu işlerken, istemci şunları gösterir: + +1. Saniyede en fazla 10 kez güncellenen ilerleme (varsayılan olarak). Hızlı sorgular için ilerleme görüntülenecek zaman olmayabilir. +2. Hata ayıklama için ayrıştırmadan sonra biçimlendirilmiş sorgu. +3. Belirtilen biçimde sonuç. +4. Sonuçtaki satır sayısı, geçen süre ve sorgu işlemenin ortalama hızı. + +Ctrl + C tuşlarına basarak uzun bir sorguyu iptal edebilirsiniz. ancak, sunucunun isteği iptal etmesi için biraz beklemeniz gerekir. Belirli aşamalarda bir sorguyu iptal etmek mümkün değildir. Beklemezseniz ve ikinci kez Ctrl + C tuşlarına basarsanız, istemci çıkacaktır. + +Komut satırı istemcisi, sorgulamak için dış verileri (dış geçici tablolar) geçirmenize izin verir. Daha fazla bilgi için bölüme bakın “External data for query processing”. + +### Parametrelerle sorgular {#cli-queries-with-parameters} + +Parametrelerle bir sorgu oluşturabilir ve istemci uygulamasından onlara değerler aktarabilirsiniz. Bu, istemci tarafında belirli dinamik değerlerle biçimlendirme sorgusunu önlemeye izin verir. Mesela: + +``` bash +$ clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" +``` + +#### Sorgu Sözdizimi {#cli-queries-with-parameters-syntax} + +Bir sorguyu her zamanki gibi biçimlendirin, ardından uygulama parametrelerinden sorguya geçirmek istediğiniz değerleri parantez içinde aşağıdaki biçimde yerleştirin: + +``` sql +{:} +``` + +- `name` — Placeholder identifier. In the console client it should be used in app parameters as `--param_ = value`. +- `data type` — [Veri türü](../sql_reference/data_types/index.md) app parametre değeri. Örneğin, aşağıdaki gibi bir veri yapısı `(integer, ('string', integer))` olabilir var `Tuple(UInt8, Tuple(String, UInt8))` veri türü (başka birini de kullanabilirsiniz [tamsayı](../sql_reference/data_types/int_uint.md) türler). + +#### Örnek {#example} + +``` bash +$ clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM table WHERE val = {tuple_in_tuple:Tuple(UInt8, Tuple(String, UInt8))}" +``` + +## Yapılandırma {#interfaces_cli_configuration} + +Parametreleri iletebilirsiniz `clickhouse-client` (tüm parametrelerin varsayılan değeri vardır) : + +- Komut satır fromından + + Komut satırı seçenekleri, yapılandırma dosyalarındaki varsayılan değerleri ve ayarları geçersiz kılar. + +- Yapılandırma dosyaları. + + Yapılandırma dosyalarındaki ayarlar varsayılan değerleri geçersiz kılar. + +### Komut Satırı Seçenekleri {#command-line-options} + +- `--host, -h` -– The server name, ‘localhost’ varsayılan olarak. Adı veya IPv4 veya IPv6 adresini kullanabilirsiniz. +- `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports. +- `--user, -u` – The username. Default value: default. +- `--password` – The password. Default value: empty string. +- `--query, -q` – The query to process when using non-interactive mode. +- `--database, -d` – Select the current default database. Default value: the current database from the server settings (‘default’ varsayılan) tarafından. +- `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter). +- `--multiquery, -n` – If specified, allow processing multiple queries separated by semicolons. +- `--format, -f` – Use the specified default format to output the result. +- `--vertical, -E` – If specified, use the Vertical format by default to output the result. This is the same as ‘–format=Vertical’. Bu biçimde, her bir değer, geniş tabloları görüntülerken yardımcı olan ayrı bir satıra yazdırılır. +- `--time, -t` – If specified, print the query execution time to ‘stderr’ etkileşimli olmayan modda. +- `--stacktrace` – If specified, also print the stack trace if an exception occurs. +- `--config-file` – The name of the configuration file. +- `--secure` – If specified, will connect to server over secure connection. +- `--param_` — Value for a [parametrelerle sorgu](#cli-queries-with-parameters). + +### Yapılandırma Dosyaları {#configuration_files} + +`clickhouse-client` aşağıdaki ilk varolan dosyayı kullanır: + +- Tanımlanan `--config-file` parametre. +- `./clickhouse-client.xml` +- `~/.clickhouse-client/config.xml` +- `/etc/clickhouse-client/config.xml` + +Bir yapılandırma dosyası örneği: + +``` xml + + username + password + False + +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/cli/) diff --git a/docs/tr/interfaces/cpp.md b/docs/tr/interfaces/cpp.md new file mode 100644 index 00000000000..9ebf93286ff --- /dev/null +++ b/docs/tr/interfaces/cpp.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 24 +toc_title: "C++ \u0130stemci Kitapl\u0131\u011F\u0131" +--- + +# C++ İstemci Kitaplığı {#c-client-library} + +Bkz. ben READİ READOKU [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp) depo. + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/cpp/) diff --git a/docs/tr/interfaces/formats.md b/docs/tr/interfaces/formats.md new file mode 100644 index 00000000000..f522d697aa1 --- /dev/null +++ b/docs/tr/interfaces/formats.md @@ -0,0 +1,1212 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 21 +toc_title: "Giri\u015F ve \xE7\u0131k\u0131\u015F bi\xE7imleri" +--- + +# Giriş ve çıkış verileri için biçimler {#formats} + +ClickHouse kabul ve çeşitli biçimlerde veri dönebilirsiniz. Giriş için desteklenen bir biçim, sağlanan verileri ayrıştırmak için kullanılabilir `INSERT`s, gerçekleştirmek için `SELECT`s dosya, URL veya HDFS gibi bir dosya destekli tablodan veya harici bir sözlük okumak için. Çıktı için desteklenen bir biçim düzenlemek için kullanılabilir +sonuçları bir `SELECT` ve gerçekleştirmek için `INSERT`s dosya destekli bir tabloya. + +Desteklenen formatlar şunlardır: + +| Biçimli | Girdi | Çıktı | +|-----------------------------------------------------------------|-------|-------| +| [TabSeparated](#tabseparated) | ✔ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ | +| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | +| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | +| [Şablon](#format-template) | ✔ | ✔ | +| [TemplateİgnoreSpaces](#templateignorespaces) | ✔ | ✗ | +| [CSV](#csv) | ✔ | ✔ | +| [CSVWithNames](#csvwithnames) | ✔ | ✔ | +| [CustomSeparated](#format-customseparated) | ✔ | ✔ | +| [Değerler](#data-format-values) | ✔ | ✔ | +| [Dikey](#vertical) | ✗ | ✔ | +| [VerticalRaw](#verticalraw) | ✗ | ✔ | +| [JSON](#json) | ✗ | ✔ | +| [JSONCompact](#jsoncompact) | ✗ | ✔ | +| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | +| [TSKV](#tskv) | ✔ | ✔ | +| [Çok](#pretty) | ✗ | ✔ | +| [PrettyCompact](#prettycompact) | ✗ | ✔ | +| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | +| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | +| [PrettySpace](#prettyspace) | ✗ | ✔ | +| [Protobuf](#protobuf) | ✔ | ✔ | +| [Avro](#data-format-avro) | ✔ | ✔ | +| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | +| [Parke](#data-format-parquet) | ✔ | ✔ | +| [ORC](#data-format-orc) | ✔ | ✗ | +| [RowBinary](#rowbinary) | ✔ | ✔ | +| [Rowbinarywithnames ve türleri](#rowbinarywithnamesandtypes) | ✔ | ✔ | +| [Yerel](#native) | ✔ | ✔ | +| [Boş](#null) | ✗ | ✔ | +| [XML](#xml) | ✗ | ✔ | +| [CapnProto](#capnproto) | ✔ | ✗ | + +ClickHouse ayarları ile bazı biçim işleme parametrelerini kontrol edebilirsiniz. Daha fazla bilgi için okuyun [Ayarlar](../operations/settings/settings.md) bölme. + +## TabSeparated {#tabseparated} + +Sekmede ayrı format, veri satır ile yazılır. Her satır sekmelerle ayrılmış değerler içerir. Her değer, satırdaki son değer dışında bir sekme tarafından takip edilir ve ardından bir satır beslemesi gelir. Kesinlikle Unix hat beslemeleri her yerde kabul edilir. Son satır ayrıca sonunda bir satır beslemesi içermelidir. Değerler metin biçiminde, tırnak işaretleri olmadan yazılır ve özel karakterler kaçtı. + +Bu biçim adı altında da kullanılabilir `TSV`. + +Bu `TabSeparated` format, özel programlar ve komut dosyaları kullanarak verileri işlemek için uygundur. Varsayılan olarak HTTP arabiriminde ve komut satırı istemcisinin toplu iş modunda kullanılır. Bu format aynı zamanda farklı Dbms'ler arasında veri aktarımı sağlar. Örneğin, Mysql'den bir dökümü alabilir ve Clickhouse'a yükleyebilirsiniz veya tam tersi. + +Bu `TabSeparated` biçim, toplam değerleri (TOPLAMLARLA birlikte kullanıldığında) ve aşırı değerleri (ne zaman ‘extremes’ 1 olarak ayarlanır). Bu durumlarda, toplam değerler ve aşırılıklar ana veriden sonra çıkar. Ana sonuç, toplam değerler ve aşırılıklar birbirinden boş bir çizgi ile ayrılır. Örnek: + +``` sql +SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` +``` + +``` text +2014-03-17 1406958 +2014-03-18 1383658 +2014-03-19 1405797 +2014-03-20 1353623 +2014-03-21 1245779 +2014-03-22 1031592 +2014-03-23 1046491 + +0000-00-00 8873898 + +2014-03-17 1031592 +2014-03-23 1406958 +``` + +### Veri Biçimlendirme {#data-formatting} + +Tamsayı sayılar ondalık biçimde yazılır. Sayılar ekstra içerebilir “+” başlangıçtaki karakter(ayrıştırma sırasında göz ardı edilir ve biçimlendirme sırasında kaydedilmez). Negatif olmayan sayılar negatif işareti içeremez. Okurken, boş bir dizeyi sıfır olarak veya (imzalı türler için) sıfır olarak sadece eksi işaretinden oluşan bir dizeyi ayrıştırmasına izin verilir. Karşılık gelen veri türüne uymayan sayılar, hata iletisi olmadan farklı bir sayı olarak ayrıştırılabilir. + +Kayan noktalı sayılar ondalık biçimde yazılır. Nokta ondalık ayırıcı olarak kullanılır. Üstel girişler olduğu gibi desteklenir ‘inf’, ‘+inf’, ‘-inf’, ve ‘nan’. Kayan noktalı sayıların bir girişi, ondalık nokta ile başlayabilir veya sona erebilir. +Biçimlendirme sırasında kayan noktalı sayılarda doğruluk kaybolabilir. +Ayrıştırma sırasında, en yakın makine temsil edilebilir numarayı okumak kesinlikle gerekli değildir. + +Tarihler YYYY-AA-DD biçiminde yazılır ve aynı biçimde ayrıştırılır, ancak ayırıcı olarak herhangi bir karakterle ayrıştırılır. +Tarihleri ile saatleri biçiminde yazılır `YYYY-MM-DD hh:mm:ss` ve aynı biçimde ayrıştırılır, ancak ayırıcı olarak herhangi bir karakterle. +Bu, tüm sistem saat diliminde istemci veya sunucu başlatıldığında (hangi veri biçimleri bağlı olarak) oluşur. Saatli tarihler için gün ışığından yararlanma saati belirtilmedi. Bu nedenle, bir dökümü gün ışığından yararlanma saati sırasında kez varsa, dökümü tümden verileri eşleşmiyor ve ayrıştırma iki kez birini seçecektir. +Bir okuma işlemi sırasında, yanlış tarih ve Tarih saatleriyle doğal taşma veya null tarih ve saat, hata iletisi olmadan ayrıştırılabilir. + +Bir istisna olarak, tam 10 ondalık basamaktan oluşuyorsa, tarihlerin zamanlarla ayrıştırılması Unix zaman damgası biçiminde de desteklenir. Sonuç, saat dilimine bağlı değildir. Yyyy-MM-DD ss:mm:ss ve nnnnnnnnnn biçimleri otomatik olarak ayırt edilir. + +Dizeler ters eğik çizgiden kaçan özel karakterlerle çıktılanır. Çıkış için aşağıdaki çıkış dizileri kullanılır: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`. Ayrıştırma da dizileri destekler `\a`, `\v`, ve `\xHH` (hxex kaçış dizileri) ve herhangi `\c` diz ,iler, nerede `c` herhangi bir karakter (bu diziler dönüştürülür `c`). Böylece, veri okuma, bir satır beslemesinin şu şekilde yazılabileceği formatları destekler `\n` veya `\`, veya bir çizgi besleme olarak. Örneğin, dize `Hello world` boşluk yerine kelimeler arasında bir çizgi beslemesi ile aşağıdaki varyasyonlardan herhangi birinde ayrıştırılabilir: + +``` text +Hello\nworld + +Hello\ +world +``` + +MySQL, sekmeyle ayrılmış dökümleri yazarken kullandığı için ikinci varyant desteklenir. + +Verileri TabSeparated biçiminde geçirirken kaçmanız gereken minimum karakter kümesi: sekme, satır besleme (LF) ve ters eğik çizgi. + +Sadece küçük bir sembol seti kaçtı. Terminalinizin çıktıda mahvedeceği bir dize değerine kolayca rastlayabilirsiniz. + +Diziler köşeli parantez içinde virgülle ayrılmış değerlerin bir listesi olarak yazılır. Dizideki sayı öğeleri normal olarak biçimlendirilir. `Date` ve `DateTime` türleri tek tırnak yazılır. Diz .eler yukarıdaki gibi aynı kural quoteslarla tek tırnak içinde yazılır. + +[NULL](../sql_reference/syntax.md) olarak format islanır `\N`. + +Her eleman [İçiçe](../sql_reference/data_types/nested_data_structures/nested.md) yapılar dizi olarak temsil edilir. + +Mesela: + +``` sql +CREATE TABLE nestedt +( + `id` UInt8, + `aux` Nested( + a UInt8, + b String + ) +) +ENGINE = TinyLog +``` + +``` sql +INSERT INTO nestedt Values ( 1, [1], ['a']) +``` + +``` sql +SELECT * FROM nestedt FORMAT TSV +``` + +``` text +1 [1] ['a'] +``` + +## TabSeparatedRaw {#tabseparatedraw} + +Farklıdır `TabSeparated` satırların kaçmadan yazıldığını biçimlendirin. +Bu biçim yalnızca bir sorgu sonucu çıktısı için uygundur, ancak ayrıştırma için değil (bir tabloya eklemek için veri alma). + +Bu biçim adı altında da kullanılabilir `TSVRaw`. + +## TabSeparatedWithNames {#tabseparatedwithnames} + +Bu farklıdır `TabSeparated` sütun adlarının ilk satırda yazıldığını biçimlendirin. +Ayrıştırma sırasında, ilk satır tamamen göz ardı edilir. Sütun adlarını konumlarını belirlemek veya doğruluğunu kontrol etmek için kullanamazsınız. +(Başlık satırı ayrıştırma desteği gelecekte eklenebilir .) + +Bu biçim adı altında da kullanılabilir `TSVWithNames`. + +## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes} + +Bu farklıdır `TabSeparated` sütun türleri ikinci satırda iken, sütun adlarının ilk satıra yazıldığını biçimlendirin. +Ayrıştırma sırasında, birinci ve ikinci satırlar tamamen göz ardı edilir. + +Bu biçim adı altında da kullanılabilir `TSVWithNamesAndTypes`. + +## Şablon {#format-template} + +Bu biçim, belirli bir kaçış kuralına sahip değerler için yer tutucularla özel bir biçim dizesinin belirtilmesine izin verir. + +Ayarları kullanır `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` kullanırken `JSON` kaçmak, daha fazla görmek) + +Ayar `format_template_row` aşağıdaki sözdizimine sahip satırlar için Biçim dizesi içeren dosya yolunu belirtir: + +`delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`, + +nerede `delimiter_i` değerler arasında bir sınırlayıcı mı (`$` sembol olarak kaçabilir `$$`), +`column_i` değerleri seçilecek veya eklenecek bir sütunun adı veya dizini (boşsa, sütun atlanır), +`serializeAs_i` sütun değerleri için kaçan bir kuraldır. Aşağıdaki kaçış kuralları desteklenir: + +- `CSV`, `JSON`, `XML` (aynı is theimlerin biçim formatslerine benzer şekilde) +- `Escaped` (aynı şekilde `TSV`) +- `Quoted` (aynı şekilde `Values`) +- `Raw` (kaç withoutmadan, benzer şekilde `TSVRaw`) +- `None` (kaçan kural yok, daha fazla görün) + +Kaçan bir kural atlanırsa, o zaman `None` kullanılacaktır. `XML` ve `Raw` sadece çıkış için uygundur. + +Yani, aşağıdaki biçim dizesi için: + + `Search phrase: ${SearchPhrase:Quoted}, count: ${c:Escaped}, ad price: $$${price:JSON};` + +değerleri `SearchPhrase`, `c` ve `price` olarak kaçır ,ılan sütunlar `Quoted`, `Escaped` ve `JSON` (select için) yazdırılacak veya (ınsert için) arasında beklenecektir `Search phrase:`, `, count:`, `, ad price: $` ve `;` sırasıyla sınırlayıcılar. Mesela: + +`Search phrase: 'bathroom interior design', count: 2166, ad price: $3;` + +Bu `format_template_rows_between_delimiter` ayar, sonuncusu hariç her satırdan sonra yazdırılan (veya beklenen) satırlar arasındaki sınırlayıcıyı belirtir (`\n` varsayılan olarak) + +Ayar `format_template_resultset` resultset için bir biçim dizesi içeren dosya yolunu belirtir. Resultset için Biçim dizesi, satır için bir biçim dizesi ile aynı sözdizimine sahiptir ve bir önek, sonek ve bazı ek bilgileri yazdırmanın bir yolunu belirtmeyi sağlar. Sütun adları yerine aşağıdaki yer tutucuları içerir: + +- `data` veri içeren satırlar mı `format_template_row` biçim, ayrılmış `format_template_rows_between_delimiter`. Bu yer tutucu, biçim dizesindeki ilk yer tutucu olmalıdır. +- `totals` toplam değerleri olan satır `format_template_row` biçim (TOPLAMLARLA birlikte kullanıldığında) +- `min` satır içinde minimum değerlere sahip mi `format_template_row` biçim (1 olarak ayarlandığında) +- `max` maksimum değerleri olan satır `format_template_row` biçim (1 olarak ayarlandığında) +- `rows` çıktı satırlarının toplam sayısıdır +- `rows_before_limit` minimum satır sayısı sınırı olmadan olurdu. Yalnızca sorgu sınırı içeriyorsa çıktı. Sorgu GROUP BY içeriyorsa, ROWS\_BEFORE\_LİMİT\_AT\_LEAST SINIRSIZDI olurdu satır tam sayısıdır. +- `time` istek yürütme süresi saniyeler içinde mi +- `rows_read` satır sayısı okun thedu mu +- `bytes_read` bayt sayısı (sıkıştırılmamış) okundu mu + +Tutucu `data`, `totals`, `min` ve `max` kaç rulema kuralı belirtilm (em (elidir (veya `None` açıkça belirtilen) olmalıdır. Kalan yer tutucuları belirtilen kaçan herhangi bir kural olabilir. +Eğer... `format_template_resultset` ayar boş bir dizedir, `${data}` varsayılan değer olarak kullanılır. +Insert sorguları biçimi için önek veya sonek varsa bazı sütunları veya bazı alanları atlamaya izin verir (örneğe bakın). + +Örnek seç: + +``` sql +SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 FORMAT Template SETTINGS +format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = '\n ' +``` + +`/some/path/resultset.format`: + +``` text + + Search phrases + + + + ${data} +
Search phrases
Search phrase Count
+ + ${max} +
Max
+ Processed ${rows_read:XML} rows in ${time:XML} sec + + +``` + +`/some/path/row.format`: + +``` text + ${0:XML} ${1:XML} +``` + +Sonuç: + +``` html + + Search phrases + + + + + + + + +
Search phrases
Search phrase Count
8267016
bathroom interior design 2166
yandex 1655
spring 2014 fashion 1549
freeform photos 1480
+ + +
Max
8873898
+ Processed 3095973 rows in 0.1569913 sec + + +``` + +Örnek Ekle: + +``` text +Some header +Page views: 5, User id: 4324182021466249494, Useless field: hello, Duration: 146, Sign: -1 +Page views: 6, User id: 4324182021466249494, Useless field: world, Duration: 185, Sign: 1 +Total rows: 2 +``` + +``` sql +INSERT INTO UserActivity FORMAT Template SETTINGS +format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format' +``` + +`/some/path/resultset.format`: + +``` text +Some header\n${data}\nTotal rows: ${:CSV}\n +``` + +`/some/path/row.format`: + +``` text +Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV} +``` + +`PageViews`, `UserID`, `Duration` ve `Sign` yer tutucular içinde tablodaki sütun adları vardır. Sonra değerler `Useless field` satır ve sonra `\nTotal rows:` sonek olarak göz ardı edilecektir. +Giriş verisindeki tüm sınırlayıcılar, belirtilen biçim dizelerindeki sınırlayıcılara kesinlikle eşit olmalıdır. + +## TemplateİgnoreSpaces {#templateignorespaces} + +Bu format sadece giriş için uygundur. +Benzer `Template`, ancak giriş akışındaki sınırlayıcılar ve değerler arasındaki boşluk karakterlerini atlar. Ancak, biçim dizeleri boşluk karakterleri içeriyorsa, bu karakterler giriş akışında beklenir. Ayrıca boş yer tutucuları belirtmek için izin verir (`${}` veya `${:None}`) aralarındaki boşlukları görmezden gelmek için bazı sınırlayıcıları ayrı parçalara ayırmak. Bu tür yer tutucular yalnızca boşluk karakterlerini atlamak için kullanılır. +Okumak mümkün `JSON` bu biçimi kullanarak, sütun değerleri tüm satırlarda aynı sıraya sahipse. Örneğin, aşağıdaki istek çıktı biçim örneğinden veri eklemek için kullanılabilir [JSON](#json): + +``` sql +INSERT INTO table_name FORMAT TemplateIgnoreSpaces SETTINGS +format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = ',' +``` + +`/some/path/resultset.format`: + +``` text +{${}"meta"${}:${:JSON},${}"data"${}:${}[${data}]${},${}"totals"${}:${:JSON},${}"extremes"${}:${:JSON},${}"rows"${}:${:JSON},${}"rows_before_limit_at_least"${}:${:JSON}${}} +``` + +`/some/path/row.format`: + +``` text +{${}"SearchPhrase"${}:${}${phrase:JSON}${},${}"c"${}:${}${cnt:JSON}${}} +``` + +## TSKV {#tskv} + +TabSeparated benzer, ancak name=value biçiminde bir değer çıkarır. Adlar tabseparated biçiminde olduğu gibi kaçtı ve = simgesi de kaçtı. + +``` text +SearchPhrase= count()=8267016 +SearchPhrase=bathroom interior design count()=2166 +SearchPhrase=yandex count()=1655 +SearchPhrase=2014 spring fashion count()=1549 +SearchPhrase=freeform photos count()=1480 +SearchPhrase=angelina jolie count()=1245 +SearchPhrase=omsk count()=1112 +SearchPhrase=photos of dog breeds count()=1091 +SearchPhrase=curtain designs count()=1064 +SearchPhrase=baku count()=1000 +``` + +[NULL](../sql_reference/syntax.md) olarak format islanır `\N`. + +``` sql +SELECT * FROM t_null FORMAT TSKV +``` + +``` text +x=1 y=\N +``` + +Çok sayıda küçük sütun olduğunda, bu biçim etkisizdir ve genellikle kullanmak için hiçbir neden yoktur. Bununla birlikte, verimlilik açısından Jsoneachrow'dan daha kötü değildir. + +Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. + +Ayrıştırma, ek alanın varlığına izin verir `tskv` eşit işareti veya bir değer olmadan. Bu alan yoksayılır. + +## CSV {#csv} + +Virgülle ayrılmış değerler biçimi ([RFC](https://tools.ietf.org/html/rfc4180)). + +Biçimlendirme yaparken, satırlar çift tırnak içine alınır. Bir dizenin içindeki çift alıntı, bir satırda iki çift tırnak olarak çıktılanır. Karakterlerden kaçmak için başka kural yoktur. Tarih ve Tarih-Saat çift tırnak içine alınır. Sayılar tırnak işaretleri olmadan çıktı. Değerler, bir sınırlayıcı karakterle ayrılır; `,` varsayılan olarak. Sınırlayıcı karakteri ayarında tanımlanır [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). Satırlar Unıx satır besleme (LF) kullanılarak ayrılır. Diziler CSV'DE aşağıdaki gibi serileştirilir: ilk olarak, dizi TabSeparated biçiminde olduğu gibi bir dizeye serileştirilir ve daha sonra ortaya çıkan dize çift tırnak içinde CSV'YE çıkarılır. CSV biçimindeki Tuples ayrı sütunlar olarak serileştirilir(yani, tuple'daki yuvalanmaları kaybolur). + +``` bash +$ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv +``` + +\* Varsayılan olarak, sınırlayıcı `,`. Görmek [format\_csv\_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) daha fazla bilgi için ayarlama. + +Ayrıştırma yaparken, tüm değerler tırnak işaretleri ile veya tırnak işaretleri olmadan ayrıştırılabilir. Hem çift hem de tek tırnak desteklenmektedir. Satırlar tırnak işaretleri olmadan da düzenlenebilir. Bu durumda, sınırlayıcı karaktere veya satır beslemesine (CR veya LF) ayrıştırılır. RFC'Yİ ihlal ederken, satırları tırnak işaretleri olmadan ayrıştırırken, önde gelen ve sondaki boşluklar ve sekmeler göz ardı edilir. Hat beslemesi için Unix (LF), Windows (CR LF) ve Mac OS Classic (CR LF) türleri desteklenir. + +Boş unquoted giriş değerleri, ilgili sütunlar için varsayılan değerlerle değiştirilir +[ınput\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) +etkindir. + +`NULL` olarak format islanır `\N` veya `NULL` veya boş bir unquoted dize (bkz. ayarlar [ınput\_format\_csv\_unquoted\_null\_literal\_as\_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null) ve [ınput\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)). + +CSV biçimi, toplamların ve aşırılıkların çıktısını aynı şekilde destekler `TabSeparated`. + +## CSVWithNames {#csvwithnames} + +Ayrıca, başlık satırını benzer şekilde yazdırır `TabSeparatedWithNames`. + +## CustomSeparated {#format-customseparated} + +Benzer [Şablon](#format-template), ancak tüm sütunları yazdırır veya okur ve ayardan kaçan kuralı kullanır `format_custom_escaping_rule` ve ayarlardan sınırlayıcılar `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` ve `format_custom_result_after_delimiter`, biçim dizelerinden değil. +Ayrıca var `CustomSeparatedIgnoreSpaces` biçim, benzer olan `TemplateIgnoreSpaces`. + +## JSON {#json} + +Verileri json formatında çıkarır. Veri tablolarının yanı sıra, bazı ek bilgilerle birlikte sütun adlarını ve türlerini de çıkarır: çıktı satırlarının toplam sayısı ve bir sınır yoksa çıktı olabilecek satır sayısı. Örnek: + +``` sql +SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON +``` + +``` json +{ + "meta": + [ + { + "name": "SearchPhrase", + "type": "String" + }, + { + "name": "c", + "type": "UInt64" + } + ], + + "data": + [ + { + "SearchPhrase": "", + "c": "8267016" + }, + { + "SearchPhrase": "bathroom interior design", + "c": "2166" + }, + { + "SearchPhrase": "yandex", + "c": "1655" + }, + { + "SearchPhrase": "spring 2014 fashion", + "c": "1549" + }, + { + "SearchPhrase": "freeform photos", + "c": "1480" + } + ], + + "totals": + { + "SearchPhrase": "", + "c": "8873898" + }, + + "extremes": + { + "min": + { + "SearchPhrase": "", + "c": "1480" + }, + "max": + { + "SearchPhrase": "", + "c": "8267016" + } + }, + + "rows": 5, + + "rows_before_limit_at_least": 141137 +} +``` + +Json JavaScript ile uyumludur. Bunu sağlamak için, bazı karakterler ek olarak kaçar: eğik çizgi `/` olarak kaç İsar `\/`; alternatif Satır sonları `U+2028` ve `U+2029`, hangi bazı tarayıcılar kırmak, olarak kaçtı `\uXXXX`. ASCII denetim karakterleri kaçtı: backspace, form besleme, satır besleme, satır başı ve yatay sekme ile değiştirilir `\b`, `\f`, `\n`, `\r`, `\t` , 00-1f aralığında kalan baytların yanı sıra `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double-quotes by default. To remove the quotes, you can set the configuration parameter [output\_format\_json\_quote\_64bit\_integers](../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) 0'a. + +`rows` – The total number of output rows. + +`rows_before_limit_at_least` Minimum satır sayısı sınırı olmadan olurdu. Yalnızca sorgu sınırı içeriyorsa çıktı. +Sorgu GROUP BY içeriyorsa, ROWS\_BEFORE\_LİMİT\_AT\_LEAST SINIRSIZDI olurdu satır tam sayısıdır. + +`totals` – Total values (when using WITH TOTALS). + +`extremes` – Extreme values (when extremes are set to 1). + +Bu biçim yalnızca bir sorgu sonucu çıktısı için uygundur, ancak ayrıştırma için değil (bir tabloya eklemek için veri alma). + +ClickHouse destekler [NULL](../sql_reference/syntax.md) olarak görüntülenen `null` JSON çıkışında. + +Ayrıca bakınız [JSONEachRow](#jsoneachrow) biçimli. + +## JSONCompact {#jsoncompact} + +Yalnızca veri satırlarında json'dan farklıdır, nesnelerde değil, dizilerde çıktıdır. + +Örnek: + +``` json +{ + "meta": + [ + { + "name": "SearchPhrase", + "type": "String" + }, + { + "name": "c", + "type": "UInt64" + } + ], + + "data": + [ + ["", "8267016"], + ["bathroom interior design", "2166"], + ["yandex", "1655"], + ["fashion trends spring 2014", "1549"], + ["freeform photo", "1480"] + ], + + "totals": ["","8873898"], + + "extremes": + { + "min": ["","1480"], + "max": ["","8267016"] + }, + + "rows": 5, + + "rows_before_limit_at_least": 141137 +} +``` + +Bu biçim yalnızca bir sorgu sonucu çıktısı için uygundur, ancak ayrıştırma için değil (bir tabloya eklemek için veri alma). +Ayrıca bakınız `JSONEachRow` biçimli. + +## JSONEachRow {#jsoneachrow} + +Bu biçimi kullanırken, ClickHouse satırları ayrılmış, yeni satırla ayrılmış JSON nesneleri olarak çıkarır, ancak bir bütün olarak veriler geçerli JSON değildir. + +``` json +{"SearchPhrase":"curtain designs","count()":"1064"} +{"SearchPhrase":"baku","count()":"1000"} +{"SearchPhrase":"","count()":"8267016"} +``` + +Verileri eklerken, her satır için ayrı bir JSON nesnesi sağlamanız gerekir. + +### Veri Ekleme {#inserting-data} + +``` sql +INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021466249494", "Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} +``` + +ClickHouse sağlar: + +- Nesnedeki herhangi bir anahtar-değer çiftleri sırası. +- Bazı değerleri atlama. + +ClickHouse, nesnelerden sonra öğeler ve virgüller arasındaki boşlukları yok sayar. Tüm nesneleri bir satırda geçirebilirsiniz. Onları Satır sonları ile ayırmak zorunda değilsiniz. + +**İhmal edilen değerler işleme** + +ClickHouse, karşılık gelen değerler için varsayılan değerlerle atlanmış değerleri değiştirir [veri türleri](../sql_reference/data_types/index.md). + +Eğer `DEFAULT expr` belirtilen, ClickHouse bağlı olarak farklı ikame kuralları kullanır [ınput\_format\_defaults\_for\_omitted\_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields) ayar. + +Aşağıdaki tabloyu düşünün: + +``` sql +CREATE TABLE IF NOT EXISTS example_table +( + x UInt32, + a DEFAULT x * 2 +) ENGINE = Memory; +``` + +- Eğer `input_format_defaults_for_omitted_fields = 0`, sonra varsayılan değer için `x` ve `a` eşitlikler `0` (varsayılan değer olarak `UInt32` veri türü). +- Eğer `input_format_defaults_for_omitted_fields = 1`, sonra varsayılan değer için `x` eşitlikler `0`, ancak varsayılan değer `a` eşitlikler `x * 2`. + +!!! note "Uyarıcı" + İle veri ek whenlerken `insert_sample_with_metadata = 1`, ClickHouse, ekleme ile karşılaştırıldığında daha fazla hesaplama kaynağı tüketir `insert_sample_with_metadata = 0`. + +### Veri Seçme {#selecting-data} + +Düşünün `UserActivity` örnek olarak tablo: + +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +Sorgu `SELECT * FROM UserActivity FORMAT JSONEachRow` dönüşler: + +``` text +{"UserID":"4324182021466249494","PageViews":5,"Duration":146,"Sign":-1} +{"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1} +``` + +Aksine [JSON](#json) biçimi, geçersiz UTF-8 dizilerinin hiçbir ikame yoktur. Değerleri için olduğu gibi aynı şekilde kaçtı `JSON`. + +!!! note "Not" + Herhangi bir bayt kümesi dizelerde çıktı olabilir. Kullan... `JSONEachRow` tablodaki verilerin herhangi bir bilgi kaybetmeden JSON olarak biçimlendirilebileceğinden eminseniz biçimlendirin. + +### İç içe yapıların kullanımı {#jsoneachrow-nested} + +İle bir tablo varsa [İçiçe](../sql_reference/data_types/nested_data_structures/nested.md) veri türü sütunları, aynı yapıya sahip json verilerini ekleyebilirsiniz. İle bu özelliği etkinleştirin [ınput\_format\_ımport\_nested\_json](../operations/settings/settings.md#settings-input_format_import_nested_json) ayar. + +Örneğin, aşağıdaki tabloyu göz önünde bulundurun: + +``` sql +CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory +``` + +Gibi görmek `Nested` veri türü açıklaması, ClickHouse, iç içe geçmiş yapının her bileşenini ayrı bir sütun olarak ele alır (`n.s` ve `n.i` sof )ram )ız için). Verileri aşağıdaki şekilde ekleyebilirsiniz: + +``` sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} +``` + +Hiyerarşik bir json nesnesi olarak veri eklemek için [input\_format\_import\_nested\_json = 1](../operations/settings/settings.md#settings-input_format_import_nested_json). + +``` json +{ + "n": { + "s": ["abc", "def"], + "i": [1, 23] + } +} +``` + +Bu ayar olmadan, ClickHouse bir özel durum atar. + +``` sql +SELECT name, value FROM system.settings WHERE name = 'input_format_import_nested_json' +``` + +``` text +┌─name────────────────────────────┬─value─┐ +│ input_format_import_nested_json │ 0 │ +└─────────────────────────────────┴───────┘ +``` + +``` sql +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +``` + +``` text +Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: n: (at row 1) +``` + +``` sql +SET input_format_import_nested_json=1 +INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n": {"s": ["abc", "def"], "i": [1, 23]}} +SELECT * FROM json_each_row_nested +``` + +``` text +┌─n.s───────────┬─n.i────┐ +│ ['abc','def'] │ [1,23] │ +└───────────────┴────────┘ +``` + +## Yerel {#native} + +En verimli biçim. Veriler ikili formatta bloklar tarafından yazılır ve okunur. Her blok için satır sayısı, sütun sayısı, sütun adları ve türleri ve bu bloktaki sütunların parçaları birbiri ardına kaydedilir. Başka bir deyişle, bu format “columnar” – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. + +Bu biçimi, yalnızca ClickHouse DBMS tarafından okunabilen dökümleri hızlı bir şekilde oluşturmak için kullanabilirsiniz. Bu formatla kendiniz çalışmak mantıklı değil. + +## Boş {#null} + +Hiçbir şey çıktı. Ancak, sorgu işlenir ve komut satırı istemcisini kullanırken, veriler istemciye iletilir. Bu, performans testi de dahil olmak üzere testler için kullanılır. +Açıkçası, bu format yalnızca ayrıştırma için değil, çıktı için uygundur. + +## Çok {#pretty} + +Verileri Unicode-art tabloları olarak çıkarır, ayrıca TERMİNALDEKİ renkleri ayarlamak için ANSI-escape dizileri kullanır. +Tablonun tam bir ızgarası çizilir ve her satır terminalde iki satır kaplar. +Her sonuç bloğu ayrı bir tablo olarak çıktı. Bu, blokların arabelleğe alma sonuçları olmadan çıkabilmesi için gereklidir (tüm değerlerin görünür genişliğini önceden hesaplamak için arabelleğe alma gerekli olacaktır). + +[NULL](../sql_reference/syntax.md) olarak çıktı `ᴺᵁᴸᴸ`. + +Örnek (gösterilen [PrettyCompact](#prettycompact) biçimli): + +``` sql +SELECT * FROM t_null +``` + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +Satırlar güzel\* biçimlerinde kaçmaz. Örnek için gösterilir [PrettyCompact](#prettycompact) biçimli: + +``` sql +SELECT 'String with \'quotes\' and \t character' AS Escaping_test +``` + +``` text +┌─Escaping_test────────────────────────┐ +│ String with 'quotes' and character │ +└──────────────────────────────────────┘ +``` + +Terminale çok fazla veri boşaltmaktan kaçınmak için yalnızca ilk 10.000 satır yazdırılır. Satır sayısı 10.000'den büyük veya eşitse, ileti “Showed first 10 000” bas .ılmıştır. +Bu biçim yalnızca bir sorgu sonucu çıktısı için uygundur, ancak ayrıştırma için değil (bir tabloya eklemek için veri alma). + +Güzel biçim, toplam değerleri (TOPLAMLARLA birlikte kullanıldığında) ve aşırılıkları (ne zaman ‘extremes’ 1 olarak ayarlanır). Bu durumlarda, toplam değerler ve aşırı değerler ana veriden sonra ayrı tablolarda çıktılanır. Örnek (gösterilen [PrettyCompact](#prettycompact) biçimli): + +``` sql +SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact +``` + +``` text +┌──EventDate─┬───────c─┐ +│ 2014-03-17 │ 1406958 │ +│ 2014-03-18 │ 1383658 │ +│ 2014-03-19 │ 1405797 │ +│ 2014-03-20 │ 1353623 │ +│ 2014-03-21 │ 1245779 │ +│ 2014-03-22 │ 1031592 │ +│ 2014-03-23 │ 1046491 │ +└────────────┴─────────┘ + +Totals: +┌──EventDate─┬───────c─┐ +│ 0000-00-00 │ 8873898 │ +└────────────┴─────────┘ + +Extremes: +┌──EventDate─┬───────c─┐ +│ 2014-03-17 │ 1031592 │ +│ 2014-03-23 │ 1406958 │ +└────────────┴─────────┘ +``` + +## PrettyCompact {#prettycompact} + +Farklıdır [Çok](#pretty) bu ızgara satırlar arasında çizilir ve sonuç daha kompakttır. +Bu biçim, etkileşimli modda komut satırı istemcisinde varsayılan olarak kullanılır. + +## PrettyCompactMonoBlock {#prettycompactmonoblock} + +Farklıdır [PrettyCompact](#prettycompact) bu kadar 10.000 satır arabelleğe alınır, daha sonra bloklarla değil, tek bir tablo olarak çıktılanır. + +## PrettyNoEscapes {#prettynoescapes} + +ANSİSİ-ESC .ape diz .ilerinin kullanıl .madığı güzel differsden farklıdır. Bu, bu formatı bir tarayıcıda görüntülemek ve aynı zamanda ‘watch’ komut satırı yardımcı programı. + +Örnek: + +``` bash +$ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FORMAT PrettyCompactNoEscapes'" +``` + +Tarayıcıda görüntülemek için HTTP arayüzünü kullanabilirsiniz. + +### PrettyCompactNoEscapes {#prettycompactnoescapes} + +Önceki ayar ile aynı. + +### PrettySpaceNoEscapes {#prettyspacenoescapes} + +Önceki ayar ile aynı. + +## PrettySpace {#prettyspace} + +Farklıdır [PrettyCompact](#prettycompact) bu boşluk (boşluk karakterleri) ızgara yerine kullanılır. + +## RowBinary {#rowbinary} + +Biçimleri ve ikili biçimde satır verileri ayrıştırır. Satırlar ve değerler, ayırıcılar olmadan ardışık olarak listelenir. +Bu biçim, satır tabanlı olduğundan yerel biçimden daha az etkilidir. + +Tamsayılar sabit uzunlukta küçük endian temsilini kullanır. Örneğin, uint64 8 bayt kullanır. +DateTime, Unix zaman damgasını değer olarak içeren Uİnt32 olarak temsil edilir. +Tarih değeri olarak 1970-01-01 yılından bu yana gün sayısını içeren bir uint16 nesnesi olarak temsil edilir. +Dize varint uzunluğu (imzasız) olarak temsil edilir [LEB128](https://en.wikipedia.org/wiki/LEB128)), ardından dizenin baytları. +FixedString sadece bir bayt dizisi olarak temsil edilir. + +Dizi varint uzunluğu (imzasız) olarak temsil edilir [LEB128](https://en.wikipedia.org/wiki/LEB128)), ardından dizinin ardışık elemanları. + +İçin [NULL](../sql_reference/syntax.md#null-literal) destek, 1 veya 0 içeren ek bir bayt her önce eklenir [Nullable](../sql_reference/data_types/nullable.md) değer. 1 ise, o zaman değer `NULL` ve bu bayt ayrı bir değer olarak yorumlanır. 0 ise, bayttan sonraki değer değil `NULL`. + +## Rowbinarywithnames ve türleri {#rowbinarywithnamesandtypes} + +Benzer [RowBinary](#rowbinary), ancak eklenen Başlık ile: + +- [LEB128](https://en.wikipedia.org/wiki/LEB128)- kodlanmış sütun sayısı (N) +- N `String`s sütun adlarını belirtme +- N `String`s sütun türlerini belirleme + +## Değerler {#data-format-values} + +Her satırı parantez içinde yazdırır. Satırlar virgülle ayrılır. Son satırdan sonra virgül yok. Parantez içindeki değerler de virgülle ayrılır. Sayılar tırnak işaretleri olmadan ondalık biçimde çıktıdır. Diziler köşeli parantez içinde çıktı. Kat tırnak içinde çıkış dizelerle, tarihleri ve tarihleri. Kaçan kurallar ve ayrıştırma benzer [TabSeparated](#tabseparated) biçimli. Biçimlendirme sırasında fazladan boşluk eklenmez, ancak ayrıştırma sırasında izin verilir ve atlanır (izin verilmeyen dizi değerleri içindeki boşluklar hariç). [NULL](../sql_reference/syntax.md) olarak temsil edilir `NULL`. + +The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes. + +Bu, kullanılan formattır `INSERT INTO t VALUES ...`, ancak sorgu sonuçlarını biçimlendirmek için de kullanabilirsiniz. + +Ayrıca bakınız: [ınput\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) ve [ınput\_format\_values\_deduce\_templates\_of\_expressions](../operations/settings/settings.md#settings-input_format_values_deduce_templates_of_expressions) ayarlar. + +## Dikey {#vertical} + +Her değeri belirtilen sütun adıyla ayrı bir satıra yazdırır. Bu biçim, her satır çok sayıda sütundan oluşuyorsa, yalnızca bir veya birkaç satır yazdırmak için uygundur. + +[NULL](../sql_reference/syntax.md) olarak çıktı `ᴺᵁᴸᴸ`. + +Örnek: + +``` sql +SELECT * FROM t_null FORMAT Vertical +``` + +``` text +Row 1: +────── +x: 1 +y: ᴺᵁᴸᴸ +``` + +Satırlar dikey biçimde kaçmadı: + +``` sql +SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical +``` + +``` text +Row 1: +────── +test: string with 'quotes' and with some special + characters +``` + +Bu biçim yalnızca bir sorgu sonucu çıktısı için uygundur, ancak ayrıştırma için değil (bir tabloya eklemek için veri alma). + +## VerticalRaw {#verticalraw} + +Benzer [Dikey](#vertical), ama kaçan engelli ile. Bu biçim, yalnızca ayrıştırma (veri alma ve tabloya ekleme) için değil, sorgu sonuçlarının çıktısı için uygundur. + +## XML {#xml} + +XML biçimi, ayrıştırma için değil, yalnızca çıktı için uygundur. Örnek: + +``` xml + + + + + + SearchPhrase + String + + + count() + UInt64 + + + + + + + 8267016 + + + bathroom interior design + 2166 + + + yandex + 1655 + + + 2014 spring fashion + 1549 + + + freeform photos + 1480 + + + angelina jolie + 1245 + + + omsk + 1112 + + + photos of dog breeds + 1091 + + + curtain designs + 1064 + + + baku + 1000 + + + 10 + 141137 + +``` + +Sütun adı kabul edilebilir bir biçime sahip değilse, sadece ‘field’ eleman adı olarak kullanılır. Genel olarak, XML yapısı JSON yapısını izler. +Just as for JSON, invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. + +Dize değerlerinde, karakterler `<` ve `&` olarak kaç arear `<` ve `&`. + +Diziler olarak çıktı `HelloWorld...`ve tuples olarak `HelloWorld...`. + +## CapnProto {#capnproto} + +Cap'n Proto, Protokol Tamponlarına ve tasarrufuna benzer, ancak JSON veya MessagePack gibi olmayan bir ikili mesaj biçimidir. + +Cap'n Proto mesajları kesinlikle yazılır ve kendi kendini tanımlamaz, yani harici bir şema açıklamasına ihtiyaç duyarlar. Şema anında uygulanır ve her sorgu için önbelleğe alınır. + +``` bash +$ cat capnproto_messages.bin | clickhouse-client --query "INSERT INTO test.hits FORMAT CapnProto SETTINGS format_schema='schema:Message'" +``` + +Nerede `schema.capnp` bu gibi görünüyor: + +``` capnp +struct Message { + SearchPhrase @0 :Text; + c @1 :Uint64; +} +``` + +Serializasyon etkilidir ve genellikle sistem yükünü arttırmaz. + +Ayrıca bakınız [Biçim Şeması](#formatschema). + +## Protobuf {#protobuf} + +Protobuf-bir [Protokol Tamp Buffonları](https://developers.google.com/protocol-buffers/) biçimli. + +Bu biçim, bir dış biçim şeması gerektirir. Şema sorgular arasında önbelleğe alınır. +ClickHouse hem destekler `proto2` ve `proto3` sözdizimiler. Tekrarlanan / isteğe bağlı / gerekli alanlar desteklenir. + +Kullanım örnekleri: + +``` sql +SELECT * FROM test.table FORMAT Protobuf SETTINGS format_schema = 'schemafile:MessageType' +``` + +``` bash +cat protobuf_messages.bin | clickhouse-client --query "INSERT INTO test.table FORMAT Protobuf SETTINGS format_schema='schemafile:MessageType'" +``` + +dosya nerede `schemafile.proto` bu gibi görünüyor: + +``` capnp +syntax = "proto3"; + +message MessageType { + string name = 1; + string surname = 2; + uint32 birthDate = 3; + repeated string phoneNumbers = 4; +}; +``` + +İletişim kuralı arabellekleri' ileti türü Tablo sütunları ve alanları arasındaki yazışmaları bulmak için clickhouse adlarını karşılaştırır. +Bu karşılaştırma büyük / küçük harf duyarsız ve karakterler `_` (alt çizgi) ve `.` (nokta) eşit olarak kabul edilir. +Bir sütun türleri ve protokol arabellekleri ileti alanı farklıysa, gerekli dönüştürme uygulanır. + +İç içe geçmiş mesajlar desteklenir. Örneğin, alan için `z` aşağıdaki ileti türünde + +``` capnp +message MessageType { + message XType { + message YType { + int32 z; + }; + repeated YType y; + }; + XType x; +}; +``` + +ClickHouse adlı bir sütun bulmaya çalışır `x.y.z` (veya `x_y_z` veya `X.y_Z` ve benzeri). +İç içe mesajlar giriş veya çıkış a için uygundur [iç içe veri yapıları](../sql_reference/data_types/nested_data_structures/nested.md). + +Böyle bir protobuf şemasında tanımlanan varsayılan değerler + +``` capnp +syntax = "proto2"; + +message MessageType { + optional int32 result_per_page = 3 [default = 10]; +} +``` + +uygulan ;mamaktadır; [tablo varsayılanları](../sql_reference/statements/create.md#create-default-values) bunların yerine kullanılır. + +ClickHouse girişleri ve çıkışları protobuf mesajları `length-delimited` biçimli. +Bu, her mesajın uzunluğunu bir olarak yazmadan önce anlamına gelir [varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). +Ayrıca bakınız [popüler dillerde uzunlukla ayrılmış protobuf mesajları nasıl okunur / yazılır](https://cwiki.apache.org/confluence/display/GEODE/Delimiting+Protobuf+Messages). + +## Avro {#data-format-avro} + +[Apache Avro](http://avro.apache.org/) Apache'nin Hadoop projesi kapsamında geliştirilen satır odaklı veri serileştirme çerçevesidir. + +ClickHouse Avro biçimi okuma ve yazma destekler [Avro veri dosyaları](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). + +### Veri Türleri Eşleştirme {#data_types-matching} + +Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse'la nasıl eşleştiğini gösterir [veri türleri](../sql_reference/data_types/index.md) içinde `INSERT` ve `SELECT` sorgular. + +| Avro veri türü `INSERT` | ClickHouse veri türü | Avro veri türü `SELECT` | +|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------| +| `boolean`, `int`, `long`, `float`, `double` | [Int(8/16/32)](../sql_reference/data_types/int_uint.md), [Uİnt(8/16/32)](../sql_reference/data_types/int_uint.md) | `int` | +| `boolean`, `int`, `long`, `float`, `double` | [Int64](../sql_reference/data_types/int_uint.md), [Uİnt64](../sql_reference/data_types/int_uint.md) | `long` | +| `boolean`, `int`, `long`, `float`, `double` | [Float32](../sql_reference/data_types/float.md) | `float` | +| `boolean`, `int`, `long`, `float`, `double` | [Float64](../sql_reference/data_types/float.md) | `double` | +| `bytes`, `string`, `fixed`, `enum` | [Dize](../sql_reference/data_types/string.md) | `bytes` | +| `bytes`, `string`, `fixed` | [FixedString(N)](../sql_reference/data_types/fixedstring.md) | `fixed(N)` | +| `enum` | [Enum (8/16)](../sql_reference/data_types/enum.md) | `enum` | +| `array(T)` | [Dizi(T)](../sql_reference/data_types/array.md) | `array(T)` | +| `union(null, T)`, `union(T, null)` | [Null (T)](../sql_reference/data_types/date.md) | `union(null, T)` | +| `null` | [Null (Hiçbir Şey)](../sql_reference/data_types/special_data_types/nothing.md) | `null` | +| `int (date)` \* | [Tarihli](../sql_reference/data_types/date.md) | `int (date)` \* | +| `long (timestamp-millis)` \* | [DateTime64 (3)](../sql_reference/data_types/datetime.md) | `long (timestamp-millis)` \* | +| `long (timestamp-micros)` \* | [DateTime64 (6)](../sql_reference/data_types/datetime.md) | `long (timestamp-micros)` \* | + +\* [Avro mantıksal türleri](http://avro.apache.org/docs/current/spec.html#Logical+Types) + +Desteklenmeyen Avro veri türleri: `record` (non-root), `map` + +Desteklenmeyen Avro mantıksal veri türleri: `uuid`, `time-millis`, `time-micros`, `duration` + +### Veri Ekleme {#inserting-data-1} + +Bir Avro dosyasından ClickHouse tablosuna veri eklemek için: + +``` bash +$ cat file.avro | clickhouse-client --query="INSERT INTO {some_table} FORMAT Avro" +``` + +Giriş Avro dosyasının kök şeması olmalıdır `record` tür. + +ClickHouse tablo sütunları ve Avro şema alanları arasındaki yazışmaları bulmak için adlarını karşılaştırır. Bu karşılaştırma büyük / küçük harf duyarlıdır. +Kullanılmayan alanlar atlanır. + +ClickHouse tablo sütunlarının veri türleri, eklenen Avro verilerinin karşılık gelen alanlarından farklı olabilir. Veri eklerken, ClickHouse veri türlerini yukarıdaki tabloya göre yorumlar ve sonra [döküm](../sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) karşılık gelen sütun türüne veri. + +### Veri Seçme {#selecting-data-1} + +ClickHouse tablosundan bir Avro dosyasına veri seçmek için: + +``` bash +$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Avro" > file.avro +``` + +Sütun adları gerekir: + +- ile başla `[A-Za-z_]` +- daha sonra sadece içerir `[A-Za-z0-9_]` + +Çıkış Avro dosya sıkıştırma ve senkronizasyon aralığı ile yapılandırılabilir [output\_format\_avro\_codec](../operations/settings/settings.md#settings-output_format_avro_codec) ve [output\_format\_avro\_sync\_interval](../operations/settings/settings.md#settings-output_format_avro_sync_interval) sırasıyla. + +## AvroConfluent {#data-format-avro-confluent} + +AvroConfluent yaygın olarak kullanılan tek nesne Avro mesajları çözme destekler [Kafka](https://kafka.apache.org/) ve [Confluent Şema Kayıt](https://docs.confluent.io/current/schema-registry/index.html). + +Her Avro iletisi, şema Kayıt defterinin yardımıyla gerçek şemaya çözülebilen bir şema kimliği gömer. + +Şemalar çözüldükten sonra önbelleğe alınır. + +Şema kayıt defteri URL'si ile yapılandırılır [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url) + +### Veri Türleri Eşleştirme {#data_types-matching-1} + +Aynı olarak [Avro](#data-format-avro) + +### Kullanma {#usage} + +Şema çözünürlüğünü hızlı bir şekilde doğrulamak için şunları kullanabilirsiniz [kafkasat](https://github.com/edenhill/kafkacat) ile [clickhouse-yerel](../operations/utilities/clickhouse-local.md): + +``` bash +$ kafkacat -b kafka-broker -C -t topic1 -o beginning -f '%s' -c 3 | clickhouse-local --input-format AvroConfluent --format_avro_schema_registry_url 'http://schema-registry' -S "field1 Int64, field2 String" -q 'select * from table' +1 a +2 b +3 c +``` + +Kullanmak `AvroConfluent` ile [Kafka](../engines/table_engines/integrations/kafka.md): + +``` sql +CREATE TABLE topic1_stream +( + field1 String, + field2 String +) +ENGINE = Kafka() +SETTINGS +kafka_broker_list = 'kafka-broker', +kafka_topic_list = 'topic1', +kafka_group_name = 'group1', +kafka_format = 'AvroConfluent'; + +SET format_avro_schema_registry_url = 'http://schema-registry'; + +SELECT * FROM topic1_stream; +``` + +!!! note "Uyarıcı" + Ayar `format_avro_schema_registry_url` yapılandırılmış olması gerekiyor `users.xml` yeniden başlattıktan sonra değerini korumak için. + +## Parke {#data-format-parquet} + +[Apache Parke](http://parquet.apache.org/) hadoop ekosisteminde yaygın bir sütunlu depolama biçimidir. ClickHouse, bu format için okuma ve yazma işlemlerini destekler. + +### Veri Türleri Eşleştirme {#data_types-matching-2} + +Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse'la nasıl eşleştiğini gösterir [veri türleri](../sql_reference/data_types/index.md) içinde `INSERT` ve `SELECT` sorgular. + +| Parke veri türü (`INSERT`) | ClickHouse veri türü | Parke veri türü (`SELECT`) | +|----------------------------|-----------------------------------------------------------|----------------------------| +| `UINT8`, `BOOL` | [Uİnt8](../sql_reference/data_types/int_uint.md) | `UINT8` | +| `INT8` | [Int8](../sql_reference/data_types/int_uint.md) | `INT8` | +| `UINT16` | [Uınt16](../sql_reference/data_types/int_uint.md) | `UINT16` | +| `INT16` | [Int16](../sql_reference/data_types/int_uint.md) | `INT16` | +| `UINT32` | [Uİnt32](../sql_reference/data_types/int_uint.md) | `UINT32` | +| `INT32` | [Int32](../sql_reference/data_types/int_uint.md) | `INT32` | +| `UINT64` | [Uİnt64](../sql_reference/data_types/int_uint.md) | `UINT64` | +| `INT64` | [Int64](../sql_reference/data_types/int_uint.md) | `INT64` | +| `FLOAT`, `HALF_FLOAT` | [Float32](../sql_reference/data_types/float.md) | `FLOAT` | +| `DOUBLE` | [Float64](../sql_reference/data_types/float.md) | `DOUBLE` | +| `DATE32` | [Tarihli](../sql_reference/data_types/date.md) | `UINT16` | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql_reference/data_types/datetime.md) | `UINT32` | +| `STRING`, `BINARY` | [Dize](../sql_reference/data_types/string.md) | `STRING` | +| — | [FixedString](../sql_reference/data_types/fixedstring.md) | `STRING` | +| `DECIMAL` | [Ondalık](../sql_reference/data_types/decimal.md) | `DECIMAL` | + +ClickHouse yapılandırılabilir hassas destekler `Decimal` tür. Bu `INSERT` sorgu parke davranır `DECIMAL` ClickHouse olarak yazın `Decimal128` tür. + +Desteklen datameyen veri türleri: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. + +ClickHouse tablo sütunlarının veri türleri, eklenen parke verilerinin ilgili alanlarından farklı olabilir. Veri eklerken, ClickHouse veri türlerini yukarıdaki tabloya göre yorumlar ve sonra [döküm](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) ClickHouse tablo sütunu için ayarlanmış olan bu veri türüne ait veriler. + +### Veri ekleme ve seçme {#inserting-and-selecting-data} + +Bir dosyadan parke verilerini ClickHouse tablosuna aşağıdaki komutla ekleyebilirsiniz: + +``` bash +$ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" +``` + +Bir ClickHouse tablosundan veri seçin ve aşağıdaki komutla parke formatında bazı dosyaya kaydedebilirsiniz: + +``` bash +$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} +``` + +Hadoop ile veri alışverişi yapmak için şunları kullanabilirsiniz [HDFS tablo motoru](../engines/table_engines/integrations/hdfs.md). + +## ORC {#data-format-orc} + +[Apache ORCC](https://orc.apache.org/) hadoop ekosisteminde yaygın bir sütunlu depolama biçimidir. Bu formatta yalnızca Clickhouse'a veri ekleyebilirsiniz. + +### Veri Türleri Eşleştirme {#data_types-matching-3} + +Aşağıdaki tablo, desteklenen veri türlerini ve Clickhouse'la nasıl eşleştiğini gösterir [veri türleri](../sql_reference/data_types/index.md) içinde `INSERT` sorgular. + +| Orc veri türü (`INSERT`) | ClickHouse veri türü | +|--------------------------|-----------------------------------------------------| +| `UINT8`, `BOOL` | [Uİnt8](../sql_reference/data_types/int_uint.md) | +| `INT8` | [Int8](../sql_reference/data_types/int_uint.md) | +| `UINT16` | [Uınt16](../sql_reference/data_types/int_uint.md) | +| `INT16` | [Int16](../sql_reference/data_types/int_uint.md) | +| `UINT32` | [Uİnt32](../sql_reference/data_types/int_uint.md) | +| `INT32` | [Int32](../sql_reference/data_types/int_uint.md) | +| `UINT64` | [Uİnt64](../sql_reference/data_types/int_uint.md) | +| `INT64` | [Int64](../sql_reference/data_types/int_uint.md) | +| `FLOAT`, `HALF_FLOAT` | [Float32](../sql_reference/data_types/float.md) | +| `DOUBLE` | [Float64](../sql_reference/data_types/float.md) | +| `DATE32` | [Tarihli](../sql_reference/data_types/date.md) | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql_reference/data_types/datetime.md) | +| `STRING`, `BINARY` | [Dize](../sql_reference/data_types/string.md) | +| `DECIMAL` | [Ondalık](../sql_reference/data_types/decimal.md) | + +ClickHouse yapılandırılabilir hassas destekler `Decimal` tür. Bu `INSERT` sorgu Orc davranır `DECIMAL` ClickHouse olarak yazın `Decimal128` tür. + +Desteklenmeyen Orc veri türleri: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. + +ClickHouse tablo sütunlarının veri türlerinin karşılık gelen ORC veri alanları ile eşleşmesi gerekmez. Veri eklerken, ClickHouse veri türlerini yukarıdaki tabloya göre yorumlar ve sonra [döküm](../sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) veri türü için veri kümesi ClickHouse tablo sütun. + +### Veri Ekleme {#inserting-data-2} + +Bir dosyadan Orc verilerini ClickHouse tablosuna aşağıdaki komutla ekleyebilirsiniz: + +``` bash +$ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" +``` + +Hadoop ile veri alışverişi yapmak için şunları kullanabilirsiniz [HDFS tablo motoru](../engines/table_engines/integrations/hdfs.md). + +## Biçim Şeması {#formatschema} + +Biçim şemasını içeren dosya adı, ayar tarafından ayarlanır `format_schema`. +Biçim onelerinden biri kullanıldığında bu ayarı ayarlamak gerekir `Cap'n Proto` ve `Protobuf`. +Biçim şeması, bir dosya adının ve bu dosyadaki bir ileti türünün adının birleşimidir ve iki nokta üst üste ile sınırlandırılmıştır, +e.g. `schemafile.proto:MessageType`. +Dosya, format için standart uzantıya sahipse (örneğin, `.proto` için `Protobuf`), +ihmal edilebilir ve bu durumda, biçim şeması şöyle görünür `schemafile:MessageType`. + +Eğer giriş veya çıkış veri üzerinden [müşteri](../interfaces/cli.md) in the [interaktif mod](../interfaces/cli.md#cli_usage) biçim şe themasında belirtilen dosya adı +mutlak bir yol veya istemci üzerinde geçerli dizine göre bir yol içerebilir. +Eğer istemci kullanıyorsanız [Toplu Modu](../interfaces/cli.md#cli_usage), şemanın yolu güvenlik nedeniyle göreceli olmalıdır. + +Eğer giriş veya çıkış veri üzerinden [HTTP arayüzü](../interfaces/http.md) biçim şemasında belirtilen dosya adı +belirtilen dizinde bulunmalıdır [format\_schema\_path](../operations/server_configuration_parameters/settings.md#server_configuration_parameters-format_schema_path) +sunucu yapılandırmasında. + +## Atlama Hataları {#skippingerrors} + +Gibi bazı format suchlar `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` ve `Protobuf` ayrıştırma hatası oluşursa kırık satırı atlayabilir ve bir sonraki satırın başından ayrıştırmaya devam edebilir. Görmek [ınput\_format\_allow\_errors\_num](../operations/settings/settings.md#settings-input_format_allow_errors_num) ve +[ınput\_format\_allow\_errors\_ratio](../operations/settings/settings.md#settings-input_format_allow_errors_ratio) ayarlar. +Sınırlamalar: +- Ayrıştırma hatası durumunda `JSONEachRow` yeni satıra (veya EOF) kadar tüm verileri atlar, bu nedenle satırlar AŞAĞIDAKİLERLE sınırlandırılmalıdır `\n` hataları doğru saymak için. +- `Template` ve `CustomSeparated` bir sonraki satırın başlangıcını bulmak için son sütundan sonra sınırlayıcı ve satırlar arasındaki sınırlayıcıyı kullanın, Bu nedenle hataları atlamak yalnızca en az biri boş değilse çalışır. + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/formats/) diff --git a/docs/tr/interfaces/http.md b/docs/tr/interfaces/http.md new file mode 100644 index 00000000000..a30e05418c8 --- /dev/null +++ b/docs/tr/interfaces/http.md @@ -0,0 +1,511 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 19 +toc_title: "HTTP aray\xFCz\xFC" +--- + +# HTTP arayüzü {#http-interface} + +HTTP arayüzü, herhangi bir programlama dilinden herhangi bir platformda Clickhouse'u kullanmanızı sağlar. Java ve Perl'den ve kabuk komut dosyalarından çalışmak için kullanıyoruz. Diğer bölümlerde, HTTP arayüzü Perl, Python ve Go'dan kullanılır. HTTP arabirimi yerel arabirimden daha sınırlıdır, ancak daha iyi uyumluluğa sahiptir. + +Varsayılan olarak, clickhouse-server, 8123 numaralı bağlantı noktasında HTTP dinler (bu, yapılandırmada değiştirilebilir). + +Parametreler olmadan bir GET / request yaparsanız, 200 yanıt kodunu ve tanımlanan dizeyi döndürür [http\_server\_default\_response](../operations/server_configuration_parameters/settings.md#server_configuration_parameters-http_server_default_response) varsayılan değer “Ok.” (sonunda bir çizgi besleme ile) + +``` bash +$ curl 'http://localhost:8123/' +Ok. +``` + +Sağlık kontrol komut GET / ping isteği kullanın. Bu işleyici her zaman döner “Ok.” (sonunda bir çizgi besleme ile). 18.12.13 sürümünden edinilebilir. + +``` bash +$ curl 'http://localhost:8123/ping' +Ok. +``` + +İsteği URL olarak gönder ‘query’ parametre veya bir POST olarak. Veya sorgunun başlangıcını gönder ‘query’ parametre ve postadaki geri kalanı (bunun neden gerekli olduğunu daha sonra açıklayacağız). URL'nin boyutu 16 KB ile sınırlıdır, bu nedenle büyük sorgular gönderirken bunu aklınızda bulundurun. + +Başarılı olursa, 200 yanıt Kodu ve yanıt gövdesinde sonucu alırsınız. +Bir hata oluşursa, 500 yanıt Kodu ve yanıt gövdesinde bir hata açıklaması metni alırsınız. + +GET yöntemini kullanırken, ‘readonly’ ayar .lanmıştır. Başka bir deyişle, verileri değiştiren sorgular için yalnızca POST yöntemini kullanabilirsiniz. Sorgunun kendisini POST gövdesinde veya URL parametresinde gönderebilirsiniz. + +Örnekler: + +``` bash +$ curl 'http://localhost:8123/?query=SELECT%201' +1 + +$ wget -O- -q 'http://localhost:8123/?query=SELECT 1' +1 + +$ echo -ne 'GET /?query=SELECT%201 HTTP/1.0\r\n\r\n' | nc localhost 8123 +HTTP/1.0 200 OK +Date: Wed, 27 Nov 2019 10:30:18 GMT +Connection: Close +Content-Type: text/tab-separated-values; charset=UTF-8 +X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal +X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f +X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} + +1 +``` + +Gördüğünüz gibi, curl, boşlukların URL'den kaçması gerektiği konusunda biraz rahatsız edici. +Her ne kadar wget her şeyden kaçsa da, onu kullanmanızı önermiyoruz çünkü keep-alive ve Transfer-Encoding: chunked kullanırken HTTP 1.1 üzerinde iyi çalışmıyor. + +``` bash +$ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- +1 + +$ echo 'SELECT 1' | curl 'http://localhost:8123/?query=' --data-binary @- +1 + +$ echo '1' | curl 'http://localhost:8123/?query=SELECT' --data-binary @- +1 +``` + +Sorgunun bir parçası parametrede gönderilirse ve gönderinin bir parçası ise, bu iki veri parçası arasına bir satır akışı eklenir. +Örnek (bu işe yaramaz): + +``` bash +$ echo 'ECT 1' | curl 'http://localhost:8123/?query=SEL' --data-binary @- +Code: 59, e.displayText() = DB::Exception: Syntax error: failed at position 0: SEL +ECT 1 +, expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception +``` + +Varsayılan olarak, veri TabSeparated biçiminde döndürülür (daha fazla bilgi için bkz: “Formats” bölme). +Başka bir biçim istemek için sorgunun biçim yan tümcesi kullanın. + +``` bash +$ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- +┏━━━┓ +┃ 1 ┃ +┡━━━┩ +│ 1 │ +└───┘ +``` + +Ekleme sorguları için veri iletmenin POST yöntemi gereklidir. Bu durumda, URL parametresinde sorgunun başlangıcını yazabilir ve eklemek için verileri iletmek için POST'u kullanabilirsiniz. Eklenecek veriler, örneğin Mysql'den sekmeyle ayrılmış bir döküm olabilir. Bu şekilde, INSERT sorgusu MYSQL'DEN load DATA LOCAL INFİLE'IN yerini alır. + +Örnekler: tablo oluşturma: + +``` bash +$ echo 'CREATE TABLE t (a UInt8) ENGINE = Memory' | curl 'http://localhost:8123/' --data-binary @- +``` + +Veri ekleme için tanıdık ekleme sorgusunu kullanma: + +``` bash +$ echo 'INSERT INTO t VALUES (1),(2),(3)' | curl 'http://localhost:8123/' --data-binary @- +``` + +Veriler sorgudan ayrı olarak gönderilebilir: + +``` bash +$ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- +``` + +Herhangi bir veri biçimini belirtebilirsiniz. Bu ‘Values’ biçim, T değerlerine INSERT yazarken kullanılanla aynıdır: + +``` bash +$ echo '(7),(8),(9)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20Values' --data-binary @- +``` + +Sekmeyle ayrılmış bir dökümden veri eklemek için ilgili biçimi belirtin: + +``` bash +$ echo -ne '10\n11\n12\n' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20FORMAT%20TabSeparated' --data-binary @- +``` + +Tablo içeriğini okuma. Paralel sorgu işleme nedeniyle veriler rastgele sırayla çıktılanır: + +``` bash +$ curl 'http://localhost:8123/?query=SELECT%20a%20FROM%20t' +7 +8 +9 +10 +11 +12 +1 +2 +3 +4 +5 +6 +``` + +Tabloyu silme. + +``` bash +$ echo 'DROP TABLE t' | curl 'http://localhost:8123/' --data-binary @- +``` + +Veri tablosu döndürmeyen başarılı istekler için boş bir yanıt gövdesi döndürülür. + +Veri iletirken dahili ClickHouse sıkıştırma formatını kullanabilirsiniz. Sıkıştırılmış veriler standart olmayan bir biçime sahiptir ve özel `clickhouse-compressor` onunla çalışmak için program (bu ile yüklü `clickhouse-client` paket). Veri ekleme verimliliğini artırmak için, sunucu tarafı sağlama toplamı doğrulamasını kullanarak devre dışı bırakabilirsiniz. [http\_native\_compression\_disable\_checksumming\_on\_decompress](../operations/settings/settings.md#settings-http_native_compression_disable_checksumming_on_decompress) ayar. + +Belirt ift ifiyseniz `compress=1` URL'de, sunucu size gönderdiği verileri sıkıştırır. +Belirt ift ifiyseniz `decompress=1` URL'de, sunucu içinde geçirdiğiniz aynı verileri açar. `POST` yöntem. + +Ayrıca kullanmayı seçebilirsiniz [HTTP sıkıştırma](https://en.wikipedia.org/wiki/HTTP_compression). Sıkıştırılmış bir göndermek için `POST` istek, istek başlığını Ekle `Content-Encoding: compression_method`. Clickhouse'un yanıtı sıkıştırması için şunları eklemelisiniz `Accept-Encoding: compression_method`. ClickHouse destekler `gzip`, `br`, ve `deflate` [sıkıştırma yöntemleri](https://en.wikipedia.org/wiki/HTTP_compression#Content-Encoding_tokens). HTTP sıkıştırmasını etkinleştirmek için Clickhouse'u kullanmanız gerekir [enable\_http\_compression](../operations/settings/settings.md#settings-enable_http_compression) ayar. Veri sıkıştırma düzeyini [http\_zlib\_compression\_level](#settings-http_zlib_compression_level) tüm sıkıştırma yöntemleri için ayarlama. + +Bunu, büyük miktarda veri iletirken ağ trafiğini azaltmak veya hemen sıkıştırılmış dökümler oluşturmak için kullanabilirsiniz. + +Sıkıştırma ile veri gönderme örnekleri: + +``` bash +#Sending data to the server: +$ curl -vsS "http://localhost:8123/?enable_http_compression=1" -d 'SELECT number FROM system.numbers LIMIT 10' -H 'Accept-Encoding: gzip' + +#Sending data to the client: +$ echo "SELECT 1" | gzip -c | curl -sS --data-binary @- -H 'Content-Encoding: gzip' 'http://localhost:8123/' +``` + +!!! note "Not" + Bazı HTTP istemcileri varsayılan olarak sunucudan verileri açabilir ( `gzip` ve `deflate`) ve sıkıştırma ayarlarını doğru kullansanız bile sıkıştırılmış veriler alabilirsiniz. + +Kullanabilirsiniz ‘database’ Varsayılan veritabanını belirtmek için URL parametresi. + +``` bash +$ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?database=system' --data-binary @- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +``` + +Varsayılan olarak, sunucu ayarlarında kayıtlı veritabanı varsayılan veritabanı olarak kullanılır. Varsayılan olarak, bu veritabanı denir ‘default’. Alternatif olarak, her zaman tablo adından önce bir nokta kullanarak veritabanını belirtebilirsiniz. + +Kullanıcı adı ve şifre üç yoldan biriyle belirtilebilir: + +1. HTTP temel kimlik doğrulamasını kullanma. Örnek: + + + +``` bash +$ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- +``` + +1. İn the ‘user’ ve ‘password’ URL parametreleri. Örnek: + + + +``` bash +$ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- +``` + +1. Kullanım ‘X-ClickHouse-User’ ve ‘X-ClickHouse-Key’ üstbilgi. Örnek: + + + +``` bash +$ echo 'SELECT 1' | curl -H 'X-ClickHouse-User: user' -H 'X-ClickHouse-Key: password' 'http://localhost:8123/' -d @- +``` + +Kullanıcı adı belirtilmemişse, `default` adı kullanılır. Parola belirtilmezse, boş parola kullanılır. +Tek bir sorguyu veya ayarların tüm profillerini işlemek için herhangi bir ayar belirtmek için URL parametrelerini de kullanabilirsiniz. Örnek: http: / / localhost: 8123/?profil = web & max\_rows\_to\_read = 1000000000 & query = seç + 1 + +Daha fazla bilgi için, bkz: [Ayarlar](../operations/settings/index.md) bölme. + +``` bash +$ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:8123/?' --data-binary @- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +``` + +Diğer parametreler hakkında bilgi için bölüme bakın “SET”. + +Benzer şekilde, http protokolünde ClickHouse oturumlarını kullanabilirsiniz. Bunu yapmak için şunları eklemeniz gerekir: `session_id` İsteğe parametre alın. Oturum kimliği olarak herhangi bir dize kullanabilirsiniz. Varsayılan olarak, oturum 60 saniye hareketsizlik sonra sonlandırılır. Bu zaman aşımını değiştirmek için, `default_session_timeout` sunucu yapılandırmasında ayarlama veya `session_timeout` İsteğe parametre alın. Oturum durumunu kontrol etmek için `session_check=1` parametre. Bir kerede yalnızca bir sorgu, tek bir oturum içinde çalıştırılabilir. + +Bir sorgunun ilerleme durumu hakkında bilgi alabilirsiniz `X-ClickHouse-Progress` yanıt başlıkları. Bunu yapmak için etkinleştir [send\_progress\_in\_http\_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Başlık dizisi örneği: + +``` text +X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"} +X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128"} +X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"} +``` + +Olası başlık alanları: + +- `read_rows` — Number of rows read. +- `read_bytes` — Volume of data read in bytes. +- `total_rows_to_read` — Total number of rows to be read. +- `written_rows` — Number of rows written. +- `written_bytes` — Volume of data written in bytes. + +Http bağlantısı kaybolursa çalışan istekler otomatik olarak durmaz. Ayrıştırma ve veri biçimlendirme sunucu tarafında gerçekleştirilir ve ağ kullanarak etkisiz olabilir. +Opsiyonel ‘query\_id’ parametre sorgu kimliği (herhangi bir dize) geçirilebilir. Daha fazla bilgi için bölüme bakın “Settings, replace\_running\_query”. + +Opsiyonel ‘quota\_key’ parametre kota anahtarı (herhangi bir dize) olarak geçirilebilir. Daha fazla bilgi için bölüme bakın “Quotas”. + +HTTP arabirimi, sorgulamak için dış verileri (dış geçici tablolar) geçirmenize izin verir. Daha fazla bilgi için bölüme bakın “External data for query processing”. + +## Yanıt Tamponlama {#response-buffering} + +Sunucu tarafında yanıt arabelleği etkinleştirebilirsiniz. Bu `buffer_size` ve `wait_end_of_query` Bu amaçla URL parametreleri sağlanmıştır. + +`buffer_size` sunucu belleğinde arabellek sonucu bayt sayısını belirler. Sonuç gövdesi bu eşikten büyükse, arabellek HTTP kanalına yazılır ve kalan veriler doğrudan HTTP kanalına gönderilir. + +Tüm yanıtın arabelleğe alındığından emin olmak için `wait_end_of_query=1`. Bu durumda, bellekte depolanan veriler geçici bir sunucu dosyasında arabelleğe alınır. + +Örnek: + +``` bash +$ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wait_end_of_query=1' -d 'SELECT toUInt8(number) FROM system.numbers LIMIT 9000000 FORMAT RowBinary' +``` + +Yanıt Kodu ve HTTP üstbilgileri istemciye gönderildikten sonra bir sorgu işleme hatası oluştu durumları önlemek için arabelleğe alma kullanın. Bu durumda, yanıt gövdesinin sonunda bir hata iletisi yazılır ve istemci tarafında hata yalnızca ayrıştırma aşamasında algılanabilir. + +### Parametrelerle sorgular {#cli-queries-with-parameters} + +Parametrelerle bir sorgu oluşturabilir ve karşılık gelen HTTP istek parametrelerinden onlar için değerler geçirebilirsiniz. Daha fazla bilgi için, bkz. [CLI için parametrelerle sorgular](cli.md#cli-queries-with-parameters). + +### Örnek {#example} + +``` bash +$ curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}" +``` + +## Önceden tanımlanmış HTTP arabirimi {#predefined_http_interface} + +ClickHouse HTTP arabirimi üzerinden belirli sorguları destekler. Örneğin, bir tabloya aşağıdaki gibi veri yazabilirsiniz: + +``` bash +$ echo '(4),(5),(6)' | curl 'http://localhost:8123/?query=INSERT%20INTO%20t%20VALUES' --data-binary @- +``` + +ClickHouse ayrıca gibi üçüncü parti araçları ile daha kolay entegrasyon yardımcı olabilir önceden tanımlanmış HTTP arayüzünü destekler [PROMETHEUS ihracatçı](https://github.com/percona-lab/clickhouse_exporter). + +Örnek: + +- Her şeyden önce, bu bölümü sunucu yapılandırma dosyasına ekleyin: + + + +``` xml + + + /metrics + GET + + SELECT * FROM system.metrics LIMIT 5 FORMAT Template SETTINGS format_template_resultset = 'prometheus_template_output_format_resultset', format_template_row = 'prometheus_template_output_format_row', format_template_rows_between_delimiter = '\n' + + + +``` + +- Artık PROMETHEUS formatında veriler için doğrudan url talep edebilirsiniz: + + + +``` bash +curl -vvv 'http://localhost:8123/metrics' +* Trying ::1... +* Connected to localhost (::1) port 8123 (#0) +> GET /metrics HTTP/1.1 +> Host: localhost:8123 +> User-Agent: curl/7.47.0 +> Accept: */* +> +< HTTP/1.1 200 OK +< Date: Wed, 27 Nov 2019 08:54:25 GMT +< Connection: Keep-Alive +< Content-Type: text/plain; charset=UTF-8 +< X-ClickHouse-Server-Display-Name: i-tl62qd0o +< Transfer-Encoding: chunked +< X-ClickHouse-Query-Id: f39235f6-6ed7-488c-ae07-c7ceafb960f6 +< Keep-Alive: timeout=3 +< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< +# HELP "Query" "Number of executing queries" +# TYPE "Query" counter +"Query" 1 + +# HELP "Merge" "Number of executing background merges" +# TYPE "Merge" counter +"Merge" 0 + +# HELP "PartMutation" "Number of mutations (ALTER DELETE/UPDATE)" +# TYPE "PartMutation" counter +"PartMutation" 0 + +# HELP "ReplicatedFetch" "Number of data parts being fetched from replica" +# TYPE "ReplicatedFetch" counter +"ReplicatedFetch" 0 + +# HELP "ReplicatedSend" "Number of data parts being sent to replicas" +# TYPE "ReplicatedSend" counter +"ReplicatedSend" 0 + +* Connection #0 to host localhost left intact +``` + +Örnekten görebileceğiniz gibi, Eğer `` yapılandırmada yapılandırılır.XML dosyası, ClickHouse önceden tanımlanmış türüne alınan HTTP istekleri eşleşecek `` Maç başarılı olursa, ClickHouse ilgili önceden tanımlanmış sorgu yürütecektir. + +Şimdi `` Yapılandır configureılabilir ``, ``, ``, `` ve `` . + +## root\_handler {#root_handler} + +`` kök yolu isteği için belirtilen içeriği döndürür. Belirli dönüş içeriği tarafından yapılandırılır `http_server_default_response` config.xml. belirtilmemişse, iade **Tamam.** + +`http_server_default_response` tanımlanmadı ve Clickhouse'a bir HTTP isteği gönderildi. Sonuç aşağıdaki gibidir: + +``` xml + + + +``` + + $ curl 'http://localhost:8123' + Ok. + +`http_server_default_response` tanımlanır ve Clickhouse'a bir HTTP isteği gönderilir. Sonuç aşağıdaki gibidir: + +``` xml +
]]> + + + + +``` + + $ curl 'http://localhost:8123' +
% + +## ping\_handler {#ping_handler} + +`` geçerli ClickHouse sunucusunun durumunu araştırmak için kullanılabilir. ClickHouse HTTP Sunucusu normal olduğunda, Clickhouse'a erişme `` dön willecektir **Tamam.**. + +Örnek: + +``` xml + + /ping + +``` + +``` bash +$ curl 'http://localhost:8123/ping' +Ok. +``` + +## replicas\_status\_handler {#replicas_status_handler} + +`` çoğaltma düğümünün durumunu algılamak ve geri dönmek için kullanılır **Tamam.** çoğaltma düğümünde gecikme yoksa. Bir gecikme varsa, belirli bir gecikmeyi iade edin. Değeri `` özelleştirme destekler. Belirt specifymezseniz ``, ClickHouse varsayılan ayarı `` oluyor **/ replicas\_status**. + +Örnek: + +``` xml + + /replicas_status + +``` + +Hiçbir gecikme durumda: + +``` bash +$ curl 'http://localhost:8123/replicas_status' +Ok. +``` + +Gecikmeli dava: + +``` bash +$ curl 'http://localhost:8123/replicas_status' +db.stats: Absolute delay: 22. Relative delay: 22. +``` + +## predefined\_query\_handler {#predefined_query_handler} + +Yapılandırabilirsiniz ``, ``, `` ve `` içinde ``. + +`` HTTP isteğinin yöntem bölümünü eşleştirmekten sorumludur. `` tam tanımına uygundur [yöntem](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) HTTP protokolünde. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin yöntem kısmıyla eşleşmez + +`` HTTP isteğinin url bölümünü eşleştirmekten sorumludur. İle uyumludur [RE2](https://github.com/google/re2)'In düzenli ifadeleri. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin url kısmıyla eşleşmez + +`` HTTP isteğinin başlık kısmını eşleştirmekten sorumludur. Bu re2 düzenli ifadeler ile uyumludur. İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanımlanmamışsa, HTTP isteğinin başlık kısmıyla eşleşmez + +`` değer, önceden tanımlanmış bir sorgudur ``, bir HTTP isteği eşleştirildiğinde ve sorgunun sonucu döndürüldüğünde ClickHouse tarafından yürütülür. Bu bir zorunluluktur yapılandırma. + +`` ayar ayarları ve query\_params değerlerini destekler. + +Aşağıdaki örnek değerleri tanımlar `max_threads` ve `max_alter_threads` ayarlar, ardından bu ayarların başarıyla ayarlanıp ayarlanmadığını kontrol etmek için sistem tablosunu sorgular. + +Örnek: + +``` xml + + + GET + + TEST_HEADER_VALUE + [^/]+)(/(?P[^/]+))?]]> + + [^/]+)(/(?P[^/]+))?]]> + + SELECT value FROM system.settings WHERE name = {name_1:String} + SELECT name, value FROM system.settings WHERE name = {name_2:String} + + + +``` + +``` bash +$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_alter_threads?max_threads=1&max_alter_threads=2' +1 +max_alter_threads 2 +``` + +!!! note "Not" + Birinde ``, biri `` sadece birini destekler `` bir ekleme türü. + +## dynamic\_query\_handler {#dynamic_query_handler} + +`` göre `` artmak `` . + +ClickHouse ayıklar ve karşılık gelen değeri yürütür `` HTTP isteğinin url'sindeki değer. +ClickHouse varsayılan ayarı `` oluyor `/query` . İsteğe bağlı bir yapılandırmadır. Yapılandırma dosyasında tanım yoksa, param iletilmez. + +Bu işlevselliği denemek için örnek max\_threads ve max\_alter\_threads değerlerini tanımlar ve ayarların başarıyla ayarlanıp ayarlanmadığını sorgular. +Fark şu ki ``, sorgu yapılandırma dosyasında yazılır. Ama içinde ``, sorgu HTTP isteğinin param şeklinde yazılır. + +Örnek: + +``` xml + + + + TEST_HEADER_VALUE_DYNAMIC + [^/]+)(/(?P[^/]+))?]]> + + query_param + + +``` + +``` bash +$ curl -H 'XXX:TEST_HEADER_VALUE_DYNAMIC' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/?query_param=SELECT%20value%20FROM%20system.settings%20where%20name%20=%20%7Bname_1:String%7D%20OR%20name%20=%20%7Bname_2:String%7D&max_threads=1&max_alter_threads=2¶m_name_2=max_alter_threads' +1 +2 +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/http_interface/) diff --git a/docs/tr/interfaces/index.md b/docs/tr/interfaces/index.md new file mode 100644 index 00000000000..6a89d8cce7e --- /dev/null +++ b/docs/tr/interfaces/index.md @@ -0,0 +1,29 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Arabirimler +toc_priority: 14 +toc_title: "Giri\u015F" +--- + +# Arabirimler {#interfaces} + +ClickHouse iki ağ arabirimi sağlar (Her ikisi de isteğe bağlı olarak ek güvenlik için TLS'YE sarılabilir): + +- [HTTP](http.md), belgelenmiş ve doğrudan kullanımı kolay olan. +- [Yerel TCP](tcp.md) daha az yükü olan. + +Çoğu durumda, doğrudan bunlarla etkileşime girmek yerine uygun araç veya kitaplık kullanılması önerilir. Resmi olarak Yandex tarafından desteklenen şunlardır: + +- [Komut satırı istemcisi](cli.md) +- [JDBC sürücüsü](jdbc.md) +- [ODBC sürücüsü](odbc.md) +- [C++ istemci kitaplığı](cpp.md) + +ClickHouse ile çalışmak için çok çeşitli üçüncü taraf kütüphaneleri de vardır: + +- [İstemci kitaplıkları](third-party/client_libraries.md) +- [Entegrasyonlar](third-party/integrations.md) +- [Görsel arayüzler](third-party/gui.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/) diff --git a/docs/tr/interfaces/jdbc.md b/docs/tr/interfaces/jdbc.md new file mode 100644 index 00000000000..a7e69550c5a --- /dev/null +++ b/docs/tr/interfaces/jdbc.md @@ -0,0 +1,15 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 22 +toc_title: "JDBC s\xFCr\xFCc\xFCs\xFC" +--- + +# JDBC sürücüsü {#jdbc-driver} + +- **[Resmi sürücü](https://github.com/ClickHouse/clickhouse-jdbc)** +- Üçüncü taraf sürücüler: + - [ClickHouse-yerli-JDBC](https://github.com/housepower/ClickHouse-Native-JDBC) + - [clickhouse4j](https://github.com/blynkkk/clickhouse4j) + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/jdbc/) diff --git a/docs/tr/interfaces/mysql.md b/docs/tr/interfaces/mysql.md new file mode 100644 index 00000000000..4e60430f554 --- /dev/null +++ b/docs/tr/interfaces/mysql.md @@ -0,0 +1,49 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 20 +toc_title: "MySQL Aray\xFCz\xFC" +--- + +# MySQL Arayüzü {#mysql-interface} + +ClickHouse MySQL Tel protokolünü destekler. Tarafından etkinleştir canilebilir [mysql\_port](../operations/server_configuration_parameters/settings.md#server_configuration_parameters-mysql_port) yapılandırma dosyasında ayarlama: + +``` xml +9004 +``` + +Komut satırı aracını kullanarak bağlanma örneği `mysql`: + +``` bash +$ mysql --protocol tcp -u default -P 9004 +``` + +Bir bağlantı başarılı olursa çıktı: + +``` text +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 4 +Server version: 20.2.1.1-ClickHouse + +Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> +``` + +Tüm MySQL istemcileri ile uyumluluk için, kullanıcı parolasını belirtmeniz önerilir [çift SHA1](../operations/settings/settings_users.md#password_double_sha1_hex) yapılandırma dosyasında. +Kullanarak kullanıcı şifresi belirt ifilirse [SHA256](../operations/settings/settings_users.md#password_sha256_hex), bazı istemciler (mysqljs ve komut satırı aracı mysql eski sürümleri) kimlik doğrulaması mümkün olmayacaktır. + +Kısıtlama: + +- hazırlanan sorgular desteklenmiyor + +- bazı veri türleri dizeleri olarak gönderilir + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/mysql/) diff --git a/docs/tr/interfaces/odbc.md b/docs/tr/interfaces/odbc.md new file mode 100644 index 00000000000..3cc0cc35700 --- /dev/null +++ b/docs/tr/interfaces/odbc.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 23 +toc_title: "ODBC s\xFCr\xFCc\xFCs\xFC" +--- + +# ODBC sürücüsü {#odbc-driver} + +- [Resmi sürücü](https://github.com/ClickHouse/clickhouse-odbc). + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/odbc/) diff --git a/docs/tr/interfaces/tcp.md b/docs/tr/interfaces/tcp.md new file mode 100644 index 00000000000..b1f712efd7d --- /dev/null +++ b/docs/tr/interfaces/tcp.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 18 +toc_title: Yerel arabirim (TCP) +--- + +# Yerel arabirim (TCP) {#native-interface-tcp} + +Yerel protokol kullanılır [komut satırı istemcisi](cli.md), dağıtılmış sorgu işleme sırasında ve ayrıca diğer C++ programlarında sunucular arası iletişim için. Ne yazık ki, yerel ClickHouse protokolü henüz resmi bir spesifikasyona sahip değildir, ancak ClickHouse kaynak kodundan ters mühendislik yapılabilir (başlangıç [bu civarda](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) ve / veya TCP trafiğini ele alarak ve analiz ederek. + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/tcp/) diff --git a/docs/tr/interfaces/third-party/client_libraries.md b/docs/tr/interfaces/third-party/client_libraries.md new file mode 100644 index 00000000000..2b89b1c4520 --- /dev/null +++ b/docs/tr/interfaces/third-party/client_libraries.md @@ -0,0 +1,59 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 26 +toc_title: "\u0130stemci Kitapl\u0131klar\u0131" +--- + +# Üçüncü taraf geliştiricilerin istemci kitaplıkları {#client-libraries-from-third-party-developers} + +!!! warning "Uyarı" + Yandex yapar **değil** Aşağıda listelenen kütüphaneleri koruyun ve kalitelerini sağlamak için kapsamlı bir test yapmadınız. + +- Piton + - [ınfi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm) + - [clickhouse-sürücü](https://github.com/mymarilyn/clickhouse-driver) + - [clickhouse-müşteri](https://github.com/yurial/clickhouse-client) + - [aiochclient](https://github.com/maximdanilchenko/aiochclient) +- PHP + - [smı2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) + - [8bitov / clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) + - [bozerkins / clickhouse-müşteri](https://packagist.org/packages/bozerkins/clickhouse-client) + - [simpod / clickhouse-müşteri](https://packagist.org/packages/simpod/clickhouse-client) + - [seva-code / php-click-house-client](https://packagist.org/packages/seva-code/php-click-house-client) + - [SeasClick c ++ istemcisi](https://github.com/SeasX/SeasClick) +- Gitmek + - [clickhouse](https://github.com/kshvakov/clickhouse/) + - [git-clickhouse](https://github.com/roistat/go-clickhouse) + - [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse) + - [golang-clickhouse](https://github.com/leprosus/golang-clickhouse) +- NodeJs + - [NodeJs) clickhouse)](https://github.com/TimonKK/clickhouse) + - [düğüm-clickhouse](https://github.com/apla/node-clickhouse) +- Per perll + - [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse) + - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) + - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) +- Yakut + - [ClickHouse (Ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) +- R + - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) +- Java + - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) + - [clickhouse-müşteri](https://github.com/Ecwid/clickhouse-client) +- SC scalaala + - [clickhouse-Scala-istemci](https://github.com/crobox/clickhouse-scala-client) +- Kotlin + - [AORM](https://github.com/TanVD/AORM) +- C\# + - [ClickHouse.Gürültü](https://github.com/killwort/ClickHouse-Net) + - [ClickHouse.Müşteri](https://github.com/DarkWanderer/ClickHouse.Client) + - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) +- İksir + - [clickhousex](https://github.com/appodeal/clickhousex/) +- Nim + - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) diff --git a/docs/tr/interfaces/third-party/gui.md b/docs/tr/interfaces/third-party/gui.md new file mode 100644 index 00000000000..cb41103563d --- /dev/null +++ b/docs/tr/interfaces/third-party/gui.md @@ -0,0 +1,152 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 28 +toc_title: "G\xF6rsel Aray\xFCzler" +--- + +# Üçüncü taraf geliştiricilerin görsel arayüzleri {#visual-interfaces-from-third-party-developers} + +## Açık Kaynak {#open-source} + +### Tabix {#tabix} + +ClickHouse için web arayüzü [Tabix](https://github.com/tabixio/tabix) projelendirmek. + +Özellikler: + +- Ek yazılım yüklemeye gerek kalmadan doğrudan tarayıcıdan ClickHouse ile çalışır. +- Sözdizimi vurgulama ile sorgu editörü. +- Komutların otomatik tamamlanması. +- Sorgu yürütme grafik analizi için araçlar. +- Renk düzeni seçenekleri. + +[Tabix belgeleri](https://tabix.io/doc/). + +### HouseOps {#houseops} + +[HouseOps](https://github.com/HouseOps/HouseOps) OSX, Linux ve Windows için bir UI / IDE. + +Özellikler: + +- Sözdizimi vurgulama ile sorgu oluşturucu. Yanıtı bir tablo veya JSON görünümünde görüntüleyin. +- CSV veya JSON olarak ihracat sorgu sonuçları. +- Açıklamaları ile süreçlerin listesi. Yazma modu. Durdurmak için yeteneği (`KILL`) işleyiş. +- Veritabanı grafiği. Tüm tabloları ve sütunlarını ek bilgilerle gösterir. +- Sütun boyutunun hızlı bir görünümü. +- Sunucu yapılandırması. + +Aşağıdaki özellikler geliştirme için planlanmıştır: + +- Veritabanı yönetimi. +- Kullanıcı yönetimi. +- Gerçek zamanlı veri analizi. +- Küme izleme. +- Küme yönetimi. +- Çoğaltılmış ve Kafka tablolarının izlenmesi. + +### Fener {#lighthouse} + +[Fener](https://github.com/VKCOM/lighthouse) ClickHouse için hafif bir web arayüzüdür. + +Özellikler: + +- Filtreleme ve meta veriler içeren tablo listesi. +- Filtreleme ve sıralama ile tablo önizleme. +- Salt okunur sorgu yürütme. + +### Redash {#redash} + +[Redash](https://github.com/getredash/redash) veri görselleştirme için bir platformdur. + +ClickHouse dahil olmak üzere birden fazla veri kaynağı için destekler, Redash bir son veri kümesi içine farklı veri kaynaklarından gelen sorguların sonuçlarını katılabilir. + +Özellikler: + +- Sorguların güçlü editörü. +- Veritabanı Gezgini. +- Verileri farklı formlarda temsil etmenize izin veren görselleştirme araçları. + +### DBeaver {#dbeaver} + +[DBeaver](https://dbeaver.io/) - ClickHouse desteği ile evrensel masaüstü veritabanı istemcisi. + +Özellikler: + +- Sözdizimi vurgulama ve otomatik tamamlama ile sorgu geliştirme. +- Filtreler ve meta arama ile tablo listesi. +- Tablo veri önizleme. +- Tam metin arama. + +### clickhouse-clı {#clickhouse-cli} + +[clickhouse-clı](https://github.com/hatarist/clickhouse-cli) Python 3 ile yazılmış ClickHouse için alternatif bir komut satırı istemcisidir. + +Özellikler: + +- Otomatik tamamlama. +- Sorgular ve veri çıkışı için sözdizimi vurgulama. +- Veri çıkışı için çağrı cihazı desteği. +- Özel PostgreSQL benzeri komutlar. + +### clickhouse-flamegraph {#clickhouse-flamegraph} + +[clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) görselleştirmek için özel bir araçtır `system.trace_log` olarak [flamegraph](http://www.brendangregg.com/flamegraphs.html). + +## Ticari {#commercial} + +### Datriagrpip {#datagrip} + +[Datriagrpip](https://www.jetbrains.com/datagrip/) ClickHouse için özel destek ile JetBrains bir veritabanı IDE mi. PyCharm, IntelliJ IDEA, GoLand, PhpStorm ve diğerleri: aynı zamanda diğer IntelliJ tabanlı araçlar gömülüdür. + +Özellikler: + +- Çok hızlı kod tamamlama. +- ClickHouse sözdizimi vurgulama. +- Clickhouse'a özgü özellikler için destek, örneğin iç içe geçmiş sütunlar, tablo motorları. +- Veri Editörü. +- Refactorings. +- Arama ve navigasyon. + +### Yandex DataLens {#yandex-datalens} + +[Yandex DataLens](https://cloud.yandex.ru/services/datalens) veri görselleştirme ve analitik bir hizmettir. + +Özellikler: + +- Basit çubuk grafiklerden karmaşık panolara kadar geniş bir yelpazede mevcut görselleştirmeler. +- Panolar kamuya açık hale getirilebilir. +- ClickHouse dahil olmak üzere birden fazla veri kaynağı için destek. +- ClickHouse dayalı hayata veri depolama. + +DataLens olduğunu [ücretsiz olarak kullanılabilir](https://cloud.yandex.com/docs/datalens/pricing) düşük yük projeleri için, ticari kullanım için bile. + +- [DataLens belgeleri](https://cloud.yandex.com/docs/datalens/). +- [Öğretici](https://cloud.yandex.com/docs/solutions/datalens/data-from-ch-visualization) bir ClickHouse veritabanından veri görselleştirme üzerinde. + +### Holistik Yazılım {#holistics-software} + +[Holistik](https://www.holistics.io/) tam yığın veri platformu ve iş zekası aracıdır. + +Özellikler: + +- Otomatik e-posta, bolluk ve raporların Google levha programları. +- Görselleştirmeler, sürüm kontrolü, Otomatik tamamlama, yeniden kullanılabilir sorgu bileşenleri ve dinamik filtreler ile SQL editörü. +- IFRAME aracılığıyla raporların ve gösterge panellerinin gömülü analitiği. +- Veri hazırlama ve ETL yetenekleri. +- Verilerin ilişkisel haritalama için SQL veri modelleme desteği. + +### Looker {#looker} + +[Looker](https://looker.com) ClickHouse dahil 50+ veritabanı lehçeleri desteği ile bir veri platformu ve iş zekası aracıdır. Looker bir SaaS platformu olarak kullanılabilir ve kendi kendine barındırılan. Kullanıcılar, verileri keşfetmek görselleştirme ve panoları, zamanlama raporları oluşturmak ve meslektaşları ile kendi görüşlerini paylaşmak için tarayıcı üzerinden Looker kullanabilirsiniz. Looker, bu özellikleri diğer uygulamalara gömmek için zengin bir araç seti ve bir API sağlar +verileri diğer uygulamalarla entegre etmek. + +Özellikler: + +- Küratörlüğünü destekleyen bir dil olan LookML kullanarak kolay ve çevik geliştirme + [Veri Modelleme](https://looker.com/platform/data-modeling) rapor yazarları ve son kullanıcıları desteklemek. +- Looker ile güçlü iş akışı entegrasyonu [Veri İşlemleri](https://looker.com/platform/actions). + +[Looker içinde ClickHouse nasıl yapılandırılır.](https://docs.looker.com/setup-and-management/database-config/clickhouse) + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/third-party/gui/) diff --git a/docs/tr/interfaces/third-party/index.md b/docs/tr/interfaces/third-party/index.md new file mode 100644 index 00000000000..d8332c00c26 --- /dev/null +++ b/docs/tr/interfaces/third-party/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "\xDC\xE7\xFCnc\xFC Taraf" +toc_priority: 24 +--- + + diff --git a/docs/tr/interfaces/third-party/integrations.md b/docs/tr/interfaces/third-party/integrations.md new file mode 100644 index 00000000000..2216e68a4c4 --- /dev/null +++ b/docs/tr/interfaces/third-party/integrations.md @@ -0,0 +1,99 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 27 +toc_title: Entegrasyonlar +--- + +# Üçüncü taraf geliştiricilerin entegrasyon kütüphaneleri {#integration-libraries-from-third-party-developers} + +!!! warning "Uyarı" + Yandex yapar **değil** Aşağıda listelenen araçları ve kütüphaneleri koruyun ve kalitelerini sağlamak için kapsamlı bir test yapmadınız. + +## Altyapı Ürünleri {#infrastructure-products} + +- İlişkisel veritabanı yönetim sistemleri + - [MySQL](https://www.mysql.com) + - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) + - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) + - [horgh-çoğaltıcı](https://github.com/larsnovikov/horgh-replicator) + - [PostgreSQL](https://www.postgresql.org) + - [clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) + - [ınfi.clickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (kullanma [ınfi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [pg2ch](https://github.com/mkabilov/pg2ch) + - [clickhouse\_fdw](https://github.com/adjust/clickhouse_fdw) + - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) + - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) +- Mesaj kuyrukları + - [Kafka](https://kafka.apache.org) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (kullanma [Go client](https://github.com/kshvakov/clickhouse/)) +- Nesne depoları + - [S3](https://en.wikipedia.org/wiki/Amazon_S3) + - [clickhouse-yedekleme](https://github.com/AlexAkulov/clickhouse-backup) +- Konteyner orkestrasyonu + - [Kubernetes](https://kubernetes.io) + - [clickhouse-operatör](https://github.com/Altinity/clickhouse-operator) +- Yapılandırma yönetimi + - [kuklacı](https://puppet.com) + - [ınnogames / clickhouse](https://forge.puppet.com/innogames/clickhouse) + - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) +- İzleme + - [Grafit](https://graphiteapp.org) + - [graphouse](https://github.com/yandex/graphouse) + - [karbon-clickhouse](https://github.com/lomik/carbon-clickhouse) + + - [grafit-clickhouse](https://github.com/lomik/graphite-clickhouse) + - [grafit-ch-doktoru](https://github.com/innogames/graphite-ch-optimizer) - staled bölümleri optimize eder [\* Graphıtemergetree](../../engines/table_engines/mergetree_family/graphitemergetree.md#graphitemergetree) eğer kurallar [toplaması yapılandırması](../../engines/table_engines/mergetree_family/graphitemergetree.md#rollup-configuration) uygulanabilir + - [Grafana](https://grafana.com/) + - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) + - [Prometheus](https://prometheus.io/) + - [clickhouse\_exporter](https://github.com/f1yegor/clickhouse_exporter) + - [PromHouse](https://github.com/Percona-Lab/PromHouse) + - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) (kullanma [Go client](https://github.com/kshvakov/clickhouse/)) + - [Nagios](https://www.nagios.org/) + - [check\_clickhouse](https://github.com/exogroup/check_clickhouse/) + - [check\_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) + - [Zabbix](https://www.zabbix.com) + - [clickhouse-zabbix-şablon](https://github.com/Altinity/clickhouse-zabbix-template) + - [Sematext](https://sematext.com/) + - [clickhouse entegrasyonu](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) +- Günlük + - [rsyslog](https://www.rsyslog.com/) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) + - [fluentd](https://www.fluentd.org) + - [loghouse](https://github.com/flant/loghouse) (içinde [Kubernetes](https://kubernetes.io)) + - [logagent](https://www.sematext.com/logagent) + - [logagent çıktı-eklenti-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) +- G geoeo + - [MaxMind](https://dev.maxmind.com/geoip/) + - [clickhouse-maxmind-geoıp](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) + +## Programlama Dili {#programming-language-ecosystems} + +- Piton + - [SQLAlchemy](https://www.sqlalchemy.org) + - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (kullanma [ınfi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [Pandalar](https://pandas.pydata.org) + - [pandahouse](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) +- R + - [dplyr](https://db.rstudio.com/dplyr/) + - [RClickHouse](https://github.com/IMSMWU/RClickHouse) (kullanma [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp)) +- Java + - [Hadoop](http://hadoop.apache.org) + - [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) (kullanma [JDBC](../../sql_reference/table_functions/jdbc.md)) +- SC scalaala + - [Akka](https://akka.io) + - [clickhouse-Scala-istemci](https://github.com/crobox/clickhouse-scala-client) +- C\# + - [ADO.NET](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/ado-net-overview) + - [ClickHouse.Gürültü](https://github.com/killwort/ClickHouse-Net) + - [ClickHouse.Müşteri](https://github.com/DarkWanderer/ClickHouse.Client) + - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) + - [ClickHouse. Net. Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations) +- İksir + - [Ecto](https://github.com/elixir-ecto/ecto) + - [clickhouse\_ecto](https://github.com/appodeal/clickhouse_ecto) + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) diff --git a/docs/tr/interfaces/third-party/proxy.md b/docs/tr/interfaces/third-party/proxy.md new file mode 100644 index 00000000000..7ff9fc3642f --- /dev/null +++ b/docs/tr/interfaces/third-party/proxy.md @@ -0,0 +1,46 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 29 +toc_title: Vekiller +--- + +# Üçüncü taraf geliştiricilerin Proxy sunucuları {#proxy-servers-from-third-party-developers} + +## chproxy {#chproxy} + +[chproxy](https://github.com/Vertamedia/chproxy), ClickHouse veritabanı için bir HTTP proxy ve yük dengeleyici. + +Özellikler: + +- Kullanıcı başına Yönlendirme ve yanıt önbelleğe alma. +- Esnek sınırlar. +- Otomatik SSL sertifikası yenileme. + +Go uygulanan. + +## KittenHouse {#kittenhouse} + +[KittenHouse](https://github.com/VKCOM/kittenhouse) ClickHouse ve uygulama sunucusu arasında yerel bir proxy olacak şekilde tasarlanmıştır. + +Özellikler: + +- Bellek içi ve diskteki veri arabelleği. +- Tablo başına yönlendirme. +- Yük dengeleme ve sağlık kontrolü. + +Go uygulanan. + +## ClickHouse-Toplu {#clickhouse-bulk} + +[ClickHouse-Toplu](https://github.com/nikepan/clickhouse-bulk) basit bir ClickHouse ekleme toplayıcı. + +Özellikler: + +- Grup istekleri ve eşik veya aralık ile gönderin. +- Birden çok uzak sunucu. +- Temel kimlik doğrulama. + +Go uygulanan. + +[Orijinal makale](https://clickhouse.tech/docs/en/interfaces/third-party/proxy/) diff --git a/docs/tr/introduction/adopters.md b/docs/tr/introduction/adopters.md new file mode 100644 index 00000000000..0e180e161ec --- /dev/null +++ b/docs/tr/introduction/adopters.md @@ -0,0 +1,83 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 8 +toc_title: Benimseyenler +--- + +# ClickHouse Benimseyenler {#clickhouse-adopters} + +!!! warning "Uyarı" + ClickHouse ve onların Başarı Öyküleri kullanarak şirketlerin aşağıdaki liste kamu kaynaklarından monte edilir, böylece mevcut gerçeklikten farklı olabilir. Şirketinizde Clickhouse'u benimseme hikayesini paylaşırsanız memnun oluruz ve [listeye Ekle](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md), ancak lütfen bunu yaparak herhangi bir NDA sorun yaşamayacağınızdan emin olun. Diğer şirketlerden gelen yayınlarla güncellemeler sağlamak da yararlıdır. + +| Şirket | Sektör | Usecase | Küme Boyutu | (Un) Sıkıştırılmış Veri Boyutu\* | Başvurma | +|----------------------------------------------------------------------------------------------------------|---------------------------------|-----------------------------|------------------------------------------------|------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [2gıs](https://2gis.ru) | Haritalar | İzleme | — | — | [Rusça konuşun, Temmuz 2019](https://youtu.be/58sPkXfq6nw) | +| [Aloha Tarayıcı](https://alohabrowser.com/) | Mobil Uygulama | Tarayıcı arka ucu | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | +| [Amadeus](https://amadeus.com/) | Seyahat | Analiz | — | — | [Basın Bülteni, Nisan 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | +| [Appsflyer](https://www.appsflyer.com) | Mobil analitik | Ana ürün | — | — | [Rusça konuşun, Temmuz 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| [ArenaData](https://arenadata.tech/) | Veri Platformu | Ana ürün | — | — | [Rusça slaytlar, Aralık 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | +| [Badoo](https://badoo.com) | Çıkma | Timeseries | — | — | [Rusça slaytlar, Aralık 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| [Benocs](https://www.benocs.com/) | Ağ telemetri ve analitik | Ana Ürün | — | — | [İngilizce slaytlar, Ekim 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | +| [Bloomberg](https://www.bloomberg.com/) | Finans, Medya | İzleme | 102 sunucular | — | [Slaytlar, Mayıs 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| [Bloxy](https://bloxy.info) | Blockchain | Analiz | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| `Dataliance/UltraPower` | Telekom | Analiz | — | — | [Çince slaytlar, Ocak 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | +| [CARTO](https://carto.com/) | İş Zek Businessası | G geoeo analyt analyticsics | — | — | [ClickHouse ile coğrafi işleme](https://carto.com/blog/geospatial-processing-with-clickhouse/) | +| [CERN](http://public.web.cern.ch/public/) | Araştırma | Deney | — | — | [Basın bülteni, Nisan 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | +| [Cisco](http://cisco.com/) | Ağ | Trafik analizi | — | — | [Yıldırım konuşması, Ekim 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| [Kale Menkul Değerler](https://www.citadelsecurities.com/) | Finansman | — | — | — | [Katkı, Mart 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| [Citymobil](https://city-mobil.ru) | Taksicilik | Analiz | — | — | [Rusça blog yazısı, Mart 2020](https://habr.com/en/company/citymobil/blog/490660/) | +| [ContentSquare](https://contentsquare.com) | Web analyt webics | Ana ürün | — | — | [Fransızca Blog yazısı, Kasım 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| [Cloudflare](https://cloudflare.com) | CDN | Trafik analizi | 36 sunucu | — | [Blog yazısı, Mayıs 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog yazısı, Mart 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | +| [Corunet](https://coru.net/) | Analiz | Ana ürün | — | — | [İngilizce slaytlar, Nisan 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| [CraiditX 氪信](https://creditx.com) | Finans AI | Analiz | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| [Criteo/Storetail](https://www.criteo.com/) | Perakendecilik | Ana ürün | — | — | [İngilizce slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| [Deut Banksche Bank](https://db.com) | Finansman | Bİ analitik | — | — | [İngilizce slaytlar, Ekim 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| [Diva-e](https://www.diva-e.com) | Dijital danışmanlık | Ana Ürün | — | — | [İngilizce slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | +| [Exness](https://www.exness.com) | Ticaret | Metrikler, Günlük Kaydı | — | — | [Rusça konuşun, Mayıs 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | +| [Geniee](https://geniee.co.jp) | Reklam Ağı | Ana ürün | — | — | [Japonca Blog yazısı, Temmuz 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | +| [HUYA](https://www.huya.com/) | Video Akışı | Analiz | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | +| [Idealista](https://www.idealista.com) | Emlak | Analiz | — | — | [İngilizce Blog yazısı, Nisan 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Infovista](https://www.infovista.com/) | Ağlar | Analiz | — | — | [İngilizce slaytlar, Ekim 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| [Innogames](https://www.innogames.com) | Oyun | Metrikler, Günlük Kaydı | — | — | [Rusça slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| [Integros](https://integros.com) | Video hizmetleri platformu | Analiz | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| [Kodiak Verileri](https://www.kodiakdata.com/) | Bulutlar | Ana ürün | — | — | [Engish slaytlar, Nisan 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | +| [Kontur](https://kontur.ru) | Yazılım Geliştirme | Metrik | — | — | [Rusça konuşma, Kasım 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | +| [LifeStreet](https://lifestreet.com/) | Reklam Ağı | Ana ürün | 75 sunucu (3 kopya) | 5.27 Pıb | [Rusça Blog yazısı, Şubat 2017](https://habr.com/en/post/322620/) | +| [Mail.ru Bulut Çözümleri](https://mcs.mail.ru/) | Bulut hizmetleri | Ana ürün | — | — | [ClickHouse örneğini Rusça olarak çalıştırma](https://mcs.mail.ru/help/db-create/clickhouse#) | +| [MessageBird](https://www.messagebird.com) | Telekomünikasyonlar | İstatistik | — | — | [İngilizce slaytlar, Kasım 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| [MGID](https://www.mgid.com/) | Reklam Ağı | Web-analyt -ics | — | — | [Analitik DBMS ClickHouse uygulama deneyimimiz, Rusça](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| [OneAPM](https://www.oneapm.com/) | İzleme ve veri analizi | Ana ürün | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | +| [Pragma Yenilik](http://www.pragma-innovation.fr/) | Telemetri ve büyük veri analizi | Ana ürün | — | — | [İngilizce slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| [QINGCLOUD](https://www.qingcloud.com/) | Bulut hizmetleri | Ana ürün | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | +| [Qrator](https://qrator.net) | DDoS koruması | Ana ürün | — | — | [Blog Yazısı, Mart 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | +| [Beijing per PERCENTC İnformationent Information Technology Co., Ltd. Ltd.Şti.](https://www.percent.cn/) | Analiz | Ana Ürün | — | — | [Çince slaytlar, Haziran 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| [Serseri](https://rambler.ru) | İnternet Hizmetleri | Analiz | — | — | [Rusça konuşma, Nisan 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | +| [Tencent](https://www.tencent.com) | Mesaj | Günlük | — | — | [Çince konuşun, Kasım 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | +| [Trafik Yıldız Starsları](https://trafficstars.com/) | Reklam Ağı | — | — | — | [Rusça slaytlar, Mayıs 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | +| [S7 Havayolları](https://www.s7.ru) | Havayolular | Metrikler, Günlük Kaydı | — | — | [Rusça konuş, Mart 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| [SEMrush](https://www.semrush.com/) | Pazarlamacı | Ana ürün | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| [scireum GmbH](https://www.scireum.de/) | e-ticaret | Ana ürün | — | — | [Almanca konuşma, Şubat 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| [Nöbet](https://sentry.io/) | Yazılımcı | Ürün için arka uç | — | — | [İngilizce Blog yazısı, Mayıs 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | Devlet Sosyal Güvenlik | Analiz | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | +| [seo.do](https://seo.do/) | Analiz | Ana ürün | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | +| [Sina](http://english.sina.com/index.html) | Haberci | — | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| [SMI2](https://smi2.ru/) | Haberci | Analiz | — | — | [Rusça blog yazısı, Kasım 2017](https://habr.com/ru/company/smi2/blog/314558/) | +| [Splunk](https://www.splunk.com/) | İş Analitiği | Ana ürün | — | — | [İngilizce slaytlar, Ocak 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | +| [Spotify](https://www.spotify.com) | Müzik | Deney | — | — | [Slaytlar, Temmuz 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| [Tencent](https://www.tencent.com) | Büyük Veri | Veri işleme | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | +| [Uber](https://www.uber.com) | Taksicilik | Günlük | — | — | [Slay ,tlar, Şubat 20 202020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | +| [VKontakte](https://vk.com) | Sosyal Ağ | İstatistik, Günlük | — | — | [Rusça slaytlar, Ağustos 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| [Wisebits](https://wisebits.com/) | BT çözümleri | Analiz | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| [Xiaoxin Tech.](https://www.xiaoheiban.cn/) | Eğitici | Ortak amaç | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| [Ximalaya](https://www.ximalaya.com/) | Ses paylaşımı | OLAP | — | — | [İngilizce slaytlar, Kasım 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | +| [Yandex Bulut](https://cloud.yandex.ru/services/managed-clickhouse) | Genel Bulut | Ana ürün | — | — | [Rusça konuşun, Aralık 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | +| [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | İş Zek Businessası | Ana ürün | — | — | [Rusça slaytlar, Aralık 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| [Yandex Pazarı](https://market.yandex.ru/) | e-ticaret | Metrikler, Günlük Kaydı | — | — | [Rusça konuşma, Ocak 2019](https://youtu.be/_l1qP0DyBcA?t=478) | +| [Yandex Metrica](https://metrica.yandex.com) | Web analyt webics | Ana ürün | Bir kümede 360 sunucu, bir bölümde 1862 sunucu | 66.41 Pıb / 5.68 Pıb | [Slay ,tlar, Şubat 20 202020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | +| [ЦВТ](https://htc-cs.ru/) | Yazılım Geliştirme | Metrikler, Günlük Kaydı | — | — | [Blog yazısı, Mart 2019, Rusça](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| [МКБ](https://mkb.ru/) | Banka | Web-sistem izleme | — | — | [Rusça slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| [金数据](https://jinshuju.net) | Bİ analitik | Ana ürün | — | — | [Çince slaytlar, Ekim 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| [Instana](https://www.instana.com) | APM Platformu | Ana ürün | — | — | [Twitter mesaj](https://twitter.com/mieldonkers/status/1248884119158882304) | + +[Orijinal makale](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/tr/introduction/distinctive_features.md b/docs/tr/introduction/distinctive_features.md new file mode 100644 index 00000000000..3b26dea645c --- /dev/null +++ b/docs/tr/introduction/distinctive_features.md @@ -0,0 +1,77 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 4 +toc_title: "Ay\u0131rt Edici \xD6zellikler" +--- + +# Clickhouse'un ayırt edici özellikleri {#distinctive-features-of-clickhouse} + +## Doğru sütun yönelimli DBMS {#true-column-oriented-dbms} + +Bir gerçek sütun yönelimli DBMS, hiçbir ek veri değerleri ile depolanır. Diğer şeylerin yanı sıra, bu, uzunluklarının saklanmasını önlemek için sabit uzunluk değerlerinin desteklenmesi gerektiği anlamına gelir “number” değerlerin yanında. Örnek olarak, bir milyar Uİnt8 tipi değerler yaklaşık 1 GB sıkıştırılmamış tüketmelidir veya bu CPU kullanımını güçlü bir şekilde etkiler. Verileri kompakt bir şekilde saklamak esastır (herhangi bir “garbage”) sıkıştırılmamış olsa bile, dekompresyon hızı (CPU kullanımı) esas olarak sıkıştırılmamış verilerin hacmine bağlıdır. + +Farklı sütunların değerlerini ayrı ayrı depolayabilen, ancak diğer senaryolar için optimizasyonları nedeniyle analitik sorguları etkili bir şekilde işleyemeyen sistemler olduğu için dikkat çekicidir. Örnekler HBase, BigTable, Cassandra ve HyperTable. Bu sistemlerde, saniyede yüz bin satır civarında verim elde edersiniz, ancak saniyede yüz milyonlarca satır olmaz. + +Clickhouse'un tek bir veritabanı değil, bir veritabanı yönetim sistemi olduğunu da belirtmek gerekir. ClickHouse, çalışma zamanında tablolar ve veritabanları oluşturmak, veri yüklemek ve sunucuyu yeniden yapılandırmadan ve yeniden başlatmadan sorguları çalıştırmaya izin verir. + +## Veri Sıkıştırma {#data-compression} + +Bazı sütun yönelimli DBMSs (InfiniDB CE ve MonetDB) veri sıkıştırma kullanmayın. Bununla birlikte, veri sıkıştırma mükemmel performans elde etmede önemli bir rol oynar. + +## Verilerin Disk Depolama {#disk-storage-of-data} + +Verileri fiziksel olarak birincil anahtara göre sıralamak, belirli değerleri veya değer aralıkları için düşük gecikme süresi ile birkaç düzine milisaniyeden daha az veri ayıklamayı mümkün kılar. Bazı sütun yönelimli Dbms'ler (SAP HANA ve Google PowerDrill gibi) yalnızca RAM'de çalışabilir. Bu yaklaşım, gerçek zamanlı analiz için gerekenden daha büyük bir donanım bütçesinin tahsisini teşvik eder. ClickHouse düzenli sabit diskler üzerinde çalışmak üzere tasarlanmıştır, bu da GB veri depolama başına maliyetin düşük olduğu anlamına gelir, ancak varsa SSD ve ek RAM de tamamen kullanılır. + +## Birden fazla çekirdekte paralel işleme {#parallel-processing-on-multiple-cores} + +Büyük sorgular, geçerli sunucuda bulunan tüm gerekli kaynakları alarak doğal olarak paralelleştirilir. + +## Birden çok sunucuda dağıtılmış işleme {#distributed-processing-on-multiple-servers} + +Yukarıda belirtilen sütunlu Dbms'lerin neredeyse hiçbiri dağıtılmış sorgu işleme desteğine sahip değildir. +Clickhouse'da, veriler farklı parçalarda bulunabilir. Her parça, hata toleransı için kullanılan bir grup kopya olabilir. Tüm kırıklar, kullanıcı için şeffaf olarak paralel bir sorgu çalıştırmak için kullanılır. + +## SQL desteği {#sql-support} + +ClickHouse, çoğu durumda SQL standardına özdeş olan sql'i temel alan bildirime dayalı bir sorgu dilini destekler. +Desteklenen sorgular arasında GROUP BY, ORDER BY, from, ın ve JOIN yan tümceleri ve skaler alt sorgular bulunur. +Bağımlı alt sorgular ve pencere işlevleri desteklenmez. + +## Vektör Motoru {#vector-engine} + +Veriler yalnızca sütunlar tarafından saklanmakla kalmaz, aynı zamanda yüksek CPU verimliliği elde etmeyi sağlayan vektörler (sütunların parçaları) tarafından işlenir. + +## Gerçek zamanlı veri güncellemeleri {#real-time-data-updates} + +ClickHouse, birincil anahtarlı tabloları destekler. Birincil anahtar aralığındaki sorguları hızlı bir şekilde gerçekleştirmek için, veriler birleştirme ağacını kullanarak aşamalı olarak sıralanır. Bu nedenle, veriler sürekli olarak tabloya eklenebilir. Yeni veri Yutulduğunda hiçbir kilit alınır. + +## Dizin {#index} + +Birincil anahtara göre fiziksel olarak sıralanmış bir veriye sahip olmak, belirli değerleri veya değer aralıkları için düşük gecikme süresi ile birkaç düzine milisaniyeden daha az veri çıkarmayı mümkün kılar. + +## Çevrimiçi sorgular için uygundur {#suitable-for-online-queries} + +Düşük gecikme süresi, kullanıcı arayüzü sayfası yüklenirken, sorguların gecikmeden ve önceden bir cevap hazırlamaya çalışmadan işlenebileceği anlamına gelir. Başka bir deyişle, çevrimiçi. + +## Yaklaşık hesaplamalar için destek {#support-for-approximated-calculations} + +ClickHouse performans için doğruluk ticaret için çeşitli yollar sağlar: + +1. Farklı değerler, medyan ve quantiles sayısı yaklaşık hesaplama için toplam işlevleri. +2. Verilerin bir bölümünü (örnek) temel alan bir sorguyu çalıştırmak ve yaklaşık bir sonuç almak. Bu durumda, diskten orantılı olarak daha az veri alınır. +3. Tüm anahtarlar yerine, sınırlı sayıda rastgele anahtar için bir toplama çalıştırma. Verilerde anahtar dağıtımı için belirli koşullar altında, bu daha az kaynak kullanırken makul derecede doğru bir sonuç sağlar. + +## Veri çoğaltma ve Veri Bütünlüğü desteği {#data-replication-and-data-integrity-support} + +ClickHouse zaman uyumsuz çoklu ana çoğaltma kullanır. Kullanılabilir herhangi bir yineleme için yazıldıktan sonra kalan tüm yinelemeler arka planda kendi kopyasını almak. Sistem, farklı yinelemelerde aynı verileri korur. Çoğu arızadan sonra kurtarma, karmaşık durumlarda otomatik olarak veya yarı otomatik olarak gerçekleştirilir. + +Daha fazla bilgi için bölüme bakın [Veri çoğaltma](../engines/table_engines/mergetree_family/replication.md). + +## Dezavantajları olarak kabul edilebilir özellikler {#clickhouse-features-that-can-be-considered-disadvantages} + +1. Tam teşekküllü işlemler yok. +2. Yüksek oranda ve düşük gecikme ile zaten eklenen verileri değiştirme veya silme yeteneği eksikliği. Verileri temizlemek veya değiştirmek için toplu silme ve güncellemeler vardır, örneğin Aşağıdakilere uymak için [GDPR](https://gdpr-info.eu). +3. Seyrek dizin, Clickhouse'u anahtarlarıyla tek satırları almak için nokta sorguları için çok uygun değildir. + +[Orijinal makale](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/tr/introduction/history.md b/docs/tr/introduction/history.md new file mode 100644 index 00000000000..cfc18df2981 --- /dev/null +++ b/docs/tr/introduction/history.md @@ -0,0 +1,56 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 7 +toc_title: "Tarih\xE7e" +--- + +# ClickHouse Geçmişi {#clickhouse-history} + +ClickHouse güç başlangıçta geliştirilmiştir [Üye.Metrica](https://metrica.yandex.com/), [dünyanın en büyük ikinci web analiz platformu](http://w3techs.com/technologies/overview/traffic_analysis/all) ve bu sistemin temel bileşeni olmaya devam ediyor. Veritabanında 13 trilyondan fazla kayıt ve günlük 20 milyardan fazla etkinlik ile ClickHouse, doğrudan toplanmamış verilerden anında özel raporlar oluşturmanıza olanak tanır. Bu makale Kısaca Clickhouse'un gelişiminin ilk aşamalarında hedeflerini kapsamaktadır. + +Üye.Metrica kullanıcı tarafından tanımlanan keyfi kesimleri ile, hit ve oturumları dayalı anında özelleştirilmiş raporlar oluşturur. Bunu sık sık yapmak, benzersiz kullanıcı sayısı gibi karmaşık agregalar oluşturmayı gerektirir. Bir rapor oluşturmak için yeni veriler gerçek zamanlı olarak gelir. + +Nisan 2014 itibariyle, Yandex.Metrica, günlük olarak yaklaşık 12 milyar olayı (sayfa görüntüleme ve tıklama) izliyordu. Tüm bu olaylar özel raporlar oluşturmak için saklanmalıdır. Tek bir sorgu, birkaç yüz milisaniye içinde milyonlarca satırı veya sadece birkaç saniye içinde yüz milyonlarca satırı taramayı gerektirebilir. + +## Yandex kullanımı.Metrica ve diğer Yandex Hizmetleri {#usage-in-yandex-metrica-and-other-yandex-services} + +ClickHouse, Yandex'te birden fazla amaca hizmet eder.Metrica. +Ana görevi, toplanmamış verileri kullanarak çevrimiçi modda raporlar oluşturmaktır. Veritabanında 20.3 trilyon satırdan fazla depolayan 374 sunucu kümesi kullanır. Sıkıştırılmış verilerin hacmi, yinelenenleri ve kopyaları hesaba katmadan yaklaşık 2 PB'DİR. Sıkıştırılmamış verilerin hacmi (TSV formatında) Yaklaşık 17 PB olacaktır. + +ClickHouse ayrıca aşağıdaki süreçlerde önemli bir rol oynar: + +- Yandex'den oturum tekrarı için veri saklama.Metrica. +- Ara veri işleme. +- Analitik ile küresel raporlar oluşturma. +- Yandex hata ayıklama için sorguları çalıştırma.Metrica motoru. +- API ve kullanıcı arayüzü günlükleri analiz. + +Günümüzde, diğer Yandex hizmetlerinde ve bölümlerinde birden fazla düzine ClickHouse kurulumu bulunmaktadır: arama dikey, e-ticaret, reklam, iş analitiği, mobil geliştirme, kişisel hizmetler ve diğerleri. + +## Toplanmış ve toplanmamış veriler {#aggregated-and-non-aggregated-data} + +İstatistikleri etkili bir şekilde hesaplamak için, veri hacmini azalttığından verileri toplamanız gerektiğine dair yaygın bir görüş vardır. + +Ancak veri toplama birçok sınırlama ile birlikte gelir: + +- Gerekli raporların önceden tanımlanmış bir listesine sahip olmanız gerekir. +- Kullanıcı özel raporlar yapamaz. +- Çok sayıda farklı anahtar üzerinde toplanırken, veri hacmi zorlukla azaltılır, bu nedenle toplama işe yaramaz. +- Çok sayıda rapor için çok fazla toplama varyasyonu vardır (kombinatoryal patlama). +- Anahtarları yüksek önemlilik (URL'ler gibi) ile toplarken, veri hacmi çok fazla azaltılmaz (iki kattan daha az). +- Bu nedenle, toplama ile veri hacmi küçültmek yerine büyüyebilir. +- Kullanıcılar onlar için oluşturduğumuz tüm raporları görüntülemez. Bu hesaplamaların büyük bir kısmı işe yaramaz. +- Verilerin mantıksal bütünlüğü, çeşitli toplamalar için ihlal edilebilir. + +Hiçbir şeyi toplamazsak ve toplanmamış verilerle çalışırsak, bu hesaplamaların hacmini azaltabilir. + +Bununla birlikte, toplama ile, çalışmanın önemli bir kısmı çevrimdışı olarak alınır ve nispeten sakin bir şekilde tamamlanır. Buna karşılık, çevrimiçi hesaplamalar, kullanıcı sonucu beklediğinden mümkün olduğunca hızlı hesaplamayı gerektirir. + +Üye.Metrica, raporların çoğunluğu için kullanılan Metrage adı verilen verileri toplamak için özel bir sisteme sahiptir. +2009'dan itibaren Yandex.Metrica, daha önce Rapor Oluşturucusu için kullanılan OLAPServer adlı toplanmamış veriler için özel bir OLAP veritabanı da kullandı. +OLAPServer, toplanmamış veriler için iyi çalıştı, ancak tüm raporlar için istenildiği gibi kullanılmasına izin vermeyen birçok kısıtlamaya sahipti. Bunlar, veri türleri için destek eksikliği (yalnızca sayılar) ve verileri gerçek zamanlı olarak aşamalı olarak güncelleyememe (yalnızca verileri günlük olarak yeniden yazarak yapılabilir) içeriyordu. OLAPServer bir DBMS değil, özel bir dB'dir. + +ClickHouse için ilk hedef OLAPServer sınırlamaları kaldırmak ve tüm raporlar için toplanmamış verilerle çalışma sorunu çözmek oldu, ama yıllar içinde, analitik görevler geniş bir yelpazede için uygun bir genel amaçlı veritabanı yönetim sistemi haline gelmiştir. + +[Orijinal makale](https://clickhouse.tech/docs/en/introduction/history/) diff --git a/docs/tr/introduction/index.md b/docs/tr/introduction/index.md new file mode 100644 index 00000000000..9691671aefa --- /dev/null +++ b/docs/tr/introduction/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Giri\u015F" +toc_priority: 1 +--- + + diff --git a/docs/tr/introduction/performance.md b/docs/tr/introduction/performance.md new file mode 100644 index 00000000000..ce54825af98 --- /dev/null +++ b/docs/tr/introduction/performance.md @@ -0,0 +1,32 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 6 +toc_title: Performans +--- + +# Performans {#performance} + +Yandex'deki dahili test sonuçlarına göre, ClickHouse, test için mevcut olan sınıfının sistemleri arasında karşılaştırılabilir işletim senaryoları için en iyi performansı (hem uzun sorgular için en yüksek verim hem de kısa sorgularda en düşük gecikme süresi) gösterir. Test sonuçlarını bir [ayrı sayfa](https://clickhouse.tech/benchmark/dbms/). + +Çok sayıda bağımsız kriterler benzer sonuçlara geldi. Bir internet araması kullanarak bulmak zor değildir veya görebilirsiniz [ilgili bağlantı ourlardan oluşan küçük koleksiyon collectionumuz](https://clickhouse.tech/#independent-benchmarks). + +## Tek bir büyük sorgu için çıktı {#throughput-for-a-single-large-query} + +Verim, saniyede satır veya saniyede megabayt olarak ölçülebilir. Veriler sayfa önbelleğine yerleştirilirse, çok karmaşık olmayan bir sorgu, modern donanım üzerinde tek bir sunucuda yaklaşık 2-10 GB/s sıkıştırılmamış veri hızında işlenir (en basit durumlar için, hız 30 GB/s'ye ulaşabilir). Veri sayfa önbelleğine yerleştirilmezse, hız disk alt sistemine ve veri sıkıştırma hızına bağlıdır. Örneğin, disk alt sistemi 400 MB/s veri okuma izin verir ve veri sıkıştırma hızı 3 ise, hız 1.2 GB / s civarında olması bekleniyor. saniyede satır hızı elde etmek için hızı saniyede bayt cinsinden sorguda kullanılan sütunların toplam boyutuna bölün. Örneğin, 10 bayt sütun ayıklanırsa, hızın saniyede yaklaşık 100-200 milyon satır olması beklenir. + +İşlem hızı, dağıtılmış işlem için neredeyse doğrusal olarak artar, ancak yalnızca toplama veya sıralamadan kaynaklanan satır sayısı çok büyük değilse. + +## Kısa Sorguları İşlerken Gecikme Süresi {#latency-when-processing-short-queries} + +Bir sorgu birincil anahtar kullanır ve çok fazla sütun ve satır (yüzbinlerce) işlemek için seçmez, veri sayfa önbelleğine yerleştirilirse, 50 milisaniyeden daha az gecikme süresi (en iyi durumda milisaniye tek basamak) bekleyebilirsiniz. Aksi takdirde, gecikme çoğunlukla arama sayısı tarafından hakimdir. Aşırı yüklenmemiş bir sistem için dönen disk sürücüleri kullanırsanız, gecikme bu formülle tahmin edilebilir: `seek time (10 ms) * count of columns queried * count of data parts`. + +## Büyük miktarda kısa sorgu işlerken verim {#throughput-when-processing-a-large-quantity-of-short-queries} + +Aynı koşullar altında, ClickHouse tek bir sunucuda saniyede birkaç yüz sorgu işleyebilir (en iyi durumda birkaç bine kadar). Bu senaryo analitik DBMSs için tipik olmadığından, saniyede en fazla 100 sorgu beklemenizi öneririz. + +## Veri Eklerken Performans {#performance-when-inserting-data} + +Verileri en az 1000 satırlık paketlere veya saniyede tek bir istekten daha fazla olmayan paketlere eklemenizi öneririz. Sekmeyle ayrılmış bir dökümden MergeTree tablosuna eklerken, ekleme hızı 50 ila 200 MB/s arasında olabilir. eklenen satırlar yaklaşık 1 Kb boyutundaysa, hız saniyede 50.000 ila 200.000 satır olacaktır. Satırlar küçükse, performans saniyede satırlarda daha yüksek olabilir (Banner sistem verileri üzerinde -`>` Saniyede 500 rows.000 satır; Graf ;it ver ;ilerinde -`>` Saniyede 1.000.000 satır). Performansı artırmak için, paralel olarak doğrusal olarak ölçeklenen birden çok ekleme sorgusu yapabilirsiniz. + +[Orijinal makale](https://clickhouse.tech/docs/en/introduction/performance/) diff --git a/docs/tr/operations/access_rights.md b/docs/tr/operations/access_rights.md new file mode 100644 index 00000000000..b270c0ecbed --- /dev/null +++ b/docs/tr/operations/access_rights.md @@ -0,0 +1,113 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 48 +toc_title: "Eri\u015Fim Haklar\u0131" +--- + +# Erişim Hakları {#access-rights} + +Kullanıcılar ve erişim hakları kullanıcı yapılandırmasında ayarlanır. Bu genellikle `users.xml`. + +Kullanıcılar kaydedilir `users` bölme. İşte bir parçası `users.xml` Dosya: + +``` xml + + + + + + + + + + + + default + + + default + + + + + + + web + default + + test + + + test + + + +``` + +İki kullanıcıdan bir bildirim görebilirsiniz: `default`ve`web`. Ek weledik `web` kullanıcı ayrı ayrı. + +Bu `default` kullanıcı adı geçilmez durumlarda kullanıcı seçilir. Bu `default` kullanıcı, sunucu veya kümenin yapılandırması, sunucu veya kümenin yapılandırılmasını belirtmezse, dağıtılmış sorgu işleme için de kullanılır. `user` ve `password` (on bölümüne bakın [Dağılı](../engines/table_engines/special/distributed.md) motor). + +The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. + +Parola, açık metin (önerilmez) veya SHA-256'da belirtilir. Haşhaş tuzlu değil. Bu bağlamda, bu şifreleri potansiyel kötü amaçlı saldırılara karşı güvenlik sağlamak olarak düşünmemelisiniz. Aksine, çalışanlardan korunmak için gereklidir. + +Erişime izin verilen ağların listesi belirtilir. Bu örnekte, her iki kullanıcı için ağ listesi ayrı bir dosyadan yüklenir (`/etc/metrika.xml`) içeren `networks` ikame. İşte bunun bir parçası: + +``` xml + + ... + + ::/64 + 203.0.113.0/24 + 2001:DB8::/32 + ... + + +``` + +Bu ağ listesini doğrudan tanımlayabilirsiniz `users.xml` veya bir dosyada `users.d` dizin (daha fazla bilgi için bölüme bakın “[Yapılandırma dosyaları](configuration_files.md#configuration_files)”). + +Yapılandırma, her yerden erişimin nasıl açılacağını açıklayan yorumları içerir. + +Üretimde kullanım için, sadece belirtin `ip` elemanları (IP adresleri ve maskeleri), kullanıl ,dığından beri `host` ve `hoost_regexp` ekstra gecikmeye neden olabilir. + +Daha sonra kullanıcı ayarları profili belirtilir (bölüme bakın “[Ayarlar profilleri](settings/settings_profiles.md)”. Varsayılan profili belirtebilirsiniz, `default'`. Profilin herhangi bir adı olabilir. Farklı kullanıcılar için aynı profili belirtebilirsiniz. Ayarlar profilinde yazabileceğiniz en önemli şey `readonly=1` sağlar okumak-sadece erişim. Ardından kullanılacak kotayı belirtin (bölüme bakın “[Kotalar](quotas.md#quotas)”). Varsayılan kotayı belirtebilirsiniz: `default`. It is set in the config by default to only count resource usage, without restricting it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually. + +İsteğe bağlı `` bölümünde, kullanıcının erişebileceği veritabanlarının bir listesini de belirtebilirsiniz. Varsayılan olarak, tüm veritabanları kullanıcı tarafından kullanılabilir. Belirtebilirsiniz `default` veritabanı. Bu durumda, kullanıcı varsayılan olarak veritabanına erişim alır. + +İsteğe bağlı `` bölümünde, kullanıcının erişebileceği sözlüklerin bir listesini de belirtebilirsiniz. Varsayılan olarak, tüm sözlükler kullanıcı tarafından kullanılabilir. + +Erişim `system` veritabanı her zaman izin verilir (bu veritabanı sorguları işlemek için kullanıldığından). + +Kullanıcı kullanarak onları tüm veritabanları ve tabloların bir listesini alabilirsiniz `SHOW` tek tek veritabanlarına erişime izin verilmese bile, sorgular veya sistem tabloları. + +Veritabanı erişimi ile ilgili değildir [readonly](settings/permissions_for_queries.md#settings_readonly) ayar. Bir veritabanına tam erişim izni veremezsiniz ve `readonly` başka birine erişim. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/tr/operations/backup.md b/docs/tr/operations/backup.md new file mode 100644 index 00000000000..5d1d806e5dd --- /dev/null +++ b/docs/tr/operations/backup.md @@ -0,0 +1,41 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 49 +toc_title: Veri Yedekleme +--- + +# Veri Yedekleme {#data-backup} + +Karşın [çoğalma](../engines/table_engines/mergetree_family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [50 GB'den fazla veri içeren MergeTree benzeri bir motorla tabloları bırakamazsınız](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Ancak, bu önlemler olası tüm davaları kapsamaz ve atlatılabilir. + +Olası insan hatalarını etkili bir şekilde azaltmak için, verilerinizi yedeklemek ve geri yüklemek için dikkatli bir şekilde bir strateji hazırlamanız gerekir **önceden**. + +Her şirketin farklı kaynakları ve iş gereksinimleri vardır, bu nedenle her duruma uyacak ClickHouse yedeklemeleri ve geri yüklemeleri için evrensel bir çözüm yoktur. Bir gigabayt veri için ne işe yarar, muhtemelen onlarca petabayt için çalışmaz. Aşağıda tartışılacak olan kendi artıları ve eksileri ile çeşitli Olası yaklaşımlar vardır. Çeşitli eksikliklerini telafi etmek için sadece bir tane yerine birkaç yaklaşım kullanmak iyi bir fikirdir. + +!!! note "Not" + Bir şeyi yedeklediyseniz ve geri yüklemeyi hiç denemediyseniz, aslında ihtiyacınız olduğunda Geri Yüklemenin düzgün çalışmayacağını (veya en azından işin tahammül edebileceğinden daha uzun süreceğini) unutmayın. Bu nedenle, seçtiğiniz yedekleme yaklaşımı ne olursa olsun, geri yükleme işlemini de otomatikleştirdiğinizden emin olun ve düzenli olarak yedek bir ClickHouse kümesinde uygulayın. + +## Kaynak Verileri Başka Bir Yerde Çoğaltma {#duplicating-source-data-somewhere-else} + +Genellikle Clickhouse'a alınan veriler, aşağıdaki gibi bir tür kalıcı sıra yoluyla teslim edilir [Apache Kafka](https://kafka.apache.org). Bu durumda, Clickhouse'a yazılırken aynı veri akışını okuyacak ve bir yerde soğuk depoda depolayacak ek bir abone kümesi yapılandırmak mümkündür. Çoğu şirket zaten bir nesne deposu veya dağıtılmış bir dosya sistemi gibi olabilecek bazı varsayılan önerilen soğuk depolamaya sahiptir [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). + +## Dosya Sistemi Anlık Görüntüleri {#filesystem-snapshots} + +Bazı yerel dosya sistemleri anlık görüntü işlevselliği sağlar (örneğin, [ZFS](https://en.wikipedia.org/wiki/ZFS)), ancak canlı sorguları sunmak için en iyi seçenek olmayabilir. Olası bir çözüm, bu tür dosya sistemi ile ek kopyalar oluşturmak ve bunları [Dağılı](../engines/table_engines/special/distributed.md) için kullanılan tablolar `SELECT` sorgular. Bu tür yinelemelerdeki anlık görüntüler, verileri değiştiren sorguların erişemeyeceği bir yerde olacaktır. Bonus olarak, bu yinelemeler, sunucu başına daha fazla disk eklenmiş özel donanım yapılandırmalarına sahip olabilir ve bu da uygun maliyetli olabilir. + +## clickhouse-fotokopi makinesi {#clickhouse-copier} + +[clickhouse-fotokopi makinesi](utilities/clickhouse-copier.md) başlangıçta yeniden shard petabyte boyutlu tablolar için oluşturulan çok yönlü bir araçtır. Ayrıca yedekleme için kullanılan ve güvenilir clickhouse tablolar ve kümeler arasında veri kopyalar çünkü amaçlar geri olabilir. + +Daha küçük veri hacimleri için, basit bir `INSERT INTO ... SELECT ...` uzak tablolara da çalışabilir. + +## Parçalar ile manipülasyonlar {#manipulations-with-parts} + +ClickHouse kullanarak sağlar `ALTER TABLE ... FREEZE PARTITION ...` tablo bölümleri yerel bir kopyasını oluşturmak için sorgu. Bu hardlinks kullanarak uygulanır `/var/lib/clickhouse/shadow/` klasör, bu yüzden genellikle eski veriler için ekstra disk alanı tüketmez. Oluşturulan dosyaların kopyaları ClickHouse server tarafından işlenmez, bu yüzden onları orada bırakabilirsiniz: herhangi bir ek harici sistem gerektirmeyen basit bir yedeklemeniz olacak, ancak yine de donanım sorunlarına eğilimli olacaktır. Bu nedenle, bunları uzaktan başka bir konuma kopyalamak ve ardından yerel kopyaları kaldırmak daha iyidir. Dağıtılmış dosya sistemleri ve nesne depoları bunun için hala iyi bir seçenektir, ancak yeterince büyük kapasiteye sahip normal ekli dosya sunucuları da işe yarayabilir (bu durumda aktarım ağ dosya sistemi veya belki de [rsync](https://en.wikipedia.org/wiki/Rsync)). + +Bölüm işlemleriyle ilgili sorgular hakkında daha fazla bilgi için bkz. [ALTER belgeleri](../sql_reference/statements/alter.md#alter_manipulations-with-partitions). + +Bu yaklaşımı otomatikleştirmek için üçüncü taraf bir araç kullanılabilir: [clickhouse-yedekleme](https://github.com/AlexAkulov/clickhouse-backup). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/backup/) diff --git a/docs/tr/operations/configuration_files.md b/docs/tr/operations/configuration_files.md new file mode 100644 index 00000000000..fda72e6ba21 --- /dev/null +++ b/docs/tr/operations/configuration_files.md @@ -0,0 +1,57 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 50 +toc_title: "Yap\u0131land\u0131rma Dosyalar\u0131" +--- + +# Yapılandırma Dosyaları {#configuration_files} + +ClickHouse Çoklu dosya yapılandırma yönetimini destekler. Ana sunucu yapılandırma dosyası `/etc/clickhouse-server/config.xml`. Diğer dosyalar içinde olmalıdır `/etc/clickhouse-server/config.d` dizin. + +!!! note "Not" + Tüm yapılandırma dosyaları XML biçiminde olmalıdır. Ayrıca, genellikle aynı kök öğeye sahip olmalıdırlar ``. + +Ana yapılandırma dosyasında belirtilen bazı ayarlar diğer yapılandırma dosyalarında geçersiz kılınabilir. Bu `replace` veya `remove` bu yapılandırma dosyalarının öğeleri için öznitelikler belirtilebilir. + +Her ikisi de belirtilmezse, yinelenen çocukların değerlerini değiştirerek öğelerin içeriğini yinelemeli olarak birleştirir. + +Eğer `replace` belirtilen, tüm öğeyi belirtilen ile değiştirir. + +Eğer `remove` belirt .ilirse, öğeyi siler. + +Yapılandırma ayrıca tanımlayabilir “substitutions”. Bir öğe varsa `incl` öznitelik, dosyadan karşılık gelen ikame değeri olarak kullanılacaktır. Varsayılan olarak, değiştirmeler ile dosyanın yolu `/etc/metrika.xml`. Bu değiştirilebilir [include\_from](server_configuration_parameters/settings.md#server_configuration_parameters-include_from) sunucu yapılandırmasında öğe. İkame değerleri belirtilen `/yandex/substitution_name` bu dosyadaki öğeler. Belirtilen bir ika Ame halinde `incl` yok, günlüğe kaydedilir. Clickhouse'un eksik değiştirmelerin günlüğe kaydedilmesini önlemek için `optional="true"` öznitelik (örneğin, ayarlar [makrolar](server_configuration_parameters/settings.md)). + +İkame da ZooKeeper yapılabilir. Bunu yapmak için özniteliği belirtin `from_zk = "/path/to/node"`. Eleman değeri, düğümün içeriği ile değiştirilir `/path/to/node` ZooKeeper. Ayrıca ZooKeeper düğümünde bir XML alt ağacının tamamını koyabilirsiniz ve kaynak öğeye tamamen eklenecektir. + +Bu `config.xml` dosya kullanıcı ayarları, profiller ve kotalar ile ayrı bir yapılandırma belirtebilirsiniz. Bu yapılandırmanın göreli yolu, `users_config` öğe. Varsayılan olarak, bu `users.xml`. Eğer `users_config` atlanır, kullanıcı ayarları, profiller ve kotalar doğrudan belirtilir `config.xml`. + +Kullanıcılar yapılandırma benzer ayrı dosyaları içine bölünmüş olabilir `config.xml` ve `config.d/`. +Dizin adı olarak tanımlanır `users_config` olmadan ayarı `.xml` postfix ile birleştirilmiş `.d`. +Dizin `users.d` varsayılan olarak kullanılır, gibi `users_config` varsayılan olarak `users.xml`. +Örneğin, bu gibi her kullanıcı için ayrı yapılandırma dosyasına sahip olabilirsiniz: + +``` bash +$ cat /etc/clickhouse-server/users.d/alice.xml +``` + +``` xml + + + + analytics + + ::/0 + + ... + analytics + + + +``` + +Her yapılandırma dosyası için sunucu da üretir `file-preprocessed.xml` başlatırken dosyalar. Bu dosyalar, tamamlanmış tüm değiştirmeleri ve geçersiz kılmaları içerir ve bunlar bilgi amaçlı kullanım içindir. Zookeeper değiştirmelerin yapılandırma dosyalarında kullanılan ancak ZooKeeper sunucu başlangıcında kullanılabilir değilse, sunucu yapılandırmayı önceden işlenmiş dosyadan yükler. + +Sunucu, yapılandırma dosyalarındaki değişikliklerin yanı sıra, değiştirmeleri ve geçersiz kılmaları gerçekleştirirken kullanılan dosya ve ZooKeeper düğümlerini izler ve anında kullanıcılar ve kümeler için ayarları yeniden yükler. Bu, sunucuyu yeniden başlatmadan kümeyi, kullanıcıları ve ayarlarını değiştirebileceğiniz anlamına gelir. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/configuration_files/) diff --git a/docs/tr/operations/index.md b/docs/tr/operations/index.md new file mode 100644 index 00000000000..e20eefc1ab0 --- /dev/null +++ b/docs/tr/operations/index.md @@ -0,0 +1,28 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Harekat +toc_priority: 41 +toc_title: "Giri\u015F" +--- + +# Harekat {#operations} + +ClickHouse işlemleri kılavuzu aşağıdaki ana bölümlerden oluşur: + +- [Gereksinimler](requirements.md) +- [İzleme](monitoring.md) +- [Arıza](troubleshooting.md) +- [Kullanım Önerileri](tips.md) +- [Güncelleme Prosedürü](update.md) +- [Erişim Hakları](access_rights.md) +- [Veri Yedekleme](backup.md) +- [Yapılandırma Dosyaları](configuration_files.md) +- [Kotalar](quotas.md) +- [Sistem Tabloları](system_tables.md) +- [Sunucu Yapılandırma Parametreleri](server_configuration_parameters/index.md) +- [Donanımınızı ClickHouse İle Test Etme](performance_test.md) +- [Ayarlar](settings/index.md) +- [Programlar](utilities/index.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/) diff --git a/docs/tr/operations/monitoring.md b/docs/tr/operations/monitoring.md new file mode 100644 index 00000000000..eb5b7bd6dc8 --- /dev/null +++ b/docs/tr/operations/monitoring.md @@ -0,0 +1,46 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 45 +toc_title: "\u0130zleme" +--- + +# İzleme {#monitoring} + +İzleyebilirsiniz: + +- Donanım kaynaklarının kullanımı. +- ClickHouse sunucu metrikleri. + +## Kaynak Kullanımı {#resource-utilization} + +ClickHouse, donanım kaynaklarının durumunu tek başına izlemez. + +İzleme ayarlamak için önerilir : + +- İşlemcilerde yük ve sıcaklık. + + Kullanabilirsiniz [dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html) ya da diğer aletler. + +- Depolama sistemi, RAM ve ağ kullanımı. + +## ClickHouse Sunucu Metrikleri {#clickhouse-server-metrics} + +ClickHouse sunucu kendini devlet izleme için araçlar gömülü vardır. + +Sunucu olaylarını izlemek için sunucu günlüklerini kullanın. Görmek [kaydedici](server_configuration_parameters/settings.md#server_configuration_parameters-logger) yapılandırma dosyasının bölümü. + +ClickHouse toplar: + +- Sunucunun hesaplama kaynaklarını nasıl kullandığına dair farklı metrikler. +- Sorgu işleme ile ilgili ortak istatistikler. + +Metrikleri şu adreste bulabilirsiniz: [sistem.metrik](../operations/system_tables.md#system_tables-metrics), [sistem.etkinlik](../operations/system_tables.md#system_tables-events), ve [sistem.asynchronous\_metrics](../operations/system_tables.md#system_tables-asynchronous_metrics) Tablolar. + +Clickhouse'u metrikleri dışa aktaracak şekilde yapılandırabilirsiniz [Grafit](https://github.com/graphite-project). Görmek [Graf sectionit bölümü](server_configuration_parameters/settings.md#server_configuration_parameters-graphite) ClickHouse sunucu yapılandırma dosyasında. Metriklerin dışa aktarımını yapılandırmadan önce, grafit'i resmi olarak takip ederek ayarlamanız gerekir [kılavuz](https://graphite.readthedocs.io/en/latest/install.html). + +Clickhouse'u metrikleri dışa aktaracak şekilde yapılandırabilirsiniz [Prometheus](https://prometheus.io). Görmek [Prometheus bölümü](server_configuration_parameters/settings.md#server_configuration_parameters-prometheus) ClickHouse sunucu yapılandırma dosyasında. Metriklerin dışa aktarılmasını yapılandırmadan önce, prometheus'u yetkililerini takip ederek ayarlamanız gerekir [kılavuz](https://prometheus.io/docs/prometheus/latest/installation/). + +Ayrıca, http API aracılığıyla sunucu kullanılabilirliğini izleyebilirsiniz. Sen sendd the `HTTP GET` istek için `/ping`. Sunucu mevcutsa, yanıt verir `200 OK`. + +Bir küme yapılandırmasındaki sunucuları izlemek için [max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) parametre ve HTTP kaynağını kullanın `/replicas_status`. Bir istek için `/replicas_status` dönüşler `200 OK` çoğaltma kullanılabilir ve diğer yinelemeler gecikmiş değil. Bir çoğaltma gecikirse, döndürür `503 HTTP_SERVICE_UNAVAILABLE` boşluk hakkında bilgi ile. diff --git a/docs/tr/operations/optimizing_performance/index.md b/docs/tr/operations/optimizing_performance/index.md new file mode 100644 index 00000000000..9d39c082a98 --- /dev/null +++ b/docs/tr/operations/optimizing_performance/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Performans\u0131 Optimize Etme" +toc_priority: 52 +--- + + diff --git a/docs/tr/operations/optimizing_performance/sampling_query_profiler.md b/docs/tr/operations/optimizing_performance/sampling_query_profiler.md new file mode 100644 index 00000000000..0d86dfad5ef --- /dev/null +++ b/docs/tr/operations/optimizing_performance/sampling_query_profiler.md @@ -0,0 +1,64 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 54 +toc_title: "Sorgu Profili Olu\u015Fturma" +--- + +# Örnekleme Sorgusu Profiler {#sampling-query-profiler} + +ClickHouse, sorgu yürütülmesini analiz etmeyi sağlayan örnekleme profiler'i çalıştırır. Profiler kullanarak sorgu yürütme sırasında en sık kullanılan kaynak kodu yordamları bulabilirsiniz. Boşta kalma süresi de dahil olmak üzere harcanan CPU zamanını ve duvar saati zamanını izleyebilirsiniz. + +Profiler kullanmak için: + +- Kurulum [trace\_log](../server_configuration_parameters/settings.md#server_configuration_parameters-trace_log) sunucu yapılandırması bölümü. + + Bu bölüm yapılandırır [trace\_log](../../operations/system_tables.md#system_tables-trace_log) profiler işleyişinin sonuçlarını içeren sistem tablosu. Varsayılan olarak yapılandırılmıştır. Bu tablodaki verilerin yalnızca çalışan bir sunucu için geçerli olduğunu unutmayın. Sunucu yeniden başlatıldıktan sonra ClickHouse tabloyu temizlemez ve depolanan tüm sanal bellek adresi geçersiz hale gelebilir. + +- Kurulum [query\_profiler\_cpu\_time\_period\_ns](../settings/settings.md#query_profiler_cpu_time_period_ns) veya [query\_profiler\_real\_time\_period\_ns](../settings/settings.md#query_profiler_real_time_period_ns) ayarlar. Her iki ayar da aynı anda kullanılabilir. + + Bu ayarlar, profiler zamanlayıcılarını yapılandırmanıza izin verir. Bunlar oturum ayarları olduğundan, tüm sunucu, bireysel kullanıcılar veya kullanıcı profilleri, etkileşimli oturumunuz ve her bir sorgu için farklı örnekleme sıklığı elde edebilirsiniz. + +Varsayılan örnekleme frekansı saniyede bir örnektir ve hem CPU hem de gerçek zamanlayıcılar etkindir. Bu frekans, ClickHouse kümesi hakkında yeterli bilgi toplamaya izin verir. Aynı zamanda, bu sıklıkla çalışan profiler, ClickHouse sunucusunun performansını etkilemez. Her bir sorguyu profillemeniz gerekiyorsa, daha yüksek örnekleme frekansı kullanmayı deneyin. + +Analiz etmek `trace_log` sistem tablosu: + +- Yüklemek `clickhouse-common-static-dbg` paket. Görmek [DEB paketlerinden yükleyin](../../getting_started/install.md#install-from-deb-packages). + +- Tarafından iç gözlem işlevlerine izin ver [allow\_introspection\_functions](../settings/settings.md#settings-allow_introspection_functions) ayar. + + Güvenlik nedenleriyle, iç gözlem işlevleri varsayılan olarak devre dışı bırakılır. + +- Kullan... `addressToLine`, `addressToSymbol` ve `demangle` [iç gözlem fonksiyonları](../../sql_reference/functions/introspection.md) ClickHouse kodu işlev adları ve konumlarını almak için. Bazı sorgu için bir profil almak için, `trace_log` Tablo. Bireysel fonksiyonları bütün yığın izleri ya da veri toplama yapabilirsiniz. + +Görselleştirmeniz gerekiyorsa `trace_log` bilgi, deneyin [flamegraph](../../interfaces/third-party/gui/#clickhouse-flamegraph) ve [speedscope](https://github.com/laplab/clickhouse-speedscope). + +## Örnek {#example} + +Bu örnekte biz: + +- Filtre `trace_log` bir sorgu tanımlayıcısı ve geçerli tarihe göre veri. + +- Yığın izleme ile toplama. + +- İç gözlem işlevlerini kullanarak, bir rapor alacağız: + + - Sembollerin isimleri ve karşılık gelen kaynak kodu işlevleri. + - Bu işlevlerin kaynak kodu konumları. + + + +``` sql +SELECT + count(), + arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym +FROM system.trace_log +WHERE (query_id = 'ebca3574-ad0a-400a-9cbc-dca382f5998c') AND (event_date = today()) +GROUP BY trace +ORDER BY count() DESC +LIMIT 10 +``` + +``` text +{% include "operations/performance/sampling_query_profiler_example_result.txt" %} +``` diff --git a/docs/tr/operations/performance/sampling_query_profiler_example_result.txt b/docs/tr/operations/performance/sampling_query_profiler_example_result.txt new file mode 120000 index 00000000000..58c5abe7122 --- /dev/null +++ b/docs/tr/operations/performance/sampling_query_profiler_example_result.txt @@ -0,0 +1 @@ +../../../en/operations/performance/sampling_query_profiler_example_result.txt \ No newline at end of file diff --git a/docs/tr/operations/performance_test.md b/docs/tr/operations/performance_test.md new file mode 100644 index 00000000000..dbfbc39998d --- /dev/null +++ b/docs/tr/operations/performance_test.md @@ -0,0 +1,82 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 54 +toc_title: "Donan\u0131m Test" +--- + +# Donanımınızı ClickHouse ile Test etme {#how-to-test-your-hardware-with-clickhouse} + +Bu talimat ile ClickHouse paketlerinin kurulumu olmadan herhangi bir sunucuda temel ClickHouse performans testi çalıştırabilirsiniz. + +1. Gitmek “commits” sayfa: https://github.com/ClickHouse/ClickHouse/commits/master + +2. Yeşil ile ilk yeşil onay işareti veya kırmızı Haç tıklayın “ClickHouse Build Check” ve tıklayın “Details” link yakın “ClickHouse Build Check”. Bazı taahhütlerde böyle bir bağlantı yoktur, örneğin belgelerle taahhüt eder. Bu durumda, bu bağlantıya sahip en yakın taahhüt seçin. + +3. Bağlantıyı kopyala “clickhouse” amd64 veya aarch64 için ikili. + +4. sunucuya ssh ve wget ile indirin: + + + + # For amd64: + wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578163263_binary/clickhouse + # For aarch64: + wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578161264_binary/clickhouse + # Then do: + chmod a+x clickhouse + +1. İndir yapılandırmaları: + + + + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.xml + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/users.xml + mkdir config.d + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml + +1. Ben benchmarkch filesmark dosyaları indir: + + + + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/benchmark-new.sh + chmod a+x benchmark-new.sh + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql + +1. İndir göre test verileri [Üye.Metrica veri kümesi](../getting_started/example_datasets/metrica.md) talimat (“hits” 100 milyon satır içeren tablo). + + + + wget https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_100m_obfuscated_v1.tar.xz + tar xvf hits_100m_obfuscated_v1.tar.xz -C . + mv hits_100m_obfuscated_v1/* . + +1. Sunucuyu Çalıştır: + + + + ./clickhouse server + +1. Verileri kontrol edin: başka bir terminaldeki sunucuya ssh + + + + ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" + 100000000 + +1. Edit the benchmark-new.sh, değişim `clickhouse-client` -e doğru `./clickhouse client` ve Ekle `–-max_memory_usage 100000000000` parametre. + + + + mcedit benchmark-new.sh + +1. Ben benchmarkch runmark Çalıştır: + + + + ./benchmark-new.sh hits_100m_obfuscated + +1. Donanım yapılandırmanız hakkındaki numaraları ve bilgileri şu adrese gönderin clickhouse-feedback@yandex-team.com + +Tüm sonuçlar burada yayınlanmaktadır: https://clickhouse.teknoloji / benchmark\_hardware.html diff --git a/docs/tr/operations/quotas.md b/docs/tr/operations/quotas.md new file mode 100644 index 00000000000..e6b1de2b7f8 --- /dev/null +++ b/docs/tr/operations/quotas.md @@ -0,0 +1,112 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 51 +toc_title: Kotalar +--- + +# Kotalar {#quotas} + +Kotalar, belirli bir süre boyunca kaynak kullanımını sınırlamanıza veya kaynak kullanımını izlemenize izin verir. +Kotalar genellikle kullanıcı yapılandırmasında ayarlanır ‘users.xml’. + +Sistem ayrıca tek bir sorgunun karmaşıklığını sınırlamak için bir özelliğe sahiptir. Bölümüne bakınız “Restrictions on query complexity”). + +Sorgu karmaşıklığı kısıtlamalarının aksine, kotalar: + +- Tek bir sorguyu sınırlamak yerine, belirli bir süre boyunca çalıştırılabilen sorgu kümesine kısıtlamalar yerleştirin. +- Dağıtılmış sorgu işleme için tüm uzak sunucularda harcanan kaynaklar için hesap. + +Bölümüne bakalım ‘users.xml’ kotaları tanımlayan dosya. + +``` xml + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + +``` + +Varsayılan olarak, kota, kullanımı sınırlamadan her saat için kaynak tüketimini izler. +Her aralık için hesaplanan kaynak tüketimi, her istekten sonra sunucu günlüğüne çıktıdır. + +``` xml + + + + + 3600 + + 1000 + 100 + 1000000000 + 100000000000 + 900 + + + + 86400 + + 10000 + 1000 + 5000000000 + 500000000000 + 7200 + + +``` + +İçin ‘statbox’ kota, kısıtlamalar her saat ve her 24 saat (86.400 saniye) için ayarlanır. Aralık saydım, bir uygulama başlangıç zamanı tamir anda tanımlanmış. Başka bir deyişle, 24 saatlik Aralık mutlaka gece yarısı başlamaz. + +Aralık sona erdiğinde, toplanan tüm değerler temizlenir. Bir sonraki saat için kota Hesaplaması başlar. + +İşte kısıt amountslan theabilecek miktar amountslar: + +`queries` – The total number of requests. + +`errors` – The number of queries that threw an exception. + +`result_rows` – The total number of rows given as a result. + +`read_rows` – The total number of source rows read from tables for running the query on all remote servers. + +`execution_time` – The total query execution time, in seconds (wall time). + +En az bir zaman aralığı için sınır aşılırsa, hangi kısıtlamanın aşıldığı, hangi aralık için ve yeni Aralık başladığında (sorgular yeniden gönderildiğinde) bir metin ile bir istisna atılır. + +Kota kullanabilirsiniz “quota key” birden fazla anahtar için kaynakları bağımsız olarak rapor etme özelliği. İşte bunun bir örneği: + +``` xml + + + + +``` + +Kota kullanıcılara atanır ‘users’ yapılandırma bölümü. Bölümüne bakınız “Access rights”. + +Dağıtılmış sorgu işleme için birikmiş tutarları istekte bulunan sunucuda depolanır. Yani kullanıcı başka bir sunucuya giderse, oradaki kota “start over”. + +Sunucu yeniden başlatıldığında, kotalar sıfırlanır. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/quotas/) diff --git a/docs/tr/operations/requirements.md b/docs/tr/operations/requirements.md new file mode 100644 index 00000000000..c77159d11d7 --- /dev/null +++ b/docs/tr/operations/requirements.md @@ -0,0 +1,61 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 44 +toc_title: Gereksinimler +--- + +# Gereksinimler {#requirements} + +## CPU {#cpu} + +Önceden oluşturulmuş deb paketlerinden kurulum için, x86\_64 mimarisine sahip bir CPU kullanın ve sse 4.2 yönergelerini destekleyin. Clickhouse'u SSE 4.2'yi desteklemeyen veya AArch64 veya PowerPC64LE mimarisine sahip işlemcilerle çalıştırmak için, kaynaklardan Clickhouse'u oluşturmanız gerekir. + +ClickHouse paralel veri işleme uygular ve mevcut tüm donanım kaynaklarını kullanır. Bir işlemci seçerken, Clickhouse'un çok sayıda çekirdeğe sahip konfigürasyonlarda daha verimli çalıştığını, ancak daha az çekirdeğe ve daha yüksek bir saat hızına sahip konfigürasyonlardan daha düşük bir saat hızına sahip olduğunu göz önünde bulundurun. Örneğin, 2600 MHz'lik 16 çekirdek, 3600 MHz'lik 8 çekirdeğe tercih edilir. + +Kullanılması tavsiye edilir **Turbo Bo Boostost** ve **hyper-thre -ading** teknolojiler. Tipik bir iş yükü ile performansı önemli ölçüde artırır. + +## RAM {#ram} + +Önemsiz olmayan sorgular gerçekleştirmek için en az 4GB RAM kullanmanızı öneririz. ClickHouse sunucusu çok daha az miktarda RAM ile çalışabilir, ancak sorguları işlemek için bellek gerektirir. + +Gerekli RAM hacmi Aşağıdakilere bağlıdır: + +- Sorguların karmaşıklığı. +- Sorgularda işlenen veri miktarı. + +Gerekli RAM hacmini hesaplamak için, aşağıdakiler için geçici verilerin boyutunu tahmin etmelisiniz [GROUP BY](../sql_reference/statements/select.md#select-group-by-clause), [DISTINCT](../sql_reference/statements/select.md#select-distinct), [JOIN](../sql_reference/statements/select.md#select-join) ve kullandığınız diğer işlemler. + +ClickHouse geçici veriler için harici bellek kullanabilirsiniz. Görmek [Harici bellekte grupla](../sql_reference/statements/select.md#select-group-by-in-external-memory) ayrıntılar için. + +## Takas Dosyası {#swap-file} + +Üretim ortamları için takas dosyasını devre dışı bırakın. + +## Depolama Alt Sistemi {#storage-subsystem} + +Clickhouse'u yüklemek için 2GB Boş disk alanına sahip olmanız gerekir. + +Verileriniz için gereken depolama hacmi ayrı ayrı hesaplanmalıdır. Değerlendirme şunları içermelidir: + +- Veri hacminin tahmini. + + Verilerin bir örneğini alabilir ve ondan bir satırın ortalama boyutunu alabilirsiniz. Ardından değeri, depolamayı planladığınız satır sayısıyla çarpın. + +- Veri sıkıştırma katsayısı. + + Veri sıkıştırma katsayısını tahmin etmek için, verilerinizin bir örneğini Clickhouse'a yükleyin ve verilerin gerçek boyutunu depolanan tablonun boyutuyla karşılaştırın. Örneğin, clickstream verileri genellikle 6-10 kez sıkıştırılır. + +Saklanacak verilerin son hacmini hesaplamak için, sıkıştırma katsayısını tahmini veri hacmine uygulayın. Verileri birkaç yinelemede depolamayı planlıyorsanız, tahmini birimi yinelemelerin sayısıyla çarpın. + +## Ağ {#network} + +Mümkünse, 10g veya daha yüksek sınıftaki ağları kullanın. + +Ağ bant genişliği, büyük miktarda Ara veriyle dağıtılmış sorguları işlemek için kritik öneme sahiptir. Ayrıca, ağ hızı çoğaltma işlemlerini etkiler. + +## Yazılım {#software} + +ClickHouse öncelikle Linux işletim sistemleri ailesi için geliştirilmiştir. Önerilen Linux dağıtımı Ubuntu'dur. Bu `tzdata` paket sisteme kurulmalıdır. + +ClickHouse diğer işletim sistemi ailelerinde de çalışabilir. Ayrıntıları görün [Başlarken](../getting_started/index.md) belgelerin bölümü. diff --git a/docs/tr/operations/server_configuration_parameters/index.md b/docs/tr/operations/server_configuration_parameters/index.md new file mode 100644 index 00000000000..f1a20b924f0 --- /dev/null +++ b/docs/tr/operations/server_configuration_parameters/index.md @@ -0,0 +1,19 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Sunucu Yap\u0131land\u0131rma Parametreleri" +toc_priority: 54 +toc_title: "Giri\u015F" +--- + +# Sunucu yapılandırma parametreleri {#server-settings} + +Bu bölüm, oturum veya sorgu düzeyinde değiştirilemeyen sunucu ayarlarının açıklamalarını içerir. + +Bu ayarlar saklanır `config.xml` ClickHouse sunucusunda dosya. + +Diğer ayarlar aşağıda açıklanmıştır “[Ayarlar](../settings/index.md#settings)” bölme. + +Ayarları incelemeden önce, [Yapılandırma dosyaları](../configuration_files.md#configuration_files) bölüm ve ikame kullanımı (not `incl` ve `optional` öznitelik). + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/) diff --git a/docs/tr/operations/server_configuration_parameters/settings.md b/docs/tr/operations/server_configuration_parameters/settings.md new file mode 100644 index 00000000000..bc0a464c511 --- /dev/null +++ b/docs/tr/operations/server_configuration_parameters/settings.md @@ -0,0 +1,896 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 57 +toc_title: "Sunucu Ayarlar\u0131" +--- + +# Sunucu Ayarları {#server-settings} + +## buıltın\_dıctıonarıes\_reload\_ınterval {#builtin-dictionaries-reload-interval} + +Dahili sözlükleri yeniden yüklemeden önce saniye cinsinden Aralık. + +ClickHouse, her x saniyede bir yerleşik sözlükleri yeniden yükler. Bu, sözlükleri düzenlemeyi mümkün kılar “on the fly” sunucuyu yeniden başlatmadan. + +Varsayılan değer: 3600. + +**Örnek** + +``` xml +3600 +``` + +## sıkıştırma {#server-settings-compression} + +İçin veri sıkıştırma ayarları [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)- motor masaları. + +!!! warning "Uyarıcı" + Sadece ClickHouse kullanmaya başladıysanız kullanmayın. + +Yapılandırma şablonu: + +``` xml + + + ... + ... + ... + + ... + +``` + +`` alanlar: + +- `min_part_size` – The minimum size of a data part. +- `min_part_size_ratio` – The ratio of the data part size to the table size. +- `method` – Compression method. Acceptable values: `lz4` veya `zstd`. + +Birden fazla yapılandırabilirsiniz `` bölmeler. + +Koşullar yerine getirildiğinde eylemler: + +- Bir veri parçası bir koşul kümesiyle eşleşirse, ClickHouse belirtilen sıkıştırma yöntemini kullanır. +- Bir veri parçası birden çok koşul kümesiyle eşleşirse, ClickHouse ilk eşleşen koşul kümesini kullanır. + +Bir veri bölümü için herhangi bir koşul karşılanmazsa, ClickHouse `lz4` sıkıştırma. + +**Örnek** + +``` xml + + + 10000000000 + 0.01 + zstd + + +``` + +## default\_database {#default-database} + +Varsayılan veritabanı. + +Veritabanlarının bir listesini almak için [SHOW DATABASES](../../sql_reference/statements/show.md#show-databases) sorgu. + +**Örnek** + +``` xml +default +``` + +## default\_profile {#default-profile} + +Varsayılan ayarlar profili. + +Ayarlar profilleri parametrede belirtilen dosyada bulunur `user_config`. + +**Örnek** + +``` xml +default +``` + +## dictionaries\_config {#server_configuration_parameters-dictionaries_config} + +Dış sözlükler için yapılandırma dosyasının yolu. + +Yol: + +- Mutlak yolu veya sunucu yapılandırma dosyasına göre yolu belirtin. +- Yol joker karakterler içerebilir \* ve ?. + +Ayrıca bakınız “[Dış söz dictionarieslükler](../../sql_reference/dictionaries/external_dictionaries/external_dicts.md)”. + +**Örnek** + +``` xml +*_dictionary.xml +``` + +## dictionaries\_lazy\_load {#server_configuration_parameters-dictionaries_lazy_load} + +Sözlüklerin tembel yüklenmesi. + +Eğer `true`, sonra her sözlük ilk kullanımda oluşturulur. Sözlük oluşturma başarısız olursa, sözlüğü kullanan işlev bir özel durum atar. + +Eğer `false`, sunucu başladığında tüm sözlükler oluşturulur ve bir hata varsa, sunucu kapanır. + +Varsayılan değer `true`. + +**Örnek** + +``` xml +true +``` + +## format\_schema\_path {#server_configuration_parameters-format_schema_path} + +Dizin için şemalar gibi giriş verileri için şemaları ile yolu [CapnProto](../../interfaces/formats.md#capnproto) biçimli. + +**Örnek** + +``` xml + + format_schemas/ +``` + +## grafit {#server_configuration_parameters-graphite} + +Veri gönderme [Grafit](https://github.com/graphite-project). + +Ayarlar: + +- host – The Graphite server. +- port – The port on the Graphite server. +- interval – The interval for sending, in seconds. +- timeout – The timeout for sending data, in seconds. +- root\_path – Prefix for keys. +- metrics – Sending data from the [sistem.metrik](../../operations/system_tables.md#system_tables-metrics) Tablo. +- events – Sending deltas data accumulated for the time period from the [sistem.etkinlik](../../operations/system_tables.md#system_tables-events) Tablo. +- events\_cumulative – Sending cumulative data from the [sistem.etkinlik](../../operations/system_tables.md#system_tables-events) Tablo. +- asynchronous\_metrics – Sending data from the [sistem.asynchronous\_metrics](../../operations/system_tables.md#system_tables-asynchronous_metrics) Tablo. + +Birden fazla yapılandırabilirsiniz `` yanlar. Örneğin, bunu farklı aralıklarla farklı veri göndermek için kullanabilirsiniz. + +**Örnek** + +``` xml + + localhost + 42000 + 0.1 + 60 + one_min + true + true + false + true + +``` + +## graphite\_rollup {#server_configuration_parameters-graphite-rollup} + +Grafit için inceltme verileri için ayarlar. + +Daha fazla ayrıntı için bkz. [Graphıtemergetree](../../engines/table_engines/mergetree_family/graphitemergetree.md). + +**Örnek** + +``` xml + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + +``` + +## http\_port/https\_port {#http-porthttps-port} + +Http(ler) üzerinden sunucuya bağlanmak için bağlantı noktası. + +Eğer `https_port` belirtilen, [openSSL](#server_configuration_parameters-openssl) yapılandırılmalıdır. + +Eğer `http_port` belirtilmişse, OpenSSL yapılandırması ayarlanmış olsa bile göz ardı edilir. + +**Örnek** + +``` xml +0000 +``` + +## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} + +ClickHouse HTTP (s) sunucusuna eriştiğinizde varsayılan olarak gösterilen sayfa. +Varsayılan değer “Ok.” (sonunda bir çizgi besleme ile) + +**Örnek** + +Açıyor `https://tabix.io/` eriş whenirken `http://localhost: http_port`. + +``` xml + +
]]> + +``` + +## include\_from {#server_configuration_parameters-include_from} + +Değiştirmeleri ile dosyanın yolu. + +Daha fazla bilgi için bölüme bakın “[Yapılandırma dosyaları](../configuration_files.md#configuration_files)”. + +**Örnek** + +``` xml +/etc/metrica.xml +``` + +## ınterserver\_http\_port {#interserver-http-port} + +ClickHouse sunucuları arasında veri alışverişi için bağlantı noktası. + +**Örnek** + +``` xml +9009 +``` + +## ınterserver\_http\_host {#interserver-http-host} + +Bu sunucuya erişmek için diğer sunucular tarafından kullanılabilecek ana bilgisayar adı. + +Eğer ihmal edilirse, aynı şekilde tanımlanır `hostname-f` komut. + +Belirli bir ağ arayüzünden kopmak için kullanışlıdır. + +**Örnek** + +``` xml +example.yandex.ru +``` + +## ınterserver\_http\_credentials {#server-settings-interserver-http-credentials} + +Sırasında kimlik doğrulaması için kullanılan kullanıcı adı ve şifre [çoğalma](../../engines/table_engines/mergetree_family/replication.md) çoğaltılan \* motorlarla. Bu kimlik bilgileri yalnızca yinelemeler arasındaki iletişim için kullanılır ve ClickHouse istemcileri için kimlik bilgileri ile ilgisizdir. Sunucu, yinelemeleri bağlamak için bu kimlik bilgilerini denetliyor ve diğer yinelemelere bağlanırken aynı kimlik bilgilerini kullanıyor. Bu nedenle, bu kimlik bilgileri kümedeki tüm yinelemeler için aynı şekilde ayarlanmalıdır. +Varsayılan olarak, kimlik doğrulama kullanılmaz. + +Bu bölüm aşağıdaki parametreleri içerir: + +- `user` — username. +- `password` — password. + +**Örnek** + +``` xml + + admin + 222 + +``` + +## keep\_alive\_timeout {#keep-alive-timeout} + +ClickHouse bağlantıyı kapatmadan önce gelen istekleri bekler saniye sayısı. Varsayılan 3 saniye. + +**Örnek** + +``` xml +3 +``` + +## listen\_host {#server_configuration_parameters-listen_host} + +İsteklerin gelebileceği ana bilgisayarlarda kısıtlama. Sunucunun hepsini yanıtlamasını istiyorsanız, belirtin `::`. + +Örnekler: + +``` xml +::1 +127.0.0.1 +``` + +## kaydedici {#server_configuration_parameters-logger} + +Günlük ayarları. + +Anahtarlar: + +- level – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`. +- log – The log file. Contains all the entries according to `level`. +- errorlog – Error log file. +- size – Size of the file. Applies to `log`ve`errorlog`. Dosya ulaştıktan sonra `size`, ClickHouse arşivleri ve yeniden adlandırır ve onun yerine yeni bir günlük dosyası oluşturur. +- count – The number of archived log files that ClickHouse stores. + +**Örnek** + +``` xml + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + +``` + +Syslog yazma da desteklenmektedir. Yapılandırma örneği: + +``` xml + + 1 + +
syslog.remote:10514
+ myhost.local + LOG_LOCAL6 + syslog + + +``` + +Anahtarlar: + +- use\_syslog — Required setting if you want to write to the syslog. +- address — The host\[:port\] of syslogd. If omitted, the local daemon is used. +- hostname — Optional. The name of the host that logs are sent from. +- facility — [Syslog tesisi anahtar sözcüğü](https://en.wikipedia.org/wiki/Syslog#Facility) ile büyük harf inlerle “LOG\_” önek: (`LOG_USER`, `LOG_DAEMON`, `LOG_LOCAL3` vb.). + Varsayılan değer: `LOG_USER` eğer `address` belirtilen, `LOG_DAEMON otherwise.` +- format – Message format. Possible values: `bsd` ve `syslog.` + +## makrolar {#macros} + +Çoğaltılmış tablolar için parametre değiştirmeleri. + +Çoğaltılmış tablolar kullanılmazsa atlanabilir. + +Daha fazla bilgi için bölüme bakın “[Çoğaltılmış tablolar oluşturma](../../engines/table_engines/mergetree_family/replication.md)”. + +**Örnek** + +``` xml + +``` + +## mark\_cache\_size {#server-mark-cache-size} + +Tablo motorları tarafından kullanılan işaretlerin önbelleğinin yaklaşık boyutu (bayt cinsinden) [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) aile. + +Önbellek sunucu için paylaşılır ve bellek gerektiği gibi ayrılır. Önbellek boyutu en az 5368709120 olmalıdır. + +**Örnek** + +``` xml +5368709120 +``` + +## max\_concurrent\_queries {#max-concurrent-queries} + +Aynı anda işlenen isteklerin maksimum sayısı. + +**Örnek** + +``` xml +100 +``` + +## max\_connections {#max-connections} + +En fazla gelen bağlantı sayısı. + +**Örnek** + +``` xml +4096 +``` + +## max\_open\_files {#max-open-files} + +Maksimum açık dosya sayısı. + +Varsayılan olarak: `maximum`. + +Biz beri Mac OS X bu seçeneği kullanmanızı öneririz `getrlimit()` işlev yanlış bir değer döndürür. + +**Örnek** + +``` xml +262144 +``` + +## max\_table\_size\_to\_drop {#max-table-size-to-drop} + +Tabloları silme konusunda kısıtlama. + +Eğer bir boyutu [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) tablo aşıyor `max_table_size_to_drop` (bayt cinsinden), bir bırakma sorgusu kullanarak silemezsiniz. + +ClickHouse sunucusunu yeniden başlatmadan tabloyu silmeniz gerekiyorsa, `/flags/force_drop_table` dosya ve bırakma sorgusunu çalıştırın. + +Varsayılan değer: 50 GB. + +0 değeri, herhangi bir kısıtlama olmaksızın tüm tabloları silebileceğiniz anlamına gelir. + +**Örnek** + +``` xml +0 +``` + +## merge\_tree {#server_configuration_parameters-merge_tree} + +Tablolar için ince ayar [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). + +Daha fazla bilgi için bkz: MergeTreeSettings.h başlık dosyası. + +**Örnek** + +``` xml + + 5 + +``` + +## openSSL {#server_configuration_parameters-openssl} + +SSL istemci / sunucu yapılandırması. + +SSL desteği tarafından sağlanmaktadır `libpoco` kitaplık. Arayüz dosyada açıklanmıştır [SSLManager.sa](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) + +Sunucu/istemci ayarları için tuşlar: + +- privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. +- certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` sertifika içerir. +- caConfig – The path to the file or directory that contains trusted root certificates. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [Bağlama](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) sınıf. Olası değerler: `none`, `relaxed`, `strict`, `once`. +- verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. +- loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \| +- cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`. +- cacheSessions – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Kabul edilebilir değerler: `true`, `false`. +- sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. Bu parametre her zaman sunucu oturumu önbelleğe alır ve istemci önbellekleme istedi, sorunları önlemek yardımcı olduğundan önerilir. Varsayılan değer: `${application.name}`. +- sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions. +- sessionTimeout – Time for caching the session on the server. +- extendedVerification – Automatically extended verification of certificates after the session ends. Acceptable values: `true`, `false`. +- requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`. +- requireTLSv1\_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. +- requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. +- fips – Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. +- privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. +- invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . +- disableProtocols – Protocols that are not allowed to use. +- preferServerCiphers – Preferred server ciphers on the client. + +**Ayarlar örneği:** + +``` xml + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + +``` + +## part\_log {#server_configuration_parameters-part-log} + +İlişkili olayları günlüğe kaydetme [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). Örneğin, veri ekleme veya birleştirme. Birleştirme algoritmalarını simüle etmek ve özelliklerini karşılaştırmak için günlüğü kullanabilirsiniz. Birleştirme işlemini görselleştirebilirsiniz. + +Sorgular günlüğe kaydedilir [sistem.part\_log](../../operations/system_tables.md#system_tables-part-log) tablo, ayrı bir dosyada değil. Bu tablonun adını aşağıdaki tabloda yapılandırabilirsiniz: `table` parametre (aşağıya bakınız). + +Günlüğü yapılandırmak için aşağıdaki parametreleri kullanın: + +- `database` – Name of the database. +- `table` – Name of the system table. +- `partition_by` – Sets a [özel bölümleme anahtarı](../../engines/table_engines/mergetree_family/custom_partitioning_key.md). +- `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. + +**Örnek** + +``` xml + + system + part_log
+ toMonday(event_date) + 7500 + +``` + +## yol {#server_configuration_parameters-path} + +Veri içeren dizinin yolu. + +!!! note "Not" + Sondaki eğik çizgi zorunludur. + +**Örnek** + +``` xml +/var/lib/clickhouse/ +``` + +## prometheus {#server_configuration_parameters-prometheus} + +Kazıma için metrik verilerini açığa çıkarma [Prometheus](https://prometheus.io). + +Ayarlar: + +- `endpoint` – HTTP endpoint for scraping metrics by prometheus server. Start from ‘/’. +- `port` – Port for `endpoint`. +- `metrics` – Flag that sets to expose metrics from the [sistem.metrik](../system_tables.md#system_tables-metrics) Tablo. +- `events` – Flag that sets to expose metrics from the [sistem.etkinlik](../system_tables.md#system_tables-events) Tablo. +- `asynchronous_metrics` – Flag that sets to expose current metrics values from the [sistem.asynchronous\_metrics](../system_tables.md#system_tables-asynchronous_metrics) Tablo. + +**Örnek** + +``` xml + + /metrics + 8001 + true + true + true + +``` + +## query\_log {#server_configuration_parameters-query-log} + +İle alınan günlük sorgu settinglarının ayarlanması [log\_queries = 1](../settings/settings.md) ayar. + +Sorgular günlüğe kaydedilir [sistem.query\_log](../../operations/system_tables.md#system_tables-query_log) tablo, ayrı bir dosyada değil. Tablonun adını değiştirebilirsiniz. `table` parametre (aşağıya bakınız). + +Günlüğü yapılandırmak için aşağıdaki parametreleri kullanın: + +- `database` – Name of the database. +- `table` – Name of the system table the queries will be logged in. +- `partition_by` – Sets a [özel bölümleme anahtarı](../../engines/table_engines/mergetree_family/custom_partitioning_key.md) bir masa için. +- `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. + +Tablo yoksa, ClickHouse bunu oluşturur. ClickHouse sunucusu güncelleştirildiğinde sorgu günlüğünün yapısı değiştiyse, eski yapıya sahip tablo yeniden adlandırılır ve otomatik olarak yeni bir tablo oluşturulur. + +**Örnek** + +``` xml + + system + query_log
+ toMonday(event_date) + 7500 + +``` + +## query\_thread\_log {#server_configuration_parameters-query-thread-log} + +İle alınan sorguların günlük iş parçacıklarının ayarlanması [log\_query\_threads = 1](../settings/settings.md#settings-log-query-threads) ayar. + +Sorgular günlüğe kaydedilir [sistem.query\_thread\_log](../../operations/system_tables.md#system_tables-query-thread-log) tablo, ayrı bir dosyada değil. Tablonun adını değiştirebilirsiniz. `table` parametre (aşağıya bakınız). + +Günlüğü yapılandırmak için aşağıdaki parametreleri kullanın: + +- `database` – Name of the database. +- `table` – Name of the system table the queries will be logged in. +- `partition_by` – Sets a [özel bölümleme anahtarı](../../engines/table_engines/mergetree_family/custom_partitioning_key.md) bir sistem tablosu için. +- `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. + +Tablo yoksa, ClickHouse bunu oluşturur. Sorgu iş parçacığı günlüğü yapısını değiştirdiyseniz ClickHouse sunucu güncelleştirildi, tablo eski yapısı ile yeniden adlandırılır ve yeni bir tablo otomatik olarak oluşturulur. + +**Örnek** + +``` xml + + system + query_thread_log
+ toMonday(event_date) + 7500 + +``` + +## trace\_log {#server_configuration_parameters-trace_log} + +İçin ayarlar [trace\_log](../../operations/system_tables.md#system_tables-trace_log) sistem tablosu çalışması. + +Parametre: + +- `database` — Database for storing a table. +- `table` — Table name. +- `partition_by` — [Özel bölümleme anahtarı](../../engines/table_engines/mergetree_family/custom_partitioning_key.md) bir sistem tablosu için. +- `flush_interval_milliseconds` — Interval for flushing data from the buffer in memory to the table. + +Varsayılan sunucu yapılandırma dosyası `config.xml` aşağıdaki ayarlar bölümünü içerir: + +``` xml + + system + trace_log
+ toYYYYMM(event_date) + 7500 + +``` + +## query\_masking\_rules {#query-masking-rules} + +Regexp tabanlı kurallar, sorgulara ve tüm günlük iletilerine sunucu günlüklerinde depolamadan önce uygulanacak, +`system.query_log`, `system.text_log`, `system.processes` tablo ve istemciye gönderilen günlüklerde. Önlem allowseyi sağlayan +SQL sorgularından hassas veri sızıntısı (isimler, e-postalar, kişisel +kimlik veya kredi kartı numaraları) günlükleri için. + +**Örnek** + +``` xml + + + hide SSN + (^|\D)\d{3}-\d{2}-\d{4}($|\D) + 000-00-0000 + + +``` + +Config alanları: +- `name` - kuralın adı (isteğe bağlı) +- `regexp` - Re2 uyumlu düzenli ifade (zorunlu) +- `replace` - hassas veriler için ikame dizesi (isteğe bağlı, varsayılan olarak-altı Yıldız İşareti) + +Maskeleme kuralları tüm sorguya uygulanır (hatalı biçimlendirilmiş / ayrıştırılamayan sorgulardan hassas verilerin sızıntılarını önlemek için). + +`system.events` tablo sayacı var `QueryMaskingRulesMatch` hangi sorgu maskeleme kuralları maçları genel bir numarası var. + +Dağıtılmış sorgular için her sunucu ayrı ayrı yapılandırılmalıdır, aksi takdirde alt sorgular diğerine iletilir +düğümler maskeleme olmadan saklanır. + +## remote\_servers {#server-settings-remote-servers} + +Tarafından kullanılan küm ofelerin yapılandırması [Dağılı](../../engines/table_engines/special/distributed.md) tablo motoru ve `cluster` tablo işlevi. + +**Örnek** + +``` xml + +``` + +Değeri için `incl` öznitelik, bölümüne bakın “[Yapılandırma dosyaları](../configuration_files.md#configuration_files)”. + +**Ayrıca Bakınız** + +- [skip\_unavailable\_shards](../settings/settings.md#settings-skip_unavailable_shards) + +## saat dilimi {#server_configuration_parameters-timezone} + +Sunucunun saat dilimi. + +UTC saat dilimi veya coğrafi konum (örneğin, Afrika / Abidjan) için bir IANA tanımlayıcısı olarak belirtilir. + +Saat dilimi, datetime alanları metin biçimine (ekranda veya dosyada yazdırıldığında) çıktığında ve datetime'ı bir dizeden alırken dize ve DateTime biçimleri arasındaki dönüşümler için gereklidir. Ayrıca, saat dilimi, giriş parametrelerinde saat dilimini almadıkları takdirde saat ve tarih ile çalışan işlevlerde kullanılır. + +**Örnek** + +``` xml +Europe/Moscow +``` + +## tcp\_port {#server_configuration_parameters-tcp_port} + +TCP protokolü üzerinden istemcilerle iletişim kurmak için bağlantı noktası. + +**Örnek** + +``` xml +9000 +``` + +## tcp\_port\_secure {#server_configuration_parameters-tcp_port-secure} + +İstemcilerle güvenli iletişim için TCP bağlantı noktası. İle kullanın [OpenSSL](#server_configuration_parameters-openssl) ayarlar. + +**Olası değerler** + +Pozitif tamsayı. + +**Varsayılan değer** + +``` xml +9440 +``` + +## mysql\_port {#server_configuration_parameters-mysql_port} + +MySQL protokolü üzerinden istemcilerle iletişim kurmak için bağlantı noktası. + +**Olası değerler** + +Pozitif tamsayı. + +Örnek + +``` xml +9004 +``` + +## tmp\_path {#server-settings-tmp_path} + +Büyük sorguları işlemek için geçici veri yolu. + +!!! note "Not" + Sondaki eğik çizgi zorunludur. + +**Örnek** + +``` xml +/var/lib/clickhouse/tmp/ +``` + +## tmp\_policy {#server-settings-tmp-policy} + +Politika dan [`storage_configuration`](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes) geçici dosyaları saklamak için. +Set değilse [`tmp_path`](#server-settings-tmp_path) kullanılır, aksi takdirde göz ardı edilir. + +!!! note "Not" + - `move_factor` göz ardı edilir +- `keep_free_space_bytes` göz ardı edilir +- `max_data_part_size_bytes` göz ardı edilir +- bu Politikada tam olarak bir cilt olmalı + +## uncompressed\_cache\_size {#server-settings-uncompressed_cache_size} + +Tablo motorları tarafından kullanılan sıkıştırılmamış veriler için önbellek boyutu (bayt cinsinden) [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). + +Sunucu için bir paylaşılan önbellek var. Bellek talep üzerine tahsis edilir. Seçenek varsa önbellek kullanılır [use\_uncompressed\_cache](../settings/settings.md#setting-use_uncompressed_cache) etkindir. + +Sıkıştırılmamış önbellek, tek tek durumlarda çok kısa sorgular için avantajlıdır. + +**Örnek** + +``` xml +8589934592 +``` + +## user\_files\_path {#server_configuration_parameters-user_files_path} + +Kullanıcı dosyaları ile dizin. Tablo işlevinde kullanılır [Dosya()](../../sql_reference/table_functions/file.md). + +**Örnek** + +``` xml +/var/lib/clickhouse/user_files/ +``` + +## users\_config {#users-config} + +İçeren dosyanın yolu: + +- Kullanıcı yapılandırmaları. +- Erişim hakları. +- Ayarlar profilleri. +- Kota ayarları. + +**Örnek** + +``` xml +users.xml +``` + +## zookeeper {#server-settings_zookeeper} + +ClickHouse ile etkileşim sağlayan ayarları içerir [ZooKeeper](http://zookeeper.apache.org/) küme. + +ClickHouse, çoğaltılmış tabloları kullanırken kopyaların meta verilerini depolamak için ZooKeeper kullanır. Çoğaltılmış tablolar kullanılmazsa, parametrelerin bu bölümü atlanabilir. + +Bu bölüm aşağıdaki parametreleri içerir: + +- `node` — ZooKeeper endpoint. You can set multiple endpoints. + + Mesela: + + + +``` xml + + example_host + 2181 + +``` + + The `index` attribute specifies the node order when trying to connect to the ZooKeeper cluster. + +- `session_timeout` — Maximum timeout for the client session in milliseconds. +- `root` — The [znode](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes) bu, ClickHouse sunucusu tarafından kullanılan znodes için kök olarak kullanılır. İsteğe bağlı. +- `identity` — User and password, that can be required by ZooKeeper to give access to requested znodes. Optional. + +**Örnek yapılandırma** + +``` xml + + + example1 + 2181 + + + example2 + 2181 + + 30000 + 10000 + + /path/to/zookeeper/node + + user:password + +``` + +**Ayrıca Bakınız** + +- [Çoğalma](../../engines/table_engines/mergetree_family/replication.md) +- [ZooKeeper programcı Kılavuzu](http://zookeeper.apache.org/doc/current/zookeeperProgrammers.html) + +## use\_minimalistic\_part\_header\_in\_zookeeper {#server-settings-use_minimalistic_part_header_in_zookeeper} + +ZooKeeper veri parçası başlıkları için depolama yöntemi. + +Bu ayar yalnızca `MergeTree` aile. Belirt specifiedilebilir: + +- Küresel olarak [merge\_tree](#server_configuration_parameters-merge_tree) bu bölüm `config.xml` Dosya. + + ClickHouse sunucudaki tüm tablolar için ayarı kullanır. Ayarı istediğiniz zaman değiştirebilirsiniz. Mevcut tablolar, ayar değiştiğinde davranışlarını değiştirir. + +- Her tablo için. + + Bir tablo oluştururken, karşılık gelen [motor ayarı](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table). Genel ayar değişse bile, bu ayara sahip varolan bir tablonun davranışı değişmez. + +**Olası değerler** + +- 0 — Functionality is turned off. +- 1 — Functionality is turned on. + +Eğer `use_minimalistic_part_header_in_zookeeper = 1`, sonraları [çoğaltıyordu](../../engines/table_engines/mergetree_family/replication.md) tablolar, veri parçalarının başlıklarını tek bir `znode`. Tablo çok sayıda sütun içeriyorsa, bu depolama yöntemi Zookeeper'da depolanan verilerin hacmini önemli ölçüde azaltır. + +!!! attention "Dikkat" + Uyguladıktan sonra `use_minimalistic_part_header_in_zookeeper = 1`, ClickHouse sunucusunu bu ayarı desteklemeyen bir sürüme düşüremezsiniz. Bir kümedeki sunucularda ClickHouse yükseltirken dikkatli olun. Tüm sunucuları bir kerede yükseltmeyin. Clickhouse'un yeni sürümlerini bir test ortamında veya bir kümenin yalnızca birkaç sunucusunda test etmek daha güvenlidir. + + Data part headers already stored with this setting can't be restored to their previous (non-compact) representation. + +**Varsayılan değer:** 0. + +## disable\_internal\_dns\_cache {#server-settings-disable-internal-dns-cache} + +İç DNS önbelleğini devre dışı bırakır. Sistemlerinde ClickHouse işletim için tavsiye +Kubernetes gibi sık sık değişen altyapı ile. + +**Varsayılan değer:** 0. + +## dns\_cache\_update\_period {#server-settings-dns-cache-update-period} + +ClickHouse iç DNS önbelleğinde saklanan IP adreslerini güncelleme süresi (saniye cinsinden). +Güncelleştirme, ayrı bir sistem iş parçacığında zaman uyumsuz olarak gerçekleştirilir. + +**Varsayılan değer**: 15. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/server_configuration_parameters/settings/) diff --git a/docs/tr/operations/settings/constraints_on_settings.md b/docs/tr/operations/settings/constraints_on_settings.md new file mode 100644 index 00000000000..a9319c2df69 --- /dev/null +++ b/docs/tr/operations/settings/constraints_on_settings.md @@ -0,0 +1,75 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 62 +toc_title: "Ayarlardaki k\u0131s\u0131tlamalar" +--- + +# Ayarlardaki kısıtlamalar {#constraints-on-settings} + +Ayarlardaki kısıtlamalar, `profiles` bu bölüm `user.xml` yapılandırma dosyası ve kullanıcıların bazı ayarları değiştirmelerini yasakla `SET` sorgu. +Kısıtlamalar aşağıdaki gibi tanımlanır: + +``` xml + + + + + lower_boundary + + + upper_boundary + + + lower_boundary + upper_boundary + + + + + + + +``` + +Kullanıcı kısıtlamaları ihlal etmeye çalışırsa, bir istisna atılır ve ayar değiştirilmez. +Desteklenen üç tür kısıtlama vardır: `min`, `max`, `readonly`. Bu `min` ve `max` kısıtlamalar, sayısal bir ayar için üst ve alt sınırları belirtir ve birlikte kullanılabilir. Bu `readonly` kısıtlama, kullanıcının karşılık gelen ayarı hiç değiştiremeyeceğini belirtir. + +**Örnek:** İzin vermek `users.xml` hatları içerir: + +``` xml + + + 10000000000 + 0 + ... + + + 5000000000 + 20000000000 + + + + + + + +``` + +Aşağıdaki sorgular tüm atma istisnaları: + +``` sql +SET max_memory_usage=20000000001; +SET max_memory_usage=4999999999; +SET force_index_by_date=1; +``` + +``` text +Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not be greater than 20000000000. +Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not be less than 5000000000. +Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should not be changed. +``` + +**Not:** bu `default` profil özel işleme sahiptir: tüm kısıtlamalar için tanımlanan `default` profil varsayılan kısıtlamalar haline gelir, bu nedenle bu kullanıcılar için açıkça geçersiz kılınana kadar tüm kullanıcıları kısıtlarlar. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/constraints_on_settings/) diff --git a/docs/tr/operations/settings/index.md b/docs/tr/operations/settings/index.md new file mode 100644 index 00000000000..72270b8a397 --- /dev/null +++ b/docs/tr/operations/settings/index.md @@ -0,0 +1,32 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Ayarlar +toc_priority: 55 +toc_title: "Giri\u015F" +--- + +# Ayarlar {#settings} + +Aşağıda açıklanan tüm ayarları yapmanın birden çok yolu vardır. +Ayarlar katmanlar halinde yapılandırılır, böylece sonraki her katman önceki ayarları yeniden tanımlar. + +Öncelik sırasına göre ayarları yapılandırma yolları: + +- Ayarlar `users.xml` sunucu yapılandırma dosyası. + + Eleman setında Set ``. + +- Oturum ayarları. + + Göndermek `SET setting=value` etkileşimli modda ClickHouse konsol istemcisinden. + Benzer şekilde, http protokolünde ClickHouse oturumlarını kullanabilirsiniz. Bunu yapmak için şunları belirtmeniz gerekir `session_id` HTTP parametresi. + +- Sorgu ayarları. + + - ClickHouse konsol istemcisini etkileşimli olmayan modda başlatırken, başlangıç parametresini ayarlayın `--setting=value`. + - HTTP API'sini kullanırken, CGI parametrelerini geçirin (`URL?setting_1=value&setting_2=value...`). + +Yalnızca sunucu yapılandırma dosyasında yapılabilecek ayarlar bu bölümde yer almaz. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/) diff --git a/docs/tr/operations/settings/permissions_for_queries.md b/docs/tr/operations/settings/permissions_for_queries.md new file mode 100644 index 00000000000..2f65630604c --- /dev/null +++ b/docs/tr/operations/settings/permissions_for_queries.md @@ -0,0 +1,61 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 58 +toc_title: "Sorgular i\xE7in izinler" +--- + +# Sorgular için izinler {#permissions_for_queries} + +Clickhouse'daki sorgular birkaç türe ayrılabilir: + +1. Veri sorgularını oku: `SELECT`, `SHOW`, `DESCRIBE`, `EXISTS`. +2. Veri sorgularını yaz: `INSERT`, `OPTIMIZE`. +3. Ayarları değiştir sorgu: `SET`, `USE`. +4. [DDL](https://en.wikipedia.org/wiki/Data_definition_language) sorgular: `CREATE`, `ALTER`, `RENAME`, `ATTACH`, `DETACH`, `DROP` `TRUNCATE`. +5. `KILL QUERY`. + +Aşağıdaki ayarlar, kullanıcı izinlerini sorgu Türüne göre düzenler: + +- [readonly](#settings_readonly) — Restricts permissions for all types of queries except DDL queries. +- [allow\_ddl](#settings_allow_ddl) — Restricts permissions for DDL queries. + +`KILL QUERY` herhangi bir ayar ile yapılabilir. + +## readonly {#settings_readonly} + +Veri okuma, veri yazma ve ayar sorgularını değiştirme izinlerini kısıtlar. + +Sorguların türlere nasıl ayrıldığını görün [üzerinde](#permissions_for_queries). + +Olası değerler: + +- 0 — All queries are allowed. +- 1 — Only read data queries are allowed. +- 2 — Read data and change settings queries are allowed. + +Sonra ayarı `readonly = 1`, kullanıcı değiştir canemez `readonly` ve `allow_ddl` geçerli oturumda ayarlar. + +Kullanırken `GET` metho thed in the [HTTP arayüzü](../../interfaces/http.md), `readonly = 1` otomatik olarak ayarlanır. Değiştirmek için veri kullanın `POST` yöntem. + +Ayar `readonly = 1` kullanıcının tüm ayarları değiştirmesini yasaklayın. Kullanıcıyı yasaklamanın bir yolu var +sadece belirli ayarları değiştirmekten, ayrıntılar için bkz [ayarlardaki kısıtlamalar](constraints_on_settings.md). + +Varsayılan değer: 0 + +## allow\_ddl {#settings_allow_ddl} + +İzin verir veya reddeder [DDL](https://en.wikipedia.org/wiki/Data_definition_language) sorgular. + +Sorguların türlere nasıl ayrıldığını görün [üzerinde](#permissions_for_queries). + +Olası değerler: + +- 0 — DDL queries are not allowed. +- 1 — DDL queries are allowed. + +Yürüt canemezsiniz `SET allow_ddl = 1` eğer `allow_ddl = 0` geçerli oturum için. + +Varsayılan değer: 1 + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/permissions_for_queries/) diff --git a/docs/tr/operations/settings/query_complexity.md b/docs/tr/operations/settings/query_complexity.md new file mode 100644 index 00000000000..54958b786ab --- /dev/null +++ b/docs/tr/operations/settings/query_complexity.md @@ -0,0 +1,302 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 59 +toc_title: "Sorgu karma\u015F\u0131kl\u0131\u011F\u0131 \xFCzerindeki k\u0131s\u0131\ + tlamalar" +--- + +# Sorgu Karmaşıklığı Üzerindeki Kısıtlamalar {#restrictions-on-query-complexity} + +Sorgu karmaşıklığı üzerindeki kısıtlamalar ayarların bir parçasıdır. +Kullanıcı arabiriminden daha güvenli yürütme sağlamak için kullanılırlar. +Hemen hemen tüm kısıtlamalar sadece aşağıdakiler için geçerlidir `SELECT`. Dağıtılmış sorgu işleme için kısıtlamalar her sunucuda ayrı ayrı uygulanır. + +ClickHouse, her satır için değil, veri bölümleri için kısıtlamaları denetler. Bu, veri parçasının boyutu ile kısıtlama değerini aşabileceğiniz anlamına gelir. + +Üzerindeki kısıtlamalar “maximum amount of something” 0 değerini alabilir, yani “unrestricted”. +Çoğu kısıtlama da bir ‘overflow\_mode’ ayar, sınır aşıldığında ne yapılması gerektiği anlamına gelir. +İki değerden birini alabilir: `throw` veya `break`. Toplama (group\_by\_overflow\_mode) üzerindeki kısıtlamalar da değere sahiptir `any`. + +`throw` – Throw an exception (default). + +`break` – Stop executing the query and return the partial result, as if the source data ran out. + +`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don't add new keys to the set. + +## max\_memory\_usage {#settings_max_memory_usage} + +Tek bir sunucuda bir sorgu çalıştırmak için kullanılacak en fazla RAM miktarı. + +Varsayılan yapılandırma dosyasında maksimum 10 GB'DİR. + +Bu ayar, kullanılabilir belleğin hacmini veya makinedeki toplam bellek hacmini dikkate almaz. +Kısıtlama, tek bir sunucu içindeki tek bir sorgu için geçerlidir. +Kullanabilirsiniz `SHOW PROCESSLIST` her sorgu için geçerli bellek tüketimini görmek için. +Ayrıca, en yüksek bellek tüketimi her sorgu için izlenir ve günlüğe yazılır. + +Bellek kullanımı, belirli toplama işlevlerinin durumları için izlenmez. + +Toplam işlevlerin durumları için bellek kullanımı tam olarak izlenmiyor `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` itibaren `String` ve `Array` değişkenler. + +Bellek tüketimi de parametrelerle sınırlıdır `max_memory_usage_for_user` ve `max_memory_usage_for_all_queries`. + +## max\_memory\_usage\_for\_user {#max-memory-usage-for-user} + +Tek bir sunucuda bir kullanıcının sorguları çalıştırmak için kullanılacak en fazla RAM miktarı. + +Varsayılan değerler [Ayarlar.sa](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L288). Varsayılan olarak, tutar sınırlı değildir (`max_memory_usage_for_user = 0`). + +Ayrıca açıklamasına bakın [max\_memory\_usage](#settings_max_memory_usage). + +## max\_memory\_usage\_for\_all\_queries {#max-memory-usage-for-all-queries} + +Tek bir sunucuda tüm sorguları çalıştırmak için kullanılacak en fazla RAM miktarı. + +Varsayılan değerler [Ayarlar.sa](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). Varsayılan olarak, tutar sınırlı değildir (`max_memory_usage_for_all_queries = 0`). + +Ayrıca açıklamasına bakın [max\_memory\_usage](#settings_max_memory_usage). + +## max\_rows\_to\_read {#max-rows-to-read} + +Aşağıdaki kısıtlamalar her blokta kontrol edilebilir (her satır yerine). Yani, kısıtlamalar biraz kırılabilir. +Birden çok iş parçacığında bir sorgu çalıştırırken, aşağıdaki kısıtlamalar her iş parçacığı için ayrı ayrı uygulanır. + +Bir sorgu çalıştırırken bir tablodan okunabilen satır sayısı. + +## max\_bytes\_to\_read {#max-bytes-to-read} + +Bir sorgu çalıştırırken bir tablodan okunabilen bayt sayısı (sıkıştırılmamış veri). + +## read\_overflow\_mode {#read-overflow-mode} + +Okunan veri hacmi sınırlardan birini aştığında ne yapmalı: ‘throw’ veya ‘break’. Varsayılan olarak, atın. + +## max\_rows\_to\_group\_by {#settings-max-rows-to-group-by} + +Toplama alınan benzersiz anahtarların maksimum sayısı. Bu ayar, toplama sırasında bellek tüketimini sınırlamanızı sağlar. + +## group\_by\_overflow\_mode {#group-by-overflow-mode} + +Toplama için benzersiz anahtarların sayısı sınırı aştığında ne yapmalı: ‘throw’, ‘break’, veya ‘any’. Varsayılan olarak, atın. +Kullanarak ‘any’ değer, GROUP BY'NİN bir yaklaşımını çalıştırmanızı sağlar. Bu yaklaşımın kalitesi, verilerin istatistiksel niteliğine bağlıdır. + +## max\_bytes\_before\_external\_group\_by {#settings-max_bytes_before_external_group_by} + +Çalıştırmayı etkinleştirir veya devre dışı bırakır `GROUP BY` harici bellekte yan tümceleri. Görmek [Harici bellekte grupla](../../sql_reference/statements/select.md#select-group-by-in-external-memory). + +Olası değerler: + +- Tek tarafından kullanılabilecek maksimum RAM hacmi (bayt cinsinden) [GROUP BY](../../sql_reference/statements/select.md#select-group-by-clause) işleyiş. +- 0 — `GROUP BY` harici bellekte devre dışı. + +Varsayılan değer: 0. + +## max\_rows\_to\_sort {#max-rows-to-sort} + +Sıralamadan önce en fazla satır sayısı. Bu, sıralama yaparken bellek tüketimini sınırlamanıza izin verir. + +## max\_bytes\_to\_sort {#max-bytes-to-sort} + +Sıralamadan önce en fazla bayt sayısı. + +## sort\_overflow\_mode {#sort-overflow-mode} + +Sıralamadan önce alınan satır sayısı sınırlardan birini aşarsa ne yapmalı: ‘throw’ veya ‘break’. Varsayılan olarak, atın. + +## max\_result\_rows {#setting-max_result_rows} + +Sonuçtaki satır sayısını sınırlayın. Ayrıca, dağıtılmış bir sorgunun parçalarını çalıştırırken alt sorgular ve uzak sunucularda da kontrol edildi. + +## max\_result\_bytes {#max-result-bytes} + +Sonuçtaki bayt sayısını sınırlayın. Önceki ayar ile aynı. + +## result\_overflow\_mode {#result-overflow-mode} + +Sonucun hacmi sınırlardan birini aşarsa ne yapmalı: ‘throw’ veya ‘break’. Varsayılan olarak, atın. + +Kullanım ‘break’ LİMİT kullanmaya benzer. `Break` yürütmeyi yalnızca blok düzeyinde keser. Bu, döndürülen satırların miktarının daha büyük olduğu anlamına gelir [max\_result\_rows](#setting-max_result_rows) birden çok [max\_block\_size](settings.md#setting-max_block_size) ve bağlıdır [max\_threads](settings.md#settings-max_threads). + +Örnek: + +``` sql +SET max_threads = 3, max_block_size = 3333; +SET max_result_rows = 3334, result_overflow_mode = 'break'; + +SELECT * +FROM numbers_mt(100000) +FORMAT Null; +``` + +Sonuç: + +``` text +6666 rows in set. ... +``` + +## max\_execution\_time {#max-execution-time} + +Saniye cinsinden maksimum sorgu yürütme süresi. +Şu anda, sıralama aşamalarından biri için veya toplama işlevlerini birleştirirken ve sonlandırırken kontrol edilmez. + +## timeout\_overflow\_mode {#timeout-overflow-mode} + +Sorgu daha uzun çalıştırılırsa ne yapmalı ‘max\_execution\_time’: ‘throw’ veya ‘break’. Varsayılan olarak, atın. + +## min\_execution\_speed {#min-execution-speed} + +Saniyede satırlarda minimum yürütme hızı. Her veri bloğunda ne zaman kontrol edildi ‘timeout\_before\_checking\_execution\_speed’ doluyor. Yürütme hızı düşükse, bir istisna atılır. + +## min\_execution\_speed\_bytes {#min-execution-speed-bytes} + +Saniyede en az yürütme bayt sayısı. Her veri bloğunda ne zaman kontrol edildi ‘timeout\_before\_checking\_execution\_speed’ doluyor. Yürütme hızı düşükse, bir istisna atılır. + +## max\_execution\_speed {#max-execution-speed} + +Saniyede en fazla yürütme satırı sayısı. Her veri bloğunda ne zaman kontrol edildi ‘timeout\_before\_checking\_execution\_speed’ doluyor. Yürütme hızı yüksekse, yürütme hızı azaltılır. + +## max\_execution\_speed\_bytes {#max-execution-speed-bytes} + +Saniyede en fazla yürütme bayt sayısı. Her veri bloğunda ne zaman kontrol edildi ‘timeout\_before\_checking\_execution\_speed’ doluyor. Yürütme hızı yüksekse, yürütme hızı azaltılır. + +## timeout\_before\_checking\_execution\_speed {#timeout-before-checking-execution-speed} + +Yürütme hızının çok yavaş olmadığını kontrol eder (en az ‘min\_execution\_speed’), saniye içinde belirtilen süre dolduktan sonra. + +## max\_columns\_to\_read {#max-columns-to-read} + +Tek bir sorguda bir tablodan okunabilen sütun sayısı. Bir sorgu daha fazla sayıda sütun okuma gerektiriyorsa, bir özel durum atar. + +## max\_temporary\_columns {#max-temporary-columns} + +Sabit sütunlar da dahil olmak üzere bir sorgu çalıştırırken aynı anda RAM'de tutulması gereken geçici sütun sayısı. Bundan daha fazla geçici sütun varsa, bir istisna atar. + +## max\_temporary\_non\_const\_columns {#max-temporary-non-const-columns} + +Aynı şey ‘max\_temporary\_columns’, ancak sabit sütunları saymadan. +Bir sorgu çalıştırırken sabit sütunların oldukça sık oluşturulduğunu, ancak yaklaşık sıfır bilgi işlem kaynağı gerektirdiğini unutmayın. + +## max\_subquery\_depth {#max-subquery-depth} + +Alt sorguların maksimum yuvalama derinliği. Alt sorgular daha derinse, bir istisna atılır. Varsayılan olarak, 100. + +## max\_pipeline\_depth {#max-pipeline-depth} + +Maksimum boru hattı derinliği. Sorgu işleme sırasında her veri bloğunun geçtiği dönüşümlerin sayısına karşılık gelir. Tek bir sunucunun sınırları içinde sayılır. Boru hattı derinliği büyükse, bir istisna atılır. Varsayılan olarak, 1000. + +## max\_ast\_depth {#max-ast-depth} + +Sorgu sözdizimsel ağacının en fazla yuvalama derinliği. Aşılırsa, bir istisna atılır. +Şu anda, ayrıştırma sırasında değil, yalnızca sorguyu ayrıştırdıktan sonra kontrol edilir. Yani, ayrıştırma sırasında çok derin bir sözdizimsel ağaç oluşturulabilir, ancak sorgu başarısız olur. Varsayılan olarak, 1000. + +## max\_ast\_elements {#max-ast-elements} + +Sorgu sözdizimsel ağacındaki en fazla öğe sayısı. Aşılırsa, bir istisna atılır. +Önceki ayarla aynı şekilde, yalnızca sorguyu ayrıştırdıktan sonra kontrol edilir. Varsayılan olarak, 50.000. + +## max\_rows\_in\_set {#max-rows-in-set} + +Bir alt sorgudan oluşturulan In yan tümcesinde bir veri kümesi için satır sayısı. + +## max\_bytes\_in\_set {#max-bytes-in-set} + +Bir alt sorgudan oluşturulan In yan tümcesinde bir set tarafından kullanılan en fazla bayt sayısı (sıkıştırılmamış veri). + +## set\_overflow\_mode {#set-overflow-mode} + +Veri miktarı sınırlardan birini aştığında ne yapmalı: ‘throw’ veya ‘break’. Varsayılan olarak, atın. + +## max\_rows\_ın\_distinct {#max-rows-in-distinct} + +DISTINCT kullanırken en fazla sayıda farklı satır. + +## max\_bytes\_ın\_distinct {#max-bytes-in-distinct} + +DISTINCT kullanırken bir karma tablo tarafından kullanılan bayt sayısı. + +## distinct\_overflow\_mode {#distinct-overflow-mode} + +Veri miktarı sınırlardan birini aştığında ne yapmalı: ‘throw’ veya ‘break’. Varsayılan olarak, atın. + +## max\_rows\_to\_transfer {#max-rows-to-transfer} + +Uzak bir sunucuya geçirilen veya GLOBAL In kullanırken geçici bir tabloya kaydedilen satır sayısı. + +## max\_bytes\_to\_transfer {#max-bytes-to-transfer} + +Uzak bir sunucuya geçirilen veya GLOBAL In kullanırken geçici bir tabloya kaydedilen bayt sayısı (sıkıştırılmamış veri). + +## transfer\_overflow\_mode {#transfer-overflow-mode} + +Veri miktarı sınırlardan birini aştığında ne yapmalı: ‘throw’ veya ‘break’. Varsayılan olarak, atın. + +## max\_rows\_in\_join {#settings-max_rows_in_join} + +Tabloları birleştirirken kullanılan karma tablodaki satır sayısını sınırlar. + +Bu ayarlar aşağıdakiler için geçerlidir [SELECT … JOIN](../../sql_reference/statements/select.md#select-join) işlemleri ve [Katmak](../../engines/table_engines/special/join.md) masa motoru. + +Bir sorgu birden çok birleşim içeriyorsa, ClickHouse her Ara sonuç için bu ayarı denetler. + +Limit ulaşıldığında ClickHouse farklı eylemlerle devam edebilirsiniz. Kullan... [join\_overflow\_mode](#settings-join_overflow_mode) eylemi seçmek için ayarlama. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Unlimited number of rows. + +Varsayılan değer: 0. + +## max\_bytes\_in\_join {#settings-max_bytes_in_join} + +Tabloları birleştirirken kullanılan karma tablonun bayt cinsinden boyutunu sınırlar. + +Bu ayarlar aşağıdakiler için geçerlidir [SELECT … JOIN](../../sql_reference/statements/select.md#select-join) işlemleri ve [Jo tablein table engine](../../engines/table_engines/special/join.md). + +Sorgu birleşimler içeriyorsa, ClickHouse her Ara sonuç için bu ayarı denetler. + +Limit ulaşıldığında ClickHouse farklı eylemlerle devam edebilirsiniz. Kullanmak [join\_overflow\_mode](#settings-join_overflow_mode) eylemi seçmek için ayarlar. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Memory control is disabled. + +Varsayılan değer: 0. + +## join\_overflow\_mode {#settings-join_overflow_mode} + +Tanımlar katılın aşağıdaki sınırlar her zaman eylem ClickHouse gerçekleştirdiği ulaştı: + +- [max\_bytes\_in\_join](#settings-max_bytes_in_join) +- [max\_rows\_in\_join](#settings-max_rows_in_join) + +Olası değerler: + +- `THROW` — ClickHouse throws an exception and breaks operation. +- `BREAK` — ClickHouse breaks operation and doesn't throw an exception. + +Varsayılan değer: `THROW`. + +**Ayrıca Bakınız** + +- [Jo](../../sql_reference/statements/select.md#select-join) +- [Jo tablein table engine](../../engines/table_engines/special/join.md) + +## max\_partitions\_per\_ınsert\_block {#max-partitions-per-insert-block} + +Eklenen tek bir bloktaki en fazla bölüm sayısını sınırlar. + +- Pozitif tamsayı. +- 0 — Unlimited number of partitions. + +Varsayılan değer: 100. + +**Ayrıntı** + +Veri eklerken, ClickHouse eklenen bloktaki bölüm sayısını hesaplar. Bölüm sayısı fazla ise `max_partitions_per_insert_block`, ClickHouse aşağıdaki metinle bir özel durum atar: + +> “Too many partitions for single INSERT block (more than” + toString (max\_parts) + “). The limit is controlled by ‘max\_partitions\_per\_insert\_block’ setting. A large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc).” + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/query_complexity/) diff --git a/docs/tr/operations/settings/settings.md b/docs/tr/operations/settings/settings.md new file mode 100644 index 00000000000..48e48ca00a6 --- /dev/null +++ b/docs/tr/operations/settings/settings.md @@ -0,0 +1,1235 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 60 +toc_title: Ayarlar +--- + +# Ayarlar {#settings} + +## distributed\_product\_mode {#distributed-product-mode} + +Davranışını değiştirir [dağıtılmış alt sorgular](../../sql_reference/statements/select.md). + +ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. + +Kısıtlama: + +- Yalnızca ın ve JOIN alt sorguları için uygulanır. +- Yalnızca FROM bölümü birden fazla parça içeren dağıtılmış bir tablo kullanıyorsa. +- Alt sorgu birden fazla parça içeren dağıtılmış bir tablo ile ilgiliyse. +- Bir tablo için kullanılmaz-değerli [uzak](../../sql_reference/table_functions/remote.md) işlev. + +Olası değerler: + +- `deny` — Default value. Prohibits using these types of subqueries (returns the “Double-distributed in/JOIN subqueries is denied” özel). +- `local` — Replaces the database and table in the subquery with local ones for the destination server (shard), leaving the normal `IN`/`JOIN.` +- `global` — Replaces the `IN`/`JOIN` ile sorgu `GLOBAL IN`/`GLOBAL JOIN.` +- `allow` — Allows the use of these types of subqueries. + +## enable\_optimize\_predicate\_expression {#enable-optimize-predicate-expression} + +Yüklemi pushdown açar `SELECT` sorgular. + +Yüklemi pushdown, dağıtılmış sorgular için ağ trafiğini önemli ölçüde azaltabilir. + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 1. + +Kullanma + +Aşağıdaki sorguları düşünün: + +1. `SELECT count() FROM test_table WHERE date = '2018-10-10'` +2. `SELECT count() FROM (SELECT * FROM test_table) WHERE date = '2018-10-10'` + +Eğer `enable_optimize_predicate_expression = 1`, daha sonra bu sorguların yürütme süresi eşittir çünkü ClickHouse geçerlidir `WHERE` işlerken alt sorguya. + +Eğer `enable_optimize_predicate_expression = 0`, daha sonra ikinci sorgunun yürütme süresi çok daha uzundur, çünkü `WHERE` yan tümcesi alt sorgu tamamlandıktan sonra tüm veriler için geçerlidir. + +## fallback\_to\_stale\_replicas\_for\_distributed\_queries {#settings-fallback_to_stale_replicas_for_distributed_queries} + +Güncelleştirilmiş veriler mevcut değilse, bir sorgu için güncel olmayan bir yineleme zorlar. Görmek [Çoğalma](../../engines/table_engines/mergetree_family/replication.md). + +ClickHouse, tablonun eski kopyalarından en alakalı olanı seçer. + +Yaparken kullanılır `SELECT` çoğaltılmış tablolara işaret eden dağıtılmış bir tablodan. + +Varsayılan olarak, 1 (etkin). + +## force\_index\_by\_date {#settings-force_index_by_date} + +Dizin tarihe göre kullanılamıyorsa, sorgu yürütülmesini devre dışı bırakır. + +MergeTree ailesindeki tablolarla çalışır. + +Eğer `force_index_by_date=1`, ClickHouse sorgunun veri aralıklarını kısıtlamak için kullanılabilecek bir tarih anahtarı koşulu olup olmadığını denetler. Uygun bir koşul yoksa, bir istisna atar. Ancak, koşul okumak için veri miktarını azaltır olup olmadığını denetlemez. Örneğin, durum `Date != ' 2000-01-01 '` tablodaki tüm verilerle eşleştiğinde bile kabul edilebilir (yani, sorguyu çalıştırmak tam bir tarama gerektirir). MergeTree tablolarındaki veri aralıkları hakkında daha fazla bilgi için bkz. [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). + +## force\_primary\_key {#force-primary-key} + +Birincil anahtar tarafından dizin oluşturma mümkün değilse, sorgu yürütülmesini devre dışı bırakır. + +MergeTree ailesindeki tablolarla çalışır. + +Eğer `force_primary_key=1`, ClickHouse, sorgunun veri aralıklarını kısıtlamak için kullanılabilecek bir birincil anahtar koşulu olup olmadığını denetler. Uygun bir koşul yoksa, bir istisna atar. Ancak, koşul okumak için veri miktarını azaltır olup olmadığını denetlemez. MergeTree tablolarındaki veri aralıkları hakkında daha fazla bilgi için bkz. [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md). + +## format\_schema {#format-schema} + +Bu parametre, aşağıdaki gibi bir şema tanımı gerektiren biçimler kullanırken kullanışlıdır [Cap'n Proto](https://capnproto.org/) veya [Protobuf](https://developers.google.com/protocol-buffers/). Değer biçime bağlıdır. + +## fsync\_metadata {#fsync-metadata} + +Etkinleştirir veya devre dışı bırakır [fsync](http://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html) yazarken `.sql` eğe. Varsayılan olarak etkin. + +Sunucu, sürekli olarak oluşturulan ve yok edilen milyonlarca küçük tabloya sahipse, onu devre dışı bırakmak mantıklıdır. + +## enable\_http\_compression {#settings-enable_http_compression} + +Bir HTTP isteğine yanıt olarak veri sıkıştırmasını etkinleştirir veya devre dışı bırakır. + +Daha fazla bilgi için, okuyun [HTTP arayüzü açıklaması](../../interfaces/http.md). + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 0. + +## http\_zlib\_compression\_level {#settings-http_zlib_compression_level} + +Eğer bir HTTP isteğine yanıt veri sıkıştırma düzeyini ayarlar [enable\_http\_compression = 1](#settings-enable_http_compression). + +Olası değerler: 1'den 9'a kadar olan sayılar. + +Varsayılan değer: 3. + +## http\_native\_compression\_disable\_checksumming\_on\_decompress {#settings-http_native_compression_disable_checksumming_on_decompress} + +İstemciden HTTP POST verilerini açarken sağlama toplamı doğrulamasını etkinleştirir veya devre dışı bırakır. Sadece ClickHouse yerel sıkıştırma formatı için kullanılır (ile kullanılmaz `gzip` veya `deflate`). + +Daha fazla bilgi için, okuyun [HTTP arayüzü açıklaması](../../interfaces/http.md). + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 0. + +## send\_progress\_in\_http\_headers {#settings-send_progress_in_http_headers} + +Etkinleştirir veya devre dışı bırakır `X-ClickHouse-Progress` HTTP yanıt başlıkları `clickhouse-server` yanıtlar. + +Daha fazla bilgi için, okuyun [HTTP arayüzü açıklaması](../../interfaces/http.md). + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 0. + +## max\_http\_get\_redirects {#setting-max_http_get_redirects} + +Maksimum http get yönlendirme atlama sayısını sınırlar [URL](../../engines/table_engines/special/url.md)- motor masaları. Ayarı tablolar iki tür tarafından oluşturulan bu geçerlidir: [CREATE TABLE](../../sql_reference/statements/create.md#create-table-query) sorgu ve [url](../../sql_reference/table_functions/url.md) tablo işlevi. + +Olası değerler: + +- Herhangi bir pozitif tamsayı şerbetçiotu sayısı. +- 0 — No hops allowed. + +Varsayılan değer: 0. + +## ınput\_format\_allow\_errors\_num {#settings-input_format_allow_errors_num} + +Metin biçimlerinden (CSV, TSV, vb.) okurken kabul edilebilir hataların maksimum sayısını ayarlar.). + +Varsayılan değer 0'dır. + +Her zaman ile eşleştirmek `input_format_allow_errors_ratio`. + +Satırları okurken bir hata oluştu, ancak hata sayacı hala daha az `input_format_allow_errors_num`, ClickHouse satırı yok sayar ve bir sonrakine geçer. + +Eğer her ikisi de `input_format_allow_errors_num` ve `input_format_allow_errors_ratio` aşıldı, ClickHouse bir istisna atar. + +## ınput\_format\_allow\_errors\_ratio {#settings-input_format_allow_errors_ratio} + +Metin biçimlerinden (CSV, TSV, vb.) okurken izin verilen maksimum hata yüzdesini ayarlar.). +Hataların yüzdesi 0 ile 1 arasında kayan nokta sayısı olarak ayarlanır. + +Varsayılan değer 0'dır. + +Her zaman ile eşleştirmek `input_format_allow_errors_num`. + +Satırları okurken bir hata oluştu, ancak hata sayacı hala daha az `input_format_allow_errors_ratio`, ClickHouse satırı yok sayar ve bir sonrakine geçer. + +Eğer her ikisi de `input_format_allow_errors_num` ve `input_format_allow_errors_ratio` aşıldı, ClickHouse bir istisna atar. + +## ınput\_format\_values\_interpret\_expressions {#settings-input_format_values_interpret_expressions} + +Hızlı akış ayrıştırıcısı verileri ayrıştıramazsa, tam SQL ayrıştırıcısını etkinleştirir veya devre dışı bırakır. Bu ayar yalnızca için kullanılır [Değerler](../../interfaces/formats.md#data-format-values) veri ekleme sırasında biçimlendirin. Sözdizimi ayrıştırma hakkında daha fazla bilgi için bkz: [Sözdizimi](../../sql_reference/syntax.md) bölme. + +Olası değerler: + +- 0 — Disabled. + + Bu durumda, biçimlendirilmiş veri sağlamanız gerekir. Görmek [Biçimliler](../../interfaces/formats.md) bölme. + +- 1 — Enabled. + + Bu durumda, bir SQL ifadesini bir değer olarak kullanabilirsiniz, ancak veri ekleme bu şekilde çok daha yavaştır. Yalnızca biçimlendirilmiş veri eklerseniz, ClickHouse ayar değeri 0 gibi davranır. + +Varsayılan değer: 1. + +Kullanım örneği + +Ekle [DateTime](../../sql_reference/data_types/datetime.md) farklı ayarlarla değer yazın. + +``` sql +SET input_format_values_interpret_expressions = 0; +INSERT INTO datetime_t VALUES (now()) +``` + +``` text +Exception on client: +Code: 27. DB::Exception: Cannot parse input: expected ) before: now()): (at row 1) +``` + +``` sql +SET input_format_values_interpret_expressions = 1; +INSERT INTO datetime_t VALUES (now()) +``` + +``` text +Ok. +``` + +Son sorgu Aşağıdakilere eşdeğerdir: + +``` sql +SET input_format_values_interpret_expressions = 0; +INSERT INTO datetime_t SELECT now() +``` + +``` text +Ok. +``` + +## ınput\_format\_values\_deduce\_templates\_of\_expressions {#settings-input_format_values_deduce_templates_of_expressions} + +SQL deyimleri için şablon kesintisini etkinleştirir veya devre dışı bırakır [Değerler](../../interfaces/formats.md#data-format-values) biçimli. Bu ayrıştırma ve ifadeleri yorumlama sağlar `Values` ardışık satırlardaki ifadeler aynı yapıya sahipse çok daha hızlı. ClickHouse, bir ifadenin şablonunu çıkarmaya, bu şablonu kullanarak aşağıdaki satırları ayrıştırmaya ve ifadeyi başarılı bir şekilde ayrıştırılmış satırların bir yığınında değerlendirmeye çalışır. + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 1. + +Aşağıdaki sorgu için: + +``` sql +INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), (upper('Values')), ... +``` + +- Eğer `input_format_values_interpret_expressions=1` ve `format_values_deduce_templates_of_expressions=0`, ifadeler her satır için ayrı ayrı yorumlanır (bu çok sayıda satır için çok yavaştır). +- Eğer `input_format_values_interpret_expressions=0` ve `format_values_deduce_templates_of_expressions=1`, birinci, ikinci ve üçüncü satırlardaki ifadeler şablon kullanılarak ayrıştırılır `lower(String)` ve birlikte yorumlanır, ileri satırdaki ifade başka bir şablonla ayrıştırılır (`upper(String)`). +- Eğer `input_format_values_interpret_expressions=1` ve `format_values_deduce_templates_of_expressions=1`, önceki durumda olduğu gibi aynı, ama aynı zamanda şablon anlamak mümkün değilse ayrı ayrı ifadeleri yorumlama geri dönüş sağlar. + +## ınput\_format\_values\_accurate\_types\_of\_literals {#settings-input-format-values-accurate-types-of-literals} + +Bu ayar yalnızca şu durumlarda kullanılır `input_format_values_deduce_templates_of_expressions = 1`. Bu, bazı sütunların ifadelerinin aynı yapıya sahip olması, ancak farklı türlerde sayısal değişmezler içermesi olabilir, örneğin + +``` sql +(..., abs(0), ...), -- UInt64 literal +(..., abs(3.141592654), ...), -- Float64 literal +(..., abs(-1), ...), -- Int64 literal +``` + +Olası değerler: + +- 0 — Disabled. + + In this case, ClickHouse may use a more general type for some literals (e.g., `Float64` veya `Int64` yerine `UInt64` için `42`), ancak taşma ve hassasiyet sorunlarına neden olabilir. + +- 1 — Enabled. + + Bu durumda, ClickHouse gerçek literal türünü denetler ve karşılık gelen türde bir ifade şablonu kullanır. Bazı durumlarda, ifade değerlendirmesini önemli ölçüde yavaşlatabilir `Values`. + +Varsayılan değer: 1. + +## ınput\_format\_defaults\_for\_omitted\_fields {#session_settings-input_format_defaults_for_omitted_fields} + +Yaparken `INSERT` sorgular, atlanmış giriş sütun değerlerini ilgili sütunların varsayılan değerleriyle değiştirin. Bu seçenek yalnızca aşağıdakiler için geçerlidir [JSONEachRow](../../interfaces/formats.md#jsoneachrow), [CSV](../../interfaces/formats.md#csv) ve [TabSeparated](../../interfaces/formats.md#tabseparated) biçimliler. + +!!! note "Not" + Bu seçenek etkinleştirildiğinde, genişletilmiş tablo meta verileri sunucudan istemciye gönderilir. Sunucuda ek bilgi işlem kaynakları tüketir ve performansı azaltabilir. + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 1. + +## ınput\_format\_tsv\_empty\_as\_default {#settings-input-format-tsv-empty-as-default} + +Etkinleştirildiğinde, TSV'DEKİ boş giriş alanlarını varsayılan değerlerle değiştirin. Karmaşık varsayılan ifadeler için `input_format_defaults_for_omitted_fields` de etkin olmalıdır. + +Varsayılan olarak devre dışı. + +## ınput\_format\_null\_as\_default {#settings-input-format-null-as-default} + +Giriş verileri içeriyorsa, varsayılan değerleri kullanarak etkinleştirir veya devre dışı bırakır `NULL`, ancak ilgili sütunun veri türü değil `Nullable(T)` (Metin Giriş biçimleri için). + +## ınput\_format\_skip\_unknown\_fields {#settings-input-format-skip-unknown-fields} + +Etkinleştirir veya ek veri ekleme atlama devre dışı bırakır. + +Veri yazarken, giriş verileri hedef tabloda bulunmayan sütunlar içeriyorsa, ClickHouse bir özel durum atar. Atlama etkinleştirilirse, ClickHouse ek veri eklemez ve bir istisna atmaz. + +Desteklenen formatlar: + +- [JSONEachRow](../../interfaces/formats.md#jsoneachrow) +- [CSVWithNames](../../interfaces/formats.md#csvwithnames) +- [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames) +- [TSKV](../../interfaces/formats.md#tskv) + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 0. + +## ınput\_format\_ımport\_nested\_json {#settings-input_format_import_nested_json} + +Json verilerinin iç içe nesnelerle eklenmesini etkinleştirir veya devre dışı bırakır. + +Desteklenen formatlar: + +- [JSONEachRow](../../interfaces/formats.md#jsoneachrow) + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 0. + +Ayrıca bakınız: + +- [İç içe yapıların kullanımı](../../interfaces/formats.md#jsoneachrow-nested) ile... `JSONEachRow` biçimli. + +## ınput\_format\_with\_names\_use\_header {#settings-input-format-with-names-use-header} + +Veri eklerken sütun sırasını denetlemeyi etkinleştirir veya devre dışı bırakır. + +Ekleme performansını artırmak için, giriş verilerinin sütun sırasının hedef tablodaki ile aynı olduğundan eminseniz, bu denetimi devre dışı bırakmanızı öneririz. + +Desteklenen formatlar: + +- [CSVWithNames](../../interfaces/formats.md#csvwithnames) +- [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames) + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 1. + +## date\_time\_input\_format {#settings-date_time_input_format} + +Tarih ve saat metin gösterimi bir ayrıştırıcı seçme sağlar. + +Ayar için geçerli değildir [tarih ve saat fonksiyonları](../../sql_reference/functions/date_time_functions.md). + +Olası değerler: + +- `'best_effort'` — Enables extended parsing. + + ClickHouse temel ayrıştırmak `YYYY-MM-DD HH:MM:SS` format ve tüm [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) tarih ve saat biçimleri. Mesela, `'2018-06-08T01:02:03.000Z'`. + +- `'basic'` — Use basic parser. + + ClickHouse sadece temel ayrıştırmak `YYYY-MM-DD HH:MM:SS` biçimli. Mesela, `'2019-08-20 10:18:56'`. + +Varsayılan değer: `'basic'`. + +Ayrıca bakınız: + +- [DateTime veri türü.](../../sql_reference/data_types/datetime.md) +- [Tarihler ve saatler ile çalışmak için fonksiyonlar.](../../sql_reference/functions/date_time_functions.md) + +## join\_default\_strictness {#settings-join_default_strictness} + +Ayarlar varsayılan strictness için [Maddeleri KATILIN ](../../sql_reference/statements/select.md#select-join). + +Olası değerler: + +- `ALL` — If the right table has several matching rows, ClickHouse creates a [Kartezyen ürün](https://en.wikipedia.org/wiki/Cartesian_product) eşleşen satırlardan. Bu normaldir `JOIN` standart SQL'DEN davranış. +- `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of `ANY` ve `ALL` aynı. +- `ASOF` — For joining sequences with an uncertain match. +- `Empty string` — If `ALL` veya `ANY` sorguda belirtilmezse, ClickHouse bir özel durum atar. + +Varsayılan değer: `ALL`. + +## join\_any\_take\_last\_row {#settings-join_any_take_last_row} + +İle birleştirme işlemlerinin davranışını değiştirir `ANY` katılık. + +!!! warning "Dikkat" + Bu ayar yalnızca aşağıdakiler için geçerlidir `JOIN` ile işlemler [Katmak](../../engines/table_engines/special/join.md) motor tabloları. + +Olası değerler: + +- 0 — If the right table has more than one matching row, only the first one found is joined. +- 1 — If the right table has more than one matching row, only the last one found is joined. + +Varsayılan değer: 0. + +Ayrıca bakınız: + +- [Jo](../../sql_reference/statements/select.md#select-join) +- [Jo tablein table engine](../../engines/table_engines/special/join.md) +- [join\_default\_strictness](#settings-join_default_strictness) + +## join\_use\_nulls {#join_use_nulls} + +Türünü ayarlar [JOIN](../../sql_reference/statements/select.md) davranış. Tabloları birleştirirken boş hücreler görünebilir. ClickHouse bu ayara göre onları farklı şekilde doldurur. + +Olası değerler: + +- 0 — The empty cells are filled with the default value of the corresponding field type. +- 1 — `JOIN` standart SQL ile aynı şekilde davranır. Karşılık gelen alanın türü dönüştürülür [Nullable](../../sql_reference/data_types/nullable.md#data_type-nullable) ve boş hücreler ile doldurulur [NULL](../../sql_reference/syntax.md). + +Varsayılan değer: 0. + +## max\_block\_size {#setting-max_block_size} + +Clickhouse'da, veriler bloklarla (sütun parçaları kümeleri) işlenir. Tek bir blok için dahili işlem döngüleri yeterince verimlidir, ancak her blokta gözle görülür harcamalar vardır. Bu `max_block_size` ayar, blokun boyutunun (satır sayımında) tablolardan yükleneceği bir öneridir. Blok boyutu çok küçük olmamalı, böylece her bloktaki harcamalar hala fark edilebilir, ancak çok büyük olmamalı, böylece ilk blok hızla işlendikten sonra tamamlanan limitli sorgu çok büyük olmamalıdır. Amaç, birden çok iş parçacığında çok sayıda sütun ayıklarken çok fazla bellek tüketmekten kaçınmak ve en azından bazı önbellek konumlarını korumaktır. + +Varsayılan değer: 65,536. + +Blok boyutu `max_block_size` her zaman tablodan yüklenmez. Daha az verinin alınması gerektiği açıksa, daha küçük bir blok işlenir. + +## preferred\_block\_size\_bytes {#preferred-block-size-bytes} + +Olarak aynı amaç için kullanılır `max_block_size`, ancak önerilen blok boyutunu bayt cinsinden, bloktaki satır sayısına uyarlayarak ayarlar. +Ancak, blok boyutu daha fazla olamaz `max_block_size` satırlar. +Varsayılan olarak: 1.000.000. Sadece MergeTree motorlarından okurken çalışır. + +## merge\_tree\_mın\_rows\_for\_concurrent\_read {#setting-merge-tree-min-rows-for-concurrent-read} + +Bir dosyadan okunacak satır sayısı ise [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) tablo aşıyor `merge_tree_min_rows_for_concurrent_read` daha sonra ClickHouse, bu dosyadan birkaç iş parçacığı üzerinde eşzamanlı bir okuma gerçekleştirmeye çalışır. + +Olası değerler: + +- Herhangi bir pozitif tamsayı. + +Varsayılan değer: 163840. + +## merge\_tree\_min\_bytes\_for\_concurrent\_read {#setting-merge-tree-min-bytes-for-concurrent-read} + +Eğer bir dosyadan okunacak bayt sayısı [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)- motor tablosu `merge_tree_min_bytes_for_concurrent_read`, daha sonra ClickHouse, bu dosyadan aynı anda birkaç iş parçacığında okumaya çalışır. + +Olası değer: + +- Herhangi bir pozitif tamsayı. + +Varsayılan değer: 251658240. + +## merge\_tree\_mın\_rows\_for\_seek {#setting-merge-tree-min-rows-for-seek} + +Bir dosyada okunacak iki veri bloğu arasındaki mesafe daha az ise `merge_tree_min_rows_for_seek` satırlar, daha sonra ClickHouse dosyayı aramaz, ancak verileri sırayla okur. + +Olası değerler: + +- Herhangi bir pozitif tamsayı. + +Varsayılan değer: 0. + +## merge\_tree\_min\_bytes\_for\_seek {#setting-merge-tree-min-bytes-for-seek} + +Bir dosyada okunacak iki veri bloğu arasındaki mesafe daha az ise `merge_tree_min_bytes_for_seek` bayt, daha sonra ClickHouse sırayla böylece ekstra arama kaçınarak, her iki blok içeren bir dosya aralığını okur. + +Olası değerler: + +- Herhangi bir pozitif tamsayı. + +Varsayılan değer: 0. + +## merge\_tree\_coarse\_index\_granularity {#setting-merge-tree-coarse-index-granularity} + +Veri ararken, ClickHouse dizin dosyasındaki veri işaretlerini denetler. ClickHouse gerekli tuşların bazı aralıklarda olduğunu bulursa, bu aralığı `merge_tree_coarse_index_granularity` subranges ve gerekli anahtarları orada yinelemeli olarak arar. + +Olası değerler: + +- Herhangi bir pozitif bile tamsayı. + +Varsayılan değer: 8. + +## merge\_tree\_max\_rows\_to\_use\_cache {#setting-merge-tree-max-rows-to-use-cache} + +ClickHouse daha fazla okumak gerekiyorsa `merge_tree_max_rows_to_use_cache` bir sorgudaki satırlar, sıkıştırılmamış blokların önbelleğini kullanmaz. + +Sıkıştırılmamış blokların önbelleği, sorgular için ayıklanan verileri depolar. ClickHouse, tekrarlanan küçük sorgulara verilen yanıtları hızlandırmak için bu önbelleği kullanır. Bu ayar, önbelleğin büyük miktarda veri okuyan sorgularla çöpe atmasını önler. Bu [uncompressed\_cache\_size](../server_configuration_parameters/settings.md#server-settings-uncompressed_cache_size) sunucu ayarı, sıkıştırılmamış blokların önbelleğinin boyutunu tanımlar. + +Olası değerler: + +- Herhangi bir pozitif tamsayı. + +Default value: 128 ✕ 8192. + +## merge\_tree\_max\_bytes\_to\_use\_cache {#setting-merge-tree-max-bytes-to-use-cache} + +ClickHouse daha fazla okumak gerekiyorsa `merge_tree_max_bytes_to_use_cache` bir sorguda bayt, sıkıştırılmamış blokların önbelleğini kullanmaz. + +Sıkıştırılmamış blokların önbelleği, sorgular için ayıklanan verileri depolar. ClickHouse, tekrarlanan küçük sorgulara verilen yanıtları hızlandırmak için bu önbelleği kullanır. Bu ayar, önbelleğin büyük miktarda veri okuyan sorgularla çöpe atmasını önler. Bu [uncompressed\_cache\_size](../server_configuration_parameters/settings.md#server-settings-uncompressed_cache_size) sunucu ayarı, sıkıştırılmamış blokların önbelleğinin boyutunu tanımlar. + +Olası değer: + +- Herhangi bir pozitif tamsayı. + +Varsayılan değer: 2013265920. + +## min\_bytes\_to\_use\_direct\_io {#settings-min-bytes-to-use-direct-io} + +Depolama diskine Doğrudan G/Ç erişimi kullanmak için gereken minimum veri hacmi. + +ClickHouse, tablolardan veri okurken bu ayarı kullanır. Okunacak tüm verilerin toplam depolama hacmi aşarsa `min_bytes_to_use_direct_io` bayt, daha sonra ClickHouse ile depolama diskinden veri okur `O_DIRECT` seçenek. + +Olası değerler: + +- 0 — Direct I/O is disabled. +- Pozitif tamsayı. + +Varsayılan değer: 0. + +## log\_queries {#settings-log-queries} + +Sorgu günlüğü ayarlama. + +Bu kurulum ile Clickhouse'a gönderilen sorgular, [query\_log](../server_configuration_parameters/settings.md#server_configuration_parameters-query-log) sunucu yapılandırma parametresi. + +Örnek: + +``` text +log_queries=1 +``` + +## log\_queries\_min\_type {#settings-log-queries-min-type} + +`query_log` giriş yapmak için en az tür. + +Olası değerler: +- `QUERY_START` (`=1`) +- `QUERY_FINISH` (`=2`) +- `EXCEPTION_BEFORE_START` (`=3`) +- `EXCEPTION_WHILE_PROCESSING` (`=4`) + +Varsayılan değer: `QUERY_START`. + +Entiries gider hangi sınırlamak için kullanılabilir `query_log`, sadece hatalarda ilginç olduğunuzu söyleyin, o zaman kullanabilirsiniz `EXCEPTION_WHILE_PROCESSING`: + +``` text +log_queries_min_type='EXCEPTION_WHILE_PROCESSING' +``` + +## log\_query\_threads {#settings-log-query-threads} + +Sorgu iş parçacığı günlüğü ayarlama. + +Bu kurulum ile ClickHouse tarafından çalıştırılan sorguların konuları, [query\_thread\_log](../server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log) sunucu yapılandırma parametresi. + +Örnek: + +``` text +log_query_threads=1 +``` + +## max\_ınsert\_block\_size {#settings-max_insert_block_size} + +Bir tabloya eklemek için oluşturulacak blokların boyutu. +Bu ayar yalnızca sunucu blokları oluşturduğu durumlarda geçerlidir. +Örneğin, HTTP arabirimi üzerinden bir ekleme için sunucu veri biçimini ayrıştırır ve belirtilen boyuttaki blokları oluşturur. +Ancak, clickhouse-client kullanırken, istemci verileri kendisi ayrıştırır ve ‘max\_insert\_block\_size’ sunucudaki ayar, eklenen blokların boyutunu etkilemez. +Veri SELECT sonra oluşturulan aynı blokları kullanarak eklendiğinden, INSERT SELECT kullanırken ayarı da bir amacı yoktur. + +Varsayılan değer: 1.048,576. + +Varsayılan biraz daha fazla `max_block_size`. Bunun nedeni, bazı tablo motorlarının (`*MergeTree`) oldukça büyük bir varlık olan eklenen her blok için diskte bir veri parçası oluşturun. Benzer bir şekilde, `*MergeTree` tablolar ekleme sırasında verileri sıralar ve yeterince büyük bir blok boyutu RAM'de daha fazla veriyi sıralamaya izin verir. + +## max\_replica\_delay\_for\_distributed\_queries {#settings-max_replica_delay_for_distributed_queries} + +Dağıtılmış sorgular için gecikmeli yinelemeleri devre dışı bırakır. Görmek [Çoğalma](../../engines/table_engines/mergetree_family/replication.md). + +Saati saniye olarak ayarlar. Bir çoğaltma ayarlanan değerden daha fazla kalıyorsa, Bu çoğaltma kullanılmaz. + +Varsayılan değer: 300. + +Yaparken kullanılır `SELECT` çoğaltılmış tablolara işaret eden dağıtılmış bir tablodan. + +## max\_threads {#settings-max_threads} + +Uzak sunuculardan veri almak için iş parçacıkları hariç olmak üzere sorgu işleme iş parçacıklarının maksimum sayısı (bkz. ‘max\_distributed\_connections’ parametre). + +Bu parametre, paralel olarak sorgu işleme ardışık düzeninin aynı aşamalarını gerçekleştiren iş parçacıkları için geçerlidir. +Örneğin, bir tablodan okurken, ifadeleri işlevlerle değerlendirmek mümkün ise, en azından paralel olarak grup için where ve pre-aggregate ile filtreleyin ‘max\_threads’ konu sayısı, daha sonra ‘max\_threads’ kullanılır. + +Varsayılan değer: fiziksel CPU çekirdeği sayısı. + +Bir kerede bir sunucuda normal olarak birden az SELECT sorgusu çalıştırılırsa, bu parametreyi gerçek işlemci çekirdeği sayısından biraz daha küçük bir değere ayarlayın. + +Bir sınır nedeniyle hızlı bir şekilde tamamlanan sorgular için, daha düşük bir ‘max\_threads’. Örneğin, gerekli sayıda giriş her blokta ve max\_threads = 8'de bulunuyorsa, sadece bir tane okumak için yeterli olsa da, 8 blok alınır. + +Daha küçük `max_threads` değer, daha az bellek tüketilir. + +## max\_ınsert\_threads {#settings-max-insert-threads} + +Çalıştırılacak maksimum iş parçacığı sayısı `INSERT SELECT` sorgu. + +Olası değerler: + +- 0 (or 1) — `INSERT SELECT` paralel infaz yok. +- Pozitif tamsayı. 1'den büyük. + +Varsayılan değer: 0. + +Paralellik `INSERT SELECT` etkisi vardır sadece eğer `SELECT` bölüm paralel olarak yürütülür, bkz [max\_threads](#settings-max_threads) ayar. +Daha yüksek değerler daha yüksek bellek kullanımına yol açacaktır. + +## max\_compress\_block\_size {#max-compress-block-size} + +Bir tabloya yazmak için sıkıştırmadan önce sıkıştırılmamış veri bloklarının en büyük boyutu. Varsayılan olarak, 1.048.576 (1 MiB). Boyut azaltılırsa, sıkıştırma oranı önemli ölçüde azalır, önbellek konumu nedeniyle sıkıştırma ve dekompresyon hızı biraz artar ve bellek tüketimi azalır. Bu ayarı değiştirmek için genellikle herhangi bir neden yoktur. + +Sıkıştırma için blokları (bayttan oluşan bir bellek yığını) sorgu işleme için bloklarla (bir tablodan satır kümesi) karıştırmayın. + +## min\_compress\_block\_size {#min-compress-block-size} + +İçin [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)" Tablolar. Sorguları işlerken gecikmeyi azaltmak için, boyutu en az bir sonraki işareti yazarken bir blok sıkıştırılır ‘min\_compress\_block\_size’. Varsayılan olarak, 65.536. + +Sıkıştırılmamış veriler daha az ise, bloğun gerçek boyutu ‘max\_compress\_block\_size’, bu değerden daha az değildir ve bir işaret için veri hacminden daha az değildir. + +Bir örneğe bakalım. Varsaymak ‘index\_granularity’ tablo oluşturma sırasında 8192 olarak ayarlandı. + +Bir uint32 tipi sütun yazıyoruz (değer başına 4 bayt). 8192 satır yazarken, toplam 32 KB veri olacaktır. Min\_compress\_block\_size = 65.536 olduğundan, her iki işaret için sıkıştırılmış bir blok oluşturulacaktır. + +Dize türüne sahip bir URL sütunu yazıyoruz (değer başına ortalama 60 bayt boyutu). 8192 satır yazarken, ortalama 500 KB veri biraz daha az olacaktır. Bu 65,536'dan fazla olduğu için, her işaret için sıkıştırılmış bir blok oluşturulacaktır. Bu durumda, diskteki verileri tek bir işaret aralığında okurken, ekstra veriler sıkıştırılmaz. + +Bu ayarı değiştirmek için genellikle herhangi bir neden yoktur. + +## max\_query\_size {#settings-max_query_size} + +SQL ayrıştırıcısı ile ayrıştırmak için RAM'e alınabilecek bir sorgunun en büyük kısmı. +INSERT sorgusu, bu kısıtlamaya dahil olmayan ayrı bir akış ayrıştırıcısı (o(1) RAM tüketir) tarafından işlenen INSERT için veri de içerir. + +Varsayılan değer: 256 KiB. + +## ınteractive\_delay {#interactive-delay} + +İstek yürütülmesinin iptal edilip edilmediğini kontrol etmek ve ilerlemeyi göndermek için mikrosaniye cinsinden Aralık. + +Varsayılan değer: 100.000 (iptal için denetler ve ilerleme saniyede on kez gönderir). + +## connect\_timeout, receıve\_tımeout, send\_timeout {#connect-timeout-receive-timeout-send-timeout} + +İstemci ile iletişim kurmak için kullanılan sokette saniye cinsinden zaman aşımları. + +Varsayılan değer: 10, 300, 300. + +## cancel\_http\_readonly\_queries\_on\_client\_close {#cancel-http-readonly-queries-on-client-close} + +Cancels HTTP read-only queries (e.g. SELECT) when a client closes the connection without waiting for the response. + +Varsayılan değer: 0 + +## poll\_interval {#poll-interval} + +Belirtilen saniye sayısı için bir bekleme döngüsünde kilitleyin. + +Varsayılan değer: 10. + +## max\_distributed\_connections {#max-distributed-connections} + +Tek bir dağıtılmış tabloya tek bir sorgunun dağıtılmış işlenmesi için uzak sunucularla eşzamanlı bağlantı sayısı. Kümedeki sunucu sayısından daha az bir değer ayarlamanızı öneririz. + +Varsayılan değer: 1024. + +Aşağıdaki parametreler yalnızca dağıtılmış tablolar oluştururken (ve bir sunucu başlatırken) kullanılır, bu nedenle bunları çalışma zamanında değiştirmek için hiçbir neden yoktur. + +## distributed\_connections\_pool\_size {#distributed-connections-pool-size} + +Tüm sorguların tek bir dağıtılmış tabloya dağıtılmış işlenmesi için uzak sunucularla eşzamanlı bağlantıların maksimum sayısı. Kümedeki sunucu sayısından daha az bir değer ayarlamanızı öneririz. + +Varsayılan değer: 1024. + +## connect\_timeout\_with\_failover\_ms {#connect-timeout-with-failover-ms} + +Dağıtılmış bir tablo altyapısı için uzak bir sunucuya bağlanmak için milisaniye cinsinden zaman aşımı ‘shard’ ve ‘replica’ bölümler küme tanımında kullanılır. +Başarısız olursa, çeşitli yinelemelere bağlanmak için birkaç deneme yapılır. + +Varsayılan değer: 50. + +## connections\_with\_failover\_max\_tries {#connections-with-failover-max-tries} + +Dağıtılmış tablo altyapısı için her yineleme ile bağlantı girişimi sayısı. + +Varsayılan değer: 3. + +## çıkmaz {#extremes} + +Aşırı değerleri (bir sorgu sonucunun sütunlarındaki minimum ve maksimum değerler) saymak ister. 0 veya 1 kabul eder. Varsayılan olarak, 0 (devre dışı). +Daha fazla bilgi için bölüme bakın “Extreme values”. + +## use\_uncompressed\_cache {#setting-use_uncompressed_cache} + +Sıkıştırılmamış blokların önbelleğinin kullanılıp kullanılmayacağı. 0 veya 1 kabul eder. Varsayılan olarak, 0 (devre dışı). +Sıkıştırılmamış önbelleği (yalnızca mergetree ailesindeki tablolar için) kullanmak, çok sayıda kısa Sorgu ile çalışırken gecikmeyi önemli ölçüde azaltabilir ve verimi artırabilir. Sık sık kısa istek Gönderen kullanıcılar için bu ayarı etkinleştirin. Ayrıca dikkat [uncompressed\_cache\_size](../server_configuration_parameters/settings.md#server-settings-uncompressed_cache_size) configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed and the least-used data is automatically deleted. + +En azından biraz büyük bir veri hacmi (bir milyon satır veya daha fazla) okuyan sorgular için sıkıştırılmamış önbellek, gerçekten küçük sorgular için yer kazanmak için otomatik olarak devre dışı bırakılır. Bu tutmak anlamına gelir ‘use\_uncompressed\_cache’ ayar her zaman 1 olarak ayarlanır. + +## replace\_running\_query {#replace-running-query} + +HTTP arayüzünü kullanırken, ‘query\_id’ parametre geçirilebilir. Bu, sorgu tanımlayıcısı olarak hizmet veren herhangi bir dizedir. +Aynı kullanıcıdan aynı sorgu varsa ‘query\_id’ zaten şu anda var, davranış bağlıdır ‘replace\_running\_query’ parametre. + +`0` (default) – Throw an exception (don't allow the query to run if a query with the same ‘query\_id’ zaten çalışan) var. + +`1` – Cancel the old query and start running the new one. + +Üye.Metrica, segmentasyon koşulları için öneriler uygulamak için 1 olarak ayarlanmış bu parametreyi kullanır. Bir sonraki karakteri girdikten sonra, eski sorgu henüz tamamlanmamışsa, iptal edilmelidir. + +## stream\_flush\_interval\_ms {#stream-flush-interval-ms} + +Bir zaman aşımı durumunda akışlı tablolar için çalışır veya bir iş parçacığı oluşturduğunda [max\_ınsert\_block\_size](#settings-max_insert_block_size) satırlar. + +Varsayılan değer 7500'dür. + +Küçük değer, daha sık veri tablosuna temizlendi. Değeri çok düşük ayarlamak, düşük performansa yol açar. + +## dengeleme {#settings-load_balancing} + +Dağıtılmış sorgu işleme için kullanılan yinelemeler seçimi algoritmasını belirtir. + +ClickHouse kopyaları seçme aşağıdaki algoritmaları destekler: + +- [Rastgele](#load_balancing-random) (varsayılan olarak) +- [En yakın hostnamename](#load_balancing-nearest_hostname) +- [Sıralı](#load_balancing-in_order) +- [İlk veya rastgele](#load_balancing-first_or_random) + +### Rastgele (varsayılan olarak) {#load_balancing-random} + +``` sql +load_balancing = random +``` + +Her yineleme için hata sayısı sayılır. Sorgu, en az hata ile çoğaltmaya gönderilir ve bunlardan birkaçı varsa, bunlardan herhangi birine gönderilir. +Dezavantajları: sunucu yakınlık hesaba değil; kopyaları farklı veri varsa, aynı zamanda farklı veri alırsınız. + +### En Yakın Hostnamename {#load_balancing-nearest_hostname} + +``` sql +load_balancing = nearest_hostname +``` + +The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server's hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). + +Örneğin, example01-01-1 ve example01-01-2.yandex.ru bir pozisyonda farklıdır, örneği01-01-1 ve örneği01-02-2 iki yerde farklılık gösterir. +Bu yöntem ilkel görünebilir, ancak ağ topolojisi hakkında harici veri gerektirmez ve IPv6 adreslerimiz için karmaşık olan IP adreslerini karşılaştırmaz. + +Bu nedenle, eşdeğer yinelemeler varsa, isme göre en yakın olanı tercih edilir. +Aynı sunucuya bir sorgu gönderirken, arızaların yokluğunda, dağıtılmış bir sorgunun da aynı sunuculara gideceğini varsayabiliriz. Bu nedenle, yinelemelere farklı veriler yerleştirilse bile, sorgu çoğunlukla aynı sonuçları döndürür. + +### Sıralı {#load_balancing-in_order} + +``` sql +load_balancing = in_order +``` + +Yapılandırmada belirtilen hataları aynı sayıda yinelemeler aynı sırayla erişilir. +Bu yöntem, tam olarak hangi kopyanın tercih edildiğini bildiğinizde uygundur. + +### İlk veya rastgele {#load_balancing-first_or_random} + +``` sql +load_balancing = first_or_random +``` + +İlk kullanılamaz, bu algoritma kümesindeki ilk yineleme veya rasgele bir yineleme seçer. Çapraz çoğaltma topolojisi kurulumlarında etkilidir, ancak diğer yapılandırmalarda işe yaramaz. + +Bu `first_or_random` algoritma sorunu çözer `in_order` algoritma. İle `in_order`, bir çoğaltma aşağı giderse, kalan yinelemeler normal trafik miktarını işlerken bir sonraki bir çift yük alır. Kullanırken `first_or_random` algoritma, yük hala mevcut olan kopyalar arasında eşit olarak dağıtılır. + +## prefer\_localhost\_replica {#settings-prefer-localhost-replica} + +Etkinleştirir / devre dışı bırakır tercih kullanarak localhost çoğaltma dağıtılmış sorguları işlerken. + +Olası değerler: + +- 1 — ClickHouse always sends a query to the localhost replica if it exists. +- 0 — ClickHouse uses the balancing strategy specified by the [dengeleme](#settings-load_balancing) ayar. + +Varsayılan değer: 1. + +!!! warning "Uyarıcı" + Kullanıyorsanız bu ayarı devre dışı bırakın [max\_parallel\_replicas](#settings-max_parallel_replicas). + +## totals\_mode {#totals-mode} + +MAX\_ROWS\_TO\_GROUP\_BY ve group\_by\_overflow\_mode = ‘any’ Bulunmak. +Bölümüne bakınız “WITH TOTALS modifier”. + +## totals\_auto\_threshold {#totals-auto-threshold} + +İçin eşik `totals_mode = 'auto'`. +Bölümüne bakınız “WITH TOTALS modifier”. + +## max\_parallel\_replicas {#settings-max_parallel_replicas} + +Bir sorgu yürütülürken her parça için en fazla yineleme sayısı. +Tutarlılık için (aynı veri bölünmesinin farklı bölümlerini elde etmek için), bu seçenek yalnızca örnekleme anahtarı ayarlandığında çalışır. +Çoğaltma gecikme denetlenmez. + +## derlemek {#compile} + +Sorguların derlenmesini etkinleştirin. Varsayılan olarak, 0 (devre dışı). + +Derleme yalnızca sorgu işleme boru hattının bir parçası için kullanılır: toplamanın ilk aşaması için (GROUP BY). +Potansiyel hattın bu bölümü derlenmişse, sorgu, kısa döngüleri ve inlining toplu işlev çağrılarının dağıtımı nedeniyle daha hızlı çalışabilir. Birden çok basit toplama işlevine sahip sorgular için maksimum performans artışı (nadir durumlarda dört kata kadar daha hızlı) görülür. Tipik olarak, performans kazancı önemsİzdİr. Çok nadir durumlarda, sorgu yürütülmesini yavaşlatabilir. + +## min\_count\_to\_compile {#min-count-to-compile} + +Derleme çalıştırmadan önce derlenmiş bir kod yığını potansiyel olarak kaç kez kullanılır. Varsayılan olarak, 3. +For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values ​​starting with 1. Compilation normally takes about 5-10 seconds. +Değer 1 veya daha fazla ise, derleme zaman uyumsuz olarak ayrı bir iş parçacığında oluşur. Sonuç, şu anda çalışmakta olan sorgular da dahil olmak üzere hazır olduğu anda kullanılacaktır. + +Derlenmiş kod, sorguda kullanılan toplama işlevlerinin her farklı birleşimi ve GROUP BY yan tümcesindeki anahtarların türü için gereklidir. +The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. + +## output\_format\_json\_quote\_64bit\_integers {#session_settings-output_format_json_quote_64bit_integers} + +Değer doğruysa, json\* Int64 ve Uİnt64 formatlarını kullanırken tamsayılar tırnak içinde görünür (çoğu JavaScript uygulamasıyla uyumluluk için); aksi takdirde, tamsayılar tırnak işaretleri olmadan çıktılanır. + +## format\_csv\_delimiter {#settings-format_csv_delimiter} + +Karakter CSV verilerinde bir sınırlayıcı olarak yorumlanır. Varsayılan olarak, sınırlayıcı `,`. + +## ınput\_format\_csv\_unquoted\_null\_literal\_as\_null {#settings-input_format_csv_unquoted_null_literal_as_null} + +CSV giriş biçimi sağlar veya unquoted ayrıştırma devre dışı bırakır için `NULL` literal olarak (eşanlamlı `\N`). + +## output\_format\_csv\_crlf\_end\_of\_line {#settings-output-format-csv-crlf-end-of-line} + +Unix stili (LF) yerine CSV'DE DOS/Windows stili çizgi ayırıcı (CRLF) kullanın. + +## output\_format\_tsv\_crlf\_end\_of\_line {#settings-output-format-tsv-crlf-end-of-line} + +Unıx stili (LF) yerine TSV'DE DOC/Windows stili çizgi ayırıcı (CRLF) kullanın. + +## insert\_quorum {#settings-insert_quorum} + +Çekirdek yazma sağlar. + +- Eğer `insert_quorum < 2`, çekirdek yazma devre dışı bırakılır. +- Eğer `insert_quorum >= 2`, çekirdek yazma etkin. + +Varsayılan değer: 0. + +Nis writesap yazar + +`INSERT` yalnızca ClickHouse verileri doğru bir şekilde yazmayı başardığında başarılı olur. `insert_quorum` sırasında kopya ofların `insert_quorum_timeout`. Herhangi bir nedenle başarılı yazma ile kopya sayısı ulaşmazsa `insert_quorum`, yazma başarısız olarak kabul edilir ve ClickHouse, verilerin zaten yazıldığı tüm kopyalardan eklenen bloğu siler. + +Nisaptaki tüm kopyalar tutarlıdır, yani önceki tüm verileri içerir `INSERT` sorgular. Bu `INSERT` sıra doğrusallaştırılmıştır. + +Yazılan verileri okurken `insert_quorum` olabilir kullanın [select\_sequential\_consistency](#settings-select_sequential_consistency) seçenek. + +ClickHouse bir istisna oluşturur + +- Sorgu sırasında kullanılabilir yinelemelerin sayısı daha az ise `insert_quorum`. +- Önceki blok henüz eklenmemiş olduğunda veri yazma girişiminde `insert_quorum` kopyaların. Bu durum, bir kullanıcı gerçekleştirmeye çalışırsa oluşabilir. `INSERT` ile bir öncekinden önce `insert_quorum` Tamam islanmıştır. + +Ayrıca bakınız: + +- [ınsert\_quorum\_timeout](#settings-insert_quorum_timeout) +- [select\_sequential\_consistency](#settings-select_sequential_consistency) + +## ınsert\_quorum\_timeout {#settings-insert_quorum-timeout} + +Çekirdek zaman aşımına saniyeler içinde yazın. Zaman aşımı geçti ve yazma henüz gerçekleşmedi, ClickHouse bir özel durum oluşturur ve istemci aynı bloğu aynı veya başka bir yineleme yazmak için sorguyu yinelemeniz gerekir. + +Varsayılan değer: 60 saniye. + +Ayrıca bakınız: + +- [insert\_quorum](#settings-insert_quorum) +- [select\_sequential\_consistency](#settings-select_sequential_consistency) + +## select\_sequential\_consistency {#settings-select_sequential_consistency} + +İçin sıralı tutarlılığı etkinleştirir veya devre dışı bırakır `SELECT` sorgular: + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 0. + +Kullanma + +Sıralı tutarlılık etkinleştirildiğinde, clickhouse istemci çalıştırmak sağlar `SELECT` yalnızca önceki tüm verileri içeren yinelemeler için sorgu `INSERT` ile yürütülen sorgular `insert_quorum`. Istemci kısmi bir yineleme başvurursa, ClickHouse bir özel durum oluşturur. SELECT sorgusu yinelemeler çekirdek için henüz yazılmamış verileri içermez. + +Ayrıca bakınız: + +- [insert\_quorum](#settings-insert_quorum) +- [ınsert\_quorum\_timeout](#settings-insert_quorum_timeout) + +## ınsert\_deduplicate {#settings-insert-deduplicate} + +Blok tekilleştirmesini etkinleştirir veya devre dışı bırakır `INSERT` (çoğaltılmış \* tablolar için). + +Olası değerler: + +- 0 — Disabled. +- 1 — Enabled. + +Varsayılan değer: 1. + +Varsayılan olarak, çoğaltılmış tablolara eklenen bloklar `INSERT` deyim tekilleştirilir (bkz. \[Data Replication\] (../engines/table\_engines/mergetree\_family/replication.md). + +## deduplicate\_blocks\_ın\_dependent\_materialized\_views {#settings-deduplicate-blocks-in-dependent-materialized-views} + +Yinelenmiş\* tablolardan veri alan materialized görünümler için tekilleştirme denetimini etkinleştirir veya devre dışı bırakır. + +Olası değerler: + + 0 — Disabled. + 1 — Enabled. + +Varsayılan değer: 0. + +Kullanma + +Varsayılan olarak, tekilleştirme materialized görünümler için gerçekleştirilmez, ancak kaynak tabloda, Yukarı akış yapılır. +Eklenen bir blok, kaynak tablodaki tekilleştirme nedeniyle atlanırsa, ekli materialized görünümlerine ekleme olmaz. Bu davranış, eklenen blokların materialized görünüm toplamasından sonra aynı olduğu, ancak kaynak tabloya farklı eklerden türetildiği durumlar için, yüksek oranda toplanmış verilerin materialized görünümlere eklenmesini sağlamak için vardır. +Aynı zamanda, bu davranış “breaks” `INSERT` idempotency. Eğer bir `INSERT` ana tabloya başarılı oldu ve `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won't receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` bu davranışı değiştirmeye izin verir. Yeniden denemede, somutlaştırılmış bir görünüm tekrar ekleme işlemini alacak ve tekilleştirme kontrolünü kendi başına gerçekleştirecektir, +kaynak tablo için onay sonucunu yoksayar ve ilk hata nedeniyle kaybedilen satırları ekler. + +## max\_network\_bytes {#settings-max-network-bytes} + +Alınan veya bir sorgu yürütülürken ağ üzerinden iletilen veri birimi (bayt cinsinden) sınırlar. Bu ayar, her bir sorgu için geçerlidir. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Data volume control is disabled. + +Varsayılan değer: 0. + +## max\_network\_bandwidth {#settings-max-network-bandwidth} + +Ağ üzerinden veri alışverişinin hızını saniyede bayt cinsinden sınırlar. Bu ayar her sorgu için geçerlidir. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Bandwidth control is disabled. + +Varsayılan değer: 0. + +## max\_network\_bandwidth\_for\_user {#settings-max-network-bandwidth-for-user} + +Ağ üzerinden veri alışverişinin hızını saniyede bayt cinsinden sınırlar. Bu ayar, tek bir kullanıcı tarafından gerçekleştirilen tüm aynı anda çalışan sorgular için geçerlidir. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Control of the data speed is disabled. + +Varsayılan değer: 0. + +## max\_network\_bandwidth\_for\_all\_users {#settings-max-network-bandwidth-for-all-users} + +Verilerin ağ üzerinden saniyede bayt olarak değiştirildiği hızı sınırlar. Bu ayar, sunucuda aynı anda çalışan tüm sorgular için geçerlidir. + +Olası değerler: + +- Pozitif tamsayı. +- 0 — Control of the data speed is disabled. + +Varsayılan değer: 0. + +## count\_distinct\_implementation {#settings-count_distinct_implementation} + +Aşağıdakilerden hang theisinin `uniq*` işlevleri gerçekleştirmek için kullanılmalıdır [COUNT(DISTINCT …)](../../sql_reference/aggregate_functions/reference.md#agg_function-count) yapma. + +Olası değerler: + +- [uniq](../../sql_reference/aggregate_functions/reference.md#agg_function-uniq) +- [uniqCombined](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqcombined) +- [uniqCombined64](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqcombined64) +- [uniqHLL12](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqhll12) +- [uniqExact](../../sql_reference/aggregate_functions/reference.md#agg_function-uniqexact) + +Varsayılan değer: `uniqExact`. + +## skip\_unavailable\_shards {#settings-skip_unavailable_shards} + +Etkinleştirir veya sessizce kullanılamaz kırıkları atlama devre dışı bırakır. + +Tüm kopyaları kullanılamıyorsa, Shard kullanılamaz olarak kabul edilir. Aşağıdaki durumlarda bir yineleme kullanılamaz: + +- ClickHouse herhangi bir nedenle kopya bağlanamıyor. + + Bir kopyaya bağlanırken, ClickHouse birkaç deneme gerçekleştirir. Tüm bu girişimler başarısız olursa, çoğaltma kullanılamaz kabul edilir. + +- Çoğaltma DNS üzerinden çözülemez. + + Çoğaltmanın ana bilgisayar adı DNS aracılığıyla çözümlenemezse, aşağıdaki durumları gösterebilir: + + - Çoğaltma ana bilgisayar DNS kaydı yok. Dinamik DNS'YE sahip sistemlerde oluşabilir, örneğin, [Kubernetes](https://kubernetes.io), burada düğümler kesinti sırasında çözülmez olabilir ve bu bir hata değildir. + + - Yapılandırma hatası. ClickHouse yapılandırma dosyası yanlış bir ana bilgisayar adı içerir. + +Olası değerler: + +- 1 — skipping enabled. + + Bir parça kullanılamıyorsa, ClickHouse kısmi verilere dayalı bir sonuç döndürür ve düğüm kullanılabilirliği sorunlarını bildirmez. + +- 0 — skipping disabled. + + Bir shard kullanılamıyorsa, ClickHouse bir özel durum atar. + +Varsayılan değer: 0. + +## optimize\_skip\_unused\_shards {#settings-optimize_skip_unused_shards} + +Prewhere/WHERE (verilerin sharding anahtarı tarafından dağıtıldığını varsayar, aksi takdirde hiçbir şey yapmaz). + +Varsayılan değer: 0 + +## force\_optimize\_skip\_unused\_shards {#settings-force_optimize_skip_unused_shards} + +Sorgu yürütülmesini etkinleştirir veya devre dışı bırakır [`optimize_skip_unused_shards`](#settings-optimize_skip_unused_shards) etkin ve kullanılmayan kırıkları atlama mümkün değildir. Atlama mümkün değilse ve ayar etkinse özel durum atılır. + +Olası değerler: + +- 0-Devre Dışı (at notmayın) +- 1-sorgu yürütülmesini yalnızca tablonun sharding anahtarı varsa devre dışı bırakın +- 2-devre dışı sorgu yürütme ne olursa olsun sharding anahtar tablo için tanımlanır + +Varsayılan değer: 0 + +## force\_optimize\_skip\_unused\_shards\_no\_nested {#settings-force_optimize_skip_unused_shards_no_nested} + +Sıfırlamak [`optimize_skip_unused_shards`](#settings-force_optimize_skip_unused_shards) iç içe geçmiş için `Distributed` Tablo + +Olası değerler: + +- 1 — Enabled. +- 0 — Disabled. + +Varsayılan değer: 0. + +## optimize\_throw\_if\_noop {#setting-optimize_throw_if_noop} + +Bir özel durum atmayı etkinleştirir veya devre dışı bırakır. [OPTIMIZE](../../sql_reference/statements/misc.md#misc_operations-optimize) sorgu birleştirme gerçekleştirmedi. + +Varsayılan olarak, `OPTIMIZE` eğer hiç bir şey yapmamış olsa bile, başarılı bir şekilde verir. Bu ayar, bu durumları ayırt etmenizi ve bir özel durum iletisinde nedeni almanızı sağlar. + +Olası değerler: + +- 1 — Throwing an exception is enabled. +- 0 — Throwing an exception is disabled. + +Varsayılan değer: 0. + +## distributed\_replica\_error\_half\_life {#settings-distributed_replica_error_half_life} + +- Türü: saniye +- Varsayılan değer: 60 saniye + +Dağıtılmış tablolardaki hataların ne kadar hızlı sıfırlandığını denetler. Bir yineleme bir süre için kullanılamıyorsa, 5 hataları biriktirir ve distributed\_replica\_error\_half\_lıfe 1 saniye olarak ayarlanır, sonra yineleme son hatadan sonra normal 3 saniye olarak kabul edilir. + +Ayrıca bakınız: + +- [Masa motoru Dağıt Distributedıldı](../../engines/table_engines/special/distributed.md) +- [distributed\_replica\_error\_cap](#settings-distributed_replica_error_cap) + +## distributed\_replica\_error\_cap {#settings-distributed_replica_error_cap} + +- Tür: imzasız int +- Varsayılan değer: 1000 + +Her yineleme hata sayısı çok fazla hata biriken tek bir yineleme engelleyerek, bu değerle kaplıdır. + +Ayrıca bakınız: + +- [Masa motoru Dağıt Distributedıldı](../../engines/table_engines/special/distributed.md) +- [distributed\_replica\_error\_half\_life](#settings-distributed_replica_error_half_life) + +## distributed\_directory\_monitor\_sleep\_time\_ms {#distributed_directory_monitor_sleep_time_ms} + +İçin taban aralığı [Dağılı](../../engines/table_engines/special/distributed.md) veri göndermek için tablo motoru. Gerçek Aralık, hatalar durumunda katlanarak büyür. + +Olası değerler: + +- Milisaniye pozitif tamsayı sayısı. + +Varsayılan değer: 100 milisaniye. + +## distributed\_directory\_monitor\_max\_sleep\_time\_ms {#distributed_directory_monitor_max_sleep_time_ms} + +İçin Maksimum Aralık [Dağılı](../../engines/table_engines/special/distributed.md) veri göndermek için tablo motoru. Sınırları içinde belirlenen Aralık üstel büyüme [distributed\_directory\_monitor\_sleep\_time\_ms](#distributed_directory_monitor_sleep_time_ms) ayar. + +Olası değerler: + +- Milisaniye pozitif tamsayı sayısı. + +Varsayılan değer: 30000 milisaniye (30 saniye). + +## distributed\_directory\_monitor\_batch\_ınserts {#distributed_directory_monitor_batch_inserts} + +Eklenen verilerin toplu olarak gönderilmesini etkinleştirir / devre dışı bırakır. + +Toplu gönderme etkinleştirildiğinde, [Dağılı](../../engines/table_engines/special/distributed.md) table engine, eklenen verilerin birden çok dosyasını ayrı ayrı göndermek yerine tek bir işlemde göndermeye çalışır. Toplu gönderme, sunucu ve ağ kaynaklarını daha iyi kullanarak küme performansını artırır. + +Olası değerler: + +- 1 — Enabled. +- 0 — Disabled. + +Varsayılan değer: 0. + +## os\_thread\_priority {#setting-os-thread-priority} + +Önceliği ayarlar ([güzel](https://en.wikipedia.org/wiki/Nice_(Unix))) sorguları yürüten iş parçacıkları için. İşletim sistemi Zamanlayıcısı, kullanılabilir her CPU çekirdeğinde çalışacak bir sonraki iş parçacığını seçerken bu önceliği dikkate alır. + +!!! warning "Uyarıcı" + Bu ayarı kullanmak için, `CAP_SYS_NICE` özellik. Bu `clickhouse-server` paket kurulum sırasında kurar. Bazı sanal ortamlar ayarlamanıza izin vermez `CAP_SYS_NICE` özellik. Bu durumda, `clickhouse-server` Başlangıçta bu konuda bir mesaj gösterir. + +Olası değerler: + +- Aralıktaki değerleri ayarlayabilirsiniz `[-20, 19]`. + +Daha düşük değerler daha yüksek öncelik anlamına gelir. Düşük olan iplikler `nice` öncelik değerleri, yüksek değerlere sahip iş parçacıklarından daha sık yürütülür. Yüksek değerler, uzun süren etkileşimli olmayan sorgular için tercih edilir, çünkü geldiklerinde kısa etkileşimli sorgular lehine kaynakları hızlı bir şekilde bırakmalarına izin verir. + +Varsayılan değer: 0. + +## query\_profiler\_real\_time\_period\_ns {#query_profiler_real_time_period_ns} + +Gerçek bir saat zamanlayıcı için süreyi ayarlar [sorgu profiler](../../operations/optimizing_performance/sampling_query_profiler.md). Gerçek saat zamanlayıcı duvar saati zaman sayar. + +Olası değerler: + +- Nanosaniye cinsinden pozitif tam sayı. + + Önerilen değerler: + + - 10000000 (100 times a second) nanoseconds and less for single queries. + - 1000000000 (once a second) for cluster-wide profiling. + +- Zamanlayıcıyı kapatmak için 0. + +Tür: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +Varsayılan değer: 1000000000 nanosaniye (saniyede bir kez). + +Ayrıca bakınız: + +- Sistem tablosu [trace\_log](../../operations/system_tables.md#system_tables-trace_log) + +## query\_profiler\_cpu\_time\_period\_ns {#query_profiler_cpu_time_period_ns} + +Bir CPU saat süreölçerinin dönemini ayarlar. [sorgu profiler](../../operations/optimizing_performance/sampling_query_profiler.md). Bu zamanlayıcı sadece CPU süresini sayar. + +Olası değerler: + +- Nanosaniye pozitif tamsayı sayısı. + + Önerilen değerler: + + - 10000000 (100 times a second) nanoseconds and more for single queries. + - 1000000000 (once a second) for cluster-wide profiling. + +- Zamanlayıcıyı kapatmak için 0. + +Tür: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +Varsayılan değer: 1000000000 nanosaniye. + +Ayrıca bakınız: + +- Sistem tablosu [trace\_log](../../operations/system_tables.md#system_tables-trace_log) + +## allow\_introspection\_functions {#settings-allow_introspection_functions} + +Devre dışı bırakmayı etkinleştirir [ıntrospections fonksiyonları](../../sql_reference/functions/introspection.md) sorgu profilleme için. + +Olası değerler: + +- 1 — Introspection functions enabled. +- 0 — Introspection functions disabled. + +Varsayılan değer: 0. + +**Ayrıca Bakınız** + +- [Örnekleme Sorgusu Profiler](../optimizing_performance/sampling_query_profiler.md) +- Sistem tablosu [trace\_log](../../operations/system_tables.md#system_tables-trace_log) + +## ınput\_format\_parallel\_parsing {#input-format-parallel-parsing} + +- Tipi: bool +- Varsayılan değer: True + +Veri biçimlerinin paralel ayrıştırma sırasını koruyarak etkinleştirin. Sadece TSV, TKSV, CSV ve JSONEachRow formatları için desteklenir. + +## min\_chunk\_bytes\_for\_parallel\_parsing {#min-chunk-bytes-for-parallel-parsing} + +- Tür: imzasız int +- Varsayılan değer: 1 MiB + +Her iş parçacığının paralel olarak ayrıştırılacağı bayt cinsinden minimum yığın boyutu. + +## output\_format\_avro\_codec {#settings-output_format_avro_codec} + +Çıkış Avro dosyası için kullanılan sıkıştırma codec ayarlar. + +Tipi: dize + +Olası değerler: + +- `null` — No compression +- `deflate` — Compress with Deflate (zlib) +- `snappy` — Compress with [Çabuk](https://google.github.io/snappy/) + +Varsayılan değer: `snappy` (varsa) veya `deflate`. + +## output\_format\_avro\_sync\_interval {#settings-output_format_avro_sync_interval} + +Çıkış Avro dosyası için senkronizasyon işaretçileri arasında minimum veri boyutunu (bayt cinsinden) ayarlar. + +Tür: imzasız int + +Olası değerler: 32 (32 bayt) - 1073741824 (1 GiB) + +Varsayılan değer: 32768 (32 KiB) + +## format\_avro\_schema\_registry\_url {#settings-format_avro_schema_registry_url} + +Sets Confluent Schema Registry URL to use with [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent) biçimli + +Type: URL + +Varsayılan değer: boş + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/tr/operations/settings/settings_profiles.md b/docs/tr/operations/settings/settings_profiles.md new file mode 100644 index 00000000000..318276ab6c8 --- /dev/null +++ b/docs/tr/operations/settings/settings_profiles.md @@ -0,0 +1,71 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 61 +toc_title: Ayarlar Profilleri +--- + +# Ayarlar Profilleri {#settings-profiles} + +Ayarlar profili, aynı ad altında gruplandırılmış ayarlar topluluğudur. Her ClickHouse kullanıcısının bir profili vardır. +Bir profildeki tüm ayarları uygulamak için `profile` ayar. + +Örnek: + +Yüklemek `web` profilli. + +``` sql +SET profile = 'web' +``` + +Ayarlar profilleri kullanıcı yapılandırma dosyasında bildirilir. Bu genellikle `users.xml`. + +Örnek: + +``` xml + + + + + + 8 + + + + + 1000000000 + 100000000000 + + 1000000 + any + + 1000000 + 1000000000 + + 100000 + 100000000 + break + + 600 + 1000000 + 15 + + 25 + 100 + 50 + + 2 + 25 + 50 + 100 + + 1 + + +``` + +Örnek iki profili belirtir: `default` ve `web`. Bu `default` profilin özel bir amacı vardır: her zaman mevcut olmalı ve sunucuyu başlatırken uygulanır. Diğer bir deyişle, `default` profil varsayılan ayarları içerir. Bu `web` profil kullanılarak ayarlanabilir düzenli bir profil `SET` sorgu veya bir HTTP sorgusunda bir URL parametresi kullanma. + +Ayarlar profilleri birbirinden miras alabilir. Kalıtım kullanmak için, bir veya birden fazla belirtiniz `profile` ayarlar profilde listelenen diğer ayarlardan önce. Farklı profillerde bir ayar tanımlandığında, en son tanımlı kullanılır. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/settings_profiles/) diff --git a/docs/tr/operations/settings/settings_users.md b/docs/tr/operations/settings/settings_users.md new file mode 100644 index 00000000000..0bc2b5ac1a5 --- /dev/null +++ b/docs/tr/operations/settings/settings_users.md @@ -0,0 +1,148 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 63 +toc_title: "Kullan\u0131c\u0131 Ayarlar\u0131" +--- + +# Kullanıcı Ayarları {#user-settings} + +Bu `users` bu bölüm `user.xml` yapılandırma dosyası kullanıcı ayarlarını içerir. + +Bu yapı `users` bölme: + +``` xml + + + + + + + + + + + profile_name + + default + + + + + expression + + + + + + +``` + +### home/şifre {#user-namepassword} + +Şifre düz metin veya SHA256 (hex formatında) belirtilebilir. + +- Düz metin içinde bir şifre atamak için (**tavsiye edilmez**bir koyun `password` öğe. + + Mesela, `qwerty`. Şifre boş bırakılabilir. + + + +- SHA256 karmasını kullanarak bir şifre atamak için, bir `password_sha256_hex` öğe. + + Mesela, `65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5`. + + Kabuktan bir parola oluşturma örneği: + + PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' + + Sonucun ilk satırı şifredir. İkinci satır karşılık gelen SHA256 karmasıdır. + + + +- MySQL istemcileri ile uyumluluk için, şifre çift SHA1 karma belirtilebilir. İçine yerleştirin `password_double_sha1_hex` öğe. + + Mesela, `08b4a0f1de6ad37da17359e592c8d74788a83eb0`. + + Kabuktan bir parola oluşturma örneği: + + PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' + + Sonucun ilk satırı şifredir. İkinci satır karşılık gelen çift SHA1 karmasıdır. + +### kullanıcı\_adı / ağlar {#user-namenetworks} + +Kullanıcının ClickHouse sunucusuna bağlanabileceği ağların listesi. + +Listenin her öğesi aşağıdaki formlardan birine sahip olabilir: + +- `` — IP address or network mask. + + Örnekler: `213.180.204.3`, `10.0.0.1/8`, `10.0.0.1/255.255.255.0`, `2a02:6b8::3`, `2a02:6b8::3/64`, `2a02:6b8::3/ffff:ffff:ffff:ffff::`. + +- `` — Hostname. + + Örnek: `example01.host.ru`. + + Erişimi denetlemek için bir DNS sorgusu gerçekleştirilir ve döndürülen tüm IP adresleri eş adresiyle karşılaştırılır. + +- `` — Regular expression for hostnames. + + Örnek, `^example\d\d-\d\d-\d\.host\.ru$` + + Erişimi kontrol etmek için, bir [DNS ptr sorgusu](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) eş adresi için gerçekleştirilir ve sonra belirtilen regexp uygulanır. Daha sonra PTR sorgusunun sonuçları için başka bir DNS sorgusu gerçekleştirilir ve alınan tüm adresler eş adresine karşılaştırılır. Regexp'nin $ile bitmesini şiddetle tavsiye ederiz. + +Sunucu yeniden başlatılıncaya kadar DNS isteklerinin tüm sonuçları önbelleğe alınır. + +**Örnekler** + +Herhangi bir ağdan kullanıcı için erişimi açmak için şunları belirtin: + +``` xml +::/0 +``` + +!!! warning "Uyarıcı" + Düzgün yapılandırılmış bir güvenlik duvarınız yoksa veya sunucu doğrudan internete bağlı değilse, herhangi bir ağdan erişimi açmak güvensizdir. + +Erişimi yalnızca localhost'tan açmak için şunları belirtin: + +``` xml +::1 +127.0.0.1 +``` + +### kullanıcı\_adı / profil {#user-nameprofile} + +Kullanıcı için bir ayarlar profili atayabilirsiniz. Ayarlar profilleri ayrı bir bölümde yapılandırılır `users.xml` Dosya. Daha fazla bilgi için, bkz. [Ayarların profilleri](settings_profiles.md). + +### user\_name / kota {#user-namequota} + +Kotalar, belirli bir süre boyunca kaynak kullanımını izlemenize veya sınırlamanıza izin verir. Kotalar yapılandırılır `quotas` +bu bölüm `users.xml` yapılandırma dosyası. + +Kullanıcı için ayarlanmış bir kotalar atayabilirsiniz. Kotalar yapılandırmasının ayrıntılı bir açıklaması için bkz. [Kotalar](../quotas.md#quotas). + +### user\_name / veritabanları {#user-namedatabases} + +Bu bölümde, ClickHouse tarafından döndürülen satırları sınırlayabilirsiniz `SELECT` geçerli kullanıcı tarafından yapılan sorgular, böylece temel satır düzeyinde güvenlik uygular. + +**Örnek** + +Aşağıdaki yapılandırma bu kullanıcıyı zorlar `user1` sadece satırları görebilirsiniz `table1` sonucu olarak `SELECT` sorgular, burada değeri `id` alan 1000'dir. + +``` xml + + + + + id = 1000 + + + + +``` + +Bu `filter` bir sonuç veren herhangi bir ifade olabilir [Uİnt8](../../sql_reference/data_types/int_uint.md)- tip değeri. Genellikle karşılaştırmalar ve mantıksal operatörler içerir. Satır fromlardan `database_name.table1` burada filtre sonuçları 0 için bu kullanıcı için döndürülür. Filtreleme ile uyumsuz `PREWHERE` işlemler ve devre dışı bırakır `WHERE→PREWHERE` optimizasyon. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/settings/settings_users/) diff --git a/docs/tr/operations/system_tables.md b/docs/tr/operations/system_tables.md new file mode 100644 index 00000000000..17fd176cc58 --- /dev/null +++ b/docs/tr/operations/system_tables.md @@ -0,0 +1,1166 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 52 +toc_title: "Sistem Tablolar\u0131" +--- + +# Sistem Tabloları {#system-tables} + +Sistem tabloları, sistemin işlevselliğinin bir kısmını uygulamak ve sistemin nasıl çalıştığı hakkında bilgilere erişim sağlamak için kullanılır. +Bir sistem tablosunu silemezsiniz (ancak ayırma işlemini gerçekleştirebilirsiniz). +Sistem tablolarında diskte veri bulunan dosyalar veya meta verilere sahip dosyalar yoktur. Sunucu, başlatıldığında tüm sistem tablolarını oluşturur. +Sistem tabloları salt okunur. +Bulun theurlar. ‘system’ veritabanı. + +## sistem.asynchronous\_metrics {#system_tables-asynchronous_metrics} + +Arka planda periyodik olarak hesaplanan metrikleri içerir. Örneğin, kullanılan RAM miktarı. + +Sütun: + +- `metric` ([Dize](../sql_reference/data_types/string.md)) — Metric name. +- `value` ([Float64](../sql_reference/data_types/float.md)) — Metric value. + +**Örnek** + +``` sql +SELECT * FROM system.asynchronous_metrics LIMIT 10 +``` + +``` text +┌─metric──────────────────────────────────┬──────value─┐ +│ jemalloc.background_thread.run_interval │ 0 │ +│ jemalloc.background_thread.num_runs │ 0 │ +│ jemalloc.background_thread.num_threads │ 0 │ +│ jemalloc.retained │ 422551552 │ +│ jemalloc.mapped │ 1682989056 │ +│ jemalloc.resident │ 1656446976 │ +│ jemalloc.metadata_thp │ 0 │ +│ jemalloc.metadata │ 10226856 │ +│ UncompressedCacheCells │ 0 │ +│ MarkCacheFiles │ 0 │ +└─────────────────────────────────────────┴────────────┘ +``` + +**Ayrıca Bakınız** + +- [İzleme](monitoring.md) — Base concepts of ClickHouse monitoring. +- [sistem.metrik](#system_tables-metrics) — Contains instantly calculated metrics. +- [sistem.etkinlik](#system_tables-events) — Contains a number of events that have occurred. +- [sistem.metric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. + +## sistem.kümeler {#system-clusters} + +Yapılandırma dosyasında bulunan kümeler ve içindeki sunucular hakkında bilgi içerir. + +Sütun: + +- `cluster` (String) — The cluster name. +- `shard_num` (UInt32) — The shard number in the cluster, starting from 1. +- `shard_weight` (UInt32) — The relative weight of the shard when writing data. +- `replica_num` (UInt32) — The replica number in the shard, starting from 1. +- `host_name` (String) — The host name, as specified in the config. +- `host_address` (String) — The host IP address obtained from DNS. +- `port` (UInt16) — The port to use for connecting to the server. +- `user` (String) — The name of the user for connecting to the server. +- `errors_count` (Uİnt32) - bu ana bilgisayarın çoğaltma ulaşamadı sayısı. +- `estimated_recovery_time` (Uİnt32) - çoğaltma hata sayısı sıfırlanana kadar saniye kaldı ve normale döndü olarak kabul edilir. + +Lütfen unutmayın `errors_count` küme için sorgu başına bir kez güncelleştirilir, ancak `estimated_recovery_time` isteğe bağlı olarak yeniden hesaplanır. Yani sıfır olmayan bir durum olabilir `errors_count` ve sıfır `estimated_recovery_time`, sonraki sorgu sıfır olacak `errors_count` ve hiçbir hata yokmuş gibi çoğaltma kullanmayı deneyin. + +**Ayrıca bakınız** + +- [Masa motoru Dağıt Distributedıldı](../engines/table_engines/special/distributed.md) +- [distributed\_replica\_error\_cap ayarı](settings/settings.md#settings-distributed_replica_error_cap) +- [distributed\_replica\_error\_half\_life ayarı](settings/settings.md#settings-distributed_replica_error_half_life) + +## sistem.sütun {#system-columns} + +Tüm tablolardaki sütunlar hakkında bilgi içerir. + +Benzer bilgileri almak için bu tabloyu kullanabilirsiniz [DESCRIBE TABLE](../sql_reference/statements/misc.md#misc-describe-table) sorgu, ancak aynı anda birden çok tablo için. + +Bu `system.columns` tablo aşağıdaki sütunları içerir (sütun türü parantez içinde gösterilir): + +- `database` (String) — Database name. +- `table` (String) — Table name. +- `name` (String) — Column name. +- `type` (String) — Column type. +- `default_kind` (String) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS`) varsayılan değer veya tanımlanmamışsa boş bir dize için. +- `default_expression` (String) — Expression for the default value, or an empty string if it is not defined. +- `data_compressed_bytes` (UInt64) — The size of compressed data, in bytes. +- `data_uncompressed_bytes` (UInt64) — The size of decompressed data, in bytes. +- `marks_bytes` (UInt64) — The size of marks, in bytes. +- `comment` (String) — Comment on the column, or an empty string if it is not defined. +- `is_in_partition_key` (UInt8) — Flag that indicates whether the column is in the partition expression. +- `is_in_sorting_key` (UInt8) — Flag that indicates whether the column is in the sorting key expression. +- `is_in_primary_key` (UInt8) — Flag that indicates whether the column is in the primary key expression. +- `is_in_sampling_key` (UInt8) — Flag that indicates whether the column is in the sampling key expression. + +## sistem.katılımcılar {#system-contributors} + +Katkıda bulunanlar hakkında bilgi içerir. Rastgele sırayla tüm constributors. Sipariş, sorgu yürütme zamanında rasgele olur. + +Sütun: + +- `name` (String) — Contributor (author) name from git log. + +**Örnek** + +``` sql +SELECT * FROM system.contributors LIMIT 10 +``` + +``` text +┌─name─────────────┐ +│ Olga Khvostikova │ +│ Max Vetrov │ +│ LiuYangkuan │ +│ svladykin │ +│ zamulla │ +│ Šimon Podlipský │ +│ BayoNet │ +│ Ilya Khomutov │ +│ Amy Krishnevsky │ +│ Loud_Scream │ +└──────────────────┘ +``` + +Tabloda kendinizi bulmak için bir sorgu kullanın: + +``` sql +SELECT * FROM system.contributors WHERE name='Olga Khvostikova' +``` + +``` text +┌─name─────────────┐ +│ Olga Khvostikova │ +└──────────────────┘ +``` + +## sistem.veritabanılar {#system-databases} + +Bu tablo, adı verilen tek bir dize sütunu içerir ‘name’ – the name of a database. +Sunucunun bildiği her veritabanı, tabloda karşılık gelen bir girdiye sahiptir. +Bu sistem tablosu uygulamak için kullanılır `SHOW DATABASES` sorgu. + +## sistem.detached\_parts {#system_tables-detached_parts} + +Müstakil parçaları hakkında bilgiler içerir [MergeTree](../engines/table_engines/mergetree_family/mergetree.md) Tablolar. Bu `reason` sütun, parçanın neden ayrıldığını belirtir. Kullanıcı tarafından ayrılmış parçalar için sebep boştur. Bu tür parçalar ile eklenebilir [ALTER TABLE ATTACH PARTITION\|PART](../sql_reference/statements/alter.md#alter_attach-partition) komut. Diğer sütunların açıklaması için bkz. [sistem.parçalar](#system_tables-parts). Bölüm adı geçersiz ise, bazı sütunların değerleri olabilir `NULL`. Bu tür parçalar ile silinebilir [ALTER TABLE DROP DETACHED PART](../sql_reference/statements/alter.md#alter_drop-detached). + +## sistem.sözlükler {#system_tables-dictionaries} + +Hakkında bilgi içerir [dış söz dictionarieslükler](../sql_reference/dictionaries/external_dictionaries/external_dicts.md). + +Sütun: + +- `database` ([Dize](../sql_reference/data_types/string.md)) — Name of the database containing the dictionary created by DDL query. Empty string for other dictionaries. +- `name` ([Dize](../sql_reference/data_types/string.md)) — [Sözlük adı](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md). +- `status` ([Enum8](../sql_reference/data_types/enum.md)) — Dictionary status. Possible values: + - `NOT_LOADED` — Dictionary was not loaded because it was not used. + - `LOADED` — Dictionary loaded successfully. + - `FAILED` — Unable to load the dictionary as a result of an error. + - `LOADING` — Dictionary is loading now. + - `LOADED_AND_RELOADING` — Dictionary is loaded successfully, and is being reloaded right now (frequent reasons: [SYSTEM RELOAD DICTIONARY](../sql_reference/statements/system.md#query_language-system-reload-dictionary) sorgu, zaman aşımı, sözlük yapılandırması değişti). + - `FAILED_AND_RELOADING` — Could not load the dictionary as a result of an error and is loading now. +- `origin` ([Dize](../sql_reference/data_types/string.md)) — Path to the configuration file that describes the dictionary. +- `type` ([Dize](../sql_reference/data_types/string.md)) — Type of a dictionary allocation. [Sözlükleri bellekte saklama](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md). +- `key` — [Anahtar tipi](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-key): Sayısal Tuş ([Uİnt64](../sql_reference/data_types/int_uint.md#uint-ranges)) or Сomposite key ([Dize](../sql_reference/data_types/string.md)) — form “(type 1, type 2, …, type n)”. +- `attribute.names` ([Dizi](../sql_reference/data_types/array.md)([Dize](../sql_reference/data_types/string.md))) — Array of [öznitelik adları](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes) sözlük tarafından sağlanmıştır. +- `attribute.types` ([Dizi](../sql_reference/data_types/array.md)([Dize](../sql_reference/data_types/string.md))) — Corresponding array of [öznitelik türleri](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes) sözlük tarafından sağlanmaktadır. +- `bytes_allocated` ([Uİnt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary. +- `query_count` ([Uİnt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot. +- `hit_rate` ([Float64](../sql_reference/data_types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache. +- `element_count` ([Uİnt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Number of items stored in the dictionary. +- `load_factor` ([Float64](../sql_reference/data_types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table). +- `source` ([Dize](../sql_reference/data_types/string.md)) — Text describing the [veri kaynağı](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md) sözlük için. +- `lifetime_min` ([Uİnt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Minimum [ömür](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md) bellekteki sözlüğün ardından ClickHouse sözlüğü yeniden yüklemeye çalışır (eğer `invalidate_query` ayarlanır, daha sonra sadece değiştiyse). Saniyeler içinde ayarlayın. +- `lifetime_max` ([Uİnt64](../sql_reference/data_types/int_uint.md#uint-ranges)) — Maximum [ömür](../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md) bellekteki sözlüğün ardından ClickHouse sözlüğü yeniden yüklemeye çalışır (eğer `invalidate_query` ayarlanır, daha sonra sadece değiştiyse). Saniyeler içinde ayarlayın. +- `loading_start_time` ([DateTime](../sql_reference/data_types/datetime.md)) — Start time for loading the dictionary. +- `last_successful_update_time` ([DateTime](../sql_reference/data_types/datetime.md)) — End time for loading or updating the dictionary. Helps to monitor some troubles with external sources and investigate causes. +- `loading_duration` ([Float32](../sql_reference/data_types/float.md)) — Duration of a dictionary loading. +- `last_exception` ([Dize](../sql_reference/data_types/string.md)) — Text of the error that occurs when creating or reloading the dictionary if the dictionary couldn't be created. + +**Örnek** + +Sözlüğü yapılandırın. + +``` sql +CREATE DICTIONARY dictdb.dict +( + `key` Int64 DEFAULT -1, + `value_default` String DEFAULT 'world', + `value_expression` String DEFAULT 'xxx' EXPRESSION 'toString(127 * 172)' +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dicttbl' DB 'dictdb')) +LIFETIME(MIN 0 MAX 1) +LAYOUT(FLAT()) +``` + +Sözlüğün yüklendiğinden emin olun. + +``` sql +SELECT * FROM system.dictionaries +``` + +``` text +┌─database─┬─name─┬─status─┬─origin──────┬─type─┬─key────┬─attribute.names──────────────────────┬─attribute.types─────┬─bytes_allocated─┬─query_count─┬─hit_rate─┬─element_count─┬───────────load_factor─┬─source─────────────────────┬─lifetime_min─┬─lifetime_max─┬──loading_start_time─┌──last_successful_update_time─┬──────loading_duration─┬─last_exception─┐ +│ dictdb │ dict │ LOADED │ dictdb.dict │ Flat │ UInt64 │ ['value_default','value_expression'] │ ['String','String'] │ 74032 │ 0 │ 1 │ 1 │ 0.0004887585532746823 │ ClickHouse: dictdb.dicttbl │ 0 │ 1 │ 2020-03-04 04:17:34 │ 2020-03-04 04:30:34 │ 0.002 │ │ +└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ +``` + +## sistem.etkinlik {#system_tables-events} + +Sistemde meydana gelen olayların sayısı hakkında bilgi içerir. Örneğin, tabloda kaç tane bulabilirsiniz `SELECT` ClickHouse sunucusu başladığından beri sorgular işlendi. + +Sütun: + +- `event` ([Dize](../sql_reference/data_types/string.md)) — Event name. +- `value` ([Uİnt64](../sql_reference/data_types/int_uint.md)) — Number of events occurred. +- `description` ([Dize](../sql_reference/data_types/string.md)) — Event description. + +**Örnek** + +``` sql +SELECT * FROM system.events LIMIT 5 +``` + +``` text +┌─event─────────────────────────────────┬─value─┬─description────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Query │ 12 │ Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries. │ +│ SelectQuery │ 8 │ Same as Query, but only for SELECT queries. │ +│ FileOpen │ 73 │ Number of files opened. │ +│ ReadBufferFromFileDescriptorRead │ 155 │ Number of reads (read/pread) from a file descriptor. Does not include sockets. │ +│ ReadBufferFromFileDescriptorReadBytes │ 9931 │ Number of bytes read from file descriptors. If the file is compressed, this will show the compressed data size. │ +└───────────────────────────────────────┴───────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [sistem.asynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. +- [sistem.metrik](#system_tables-metrics) — Contains instantly calculated metrics. +- [sistem.metric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. +- [İzleme](monitoring.md) — Base concepts of ClickHouse monitoring. + +## sistem.işlevler {#system-functions} + +Normal ve toplama işlevleri hakkında bilgi içerir. + +Sütun: + +- `name`(`String`) – The name of the function. +- `is_aggregate`(`UInt8`) — Whether the function is aggregate. + +## sistem.graphite\_retentions {#system-graphite-retentions} + +Parametreleri hakkında bilgi içerir [graphite\_rollup](server_configuration_parameters/settings.md#server_configuration_parameters-graphite_rollup) tablo usedlarında kullanılan [\* Graphıtemergetree](../engines/table_engines/mergetree_family/graphitemergetree.md) motorlar. + +Sütun: + +- `config_name` (Dize) - `graphite_rollup` parametre adı. +- `regexp` (String) - metrik adı için bir desen. +- `function` (String) - toplama işlevinin adı. +- `age` (Uint64) - saniye cinsinden verilerin minimum yaş. +- `precision` (Uİnt64) - verilerin yaşını saniyeler içinde tam olarak tanımlamak için. +- `priority` (Uİnt16) - desen önceliği. +- `is_default` (Uİnt8) - desenin varsayılan olup olmadığı. +- `Tables.database` (Array (String)) - kullanılan veritabanı tablolarının adlarının dizisi `config_name` parametre. +- `Tables.table` (Array (String)) - kullanılan tablo adları dizisi `config_name` parametre. + +## sistem.birleştiriyor {#system-merges} + +Mergetree ailesindeki tablolar için şu anda işlemde olan birleştirme ve parça mutasyonları hakkında bilgi içerir. + +Sütun: + +- `database` (String) — The name of the database the table is in. +- `table` (String) — Table name. +- `elapsed` (Float64) — The time elapsed (in seconds) since the merge started. +- `progress` (Float64) — The percentage of completed work from 0 to 1. +- `num_parts` (UInt64) — The number of pieces to be merged. +- `result_part_name` (String) — The name of the part that will be formed as the result of merging. +- `is_mutation` (Uİnt8 ) - 1 Bu işlem bir parça mutasyonu ise. +- `total_size_bytes_compressed` (UInt64) — The total size of the compressed data in the merged chunks. +- `total_size_marks` (UInt64) — The total number of marks in the merged parts. +- `bytes_read_uncompressed` (UInt64) — Number of bytes read, uncompressed. +- `rows_read` (UInt64) — Number of rows read. +- `bytes_written_uncompressed` (UInt64) — Number of bytes written, uncompressed. +- `rows_written` (UInt64) — Number of rows written. + +## sistem.metrik {#system_tables-metrics} + +Anında hesaplanan veya geçerli bir değere sahip olabilir metrikleri içerir. Örneğin, aynı anda işlenen sorguların sayısı veya geçerli yineleme gecikmesi. Bu tablo her zaman güncel. + +Sütun: + +- `metric` ([Dize](../sql_reference/data_types/string.md)) — Metric name. +- `value` ([Int64](../sql_reference/data_types/int_uint.md)) — Metric value. +- `description` ([Dize](../sql_reference/data_types/string.md)) — Metric description. + +Desteklenen metriklerin listesi [src / ortak / CurrentMetrics.cpp](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/CurrentMetrics.cpp) ClickHouse kaynak dosyası. + +**Örnek** + +``` sql +SELECT * FROM system.metrics LIMIT 10 +``` + +``` text +┌─metric─────────────────────┬─value─┬─description──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Query │ 1 │ Number of executing queries │ +│ Merge │ 0 │ Number of executing background merges │ +│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │ +│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │ +│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │ +│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │ +│ BackgroundPoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping) │ +│ BackgroundSchedulePoolTask │ 0 │ Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc. │ +│ DiskSpaceReservedForMerge │ 0 │ Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts. │ +│ DistributedSend │ 0 │ Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode. │ +└────────────────────────────┴───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [sistem.asynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. +- [sistem.etkinlik](#system_tables-events) — Contains a number of events that occurred. +- [sistem.metric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. +- [İzleme](monitoring.md) — Base concepts of ClickHouse monitoring. + +## sistem.metric\_log {#system_tables-metric_log} + +Tablolardan metrik değerlerinin geçmişini içerir `system.metrics` ve `system.events`, periyodik olarak diske boşaltılır. +Metrik geçmişi koleksiyonunu açmak için `system.metric_log`, oluşturmak `/etc/clickhouse-server/config.d/metric_log.xml` aşağıdaki içerik ile: + +``` xml + + + system + metric_log
+ 7500 + 1000 + + +``` + +**Örnek** + +``` sql +SELECT * FROM system.metric_log LIMIT 1 FORMAT Vertical; +``` + +``` text +Row 1: +────── +event_date: 2020-02-18 +event_time: 2020-02-18 07:15:33 +milliseconds: 554 +ProfileEvent_Query: 0 +ProfileEvent_SelectQuery: 0 +ProfileEvent_InsertQuery: 0 +ProfileEvent_FileOpen: 0 +ProfileEvent_Seek: 0 +ProfileEvent_ReadBufferFromFileDescriptorRead: 1 +ProfileEvent_ReadBufferFromFileDescriptorReadFailed: 0 +ProfileEvent_ReadBufferFromFileDescriptorReadBytes: 0 +ProfileEvent_WriteBufferFromFileDescriptorWrite: 1 +ProfileEvent_WriteBufferFromFileDescriptorWriteFailed: 0 +ProfileEvent_WriteBufferFromFileDescriptorWriteBytes: 56 +... +CurrentMetric_Query: 0 +CurrentMetric_Merge: 0 +CurrentMetric_PartMutation: 0 +CurrentMetric_ReplicatedFetch: 0 +CurrentMetric_ReplicatedSend: 0 +CurrentMetric_ReplicatedChecks: 0 +... +``` + +**Ayrıca bakınız** + +- [sistem.asynchronous\_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics. +- [sistem.etkinlik](#system_tables-events) — Contains a number of events that occurred. +- [sistem.metrik](#system_tables-metrics) — Contains instantly calculated metrics. +- [İzleme](monitoring.md) — Base concepts of ClickHouse monitoring. + +## sistem.şiir {#system-numbers} + +Bu tablo adında tek bir uint64 sütunu içerir ‘number’ bu sıfırdan başlayarak hemen hemen tüm doğal sayıları içerir. +Bu tabloyu testler için veya kaba kuvvet araması yapmanız gerekiyorsa kullanabilirsiniz. +Bu tablodan okumalar parallelized değil. + +## sistem.numbers\_mt {#system-numbers-mt} + +Olarak aynı ‘system.numbers’ ancak okumalar paralelleştirilmiştir. Sayılar herhangi bir sırayla iade edilebilir. +Testler için kullanılır. + +## sistem.bir {#system-one} + +Bu tablo, tek bir satır içeren tek bir satır içerir ‘dummy’ 0 değerini içeren uint8 sütunu. +SELECT sorgusu FROM yan tümcesi belirtmezse, bu tablo kullanılır. +Bu, diğer Dbms'lerde bulunan ikili tabloya benzer. + +## sistem.parçalar {#system_tables-parts} + +Bölümleri hakkında bilgi içerir [MergeTree](../engines/table_engines/mergetree_family/mergetree.md) Tablolar. + +Her satır bir veri bölümünü açıklar. + +Sütun: + +- `partition` (String) – The partition name. To learn what a partition is, see the description of the [ALTER](../sql_reference/statements/alter.md#query_language_queries_alter) sorgu. + + Biçimliler: + + - `YYYYMM` ay otomatik bölümleme için. + - `any_string` el ile bölümleme yaparken. + +- `name` (`String`) – Name of the data part. + +- `active` (`UInt8`) – Flag that indicates whether the data part is active. If a data part is active, it's used in a table. Otherwise, it's deleted. Inactive data parts remain after merging. + +- `marks` (`UInt64`) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` dizin ayrıntısına göre (genellikle 8192) (bu ipucu uyarlanabilir ayrıntı için çalışmaz). + +- `rows` (`UInt64`) – The number of rows. + +- `bytes_on_disk` (`UInt64`) – Total size of all the data part files in bytes. + +- `data_compressed_bytes` (`UInt64`) – Total size of compressed data in the data part. All the auxiliary files (for example, files with marks) are not included. + +- `data_uncompressed_bytes` (`UInt64`) – Total size of uncompressed data in the data part. All the auxiliary files (for example, files with marks) are not included. + +- `marks_bytes` (`UInt64`) – The size of the file with marks. + +- `modification_time` (`DateTime`) – The time the directory with the data part was modified. This usually corresponds to the time of data part creation.\| + +- `remove_time` (`DateTime`) – The time when the data part became inactive. + +- `refcount` (`UInt32`) – The number of places where the data part is used. A value greater than 2 indicates that the data part is used in queries or merges. + +- `min_date` (`Date`) – The minimum value of the date key in the data part. + +- `max_date` (`Date`) – The maximum value of the date key in the data part. + +- `min_time` (`DateTime`) – The minimum value of the date and time key in the data part. + +- `max_time`(`DateTime`) – The maximum value of the date and time key in the data part. + +- `partition_id` (`String`) – ID of the partition. + +- `min_block_number` (`UInt64`) – The minimum number of data parts that make up the current part after merging. + +- `max_block_number` (`UInt64`) – The maximum number of data parts that make up the current part after merging. + +- `level` (`UInt32`) – Depth of the merge tree. Zero means that the current part was created by insert rather than by merging other parts. + +- `data_version` (`UInt64`) – Number that is used to determine which mutations should be applied to the data part (mutations with a version higher than `data_version`). + +- `primary_key_bytes_in_memory` (`UInt64`) – The amount of memory (in bytes) used by primary key values. + +- `primary_key_bytes_in_memory_allocated` (`UInt64`) – The amount of memory (in bytes) reserved for primary key values. + +- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn't exist. For more details, see [FREEZE PARTITION](../sql_reference/statements/alter.md#alter_freeze-partition) + +- `database` (`String`) – Name of the database. + +- `table` (`String`) – Name of the table. + +- `engine` (`String`) – Name of the table engine without parameters. + +- `path` (`String`) – Absolute path to the folder with data part files. + +- `disk` (`String`) – Name of a disk that stores the data part. + +- `hash_of_all_files` (`String`) – [sifash128](../sql_reference/functions/hash_functions.md#hash_functions-siphash128) sıkıştırılmış dosyaların. + +- `hash_of_uncompressed_files` (`String`) – [sifash128](../sql_reference/functions/hash_functions.md#hash_functions-siphash128) sıkıştırılmamış dosyaların (işaretli dosyalar, dizin dosyası vb.)). + +- `uncompressed_hash_of_compressed_files` (`String`) – [sifash128](../sql_reference/functions/hash_functions.md#hash_functions-siphash128) sıkıştırılmış dosyalardaki verilerin sıkıştırılmamış gibi. + +- `bytes` (`UInt64`) – Alias for `bytes_on_disk`. + +- `marks_size` (`UInt64`) – Alias for `marks_bytes`. + +## sistem.part\_log {#system_tables-part-log} + +Bu `system.part_log` tablo yalnızca aşağıdaki durumlarda oluşturulur: [part\_log](server_configuration_parameters/settings.md#server_configuration_parameters-part-log) sunucu ayarı belirtilir. + +Bu tablo ile oluşan olaylar hakkında bilgi içerir [veri parçaları](../engines/table_engines/mergetree_family/custom_partitioning_key.md) in the [MergeTree](../engines/table_engines/mergetree_family/mergetree.md) veri ekleme veya birleştirme gibi aile tabloları. + +Bu `system.part_log` tablo aşağıdaki sütunları içerir: + +- `event_type` (Enum) — Type of the event that occurred with the data part. Can have one of the following values: + - `NEW_PART` — Inserting of a new data part. + - `MERGE_PARTS` — Merging of data parts. + - `DOWNLOAD_PART` — Downloading a data part. + - `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../sql_reference/statements/alter.md#alter_detach-partition). + - `MUTATE_PART` — Mutating of a data part. + - `MOVE_PART` — Moving the data part from the one disk to another one. +- `event_date` (Date) — Event date. +- `event_time` (DateTime) — Event time. +- `duration_ms` (UInt64) — Duration. +- `database` (String) — Name of the database the data part is in. +- `table` (String) — Name of the table the data part is in. +- `part_name` (String) — Name of the data part. +- `partition_id` (String) — ID of the partition that the data part was inserted to. The column takes the ‘all’ bölümleme tarafından ise değer `tuple()`. +- `rows` (UInt64) — The number of rows in the data part. +- `size_in_bytes` (UInt64) — Size of the data part in bytes. +- `merged_from` (Array(String)) — An array of names of the parts which the current part was made up from (after the merge). +- `bytes_uncompressed` (UInt64) — Size of uncompressed bytes. +- `read_rows` (UInt64) — The number of rows was read during the merge. +- `read_bytes` (UInt64) — The number of bytes was read during the merge. +- `error` (UInt16) — The code number of the occurred error. +- `exception` (String) — Text message of the occurred error. + +Bu `system.part_log` tablo ilk veri ekleme sonra oluşturulur `MergeTree` Tablo. + +## sistem.işleyişler {#system_tables-processes} + +Bu sistem tablosu uygulamak için kullanılır `SHOW PROCESSLIST` sorgu. + +Sütun: + +- `user` (String) – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the `default` kullanan. Alan, bu sorgunun başlattığı bir sorgu için değil, belirli bir sorgunun kullanıcı adını içerir. +- `address` (String) – The IP address the request was made from. The same for distributed processing. To track where a distributed query was originally made from, look at `system.processes` sorgu istek sahibi sunucuda. +- `elapsed` (Float64) – The time in seconds since request execution started. +- `rows_read` (UInt64) – The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. +- `bytes_read` (UInt64) – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. +- `total_rows_approx` (UInt64) – The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. +- `memory_usage` (UInt64) – Amount of RAM the request uses. It might not include some types of dedicated memory. See the [max\_memory\_usage](../operations/settings/query_complexity.md#settings_max_memory_usage) ayar. +- `query` (String) – The query text. For `INSERT`, eklemek için veri içermez. +- `query_id` (String) – Query ID, if defined. + +## sistem.text\_log {#system-tables-text-log} + +Günlük girişleri içerir. Bu tabloya giden günlük seviyesi ile sınırlı olabilir `text_log.level` sunucu ayarı. + +Sütun: + +- `event_date` (`Date`)- Giriş tarihi. +- `event_time` (`DateTime`)- Giriş zamanı. +- `microseconds` (`UInt32`)- Girişin mikrosaniye. +- `thread_name` (String) — Name of the thread from which the logging was done. +- `thread_id` (UInt64) — OS thread ID. +- `level` (`Enum8`)- Giriş seviyesi. + - `'Fatal' = 1` + - `'Critical' = 2` + - `'Error' = 3` + - `'Warning' = 4` + - `'Notice' = 5` + - `'Information' = 6` + - `'Debug' = 7` + - `'Trace' = 8` +- `query_id` (`String`)- Sorgunun kimliği. +- `logger_name` (`LowCardinality(String)`) - Name of the logger (i.e. `DDLWorker`) +- `message` (`String`)- Mesajın kendisi . +- `revision` (`UInt32`)- ClickHouse revizyon. +- `source_file` (`LowCardinality(String)`)- Günlüğü yapıldığı kaynak dosya. +- `source_line` (`UInt64`)- Kaynak satır hangi günlüğü yapıldı. + +## sistem.query\_log {#system_tables-query_log} + +Sorguların yürütülmesi hakkında bilgi içerir. Her sorgu için, işlem başlangıç saatini, işlem süresini, hata mesajlarını ve diğer bilgileri görebilirsiniz. + +!!! note "Not" + Tablo için giriş verileri içermiyor `INSERT` sorgular. + +ClickHouse bu tabloyu yalnızca [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) sunucu parametresi belirtilir. Bu parametre, günlük aralığı veya sorguların oturum açacağı tablonun adı gibi günlük kurallarını ayarlar. + +Sorgu günlüğünü etkinleştirmek için, [log\_queries](settings/settings.md#settings-log-queries) parametre 1. Ayrıntılar için, bkz. [Ayarlar](settings/settings.md) bölme. + +Bu `system.query_log` tablo iki tür sorgu kaydeder: + +1. Doğrudan istemci tarafından çalıştırılan ilk sorgular. +2. Diğer sorgular tarafından başlatılan alt sorgular (dağıtılmış sorgu yürütme için). Bu tür sorgular için, üst sorgular hakkında bilgi `initial_*` sütun. + +Sütun: + +- `type` (`Enum8`) — Type of event that occurred when executing the query. Values: + - `'QueryStart' = 1` — Successful start of query execution. + - `'QueryFinish' = 2` — Successful end of query execution. + - `'ExceptionBeforeStart' = 3` — Exception before the start of query execution. + - `'ExceptionWhileProcessing' = 4` — Exception during the query execution. +- `event_date` (Date) — Query starting date. +- `event_time` (DateTime) — Query starting time. +- `query_start_time` (DateTime) — Start time of query execution. +- `query_duration_ms` (UInt64) — Duration of query execution. +- `read_rows` (UInt64) — Number of read rows. +- `read_bytes` (UInt64) — Number of read bytes. +- `written_rows` (UInt64) — For `INSERT` sorgular, yazılı satır sayısı. Diğer sorgular için sütun değeri 0'dır. +- `written_bytes` (UInt64) — For `INSERT` sorgular, yazılı bayt sayısı. Diğer sorgular için sütun değeri 0'dır. +- `result_rows` (UInt64) — Number of rows in the result. +- `result_bytes` (UInt64) — Number of bytes in the result. +- `memory_usage` (UInt64) — Memory consumption by the query. +- `query` (String) — Query string. +- `exception` (String) — Exception message. +- `stack_trace` (String) — Stack trace (a list of methods called before the error occurred). An empty string, if the query is completed successfully. +- `is_initial_query` (UInt8) — Query type. Possible values: + - 1 — Query was initiated by the client. + - 0 — Query was initiated by another query for distributed query execution. +- `user` (String) — Name of the user who initiated the current query. +- `query_id` (String) — ID of the query. +- `address` (IPv6) — IP address that was used to make the query. +- `port` (UInt16) — The client port that was used to make the query. +- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). +- `initial_query_id` (String) — ID of the initial query (for distributed query execution). +- `initial_address` (IPv6) — IP address that the parent query was launched from. +- `initial_port` (UInt16) — The client port that was used to make the parent query. +- `interface` (UInt8) — Interface that the query was initiated from. Possible values: + - 1 — TCP. + - 2 — HTTP. +- `os_user` (String) — OS's username who runs [clickhouse-müşteri](../interfaces/cli.md). +- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi çalıştırılır. +- `client_name` (String) — The [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemci adı. +- `client_revision` (UInt32) — Revision of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. +- `client_version_major` (UInt32) — Major version of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. +- `client_version_minor` (UInt32) — Minor version of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. +- `client_version_patch` (UInt32) — Patch component of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemci sürümü. +- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: + - 0 — The query was launched from the TCP interface. + - 1 — `GET` yöntem kullanılmıştır. + - 2 — `POST` yöntem kullanılmıştır. +- `http_user_agent` (String) — The `UserAgent` başlık http isteğinde geçti. +- `quota_key` (String) — The “quota key” belirtilen [kotalar](quotas.md) ayarı (bakınız `keyed`). +- `revision` (UInt32) — ClickHouse revision. +- `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution. +- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [sistem.etkinlik](#system_tables-events) +- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` sütun. +- `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parametre 1. +- `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` sütun. + +Her sorgu bir veya iki satır oluşturur `query_log` tablo, sorgunun durumuna bağlı olarak: + +1. Sorgu yürütme başarılı olursa, tip 1 ve 2 ile iki olay oluşturulur (bkz. `type` sütun). +2. Sorgu işleme sırasında bir hata oluştu, iki olay türleri 1 ve 4 oluşturulur. +3. Sorguyu başlatmadan önce bir hata oluşmuşsa, 3 tipi olan tek bir olay oluşturulur. + +Varsayılan olarak, günlükleri 7.5 saniye aralıklarla tabloya eklenir. Bu aralığı ayarlayabilirsiniz [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) sunucu ayarı (bkz. `flush_interval_milliseconds` parametre). Günlükleri zorla bellek arabelleğinden tabloya temizlemek için `SYSTEM FLUSH LOGS` sorgu. + +Tablo elle silindiğinde, otomatik olarak anında oluşturulur. Önceki tüm günlüklerin silineceğini unutmayın. + +!!! note "Not" + Günlüklerin depolama süresi sınırsızdır. Günlükler tablodan otomatik olarak silinmez. Eski günlüklerin kaldırılmasını kendiniz düzenlemeniz gerekir. + +İçin keyfi bir bölümleme anahtarı belirtebilirsiniz `system.query_log` tablo içinde [query\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-log) sunucu ayarı (bkz. `partition_by` parametre). + +## sistem.query\_thread\_log {#system_tables-query-thread-log} + +Tablo, her sorgu yürütme iş parçacığı hakkında bilgi içerir. + +ClickHouse bu tabloyu yalnızca [query\_thread\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log) sunucu parametresi belirtilir. Bu parametre, günlük aralığı veya sorguların oturum açacağı tablonun adı gibi günlük kurallarını ayarlar. + +Sorgu günlüğünü etkinleştirmek için, [log\_query\_threads](settings/settings.md#settings-log-query-threads) parametre 1. Ayrıntılar için, bkz. [Ayarlar](settings/settings.md) bölme. + +Sütun: + +- `event_date` (Date) — the date when the thread has finished execution of the query. +- `event_time` (DateTime) — the date and time when the thread has finished execution of the query. +- `query_start_time` (DateTime) — Start time of query execution. +- `query_duration_ms` (UInt64) — Duration of query execution. +- `read_rows` (UInt64) — Number of read rows. +- `read_bytes` (UInt64) — Number of read bytes. +- `written_rows` (UInt64) — For `INSERT` sorgular, yazılı satır sayısı. Diğer sorgular için sütun değeri 0'dır. +- `written_bytes` (UInt64) — For `INSERT` sorgular, yazılı bayt sayısı. Diğer sorgular için sütun değeri 0'dır. +- `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread. +- `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread. +- `thread_name` (String) — Name of the thread. +- `thread_number` (UInt32) — Internal thread ID. +- `os_thread_id` (Int32) — OS thread ID. +- `master_thread_id` (UInt64) — OS initial ID of initial thread. +- `query` (String) — Query string. +- `is_initial_query` (UInt8) — Query type. Possible values: + - 1 — Query was initiated by the client. + - 0 — Query was initiated by another query for distributed query execution. +- `user` (String) — Name of the user who initiated the current query. +- `query_id` (String) — ID of the query. +- `address` (IPv6) — IP address that was used to make the query. +- `port` (UInt16) — The client port that was used to make the query. +- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). +- `initial_query_id` (String) — ID of the initial query (for distributed query execution). +- `initial_address` (IPv6) — IP address that the parent query was launched from. +- `initial_port` (UInt16) — The client port that was used to make the parent query. +- `interface` (UInt8) — Interface that the query was initiated from. Possible values: + - 1 — TCP. + - 2 — HTTP. +- `os_user` (String) — OS's username who runs [clickhouse-müşteri](../interfaces/cli.md). +- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi çalıştırılır. +- `client_name` (String) — The [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemci adı. +- `client_revision` (UInt32) — Revision of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. +- `client_version_major` (UInt32) — Major version of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. +- `client_version_minor` (UInt32) — Minor version of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemcisi. +- `client_version_patch` (UInt32) — Patch component of the [clickhouse-müşteri](../interfaces/cli.md) veya başka bir TCP istemci sürümü. +- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: + - 0 — The query was launched from the TCP interface. + - 1 — `GET` yöntem kullanılmıştır. + - 2 — `POST` yöntem kullanılmıştır. +- `http_user_agent` (String) — The `UserAgent` başlık http isteğinde geçti. +- `quota_key` (String) — The “quota key” belirtilen [kotalar](quotas.md) ayarı (bakınız `keyed`). +- `revision` (UInt32) — ClickHouse revision. +- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [sistem.etkinlik](#system_tables-events) +- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` sütun. + +Varsayılan olarak, günlükleri 7.5 saniye aralıklarla tabloya eklenir. Bu aralığı ayarlayabilirsiniz [query\_thread\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log) sunucu ayarı (bkz. `flush_interval_milliseconds` parametre). Günlükleri zorla bellek arabelleğinden tabloya temizlemek için `SYSTEM FLUSH LOGS` sorgu. + +Tablo elle silindiğinde, otomatik olarak anında oluşturulur. Önceki tüm günlüklerin silineceğini unutmayın. + +!!! note "Not" + Günlüklerin depolama süresi sınırsızdır. Günlükler tablodan otomatik olarak silinmez. Eski günlüklerin kaldırılmasını kendiniz düzenlemeniz gerekir. + +İçin keyfi bir bölümleme anahtarı belirtebilirsiniz `system.query_thread_log` tablo içinde [query\_thread\_log](server_configuration_parameters/settings.md#server_configuration_parameters-query-thread-log) sunucu ayarı (bkz. `partition_by` parametre). + +## sistem.trace\_log {#system_tables-trace_log} + +Örnekleme sorgusu profiler tarafından toplanan yığın izlemeleri içerir. + +ClickHouse bu tabloyu oluşturduğunda [trace\_log](server_configuration_parameters/settings.md#server_configuration_parameters-trace_log) sunucu yapılandırma bölümü ayarlanır. Ayrıca [query\_profiler\_real\_time\_period\_ns](settings/settings.md#query_profiler_real_time_period_ns) ve [query\_profiler\_cpu\_time\_period\_ns](settings/settings.md#query_profiler_cpu_time_period_ns) ayarlar ayarlan .malıdır. + +Günlükleri analiz etmek için `addressToLine`, `addressToSymbol` ve `demangle` iç gözlem fonksiyonları. + +Sütun: + +- `event_date`([Tarihli](../sql_reference/data_types/date.md)) — Date of sampling moment. + +- `event_time`([DateTime](../sql_reference/data_types/datetime.md)) — Timestamp of sampling moment. + +- `revision`([Uİnt32](../sql_reference/data_types/int_uint.md)) — ClickHouse server build revision. + + Tarafından sunucuya Bağlan byırken `clickhouse-client`, benzer diz theg seeeyi görüyorsunuz `Connected to ClickHouse server version 19.18.1 revision 54429.`. Bu alan şunları içerir `revision` ama `version` bir sunucunun. + +- `timer_type`([Enum8](../sql_reference/data_types/enum.md)) — Timer type: + + - `Real` duvar saati zamanını temsil eder. + - `CPU` CPU süresini temsil eder. + +- `thread_number`([Uİnt32](../sql_reference/data_types/int_uint.md)) — Thread identifier. + +- `query_id`([Dize](../sql_reference/data_types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query\_log](#system_tables-query_log) sistem tablosu. + +- `trace`([Dizi (Uİnt64)](../sql_reference/data_types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. + +**Örnek** + +``` sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` + +``` text +Row 1: +────── +event_date: 2019-11-15 +event_time: 2019-11-15 15:09:38 +revision: 54428 +timer_type: Real +thread_number: 48 +query_id: acc4d61f-5bd1-4a3e-bc91-2180be37c915 +trace: [94222141367858,94222152240175,94222152325351,94222152329944,94222152330796,94222151449980,94222144088167,94222151682763,94222144088167,94222151682763,94222144088167,94222144058283,94222144059248,94222091840750,94222091842302,94222091831228,94222189631488,140509950166747,140509942945935] +``` + +## sistem.yinelemeler {#system_tables-replicas} + +Yerel sunucuda bulunan çoğaltılmış tablolar için bilgi ve durum içerir. +Bu tablo izleme için kullanılabilir. Tablo, her çoğaltılmış \* tablo için bir satır içerir. + +Örnek: + +``` sql +SELECT * +FROM system.replicas +WHERE table = 'visits' +FORMAT Vertical +``` + +``` text +Row 1: +────── +database: merge +table: visits +engine: ReplicatedCollapsingMergeTree +is_leader: 1 +can_become_leader: 1 +is_readonly: 0 +is_session_expired: 0 +future_parts: 1 +parts_to_check: 0 +zookeeper_path: /clickhouse/tables/01-06/visits +replica_name: example01-06-1.yandex.ru +replica_path: /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru +columns_version: 9 +queue_size: 1 +inserts_in_queue: 0 +merges_in_queue: 1 +part_mutations_in_queue: 0 +queue_oldest_time: 2020-02-20 08:34:30 +inserts_oldest_time: 0000-00-00 00:00:00 +merges_oldest_time: 2020-02-20 08:34:30 +part_mutations_oldest_time: 0000-00-00 00:00:00 +oldest_part_to_get: +oldest_part_to_merge_to: 20200220_20284_20840_7 +oldest_part_to_mutate_to: +log_max_index: 596273 +log_pointer: 596274 +last_queue_update: 2020-02-20 08:34:32 +absolute_delay: 0 +total_replicas: 2 +active_replicas: 2 +``` + +Sütun: + +- `database` (`String`)- Veritabanı adı +- `table` (`String`)- Tablo adı +- `engine` (`String`)- Tablo motor adı +- `is_leader` (`UInt8`)- Kopya lider olup olmadığı. + Bir seferde sadece bir kopya lider olabilir. Lider, gerçekleştirmek için arka plan birleştirmelerini seçmekten sorumludur. + Yazma kullanılabilir ve bir oturum ZK, bir lider olup olmadığına bakılmaksızın olan herhangi bir yineleme için gerçekleştirilebilir unutmayın. +- `can_become_leader` (`UInt8`)- Rep .lik leaderanın lider olarak seçil .ip seçil .emeyeceği. +- `is_readonly` (`UInt8`)- Yinelemenin salt okunur modda olup olmadığı. + Yapılandırmanın ZooKeeper ile bölümleri yoksa, zookeeper'daki oturumları yeniden başlatırken ve Zookeeper'daki oturum yeniden başlatılırken bilinmeyen bir hata oluşmuşsa bu mod açılır. +- `is_session_expired` (`UInt8`)- ZooKeeper ile oturum süresi doldu. Temelde aynı `is_readonly`. +- `future_parts` (`UInt32`)- Henüz yapılmamış ekler veya birleştirmelerin sonucu olarak görünecek veri parçalarının sayısı. +- `parts_to_check` (`UInt32`)- Doğrulama için kuyruktaki veri parçalarının sayısı. Hasar görebileceğinden şüphe varsa, bir parça doğrulama kuyruğuna konur. +- `zookeeper_path` (`String`)- ZooKeeper tablo verilerine yolu. +- `replica_name` (`String`)- Zookeeper çoğaltma adı. Aynı tablonun farklı kopyaları farklı adlara sahiptir. +- `replica_path` (`String`)- ZooKeeper çoğaltma veri yolu. Birleştirme ile aynı ‘zookeeper\_path/replicas/replica\_path’. +- `columns_version` (`Int32`)- Tablo yapısının sürüm numarası. ALTER kaç kez gerçekleştirildiğini gösterir. Kopyaların farklı sürümleri varsa, bazı kopyaların tüm değişiklikleri henüz yapmadığı anlamına gelir. +- `queue_size` (`UInt32`)- Yapılması beklenen işlemler için sıranın büyüklüğü. İşlemler, veri bloklarını, birleştirmeleri ve diğer bazı eylemleri eklemeyi içerir. Genellikle ile çakışmaktadır `future_parts`. +- `inserts_in_queue` (`UInt32`)- Yapılması gereken veri bloklarının eklerinin sayısı. Eklemeler genellikle oldukça hızlı bir şekilde çoğaltılır. Bu sayı büyükse, bir şeylerin yanlış olduğu anlamına gelir. +- `merges_in_queue` (`UInt32`)- Yapılmasını bekleyen birleştirme sayısı. Bazen birleştirmeler uzundur, bu nedenle bu değer uzun süre sıfırdan büyük olabilir. +- `part_mutations_in_queue` (`UInt32`)- Yapılması beklenen Mut numberasyon sayısı. +- `queue_oldest_time` (`DateTime`) - Eğer `queue_size` daha büyük 0, en eski işlem sıraya eklendiğinde gösterir. +- `inserts_oldest_time` (`DateTime`) - Görmek `queue_oldest_time` +- `merges_oldest_time` (`DateTime`) - Görmek `queue_oldest_time` +- `part_mutations_oldest_time` (`DateTime`) - Görmek `queue_oldest_time` + +Sonraki 4 sütun, yalnızca ZK ile aktif bir oturumun olduğu sıfır olmayan bir değere sahiptir. + +- `log_max_index` (`UInt64`)- Genel faaliyet günlüğüne maksimum giriş numarası. +- `log_pointer` (`UInt64`)- Çoğaltma yürütme kuyruğuna kopyalanan genel faaliyet günlüğüne maksimum giriş numarası, artı bir. Eğer `log_pointer` daha küçük `log_max_index` yanlış bir şey olduğunu. +- `last_queue_update` (`DateTime`)- Kuyruk son kez güncellendiğinde. +- `absolute_delay` (`UInt64`)- Geçerli kopyanın saniyeler içinde ne kadar büyük gecikme var. +- `total_replicas` (`UInt8`)- Bu tablonun bilinen kopyalarının toplam sayısı. +- `active_replicas` (`UInt8`)- ZooKeeper bir oturum var bu tablonun kopyaları sayısı (yani, işleyen kopyaları sayısı). + +Tüm sütunları talep ederseniz, Tablo biraz yavaş çalışabilir, çünkü ZooKeeper birkaç okuma her satır için yapılır. +Son 4 sütun (log\_max\_ındex, log\_pointer, total\_replicas, active\_replicas) istemiyorsanız, tablo hızlı bir şekilde çalışır. + +Örneğin, her şeyin böyle düzgün çalıştığını kontrol edebilirsiniz: + +``` sql +SELECT + database, + table, + is_leader, + is_readonly, + is_session_expired, + future_parts, + parts_to_check, + columns_version, + queue_size, + inserts_in_queue, + merges_in_queue, + log_max_index, + log_pointer, + total_replicas, + active_replicas +FROM system.replicas +WHERE + is_readonly + OR is_session_expired + OR future_parts > 20 + OR parts_to_check > 10 + OR queue_size > 20 + OR inserts_in_queue > 10 + OR log_max_index - log_pointer > 10 + OR total_replicas < 2 + OR active_replicas < total_replicas +``` + +Bu sorgu hiçbir şey döndürmezse, her şeyin yolunda olduğu anlamına gelir. + +## sistem.ayarlar {#system-tables-system-settings} + +Geçerli kullanıcı için oturum ayarları hakkında bilgi içerir. + +Sütun: + +- `name` ([Dize](../sql_reference/data_types/string.md)) — Setting name. +- `value` ([Dize](../sql_reference/data_types/string.md)) — Setting value. +- `changed` ([Uİnt8](../sql_reference/data_types/int_uint.md#uint-ranges)) — Shows whether a setting is changed from its default value. +- `description` ([Dize](../sql_reference/data_types/string.md)) — Short setting description. +- `min` ([Nullable](../sql_reference/data_types/nullable.md)([Dize](../sql_reference/data_types/string.md))) — Minimum value of the setting, if any is set via [kısıtlamalar](settings/constraints_on_settings.md#constraints-on-settings). Ayarın minimum değeri yoksa, şunları içerir [NULL](../sql_reference/syntax.md#null-literal). +- `max` ([Nullable](../sql_reference/data_types/nullable.md)([Dize](../sql_reference/data_types/string.md))) — Maximum value of the setting, if any is set via [kısıtlamalar](settings/constraints_on_settings.md#constraints-on-settings). Ayarın maksimum değeri yoksa, şunları içerir [NULL](../sql_reference/syntax.md#null-literal). +- `readonly` ([Uİnt8](../sql_reference/data_types/int_uint.md#uint-ranges)) — Shows whether the current user can change the setting: + - `0` — Current user can change the setting. + - `1` — Current user can't change the setting. + +**Örnek** + +Aşağıdaki örnek, adı içeren ayarlar hakkında bilgi almak gösterilmiştir `min_i`. + +``` sql +SELECT * +FROM system.settings +WHERE name LIKE '%min_i%' +``` + +``` text +┌─name────────────────────────────────────────┬─value─────┬─changed─┬─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─min──┬─max──┬─readonly─┐ +│ min_insert_block_size_rows │ 1048576 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ min_insert_block_size_bytes │ 268435456 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ +└─────────────────────────────────────────────┴───────────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────┴──────┴──────────┘ +``` + +Kullanımı `WHERE changed` örneğin, kontrol etmek istediğinizde yararlı olabilir: + +- Olsun yapılandırma dosyaları, ayarları doğru şekilde yüklenmiş ve kullanımdadır. +- Geçerli oturumda değişen ayarlar. + + + +``` sql +SELECT * FROM system.settings WHERE changed AND name='load_balancing' +``` + +**Ayrıca bakınız** + +- [Ayarlar](settings/index.md#settings) +- [Sorgular için izinler](settings/permissions_for_queries.md#settings_readonly) +- [Ayarlardaki kısıtlamalar](settings/constraints_on_settings.md) + +## sistem.table\_engines {#system.table_engines} + +``` text +┌─name───────────────────┬─value───────┐ +│ max_threads │ 8 │ +│ use_uncompressed_cache │ 0 │ +│ load_balancing │ random │ +│ max_memory_usage │ 10000000000 │ +└────────────────────────┴─────────────┘ +``` + +## sistem.merge\_tree\_settings {#system-merge_tree_settings} + +İçin ayarlar hakkında bilgi içerir `MergeTree` Tablolar. + +Sütun: + +- `name` (String) — Setting name. +- `value` (String) — Setting value. +- `description` (String) — Setting description. +- `type` (String) — Setting type (implementation specific string value). +- `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed. + +## sistem.table\_engines {#system-table-engines} + +Sunucu tarafından desteklenen tablo motorlarının açıklamasını ve özellik destek bilgilerini içerir. + +Bu tablo aşağıdaki sütunları içerir (sütun türü parantez içinde gösterilir): + +- `name` (String) — The name of table engine. +- `supports_settings` (UInt8) — Flag that indicates if table engine supports `SETTINGS` yan. +- `supports_skipping_indices` (UInt8) — Flag that indicates if table engine supports [endeksleri atlama](../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-data_skipping-indexes). +- `supports_ttl` (UInt8) — Flag that indicates if table engine supports [TTL](../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-ttl). +- `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` ve `SAMPLE_BY`. +- `supports_replication` (UInt8) — Flag that indicates if table engine supports [veri çoğaltma](../engines/table_engines/mergetree_family/replication.md). +- `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. + +Örnek: + +``` sql +SELECT * +FROM system.table_engines +WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') +``` + +``` text +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ +``` + +**Ayrıca bakınız** + +- MergeTree ailesi [sorgu yan tümceleri](../engines/table_engines/mergetree_family/mergetree.md#mergetree-query-clauses) +- Kafka [ayarlar](../engines/table_engines/integrations/kafka.md#table_engine-kafka-creating-a-table) +- Katmak [ayarlar](../engines/table_engines/special/join.md#join-limitations-and-settings) + +## sistem.Tablolar {#system-tables} + +Sunucunun bildiği her tablonun meta verilerini içerir. Müstakil tablolar gösterilmez `system.tables`. + +Bu tablo aşağıdaki sütunları içerir (sütun türü parantez içinde gösterilir): + +- `database` (String) — The name of the database the table is in. + +- `name` (String) — Table name. + +- `engine` (String) — Table engine name (without parameters). + +- `is_temporary` (Uİnt8) - tablonun geçici olup olmadığını gösteren bayrak. + +- `data_path` (String) - dosya sistemindeki tablo verilerinin yolu. + +- `metadata_path` (String) - dosya sistemindeki tablo Meta Veri Yolu. + +- `metadata_modification_time` (DateTime) - tablo meta son değişiklik zamanı. + +- `dependencies_database` (Array (String)) - veritabanı bağımlılıkları. + +- `dependencies_table` (Array (String)) - Tablo bağımlılıkları ([MaterializedView](../engines/table_engines/special/materializedview.md) geçerli tabloya dayalı tablolar). + +- `create_table_query` (String) - tablo oluşturmak için kullanılan sorgu. + +- `engine_full` (String) - tablo motorunun parametreleri. + +- `partition_key` (String) - tabloda belirtilen bölüm anahtarı ifadesi. + +- `sorting_key` (String) - tabloda belirtilen sıralama anahtarı ifadesi. + +- `primary_key` (String) - tabloda belirtilen birincil anahtar ifadesi. + +- `sampling_key` (String) - tabloda belirtilen örnekleme anahtar ifadesi. + +- `storage_policy` (String) - depolama politikası: + + - [MergeTree](../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes) + - [Dağılı](../engines/table_engines/special/distributed.md#distributed) + +- `total_rows` (Nullable (Uİnt64)) - tablodaki tam satır sayısını hızlı bir şekilde belirlemek mümkün ise, toplam satır sayısı `Null` (underying dahil `Buffer` Tablo). + +- `total_bytes` (Nullable (Uİnt64)) - toplam bayt sayısı, eğer depolama alanındaki tablo için tam bayt sayısını hızlı bir şekilde belirlemek mümkün ise, aksi takdirde `Null` (**do Notes not** herhangi bir temel depolama içerir). + + - If the table stores data on disk, returns used space on disk (i.e. compressed). + - Tablo verileri bellekte depolarsa, bellekte kullanılan bayt sayısını yaklaşık olarak döndürür. + +Bu `system.tables` tablo kullanılır `SHOW TABLES` sorgu uygulaması. + +## sistem.zookeeper {#system-zookeeper} + +ZooKeeper yapılandırılmamışsa, tablo yok. Yapılandırmada tanımlanan ZooKeeper kümesinden veri okumayı sağlar. +Sorgu bir olmalıdır ‘path’ WH .ere madd .esindeki eşitlik koşulu. Bu veri almak istediğiniz çocuklar için ZooKeeper yoludur. + +Sorgu `SELECT * FROM system.zookeeper WHERE path = '/clickhouse'` tüm çocuklar için veri çıkışı `/clickhouse` düğümlü. +Tüm kök düğümler için veri çıkışı yapmak için, path = yazın ‘/’. +Belirtilen yol ise ‘path’ yok, bir istisna atılır. + +Sütun: + +- `name` (String) — The name of the node. +- `path` (String) — The path to the node. +- `value` (String) — Node value. +- `dataLength` (Int32) — Size of the value. +- `numChildren` (Int32) — Number of descendants. +- `czxid` (Int64) — ID of the transaction that created the node. +- `mzxid` (Int64) — ID of the transaction that last changed the node. +- `pzxid` (Int64) — ID of the transaction that last deleted or added descendants. +- `ctime` (DateTime) — Time of node creation. +- `mtime` (DateTime) — Time of the last modification of the node. +- `version` (Int32) — Node version: the number of times the node was changed. +- `cversion` (Int32) — Number of added or removed descendants. +- `aversion` (Int32) — Number of changes to the ACL. +- `ephemeralOwner` (Int64) — For ephemeral nodes, the ID of the session that owns this node. + +Örnek: + +``` sql +SELECT * +FROM system.zookeeper +WHERE path = '/clickhouse/tables/01-08/visits/replicas' +FORMAT Vertical +``` + +``` text +Row 1: +────── +name: example01-08-1.yandex.ru +value: +czxid: 932998691229 +mzxid: 932998691229 +ctime: 2015-03-27 16:49:51 +mtime: 2015-03-27 16:49:51 +version: 0 +cversion: 47 +aversion: 0 +ephemeralOwner: 0 +dataLength: 0 +numChildren: 7 +pzxid: 987021031383 +path: /clickhouse/tables/01-08/visits/replicas + +Row 2: +────── +name: example01-08-2.yandex.ru +value: +czxid: 933002738135 +mzxid: 933002738135 +ctime: 2015-03-27 16:57:01 +mtime: 2015-03-27 16:57:01 +version: 0 +cversion: 37 +aversion: 0 +ephemeralOwner: 0 +dataLength: 0 +numChildren: 7 +pzxid: 987021252247 +path: /clickhouse/tables/01-08/visits/replicas +``` + +## sistem.mutasyonlar {#system_tables-mutations} + +Tablo hakkında bilgi içerir [mutasyonlar](../sql_reference/statements/alter.md#alter-mutations) MergeTree tabloları ve bunların ilerleme. Her mutasyon komutu tek bir satırla temsil edilir. Tablo aşağıdaki sütunlara sahiptir: + +**veritabanı**, **Tablo** - Mutasyonun uygulandığı veritabanı ve tablonun adı. + +**mutation\_id** - Mutasyonun kimliği. Çoğaltılmış tablolar için bu kimlikler znode adlarına karşılık gelir `/mutations/` ZooKeeper dizin. Yinelenmemiş tablolar için kimlikler, tablonun veri dizinindeki dosya adlarına karşılık gelir. + +**komut** - Mut commandasyon komut diz (gesi (sorgu afterdan sonra `ALTER TABLE [db.]table`). + +**create\_time** - Bu mutasyon komutu idam için sunulduğunda. + +**block\_numbers.partition\_id**, **block\_numbers.numara** - İç içe geçmiş bir sütun. Çoğaltılmış tabloların mutasyonları için, her bölüm için bir kayıt içerir: bölüm kimliği ve mutasyon tarafından elde edilen blok numarası (her bölümde, yalnızca bu bölümdeki mutasyon tarafından elde edilen blok sayısından daha az sayıda blok içeren parçalar mutasyona uğrayacaktır). Çoğaltılmamış tablolarda, tüm bölümlerdeki blok numaraları tek bir sıra oluşturur. Bu, çoğaltılmamış tabloların mutasyonları için, sütunun mutasyon tarafından elde edilen tek bir blok numarasına sahip bir kayıt içereceği anlamına gelir. + +**parts\_to\_do** - Mutasyonun bitmesi için mutasyona uğraması gereken veri parçalarının sayısı. + +**is\_done** - Mutasyon bitti mi? Not bile `parts_to_do = 0` çoğaltılmış bir tablonun mutasyonu, mutasyona uğraması gereken yeni bir veri parçası yaratacak uzun süren bir ekleme nedeniyle henüz yapılmamıştır. + +Bazı bölümleri mutasyon ile ilgili sorunlar varsa, aşağıdaki sütunlar ek bilgi içerir: + +**latest\_failed\_part** - Mutasyona uğramayan en son bölümün adı. + +**latest\_fail\_time** - En son bölüm mutasyon başarısızlığı zamanı. + +**latest\_fail\_reason** - En son bölüm mutasyon başarısızlığına neden olan istisna mesajı. + +## sistem.diskler {#system_tables-disks} + +İçinde tanımlanan diskler hakkında bilgi içerir [sunucu yapılandırması](../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). + +Sütun: + +- `name` ([Dize](../sql_reference/data_types/string.md)) — Name of a disk in the server configuration. +- `path` ([Dize](../sql_reference/data_types/string.md)) — Path to the mount point in the file system. +- `free_space` ([Uİnt64](../sql_reference/data_types/int_uint.md)) — Free space on disk in bytes. +- `total_space` ([Uİnt64](../sql_reference/data_types/int_uint.md)) — Disk volume in bytes. +- `keep_free_space` ([Uİnt64](../sql_reference/data_types/int_uint.md)) — Amount of disk space that should stay free on disk in bytes. Defined in the `keep_free_space_bytes` disk yapılandırması parametresi. + +## sistem.storage\_policies {#system_tables-storage_policies} + +Depolama ilkeleri ve birimlerinde tanımlanan bilgiler içerir. [sunucu yapılandırması](../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes_configure). + +Sütun: + +- `policy_name` ([Dize](../sql_reference/data_types/string.md)) — Name of the storage policy. +- `volume_name` ([Dize](../sql_reference/data_types/string.md)) — Volume name defined in the storage policy. +- `volume_priority` ([Uİnt64](../sql_reference/data_types/int_uint.md)) — Volume order number in the configuration. +- `disks` ([Ar Arrayray (String)](../sql_reference/data_types/array.md)) — Disk names, defined in the storage policy. +- `max_data_part_size` ([Uİnt64](../sql_reference/data_types/int_uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit). +- `move_factor` ([Float64](../sql_reference/data_types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order. + +Depolama ilkesi birden fazla birim içeriyorsa, her birim için bilgiler tablonun tek tek satırında saklanır. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/system_tables/) diff --git a/docs/tr/operations/tips.md b/docs/tr/operations/tips.md new file mode 100644 index 00000000000..e6b02a239c5 --- /dev/null +++ b/docs/tr/operations/tips.md @@ -0,0 +1,251 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 58 +toc_title: "Kullan\u0131m \xD6nerileri" +--- + +# Kullanım Önerileri {#usage-recommendations} + +## CPU Ölçekleme Vali {#cpu-scaling-governor} + +Her zaman kullanın `performance` Ölçekleme Valisi. Bu `on-demand` ölçeklendirme Valisi sürekli yüksek talep ile çok daha kötü çalışır. + +``` bash +$ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +``` + +## CPU sınırlamaları {#cpu-limitations} + +İşlemciler aşırı ısınabilir. Kullanmak `dmesg` aşırı ısınma nedeniyle CPU'nun saat hızının sınırlı olup olmadığını görmek için. +Kısıtlama, veri merkezi düzeyinde harici olarak da ayarlanabilir. Kullanabilirsiniz `turbostat` bir yük altında izlemek için. + +## RAM {#ram} + +Küçük miktarlarda veri için (~200 GB'a kadar sıkıştırılmış), veri hacmi kadar bellek kullanmak en iyisidir. +Büyük miktarda veri için ve etkileşimli (çevrimiçi) sorguları işlerken, sıcak veri alt kümesi sayfaların önbelleğine sığacak şekilde makul miktarda RAM (128 GB veya daha fazla) kullanmalısınız. +Sunucu başına ~50 TB veri hacimleri için bile, 128 GB RAM kullanmak, 64 GB'ye kıyasla sorgu performansını önemli ölçüde artırır. + +Overcommit devre dışı bırakmayın. Değer `cat /proc/sys/vm/overcommit_memory` 0 veya 1 olmalıdır. Koşmak + +``` bash +$ echo 0 | sudo tee /proc/sys/vm/overcommit_memory +``` + +## Büyük Sayfalar {#huge-pages} + +Her zaman şeffaf büyük sayfaları devre dışı bırakın. Önemli performans düşmesine neden olan bellek yöneticileri, engel oluyor. + +``` bash +$ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled +``` + +Kullanmak `perf top` bellek yönetimi için çekirdekte harcanan zamanı izlemek için. +Kalıcı büyük sayfaların da tahsis edilmesine gerek yoktur. + +## Depolama Alt Sistemi {#storage-subsystem} + +Bütçeniz SSD kullanmanıza izin veriyorsa, SSD kullanın. +Eğer değilse, sabit disk kullanın. SATA HDD'ler 7200 RPM yapacak. + +Bağlı disk raflarına sahip daha az sayıda sunucu üzerinde yerel sabit disklere sahip birçok sunucuyu tercih edin. +Ancak nadir sorguları olan arşivleri saklamak için raflar çalışacaktır. + +## RAID {#raid} + +HDD kullanırken, RAID-10, RAID-5, RAID-6 veya RAID-50'yi birleştirebilirsiniz. +Linux için, yazılım RAID daha iyidir (ile `mdadm`). LVM'Yİ kullanmanızı önermiyoruz. +RAID-10 oluştururken, `far` düzen. +Bütçeniz izin veriyorsa, RAID-10'u seçin. + +4'ten fazla diskiniz varsa, RAID-5 yerine RAID-6 (tercih edilen) veya RAID-50 kullanın. +RAID-5, RAID-6 veya RAID-50 kullanırken, varsayılan değer genellikle en iyi seçenek olmadığından daima stripe\_cache\_size değerini artırın. + +``` bash +$ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size +``` + +Formülü kullanarak cihaz sayısından ve blok boyutundan tam sayıyı hesaplayın: `2 * num_devices * chunk_size_in_bytes / 4096`. + +Tüm RAID yapılandırmaları için 1024 KB blok boyutu yeterlidir. +Blok boyutunu asla çok küçük veya çok büyük ayarlamayın. + +SSD'DE RAID-0 kullanabilirsiniz. +RAID kullanımı ne olursa olsun, her zaman veri güvenliği için çoğaltma kullanın. + +Uzun bir kuyruk ile NCQ etkinleştirin. HDD için CFQ zamanlayıcısını seçin ve SSD için noop'u seçin. Azalt themayın ‘readahead’ ayar. +HDD için yazma önbelleğini etkinleştirin. + +## Dosya Sistemi {#file-system} + +Ext4 en güvenilir seçenektir. Bağlama seçeneklerini ayarlama `noatime, nobarrier`. +XFS de uygundur, ancak ClickHouse ile iyice test edilmemiştir. +Diğer çoğu dosya sistemi de iyi çalışmalıdır. Gecikmeli tahsisli dosya sistemleri daha iyi çalışır. + +## Linux Çekirdeği {#linux-kernel} + +Eski bir Linux çekirdeği kullanmayın. + +## Ağ {#network} + +IPv6 kullanıyorsanız, rota önbelleğinin boyutunu artırın. +3.2 öncesinde Linux çekirdeği IPv6 uygulaması ile ilgili sorunlar çok sayıda vardı. + +Mümkünse en az 10 GB ağ kullanın. 1 Gb de çalışacak, ancak onlarca terabayt veri içeren kopyaları yamalamak veya büyük miktarda Ara veriyle dağıtılmış sorguları işlemek için çok daha kötü olacaktır. + +## ZooKeeper {#zookeeper} + +Muhtemelen zaten başka amaçlar için ZooKeeper kullanıyor. Zaten aşırı değilse, aynı ZooKeeper kurulumunu kullanabilirsiniz. + +It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. + +Sonuç sıralı düğümler için yanlış olacağından, farklı ZooKeeper kümeleri arasında veri aktarmak için el ile yazılmış komut dosyalarını asla kullanmamalısınız. Asla kullanmayın “zkcopy” aynı nedenle yardımcı program: https://github.com/ksprojects/zkcopy/issues/15 + +Varolan bir ZooKeeper kümesini ikiye bölmek istiyorsanız, doğru yol, yinelemelerinin sayısını artırmak ve sonra iki bağımsız küme olarak yeniden yapılandırmaktır. + +Zookeeper ClickHouse aynı sunucularda çalıştırmayın. ZooKeeper gecikme için çok hassas olduğundan ve ClickHouse mevcut tüm sistem kaynaklarını kullanabilir. + +Varsayılan ayarlarla, ZooKeeper bir saatli bomba: + +> ZooKeeper sunucusu, varsayılan yapılandırmayı kullanırken eski anlık görüntülerden ve günlüklerden dosyaları silmez (bkz.autopurge) ve bu operatörün sorumluluğundadır. + +Bu bomba etkisiz hale getirilmeli. + +Aşağıdaki ZooKeeper (3.5.1) yapılandırması Yandex'te kullanılmaktadır.20 Mayıs 2017 tarihi itibariyle Metrica üretim ortamı: + +hayvanat bahçesi.cfg: + +``` bash +# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html + +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=30000 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=10 + +maxClientCnxns=2000 + +maxSessionTimeout=60000000 +# the directory where the snapshot is stored. +dataDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '}}' }}/data +# Place the dataLogDir to a separate physical disc for better performance +dataLogDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '}}' }}/logs + +autopurge.snapRetainCount=10 +autopurge.purgeInterval=1 + + +# To avoid seeks ZooKeeper allocates space in the transaction log file in +# blocks of preAllocSize kilobytes. The default block size is 64M. One reason +# for changing the size of the blocks is to reduce the block size if snapshots +# are taken more often. (Also, see snapCount). +preAllocSize=131072 + +# Clients can submit requests faster than ZooKeeper can process them, +# especially if there are a lot of clients. To prevent ZooKeeper from running +# out of memory due to queued requests, ZooKeeper will throttle clients so that +# there is no more than globalOutstandingLimit outstanding requests in the +# system. The default limit is 1,000.ZooKeeper logs transactions to a +# transaction log. After snapCount transactions are written to a log file a +# snapshot is started and a new transaction log file is started. The default +# snapCount is 10,000. +snapCount=3000000 + +# If this option is defined, requests will be will logged to a trace file named +# traceFile.year.month.day. +#traceFile= + +# Leader accepts client connections. Default value is "yes". The leader machine +# coordinates updates. For higher update throughput at thes slight expense of +# read throughput the leader can be configured to not accept clients and focus +# on coordination. +leaderServes=yes + +standaloneEnabled=false +dynamicConfigFile=/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/zoo.cfg.dynamic +``` + +Java sürümü: + +``` text +Java(TM) SE Runtime Environment (build 1.8.0_25-b17) +Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) +``` + +JVM parametreleri: + +``` bash +NAME=zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }} +ZOOCFGDIR=/etc/$NAME/conf + +# TODO this is really ugly +# How to find out, which jars are needed? +# seems, that log4j requires the log4j.properties file to be in the classpath +CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper" + +ZOOCFG="$ZOOCFGDIR/zoo.cfg" +ZOO_LOG_DIR=/var/log/$NAME +USER=zookeeper +GROUP=zookeeper +PIDDIR=/var/run/$NAME +PIDFILE=$PIDDIR/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +JAVA=/usr/bin/java +ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" +ZOO_LOG4J_PROP="INFO,ROLLINGFILE" +JMXLOCALONLY=false +JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '}}' }} \ + -Xmx{{ '{{' }} cluster.get('xmx','1G') {{ '}}' }} \ + -Xloggc:/var/log/$NAME/zookeeper-gc.log \ + -XX:+UseGCLogFileRotation \ + -XX:NumberOfGCLogFiles=16 \ + -XX:GCLogFileSize=16M \ + -verbose:gc \ + -XX:+PrintGCTimeStamps \ + -XX:+PrintGCDateStamps \ + -XX:+PrintGCDetails + -XX:+PrintTenuringDistribution \ + -XX:+PrintGCApplicationStoppedTime \ + -XX:+PrintGCApplicationConcurrentTime \ + -XX:+PrintSafepointStatistics \ + -XX:+UseParNewGC \ + -XX:+UseConcMarkSweepGC \ +-XX:+CMSParallelRemarkEnabled" +``` + +Tuz ve Karab saltiber: + +``` text +description "zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }} centralized coordination service" + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +limit nofile 8192 8192 + +pre-start script + [ -r "/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/environment" ] || exit 0 + . /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/environment + [ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR + chown $USER:$GROUP $ZOO_LOG_DIR +end script + +script + . /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/environment + [ -r /etc/default/zookeeper ] && . /etc/default/zookeeper + if [ -z "$JMXDISABLE" ]; then + JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" + fi + exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }} \ + -- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \ + -Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG +end script +``` + +{## [Orijinal makale](https://clickhouse.tech/docs/en/operations/tips/) ##} diff --git a/docs/tr/operations/troubleshooting.md b/docs/tr/operations/troubleshooting.md new file mode 100644 index 00000000000..f16a59767d6 --- /dev/null +++ b/docs/tr/operations/troubleshooting.md @@ -0,0 +1,146 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 46 +toc_title: "Ar\u0131za" +--- + +# Arıza {#troubleshooting} + +- [Kurulum](#troubleshooting-installation-errors) +- [Sunucuya bağlanma](#troubleshooting-accepts-no-connections) +- [Sorgu işleme](#troubleshooting-does-not-process-queries) +- [Sorgu işleme verimliliği](#troubleshooting-too-slow) + +## Kurulum {#troubleshooting-installation-errors} + +### Apt-get ile ClickHouse deposundan Deb paketleri alınamıyor {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get} + +- Güvenlik Duvarı ayarlarını kontrol edin. +- Depoya herhangi bir nedenle erişemiyorsanız, paketleri aşağıda açıklandığı gibi indirin [Başlarken](../getting_started/index.md) makale ve bunları kullanarak manuel olarak yükleyin `sudo dpkg -i ` komut. Ayrıca ihtiyacınız olacak `tzdata` paket. + +## Sunucuya bağlanma {#troubleshooting-accepts-no-connections} + +Olası sorunlar: + +- Sunucu çalışmıyor. +- Beklenmeyen veya yanlış yapılandırma parametreleri. + +### Sunucu Çalışmıyor {#server-is-not-running} + +**Sunucu runnnig olup olmadığını kontrol edin** + +Komut: + +``` bash +$ sudo service clickhouse-server status +``` + +Sunucu çalışmıyorsa, komutla başlatın: + +``` bash +$ sudo service clickhouse-server start +``` + +**Günlükleri kontrol et** + +Ana günlüğü `clickhouse-server` içinde `/var/log/clickhouse-server/clickhouse-server.log` varsayılan olarak. + +Sunucu başarıyla başlatıldıysa, dizeleri görmelisiniz: + +- ` Application: starting up.` — Server started. +- ` Application: Ready for connections.` — Server is running and ready for connections. + +Eğer `clickhouse-server` Başlat bir yapılandırma hatası ile başarısız oldu, görmelisiniz `` bir hata açıklaması ile dize. Mesela: + +``` text +2019.01.11 15:23:25.549505 [ 45 ] {} ExternalDictionaries: Failed reloading 'event2id' external dictionary: Poco::Exception. Code: 1000, e.code() = 111, e.displayText() = Connection refused, e.what() = Connection refused +``` + +Dosyanın sonunda bir hata görmüyorsanız, dizeden başlayarak tüm dosyaya bakın: + +``` text + Application: starting up. +``` + +İkinci bir örneğini başlatmaya çalışırsanız `clickhouse-server` sunucuda, aşağıdaki günlük bakın: + +``` text +2019.01.11 15:25:11.151730 [ 1 ] {} : Starting ClickHouse 19.1.0 with revision 54413 +2019.01.11 15:25:11.154578 [ 1 ] {} Application: starting up +2019.01.11 15:25:11.156361 [ 1 ] {} StatusFile: Status file ./status already exists - unclean restart. Contents: +PID: 8510 +Started at: 2019-01-11 15:24:23 +Revision: 54413 + +2019.01.11 15:25:11.156673 [ 1 ] {} Application: DB::Exception: Cannot lock file ./status. Another server instance in same directory is already running. +2019.01.11 15:25:11.156682 [ 1 ] {} Application: shutting down +2019.01.11 15:25:11.156686 [ 1 ] {} Application: Uninitializing subsystem: Logging Subsystem +2019.01.11 15:25:11.156716 [ 2 ] {} BaseDaemon: Stop SignalListener thread +``` + +**Bkz. sistem.d günlükleri** + +Eğer herhangi bir yararlı bilgi bulamazsanız `clickhouse-server` günlükler veya herhangi bir günlük yok, görüntüleyebilirsiniz `system.d` komutu kullanarak günlükleri: + +``` bash +$ sudo journalctl -u clickhouse-server +``` + +**Clickhouse-Server'ı etkileşimli modda Başlat** + +``` bash +$ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-server/config.xml +``` + +Bu komut, sunucuyu otomatik başlatma komut dosyasının standart parametreleriyle etkileşimli bir uygulama olarak başlatır. Bu modda `clickhouse-server` konsoldaki tüm olay iletilerini yazdırır. + +### Yapılandırma Parametreleri {#configuration-parameters} + +Kontrol: + +- Docker ayarları. + + Bir IPv6 ağında Docker'da ClickHouse çalıştırırsanız, `network=host` ayar .lanmıştır. + +- Bitiş noktası ayarları. + + Kontrol [listen\_host](server_configuration_parameters/settings.md#server_configuration_parameters-listen_host) ve [tcp\_port](server_configuration_parameters/settings.md#server_configuration_parameters-tcp_port) ayarlar. + + ClickHouse server, yalnızca varsayılan olarak localhost bağlantılarını kabul eder. + +- HTTP protokolü ayarları. + + HTTP API protokol ayarlarını denetleyin. + +- Güvenli bağlantı ayarları. + + Kontrol: + + - Bu [tcp\_port\_secure](server_configuration_parameters/settings.md#server_configuration_parameters-tcp_port_secure) ayar. + - İçin ayarlar [SSL sertifikaları](server_configuration_parameters/settings.md#server_configuration_parameters-openssl). + + Bağlanırken uygun parametreleri kullanın. Örneğin, kullanın `port_secure` parametre ile `clickhouse_client`. + +- Kullanıcı ayarları. + + Yanlış kullanıcı adı veya parola kullanıyor olabilirsiniz. + +## Sorgu İşleme {#troubleshooting-does-not-process-queries} + +ClickHouse sorguyu işlemek mümkün değilse, istemciye bir hata açıklaması gönderir. İn the `clickhouse-client` konsoldaki hatanın bir açıklamasını alırsınız. Http arabirimini kullanıyorsanız, ClickHouse yanıt gövdesinde hata açıklamasını gönderir. Mesela: + +``` bash +$ curl 'http://localhost:8123/' --data-binary "SELECT a" +Code: 47, e.displayText() = DB::Exception: Unknown identifier: a. Note that there are no tables (FROM clause) in your query, context: required_names: 'a' source_tables: table_aliases: private_aliases: column_aliases: public_columns: 'a' masked_columns: array_join_columns: source_columns: , e.what() = DB::Exception +``` + +Eğer başlarsanız `clickhouse-client` ile... `stack-trace` parametre, ClickHouse bir hata açıklaması ile sunucu yığın izleme döndürür. + +Bozuk bir bağlantı hakkında bir mesaj görebilirsiniz. Bu durumda, sorguyu tekrarlayabilirsiniz. Sorguyu her gerçekleştirdiğinizde bağlantı kesilirse, sunucu günlüklerini hatalar için denetleyin. + +## Sorgu işleme verimliliği {#troubleshooting-too-slow} + +Clickhouse'un çok yavaş çalıştığını görürseniz, sorgularınız için sunucu kaynakları ve ağdaki yükü profillemeniz gerekir. + +Profil sorguları için clickhouse-benchmark yardımcı programını kullanabilirsiniz. Saniyede işlenen sorgu sayısını, saniyede işlenen satır sayısını ve sorgu işleme sürelerinin yüzdelerini gösterir. diff --git a/docs/tr/operations/update.md b/docs/tr/operations/update.md new file mode 100644 index 00000000000..1529729321e --- /dev/null +++ b/docs/tr/operations/update.md @@ -0,0 +1,20 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 47 +toc_title: "ClickHouse G\xFCncelleme" +--- + +# ClickHouse Güncelleme {#clickhouse-update} + +ClickHouse DEB paketlerinden yüklüyse, sunucuda aşağıdaki komutları çalıştırın: + +``` bash +$ sudo apt-get update +$ sudo apt-get install clickhouse-client clickhouse-server +$ sudo service clickhouse-server restart +``` + +Önerilen deb paketleri dışında bir şey kullanarak ClickHouse yüklediyseniz, uygun güncelleştirme yöntemini kullanın. + +ClickHouse dağıtılmış bir güncelleştirmeyi desteklemiyor. İşlem, her ayrı sunucuda ardışık olarak gerçekleştirilmelidir. Bir kümedeki tüm sunucuları aynı anda güncelleştirmeyin veya küme Bir süre kullanılamaz. diff --git a/docs/tr/operations/utilities/clickhouse-benchmark.md b/docs/tr/operations/utilities/clickhouse-benchmark.md new file mode 100644 index 00000000000..952852d880e --- /dev/null +++ b/docs/tr/operations/utilities/clickhouse-benchmark.md @@ -0,0 +1,156 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 61 +toc_title: clickhouse-benchmark +--- + +# clickhouse-benchmark {#clickhouse-benchmark} + +Bir ClickHouse sunucusuna bağlanır ve art arda belirtilen sorguları gönderir. + +Sözdizimi: + +``` bash +$ echo "single query" | clickhouse-benchmark [keys] +``` + +veya + +``` bash +$ clickhouse-benchmark [keys] <<< "single query" +``` + +Bir dizi sorgu göndermek istiyorsanız, Bir metin dosyası oluşturun ve her sorguyu bu dosyadaki tek tek dizeye yerleştirin. Mesela: + +``` sql +SELECT * FROM system.numbers LIMIT 10000000 +SELECT 1 +``` + +Sonra bu dosyayı standart bir girişe geçirin `clickhouse-benchmark`. + +``` bash +clickhouse-benchmark [keys] < queries_file +``` + +## Anahtarlar {#clickhouse-benchmark-keys} + +- `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` aynı anda gönderir. Varsayılan değer: 1. +- `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1. +- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. İçin [karşılaştırma modu](#clickhouse-benchmark-comparison-mode) birden fazla kullanabilirsiniz `-h` anahtarlar. +- `-p N`, `--port=N` — Server port. Default value: 9000. For the [karşılaştırma modu](#clickhouse-benchmark-comparison-mode) birden fazla kullanabilirsiniz `-p` anahtarlar. +- `-i N`, `--iterations=N` — Total number of queries. Default value: 0. +- `-r`, `--randomize` — Random order of queries execution if there is more then one input query. +- `-s`, `--secure` — Using TLS connection. +- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` belirtilen zaman sınırına ulaşıldığında sorgu göndermeyi durdurur. Varsayılan değer: 0 (zaman sınırı devre dışı). +- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [karşılaştırma modu](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` gerçekleştirir [Bağımsız iki örnek öğrencinin t-testi](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) iki dağıtımın seçilen güven düzeyi ile farklı olup olmadığını belirlemek için sınayın. +- `--cumulative` — Printing cumulative data instead of data per interval. +- `--database=DATABASE_NAME` — ClickHouse database name. Default value: `default`. +- `--json=FILEPATH` — JSON output. When the key is set, `clickhouse-benchmark` belirtilen json dosyasına bir rapor verir. +- `--user=USERNAME` — ClickHouse user name. Default value: `default`. +- `--password=PSWD` — ClickHouse user password. Default value: empty string. +- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` çıkışlar özel durumların izlerini yığın. +- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` belirtilen aşamada. Olası değerler: `complete`, `fetch_columns`, `with_mergeable_state`. Varsayılan değer: `complete`. +- `--help` — Shows the help message. + +Bazı uygulamak istiyorsanız [ayarlar](../../operations/settings/index.md) sorgular için bunları bir anahtar olarak geçirin `--= SETTING_VALUE`. Mesela, `--max_memory_usage=1048576`. + +## Çıktı {#clickhouse-benchmark-output} + +Varsayılan olarak, `clickhouse-benchmark` her biri için raporlar `--delay` aralıklı. + +Rapor örneği: + +``` text +Queries executed: 10. + +localhost:9000, queries 10, QPS: 6.772, RPS: 67904487.440, MiB/s: 518.070, result RPS: 67721584.984, result MiB/s: 516.675. + +0.000% 0.145 sec. +10.000% 0.146 sec. +20.000% 0.146 sec. +30.000% 0.146 sec. +40.000% 0.147 sec. +50.000% 0.148 sec. +60.000% 0.148 sec. +70.000% 0.148 sec. +80.000% 0.149 sec. +90.000% 0.150 sec. +95.000% 0.150 sec. +99.000% 0.150 sec. +99.900% 0.150 sec. +99.990% 0.150 sec. +``` + +Raporda bulabilirsiniz: + +- Sorgu sayısı `Queries executed:` alan. + +- İçeren durum dizesi (sırayla): + + - ClickHouse sunucusunun bitiş noktası. + - İşlenen sorgu sayısı. + - QPS: qps: kaç sorgu sunucusu saniyede belirtilen bir süre boyunca gerçekleştirilen `--delay` değişken. + - RPS: kaç satır sunucu saniyede belirtilen bir süre boyunca okuma `--delay` değişken. + - MıB / s: kaç mebibytes sunucu saniyede belirtilen bir süre boyunca okuma `--delay` değişken. + - sonuç RPS: sunucu tarafından belirtilen bir süre boyunca saniyede bir sorgunun sonucuna kaç satır yerleştirilir `--delay` değişken. + - sonuç MıB / s. kaç mebibytes sunucu tarafından belirtilen bir dönemde saniyede bir sorgu sonucu yerleştirilir `--delay` değişken. + +- Sorgu yürütme süresi yüzdelik. + +## Karşılaştırma modu {#clickhouse-benchmark-comparison-mode} + +`clickhouse-benchmark` iki çalışan ClickHouse sunucuları için performansları karşılaştırabilirsiniz. + +Karşılaştırma modunu kullanmak için, her iki sunucunun bitiş noktalarını iki çift `--host`, `--port` anahtarlar. Anahtarlar argüman listesindeki konuma göre eşleşti, ilk `--host` ilk ile eşleştirilir `--port` ve böyle devam eder. `clickhouse-benchmark` her iki sunucuya da bağlantılar kurar, sonra sorgular gönderir. Her sorgu rastgele seçilen bir sunucuya gönderilir. Sonuçlar her sunucu için ayrı ayrı gösterilir. + +## Örnek {#clickhouse-benchmark-example} + +``` bash +$ echo "SELECT * FROM system.numbers LIMIT 10000000 OFFSET 10000000" | clickhouse-benchmark -i 10 +``` + +``` text +Loaded 1 queries. + +Queries executed: 6. + +localhost:9000, queries 6, QPS: 6.153, RPS: 123398340.957, MiB/s: 941.455, result RPS: 61532982.200, result MiB/s: 469.459. + +0.000% 0.159 sec. +10.000% 0.159 sec. +20.000% 0.159 sec. +30.000% 0.160 sec. +40.000% 0.160 sec. +50.000% 0.162 sec. +60.000% 0.164 sec. +70.000% 0.165 sec. +80.000% 0.166 sec. +90.000% 0.166 sec. +95.000% 0.167 sec. +99.000% 0.167 sec. +99.900% 0.167 sec. +99.990% 0.167 sec. + + + +Queries executed: 10. + +localhost:9000, queries 10, QPS: 6.082, RPS: 121959604.568, MiB/s: 930.478, result RPS: 60815551.642, result MiB/s: 463.986. + +0.000% 0.159 sec. +10.000% 0.159 sec. +20.000% 0.160 sec. +30.000% 0.163 sec. +40.000% 0.164 sec. +50.000% 0.165 sec. +60.000% 0.166 sec. +70.000% 0.166 sec. +80.000% 0.167 sec. +90.000% 0.167 sec. +95.000% 0.170 sec. +99.000% 0.172 sec. +99.900% 0.172 sec. +99.990% 0.172 sec. +``` diff --git a/docs/tr/operations/utilities/clickhouse-copier.md b/docs/tr/operations/utilities/clickhouse-copier.md new file mode 100644 index 00000000000..932519086c1 --- /dev/null +++ b/docs/tr/operations/utilities/clickhouse-copier.md @@ -0,0 +1,176 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 59 +toc_title: clickhouse-fotokopi makinesi +--- + +# clickhouse-fotokopi makinesi {#clickhouse-copier} + +Bir kümedeki tablolardan başka bir (veya aynı) kümedeki tablolara veri kopyalar. + +Birden fazla çalıştırabilirsiniz `clickhouse-copier` aynı işi gerçekleştirmek için farklı sunuculardaki örnekler. ZooKeeper süreçleri senkronize etmek için kullanılır. + +Başladıktan sonra, `clickhouse-copier`: + +- ZooKeeper bağlanır ve alır: + + - Kopyalama işleri. + - Kopyalama işlerinin durumu. + +- İşleri gerçekleştirir. + + Her çalışan işlem seçer “closest” kaynak kümenin parçası ve verileri hedef kümeye kopyalar, gerekirse verileri yeniden şekillendirir. + +`clickhouse-copier` ZooKeeper değişiklikleri izler ve anında uygular. + +Ağ trafiğini azaltmak için çalıştırmanızı öneririz `clickhouse-copier` kaynak verilerin bulunduğu aynı sunucuda. + +## Clickhouse çalışan-fotokopi {#running-clickhouse-copier} + +Yardımcı program el ile çalıştırılmalıdır: + +``` bash +$ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir +``` + +Parametre: + +- `daemon` — Starts `clickhouse-copier` daemon modunda. +- `config` — The path to the `zookeeper.xml` ZooKeeper bağlantı parametreleri ile dosya. +- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` süreçleri ve depolama görevleri. Görevler saklanır `$task-path/description`. +- `task-file` — Optional path to file with task configuration for initial upload to ZooKeeper. +- `task-upload-force` — Force upload `task-file` düğüm zaten var olsa bile. +- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` oluşturuyor `clickhouse-copier_YYYYMMHHSS_` içinde alt dizinler `$base-dir`. Bu parametre atlanırsa, dizinler aşağıdaki dizinde oluşturulur `clickhouse-copier` başlatıldı. + +## Zookeeper biçimi.xml {#format-of-zookeeper-xml} + +``` xml + + + trace + 100M + 3 + + + + + 127.0.0.1 + 2181 + + + +``` + +## Kopyalama görevlerinin yapılandırması {#configuration-of-copying-tasks} + +``` xml + + + + + + false + + 127.0.0.1 + 9000 + + + ... + + + + ... + + + + + 2 + + + + 1 + + + + + 0 + + + + + 3 + + 1 + + + + + + + + source_cluster + test + hits + + + destination_cluster + test + hits2 + + + + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') + PARTITION BY toMonday(date) + ORDER BY (CounterID, EventDate) + + + + jumpConsistentHash(intHash64(UserID), 2) + + + CounterID != 0 + + + + '2018-02-26' + '2018-03-05' + ... + + + + + + ... + + ... + + +``` + +`clickhouse-copier` değişiklikleri izler `/task/path/description` ve onları anında uygular. Örneğin, değerini değiştirirseniz `max_workers`, görevleri çalıştıran süreçlerin sayısı da değişecektir. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/utils/clickhouse-copier/) diff --git a/docs/tr/operations/utilities/clickhouse-local.md b/docs/tr/operations/utilities/clickhouse-local.md new file mode 100644 index 00000000000..5e47459b670 --- /dev/null +++ b/docs/tr/operations/utilities/clickhouse-local.md @@ -0,0 +1,81 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 60 +toc_title: clickhouse-yerel +--- + +# clickhouse-yerel {#clickhouse-local} + +Bu `clickhouse-local` program, ClickHouse sunucusunu dağıtmak ve yapılandırmak zorunda kalmadan yerel dosyalar üzerinde hızlı işlem yapmanızı sağlar. + +Tabloları temsil eden verileri kabul eder ve bunları kullanarak sorgular [ClickHouse SQL lehçesi](../../sql_reference/index.md). + +`clickhouse-local` ClickHouse server ile aynı çekirdeği kullanır, bu nedenle özelliklerin çoğunu ve aynı format ve tablo motorlarını destekler. + +Varsayılan olarak `clickhouse-local` aynı ana bilgisayarda verilere erişimi yok, ancak kullanarak yükleme sunucu yapılandırmasını destekler `--config-file` değişken. + +!!! warning "Uyarıcı" + İçine üretim sunucusu yapılandırmasını yüklemek için tavsiye edilmez `clickhouse-local` çünkü insan hatası durumunda veriler zarar görebilir. + +## Kullanma {#usage} + +Temel kullanım: + +``` bash +$ clickhouse-local --structure "table_structure" --input-format "format_of_incoming_data" -q "query" +``` + +Değişkenler: + +- `-S`, `--structure` — table structure for input data. +- `-if`, `--input-format` — input format, `TSV` varsayılan olarak. +- `-f`, `--file` — path to data, `stdin` varsayılan olarak. +- `-q` `--query` — queries to execute with `;` delimeter olarak. +- `-N`, `--table` — table name where to put output data, `table` varsayılan olarak. +- `-of`, `--format`, `--output-format` — output format, `TSV` varsayılan olarak. +- `--stacktrace` — whether to dump debug output in case of exception. +- `--verbose` — more details on query execution. +- `-s` — disables `stderr` günlük. +- `--config-file` — path to configuration file in same format as for ClickHouse server, by default the configuration empty. +- `--help` — arguments references for `clickhouse-local`. + +Ayrıca, bunun yerine daha yaygın olarak kullanılan her ClickHouse yapılandırma değişkeni için argümanlar vardır `--config-file`. + +## Örnekler {#examples} + +``` bash +$ echo -e "1,2\n3,4" | clickhouse-local -S "a Int64, b Int64" -if "CSV" -q "SELECT * FROM table" +Read 2 rows, 32.00 B in 0.000 sec., 5182 rows/sec., 80.97 KiB/sec. +1 2 +3 4 +``` + +Önceki örnek aynıdır: + +``` bash +$ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); SELECT a, b FROM table; DROP TABLE table" +Read 2 rows, 32.00 B in 0.000 sec., 4987 rows/sec., 77.93 KiB/sec. +1 2 +3 4 +``` + +Şimdi her Unix kullanıcısı için bellek kullanıcısını çıkaralım: + +``` bash +$ ps aux | tail -n +2 | awk '{ printf("%s\t%s\n", $1, $4) }' | clickhouse-local -S "user String, mem Float64" -q "SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty" +``` + +``` text +Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec. +┏━━━━━━━━━━┳━━━━━━━━━━┓ +┃ user ┃ memTotal ┃ +┡━━━━━━━━━━╇━━━━━━━━━━┩ +│ bayonet │ 113.5 │ +├──────────┼──────────┤ +│ root │ 8.8 │ +├──────────┼──────────┤ +... +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/utils/clickhouse-local/) diff --git a/docs/tr/operations/utilities/index.md b/docs/tr/operations/utilities/index.md new file mode 100644 index 00000000000..17d3849cc6a --- /dev/null +++ b/docs/tr/operations/utilities/index.md @@ -0,0 +1,15 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Programlar +toc_priority: 56 +toc_title: "Genel bak\u0131\u015F" +--- + +# ClickHouse Programı {#clickhouse-utility} + +- [clickhouse-yerel](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` bunu yapar. +- [clickhouse-fotokopi makinesi](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster. +- [clickhouse-benchmark](clickhouse-benchmark.md) — Loads server with the custom queries and settings. + +[Orijinal makale](https://clickhouse.tech/docs/en/operations/utils/) diff --git a/docs/tr/sql_reference/aggregate_functions/combinators.md b/docs/tr/sql_reference/aggregate_functions/combinators.md new file mode 100644 index 00000000000..693bf5e0348 --- /dev/null +++ b/docs/tr/sql_reference/aggregate_functions/combinators.md @@ -0,0 +1,166 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 37 +toc_title: "Toplama fonksiyonu birle\u015Ftiriciler" +--- + +# Toplama Fonksiyonu Birleştiriciler {#aggregate_functions_combinators} + +Bir toplama işlevinin adı, ona eklenmiş bir sonek olabilir. Bu, toplama işlevinin çalışma şeklini değiştirir. + +## -Eğer {#agg-functions-combinator-if} + +The suffix -If can be appended to the name of any aggregate function. In this case, the aggregate function accepts an extra argument – a condition (Uint8 type). The aggregate function processes only the rows that trigger the condition. If the condition was not triggered even once, it returns a default value (usually zeros or empty strings). + +Örnekler: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` ve böyle devam eder. + +Koşullu toplama işlevleriyle, alt sorgular kullanmadan aynı anda birkaç koşul için toplamları hesaplayabilirsiniz ve `JOIN`s. Örneğin, Üye olarak.Metrica, koşullu toplama işlevleri segment karşılaştırma işlevselliğini uygulamak için kullanılır. + +## -Dizi {#agg-functions-combinator-array} + +\- Array soneki herhangi bir toplama işlevine eklenebilir. Bu durumda, toplama işlevi, ‘Array(T)’ type (ar arraysra )ys) yerine ‘T’ bağımsız değişkenleri yazın. Toplama işlevi birden çok bağımsız değişken kabul ederse, bu eşit uzunlukta diziler olmalıdır. Dizileri işlerken, toplama işlevi tüm dizi öğelerinde orijinal toplama işlevi gibi çalışır. + +Örnek 1: `sumArray(arr)` - Tüm unsurları toplamları ‘arr’ diziler. Bu örnekte, daha basit yazılmış olabilir: `sum(arraySum(arr))`. + +Örnek 2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ diziler. Bu daha kolay bir şekilde yapılabilir: `uniq(arrayJoin(arr))`, ancak eklemek her zaman mümkün değildir ‘arrayJoin’ bir sorguya. + +\- Eğer ve-dizi kombine edilebilir. Ancak, ‘Array’ önce gel mustmeli, sonra ‘If’. Örnekler: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. Nedeniyle bu sipariş için, ‘cond’ argüman bir dizi olmayacak. + +## -Devlet {#agg-functions-combinator-state} + +Bu birleştiriciyi uygularsanız, toplama işlevi elde edilen değeri döndürmez (örneğin, [uniq](reference.md#agg_function-uniq) fonksiyonu), ancak top aggreglamanın bir ara durumu (for `uniq`, bu benzersiz değerlerin sayısını hesaplamak için karma tablodur). Bu bir `AggregateFunction(...)` bu, daha fazla işlem için kullanılabilir veya daha sonra toplanmayı bitirmek için bir tabloda saklanabilir. + +Bu durumlarla çalışmak için şunları kullanın: + +- [AggregatingMergeTree](../../engines/table_engines/mergetree_family/aggregatingmergetree.md) masa motoru. +- [finalizeAggregation](../../sql_reference/functions/other_functions.md#function-finalizeaggregation) işlev. +- [runningAccumulate](../../sql_reference/functions/other_functions.md#function-runningaccumulate) işlev. +- [-Birleştirmek](#aggregate_functions_combinators_merge) birleştirici. +- [- MergeState](#aggregate_functions_combinators_mergestate) birleştirici. + +## -Birleştirmek {#aggregate_functions_combinators-merge} + +Bu birleştiriciyi uygularsanız, toplama işlevi Ara toplama durumunu bağımsız değişken olarak alır, toplama işlemini tamamlamak için durumları birleştirir ve elde edilen değeri döndürür. + +## - MergeState {#aggregate_functions_combinators-mergestate} + +Ara toplama durumlarını-birleştirme Birleştiricisi ile aynı şekilde birleştirir. Bununla birlikte, elde edilen değeri döndürmez, ancak-State combinator'a benzer bir ara toplama durumu döndürür. + +## - ForEach {#agg-functions-combinator-foreach} + +Tablolar için bir toplama işlevi, karşılık gelen dizi öğelerini toplayan ve bir dizi sonuç döndüren diziler için bir toplama işlevine dönüştürür. Mesela, `sumForEach` diz theiler için `[1, 2]`, `[3, 4, 5]`ve`[6, 7]`sonucu döndürür `[10, 13, 5]` karşılık gelen dizi öğelerini bir araya getirdikten sonra. + +## - OrDefault {#agg-functions-combinator-ordefault} + +Toplamak için hiçbir şey yoksa, toplama işlevinin dönüş türünün Varsayılan değerini doldurur. + +``` sql +SELECT avg(number), avgOrDefault(number) FROM numbers(0) +``` + +``` text +┌─avg(number)─┬─avgOrDefault(number)─┐ +│ nan │ 0 │ +└─────────────┴──────────────────────┘ +``` + +## - OrNull {#agg-functions-combinator-ornull} + +Doldurmalar `null` toplamak için hiçbir şey varsa. Dönüş sütun null olur. + +``` sql +SELECT avg(number), avgOrNull(number) FROM numbers(0) +``` + +``` text +┌─avg(number)─┬─avgOrNull(number)─┐ +│ nan │ ᴺᵁᴸᴸ │ +└─────────────┴───────────────────┘ +``` + +\- OrDefault ve-OrNull diğer birleştiriciler ile kombine edilebilir. Toplama işlevi boş girişi kabul etmediğinde yararlıdır. + +``` sql +SELECT avgOrNullIf(x, x > 10) +FROM +( + SELECT toDecimal32(1.23, 2) AS x +) +``` + +``` text +┌─avgOrNullIf(x, greater(x, 10))─┐ +│ ᴺᵁᴸᴸ │ +└────────────────────────────────┘ +``` + +## - Resample {#agg-functions-combinator-resample} + +Verileri gruplara ayırmanızı sağlar ve ardından bu gruplardaki verileri ayrı ayrı toplar. Gruplar, değerleri bir sütundan aralıklara bölerek oluşturulur. + +``` sql +Resample(start, end, step)(, resampling_key) +``` + +**Parametre** + +- `start` — Starting value of the whole required interval for `resampling_key` değerler. +- `stop` — Ending value of the whole required interval for `resampling_key` değerler. Tüm Aralık içermez `stop` değer `[start, stop)`. +- `step` — Step for separating the whole interval into subintervals. The `aggFunction` bu alt aralıkların her biri üzerinde bağımsız olarak yürütülür. +- `resampling_key` — Column whose values are used for separating data into intervals. +- `aggFunction_params` — `aggFunction` parametre. + +**Döndürülen değerler** + +- Ar arrayray of `aggFunction` her subinterval için sonuçlar. + +**Örnek** + +Düşünün `people` aşağıdaki verilerle tablo: + +``` text +┌─name───┬─age─┬─wage─┐ +│ John │ 16 │ 10 │ +│ Alice │ 30 │ 15 │ +│ Mary │ 35 │ 8 │ +│ Evelyn │ 48 │ 11.5 │ +│ David │ 62 │ 9.9 │ +│ Brian │ 60 │ 16 │ +└────────┴─────┴──────┘ +``` + +Yaş aralığı içinde olan kişilerin isimlerini alalım `[30,60)` ve `[60,75)`. Yaş için tamsayı temsilini kullandığımızdan, yaşları `[30, 59]` ve `[60,74]` aralıklılar. + +Bir dizideki isimleri toplamak için, [groupArray](reference.md#agg_function-grouparray) toplama işlevi. Bir argüman alır. Bizim durumumuzda, bu `name` sütun. Bu `groupArrayResample` fonksiyon kullanmalıdır `age` yaşlara göre isimleri toplamak için sütun. Gerekli aralıkları tanımlamak için `30, 75, 30` argü themanlar içine `groupArrayResample` işlev. + +``` sql +SELECT groupArrayResample(30, 75, 30)(name, age) FROM people +``` + +``` text +┌─groupArrayResample(30, 75, 30)(name, age)─────┐ +│ [['Alice','Mary','Evelyn'],['David','Brian']] │ +└───────────────────────────────────────────────┘ +``` + +Sonuçları düşünün. + +`Jonh` çok genç olduğu için numunenin dışında. Diğer insanlar belirtilen yaş aralıklarına göre dağıtılır. + +Şimdi toplam insan sayısını ve ortalama ücretlerini belirtilen yaş aralıklarında sayalım. + +``` sql +SELECT + countResample(30, 75, 30)(name, age) AS amount, + avgResample(30, 75, 30)(wage, age) AS avg_wage +FROM people +``` + +``` text +┌─amount─┬─avg_wage──────────────────┐ +│ [3,2] │ [11.5,12.949999809265137] │ +└────────┴───────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/agg_functions/combinators/) diff --git a/docs/tr/sql_reference/aggregate_functions/index.md b/docs/tr/sql_reference/aggregate_functions/index.md new file mode 100644 index 00000000000..b1699ab5b62 --- /dev/null +++ b/docs/tr/sql_reference/aggregate_functions/index.md @@ -0,0 +1,62 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Toplama Fonksiyonlar\u0131" +toc_priority: 33 +toc_title: "Giri\u015F" +--- + +# Toplama fonksiyonları {#aggregate-functions} + +Toplama fonksiyonları [normal](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) veritabanı uzmanları tarafından beklendiği gibi. + +ClickHouse da destekler: + +- [Parametrik agrega fonksiyonları](parametric_functions.md#aggregate_functions_parametric), sütunlara ek olarak diğer parametreleri kabul eder. +- [Birleştiriciler](combinators.md#aggregate_functions_combinators) toplama işlevlerinin davranışını değiştiren. + +## NULL işleme {#null-processing} + +Toplama sırasında, tüm `NULL`s atlanır. + +**Örnekler:** + +Bu tabloyu düşünün: + +``` text +┌─x─┬────y─┐ +│ 1 │ 2 │ +│ 2 │ ᴺᵁᴸᴸ │ +│ 3 │ 2 │ +│ 3 │ 3 │ +│ 3 │ ᴺᵁᴸᴸ │ +└───┴──────┘ +``` + +Diyelim ki değerleri toplamanız gerekiyor `y` sütun: + +``` sql +SELECT sum(y) FROM t_null_big +``` + + ┌─sum(y)─┐ + │ 7 │ + └────────┘ + +Bu `sum` fonksiyon yorumlar `NULL` olarak `0`. Özellikle, bu, işlevin tüm değerlerin bulunduğu bir seçimin girişini aldığı anlamına gelir `NULL`, sonra sonuç olacak `0`, değil `NULL`. + +Şimdi kullanabilirsiniz `groupArray` bir dizi oluşturmak için işlev `y` sütun: + +``` sql +SELECT groupArray(y) FROM t_null_big +``` + +``` text +┌─groupArray(y)─┐ +│ [2,2,3] │ +└───────────────┘ +``` + +`groupArray` içermez `NULL` elde edilen dizi. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/agg_functions/) diff --git a/docs/tr/sql_reference/aggregate_functions/parametric_functions.md b/docs/tr/sql_reference/aggregate_functions/parametric_functions.md new file mode 100644 index 00000000000..557b844ef00 --- /dev/null +++ b/docs/tr/sql_reference/aggregate_functions/parametric_functions.md @@ -0,0 +1,499 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 38 +toc_title: "Parametrik agrega fonksiyonlar\u0131" +--- + +# Parametrik Agrega Fonksiyonları {#aggregate_functions_parametric} + +Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. + +## çubuk {#histogram} + +Uyarlanabilir bir histogram hesaplar. Kesin sonuçları garanti etmez. + +``` sql +histogram(number_of_bins)(values) +``` + +İşlevleri kullanır [Bir Akış Paralel Karar Ağacı Algoritması](http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf). Histogram kutularının sınırları, yeni veriler bir işleve girdiğinde ayarlanır. Ortak durumda, kutu genişlikleri eşit değildir. + +**Parametre** + +`number_of_bins` — Upper limit for the number of bins in the histogram. The function automatically calculates the number of bins. It tries to reach the specified number of bins, but if it fails, it uses fewer bins. +`values` — [İfade](../syntax.md#syntax-expressions) giriş değerleri ile sonuçlanır. + +**Döndürülen değerler** + +- [Dizi](../../sql_reference/data_types/array.md) -den [Demetler](../../sql_reference/data_types/tuple.md) aşağıdaki format oftan: + + ``` + [(lower_1, upper_1, height_1), ... (lower_N, upper_N, height_N)] + ``` + + - `lower` — Lower bound of the bin. + - `upper` — Upper bound of the bin. + - `height` — Calculated height of the bin. + +**Örnek** + +``` sql +SELECT histogram(5)(number + 1) +FROM ( + SELECT * + FROM system.numbers + LIMIT 20 +) +``` + +``` text +┌─histogram(5)(plus(number, 1))───────────────────────────────────────────┐ +│ [(1,4.5,4),(4.5,8.5,4),(8.5,12.75,4.125),(12.75,17,4.625),(17,20,3.25)] │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +Bir histogram ile görselleştirebilirsiniz [bar](../../sql_reference/functions/other_functions.md#function-bar) fonksiyon, örneğin: + +``` sql +WITH histogram(5)(rand() % 100) AS hist +SELECT + arrayJoin(hist).3 AS height, + bar(height, 0, 6, 5) AS bar +FROM +( + SELECT * + FROM system.numbers + LIMIT 20 +) +``` + +``` text +┌─height─┬─bar───┐ +│ 2.125 │ █▋ │ +│ 3.25 │ ██▌ │ +│ 5.625 │ ████▏ │ +│ 5.625 │ ████▏ │ +│ 3.375 │ ██▌ │ +└────────┴───────┘ +``` + +Bu durumda, histogram kutusu kenarlıklarını bilmediğinizi unutmamalısınız. + +## sequenceMatch(pattern)(timestamp, cond1, cond2, …) {#function-sequencematch} + +Dizinin desenle eşleşen bir olay zinciri içerip içermediğini denetler. + +``` sql +sequenceMatch(pattern)(timestamp, cond1, cond2, ...) +``` + +!!! warning "Uyarıcı" + Aynı saniyede meydana gelen olaylar sonucu etkileyen tanımsız bir sırada sırayla yatıyordu. + +**Parametre** + +- `pattern` — Pattern string. See [Desen sözdizimi](#sequence-function-pattern-syntax). + +- `timestamp` — Column considered to contain time data. Typical data types are `Date` ve `DateTime`. Ayrıca desteklenen herhangi birini kullanabilirsiniz [Uİnt](../../sql_reference/data_types/int_uint.md) veri türleri. + +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. En fazla 32 koşul argümanını iletebilirsiniz. İşlev yalnızca bu koşullarda açıklanan olayları dikkate alır. Sıra, bir koşulda açıklanmayan veriler içeriyorsa, işlev bunları atlar. + +**Döndürülen değerler** + +- 1, Eğer desen eşleşti. +- Desen eşleşmezse 0. + +Tür: `UInt8`. + + +**Desen sözdizimi** + +- `(?N)` — Matches the condition argument at position `N`. Şartlar numaralandırılmıştır `[1, 32]` Aralık. Mesela, `(?1)` argü theman thela eşleş their `cond1` parametre. + +- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. + +- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` birbirinden 1800 saniyeden fazla meydana gelen olayları eşleşir. Bu olaylar arasında herhangi bir olayın keyfi bir sayısı olabilir. Kullanabilirsiniz `>=`, `>`, `<`, `<=` operatörler. + +**Örnekler** + +Verileri göz önünde bulundurun `t` Tablo: + +``` text +┌─time─┬─number─┐ +│ 1 │ 1 │ +│ 2 │ 3 │ +│ 3 │ 2 │ +└──────┴────────┘ +``` + +Sorguyu gerçekleştir: + +``` sql +SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2) FROM t +``` + +``` text +┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2))─┐ +│ 1 │ +└───────────────────────────────────────────────────────────────────────┘ +``` + +İşlev, 2 numarasının 1 numarayı takip ettiği olay zincirini buldu. Sayı bir olay olarak tanımlanmadığı için aralarında 3 sayısını atladı. Örnekte verilen olay zincirini ararken bu numarayı dikkate almak istiyorsak, bunun için bir koşul oluşturmalıyız. + +``` sql +SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM t +``` + +``` text +┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2), equals(number, 3))─┐ +│ 0 │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +Bu durumda, işlev desenle eşleşen olay zincirini bulamadı, çünkü 3 numaralı olay 1 ile 2 arasında gerçekleşti. Aynı durumda 4 numaralı koşulu kontrol edersek, sıra desenle eşleşir. + +``` sql +SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM t +``` + +``` text +┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2), equals(number, 4))─┐ +│ 1 │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [sequenceCount](#function-sequencecount) + +## sequenceCount(pattern)(time, cond1, cond2, …) {#function-sequencecount} + +Desenle eşleşen olay zincirlerinin sayısını sayar. İşlev, çakışmayan olay zincirlerini arar. Geçerli zincir eşleştirildikten sonra bir sonraki zinciri aramaya başlar. + +!!! warning "Uyarıcı" + Aynı saniyede meydana gelen olaylar sonucu etkileyen tanımsız bir sırada sırayla yatıyordu. + +``` sql +sequenceCount(pattern)(timestamp, cond1, cond2, ...) +``` + +**Parametre** + +- `pattern` — Pattern string. See [Desen sözdizimi](#sequence-function-pattern-syntax). + +- `timestamp` — Column considered to contain time data. Typical data types are `Date` ve `DateTime`. Ayrıca desteklenen herhangi birini kullanabilirsiniz [Uİnt](../../sql_reference/data_types/int_uint.md) veri türleri. + +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. En fazla 32 koşul argümanını iletebilirsiniz. İşlev yalnızca bu koşullarda açıklanan olayları dikkate alır. Sıra, bir koşulda açıklanmayan veriler içeriyorsa, işlev bunları atlar. + +**Döndürülen değerler** + +- Eşleşen çakışmayan olay zincirlerinin sayısı. + +Tür: `UInt64`. + +**Örnek** + +Verileri göz önünde bulundurun `t` Tablo: + +``` text +┌─time─┬─number─┐ +│ 1 │ 1 │ +│ 2 │ 3 │ +│ 3 │ 2 │ +│ 4 │ 1 │ +│ 5 │ 3 │ +│ 6 │ 2 │ +└──────┴────────┘ +``` + +2 numara aralarında diğer sayıların herhangi bir miktarda 1 numaradan sonra meydana kaç kez Sayın: + +``` sql +SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t +``` + +``` text +┌─sequenceCount('(?1).*(?2)')(time, equals(number, 1), equals(number, 2))─┐ +│ 2 │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [sequenceMatch](#function-sequencematch) + +## windowFunnel {#windowfunnel} + +Kayan bir zaman penceresinde olay zincirlerini arar ve zincirden meydana gelen en fazla olay sayısını hesaplar. + +Fonksiyon algoritmaya göre çalışır: + +- İşlev, zincirdeki ilk koşulu tetikleyen ve olay sayacını 1'e ayarlayan verileri arar. Sürgülü pencerenin başladığı an budur. + +- Zincirdeki olaylar pencerede sırayla gerçekleşirse, sayaç artırılır. Olayların sırası bozulursa, sayaç artırılmaz. + +- Verilerin çeşitli tamamlanma noktalarında birden çok olay zinciri varsa, işlev yalnızca en uzun zincirin boyutunu çıkarır. + +**Sözdizimi** + +``` sql +windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) +``` + +**Parametre** + +- `window` — Length of the sliding window in seconds. +- `mode` - Bu isteğe bağlı bir argüman. + - `'strict'` - Zaman `'strict'` ayarlanırsa, windowFunnel () yalnızca benzersiz değerler için koşullar uygular. +- `timestamp` — Name of the column containing the timestamp. Data types supported: [Tarihli](../../sql_reference/data_types/date.md), [DateTime](../../sql_reference/data_types/datetime.md#data_type-datetime) ve diğer imzasız tamsayı türleri (timestamp'ın `UInt64` yazın, değeri 2^63 - 1 olan Int64 maksimum değerini aşamaz). +- `cond` — Conditions or data describing the chain of events. [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değer** + +Sürgülü zaman penceresi içindeki zincirden ardışık tetiklenen koşulların maksimum sayısı. +Seçimdeki tüm zincirler analiz edilir. + +Tür: `Integer`. + +**Örnek** + +Kullanıcının bir telefon seçmesi ve çevrimiçi mağazada iki kez satın alması için belirli bir süre yeterli olup olmadığını belirleyin. + +Aşağıdaki olaylar zincirini ayarlayın: + +1. Mağaz theadaki Hesabına giriş yapan kullanıcı (`eventID = 1003`). +2. Kullanıcı bir telefon arar (`eventID = 1007, product = 'phone'`). +3. Kullanıcı sipariş verdi (`eventID = 1009`). +4. Kullanıcı tekrar sipariş yaptı (`eventID = 1010`). + +Giriş tablosu: + +``` text +┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ +│ 2019-01-28 │ 1 │ 2019-01-29 10:00:00 │ 1003 │ phone │ +└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ +┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ +│ 2019-01-31 │ 1 │ 2019-01-31 09:00:00 │ 1007 │ phone │ +└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ +┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ +│ 2019-01-30 │ 1 │ 2019-01-30 08:00:00 │ 1009 │ phone │ +└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ +┌─event_date─┬─user_id─┬───────────timestamp─┬─eventID─┬─product─┐ +│ 2019-02-01 │ 1 │ 2019-02-01 08:00:00 │ 1010 │ phone │ +└────────────┴─────────┴─────────────────────┴─────────┴─────────┘ +``` + +Kullanıcının ne kadar uzakta olduğunu öğrenin `user_id` 2019 yılının Ocak-Şubat aylarında bir dönemde zincirden geçebilir. + +Sorgu: + +``` sql +SELECT + level, + count() AS c +FROM +( + SELECT + user_id, + windowFunnel(6048000000000000)(timestamp, eventID = 1003, eventID = 1009, eventID = 1007, eventID = 1010) AS level + FROM trend + WHERE (event_date >= '2019-01-01') AND (event_date <= '2019-02-02') + GROUP BY user_id +) +GROUP BY level +ORDER BY level ASC +``` + +Sonuç: + +``` text +┌─level─┬─c─┐ +│ 4 │ 1 │ +└───────┴───┘ +``` + +## saklama {#retention} + +İşlev, bağımsız değişken olarak 1'den 32'ye kadar bir dizi koşul türünü alır `UInt8` bu, etkinlik için belirli bir koşulun karşılanıp karşılanmadığını gösterir. +Herhangi bir koşul bir argüman olarak belirtilebilir (aşağıdaki gibi [WHERE](../../sql_reference/statements/select.md#select-where)). + +İlk hariç, koşullar çiftler halinde geçerlidir: birinci ve ikinci doğruysa, ikincinin sonucu, birinci ve fird doğruysa, üçüncüsü doğru olacaktır. + +**Sözdizimi** + +``` sql +retention(cond1, cond2, ..., cond32); +``` + +**Parametre** + +- `cond` — an expression that returns a `UInt8` sonuç (1 veya 0). + +**Döndürülen değer** + +1 veya 0 dizisi. + +- 1 — condition was met for the event. +- 0 — condition wasn't met for the event. + +Tür: `UInt8`. + +**Örnek** + +Hesaplamanın bir örneğini düşünelim `retention` site trafiğini belirlemek için işlev. + +**1.** Сreate a table to illustrate an example. + +``` sql +CREATE TABLE retention_test(date Date, uid Int32) ENGINE = Memory; + +INSERT INTO retention_test SELECT '2020-01-01', number FROM numbers(5); +INSERT INTO retention_test SELECT '2020-01-02', number FROM numbers(10); +INSERT INTO retention_test SELECT '2020-01-03', number FROM numbers(15); +``` + +Giriş tablosu: + +Sorgu: + +``` sql +SELECT * FROM retention_test +``` + +Sonuç: + +``` text +┌───────date─┬─uid─┐ +│ 2020-01-01 │ 0 │ +│ 2020-01-01 │ 1 │ +│ 2020-01-01 │ 2 │ +│ 2020-01-01 │ 3 │ +│ 2020-01-01 │ 4 │ +└────────────┴─────┘ +┌───────date─┬─uid─┐ +│ 2020-01-02 │ 0 │ +│ 2020-01-02 │ 1 │ +│ 2020-01-02 │ 2 │ +│ 2020-01-02 │ 3 │ +│ 2020-01-02 │ 4 │ +│ 2020-01-02 │ 5 │ +│ 2020-01-02 │ 6 │ +│ 2020-01-02 │ 7 │ +│ 2020-01-02 │ 8 │ +│ 2020-01-02 │ 9 │ +└────────────┴─────┘ +┌───────date─┬─uid─┐ +│ 2020-01-03 │ 0 │ +│ 2020-01-03 │ 1 │ +│ 2020-01-03 │ 2 │ +│ 2020-01-03 │ 3 │ +│ 2020-01-03 │ 4 │ +│ 2020-01-03 │ 5 │ +│ 2020-01-03 │ 6 │ +│ 2020-01-03 │ 7 │ +│ 2020-01-03 │ 8 │ +│ 2020-01-03 │ 9 │ +│ 2020-01-03 │ 10 │ +│ 2020-01-03 │ 11 │ +│ 2020-01-03 │ 12 │ +│ 2020-01-03 │ 13 │ +│ 2020-01-03 │ 14 │ +└────────────┴─────┘ +``` + +**2.** Kullanıcıları benzersiz kimliğe göre grupla `uid` kullanarak `retention` işlev. + +Sorgu: + +``` sql +SELECT + uid, + retention(date = '2020-01-01', date = '2020-01-02', date = '2020-01-03') AS r +FROM retention_test +WHERE date IN ('2020-01-01', '2020-01-02', '2020-01-03') +GROUP BY uid +ORDER BY uid ASC +``` + +Sonuç: + +``` text +┌─uid─┬─r───────┐ +│ 0 │ [1,1,1] │ +│ 1 │ [1,1,1] │ +│ 2 │ [1,1,1] │ +│ 3 │ [1,1,1] │ +│ 4 │ [1,1,1] │ +│ 5 │ [0,0,0] │ +│ 6 │ [0,0,0] │ +│ 7 │ [0,0,0] │ +│ 8 │ [0,0,0] │ +│ 9 │ [0,0,0] │ +│ 10 │ [0,0,0] │ +│ 11 │ [0,0,0] │ +│ 12 │ [0,0,0] │ +│ 13 │ [0,0,0] │ +│ 14 │ [0,0,0] │ +└─────┴─────────┘ +``` + +**3.** Günde toplam site ziyaret sayısını hesaplayın. + +Sorgu: + +``` sql +SELECT + sum(r[1]) AS r1, + sum(r[2]) AS r2, + sum(r[3]) AS r3 +FROM +( + SELECT + uid, + retention(date = '2020-01-01', date = '2020-01-02', date = '2020-01-03') AS r + FROM retention_test + WHERE date IN ('2020-01-01', '2020-01-02', '2020-01-03') + GROUP BY uid +) +``` + +Sonuç: + +``` text +┌─r1─┬─r2─┬─r3─┐ +│ 5 │ 5 │ 5 │ +└────┴────┴────┘ +``` + +Nerede: + +- `r1`- 2020-01-01 sırasında siteyi ziyaret eden tekil ziyaretçi sayısı ( `cond1` koşul). +- `r2`- 2020-01-01 ve 2020-01-02 arasında belirli bir süre boyunca siteyi ziyaret eden tekil ziyaretçi sayısı (`cond1` ve `cond2` şartlar). +- `r3`- 2020-01-01 ve 2020-01-03 arasında belirli bir süre boyunca siteyi ziyaret eden tekil ziyaretçi sayısı (`cond1` ve `cond3` şartlar). + +## uniqUpTo(N) (x) {#uniquptonx} + +Calculates the number of different argument values ​​if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1. + +Küçük Ns ile kullanım için tavsiye, kadar 10. N'nin maksimum değeri 100'dür. + +Bir toplama işlevinin durumu için, 1 + n \* bir bayt değerinin boyutuna eşit bellek miktarını kullanır. +Dizeler için, 8 baytlık kriptografik olmayan bir karma saklar. Yani, hesaplama dizeler için yaklaşık olarak hesaplanır. + +İşlev ayrıca birkaç argüman için de çalışır. + +Büyük bir N değeri kullanıldığında ve benzersiz değerlerin sayısı n'den biraz daha az olduğu durumlar dışında mümkün olduğunca hızlı çalışır. + +Kullanım örneği: + +``` text +Problem: Generate a report that shows only keywords that produced at least 5 unique users. +Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5 +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/agg_functions/parametric_functions/) + +## sumMapFiltered (keys\_to\_keep) (anahtarlar, değerler) {#summapfilteredkeys-to-keepkeys-values} + +Aynı davranış [sumMap](reference.md#agg_functions-summap) dışında bir dizi anahtar parametre olarak geçirilir. Bu, özellikle yüksek bir Anahtarlık ile çalışırken yararlı olabilir. diff --git a/docs/tr/sql_reference/aggregate_functions/reference.md b/docs/tr/sql_reference/aggregate_functions/reference.md new file mode 100644 index 00000000000..d52b82be996 --- /dev/null +++ b/docs/tr/sql_reference/aggregate_functions/reference.md @@ -0,0 +1,1878 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 36 +toc_title: "Ba\u015Fvurma" +--- + +# Fonksiyon Referans {#function-reference} + +## sayma {#agg_function-count} + +Satır veya NOT-NULL değerleri sayar. + +ClickHouse için aşağıdaki sözdizimleri destekler `count`: +- `count(expr)` veya `COUNT(DISTINCT expr)`. +- `count()` veya `COUNT(*)`. Bu `count()` sözdizimi ClickHouse özeldir. + +**Parametre** + +Fonksiyon alabilir: + +- Sıfır parametreler. +- Bir [ifade](../syntax.md#syntax-expressions). + +**Döndürülen değer** + +- Fonksiyon parametreleri olmadan çağrılırsa, satır sayısını sayar. +- Eğer... [ifade](../syntax.md#syntax-expressions) geçirilir, daha sonra işlev bu ifadenin kaç kez NOT null döndürdüğünü sayar. İfad aede bir [Nullable](../../sql_reference/data_types/nullable.md)- type değeri, sonra sonucu `count` kalır değil `Nullable`. İfade döndürülürse işlev 0 döndürür `NULL` tüm satırlar için. + +Her iki durumda da döndürülen değerin türü [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Ayrıntı** + +ClickHouse destekler `COUNT(DISTINCT ...)` sözdizimi. Bu yapının davranışı Aşağıdakilere bağlıdır [count\_distinct\_implementation](../../operations/settings/settings.md#settings-count_distinct_implementation) ayar. Aşağıdakilerden hang theisini tanımlar [uniq\*](#agg_function-uniq) fonksiyonlar işlemi gerçekleştirmek için kullanılır. Varsayılan değer [uniqExact](#agg_function-uniqexact) işlev. + +Bu `SELECT count() FROM table` tablodaki girdi sayısı ayrı olarak depolanmadığı için sorgu en iyi duruma getirilmez. Tablodan küçük bir sütun seçer ve içindeki değerlerin sayısını sayar. + +**Örnekler** + +Örnek 1: + +``` sql +SELECT count() FROM t +``` + +``` text +┌─count()─┐ +│ 5 │ +└─────────┘ +``` + +Örnek 2: + +``` sql +SELECT name, value FROM system.settings WHERE name = 'count_distinct_implementation' +``` + +``` text +┌─name──────────────────────────┬─value─────┐ +│ count_distinct_implementation │ uniqExact │ +└───────────────────────────────┴───────────┘ +``` + +``` sql +SELECT count(DISTINCT num) FROM t +``` + +``` text +┌─uniqExact(num)─┐ +│ 3 │ +└────────────────┘ +``` + +Bu örnek gösteriyor ki `count(DISTINCT num)` tarafından gerçekleştirilir `uniqExact` fonksiyonu göre `count_distinct_implementation` ayar değeri. + +## herhangi(x) {#agg_function-any} + +İlk karşılaşılan değeri seçer. +Sorgu herhangi bir sırada ve hatta her seferinde farklı bir sırada çalıştırılabilir, bu nedenle bu işlevin sonucu belirsizdir. +Belirli bir sonuç elde etmek için ‘min’ veya ‘max’ fonksiyon yerine ‘any’. + +Bazı durumlarda, yürütme sırasına güvenebilirsiniz. Bu, select ORDER BY kullanan bir alt sorgudan geldiğinde durumlar için geçerlidir. + +Ne zaman bir `SELECT` sorgu vardır `GROUP BY` yan tümce veya en az bir toplama işlevi, ClickHouse (Mysql'in aksine), tüm ifadelerin `SELECT`, `HAVING`, ve `ORDER BY` anahtar functionslardan veya toplama işlev .lerinden hesaplan .malıdır. Başka bir deyişle, tablodan seçilen her sütun, anahtarlarda veya toplama işlevlerinde kullanılmalıdır. Mysql'de olduğu gibi davranış elde etmek için, diğer sütunları `any` toplama işlevi. + +## anyHeavy (x) {#anyheavyx} + +Kullanarak sık oluşan bir değer seçer [ağır vurucular](http://www.cs.umd.edu/~samir/498/karp.pdf) algoritma. Sorgunun yürütme iş parçacığı her durumda yarısından fazlasını oluşan bir değer varsa, bu değer döndürülür. Normalde, sonuç belirsizdir. + +``` sql +anyHeavy(column) +``` + +**Değişkenler** + +- `column` – The column name. + +**Örnek** + +Tak thee the [OnTime](../../getting_started/example_datasets/ontime.md) veri kümesi ve herhangi bir sık oluşan değeri seçin `AirlineID` sütun. + +``` sql +SELECT anyHeavy(AirlineID) AS res +FROM ontime +``` + +``` text +┌───res─┐ +│ 19690 │ +└───────┘ +``` + +## anyLast(x) {#anylastx} + +Karşılaşılan son değeri seçer. +Sonuç için olduğu kadar belirsiz `any` işlev. + +## groupBitAnd {#groupbitand} + +Bitwise uygular `AND` sayı serisi için. + +``` sql +groupBitAnd(expr) +``` + +**Parametre** + +`expr` – An expression that results in `UInt*` tür. + +**Dönüş değeri** + +Bu değer `UInt*` tür. + +**Örnek** + +Test verileri: + +``` text +binary decimal +00101100 = 44 +00011100 = 28 +00001101 = 13 +01010101 = 85 +``` + +Sorgu: + +``` sql +SELECT groupBitAnd(num) FROM t +``` + +Nerede `num` test verileri ile sütundur. + +Sonuç: + +``` text +binary decimal +00000100 = 4 +``` + +## groupBitOr {#groupbitor} + +Bitwise uygular `OR` sayı serisi için. + +``` sql +groupBitOr(expr) +``` + +**Parametre** + +`expr` – An expression that results in `UInt*` tür. + +**Dönüş değeri** + +Bu değer `UInt*` tür. + +**Örnek** + +Test verileri: + +``` text +binary decimal +00101100 = 44 +00011100 = 28 +00001101 = 13 +01010101 = 85 +``` + +Sorgu: + +``` sql +SELECT groupBitOr(num) FROM t +``` + +Nerede `num` test verileri ile sütundur. + +Sonuç: + +``` text +binary decimal +01111101 = 125 +``` + +## groupBitXor {#groupbitxor} + +Bitwise uygular `XOR` sayı serisi için. + +``` sql +groupBitXor(expr) +``` + +**Parametre** + +`expr` – An expression that results in `UInt*` tür. + +**Dönüş değeri** + +Bu değer `UInt*` tür. + +**Örnek** + +Test verileri: + +``` text +binary decimal +00101100 = 44 +00011100 = 28 +00001101 = 13 +01010101 = 85 +``` + +Sorgu: + +``` sql +SELECT groupBitXor(num) FROM t +``` + +Nerede `num` test verileri ile sütundur. + +Sonuç: + +``` text +binary decimal +01101000 = 104 +``` + +## groupBitmap {#groupbitmap} + +İşaretsiz tamsayı sütun, Uınt64 tür iade önem, gelen bit eşlem veya Toplama hesaplamaları suffix ekleme -Devlet, sonra iade [bitmap nesnesi](../../sql_reference/functions/bitmap_functions.md). + +``` sql +groupBitmap(expr) +``` + +**Parametre** + +`expr` – An expression that results in `UInt*` tür. + +**Dönüş değeri** + +Bu değer `UInt64` tür. + +**Örnek** + +Test verileri: + +``` text +UserID +1 +1 +2 +3 +``` + +Sorgu: + +``` sql +SELECT groupBitmap(UserID) as num FROM t +``` + +Sonuç: + +``` text +num +3 +``` + +## min (x) {#agg_function-min} + +Minimum hesaplar. + +## max (x) {#agg_function-max} + +Maksimum hesaplar. + +## argMin (arg, val) {#agg-function-argmin} + +Hesaplar ‘arg’ minimum değer ‘val’ değer. Birkaç farklı değer varsa ‘arg’ minimum değerler için ‘val’, karşılaşılan bu değerlerin ilki çıktıdır. + +**Örnek:** + +``` text +┌─user─────┬─salary─┐ +│ director │ 5000 │ +│ manager │ 3000 │ +│ worker │ 1000 │ +└──────────┴────────┘ +``` + +``` sql +SELECT argMin(user, salary) FROM salary +``` + +``` text +┌─argMin(user, salary)─┐ +│ worker │ +└──────────────────────┘ +``` + +## argMax (arg, val) {#agg-function-argmax} + +Hesaplar ‘arg’ maksimum değer ‘val’ değer. Birkaç farklı değer varsa ‘arg’ maksimum değerler için ‘val’, karşılaşılan bu değerlerin ilki çıktıdır. + +## s (um (x) {#agg_function-sum} + +Toplamı hesaplar. +Sadece sayılar için çalışır. + +## sumWithOverflow(x) {#sumwithoverflowx} + +Giriş parametreleri için olduğu gibi sonuç için aynı veri türünü kullanarak sayıların toplamını hesaplar. Toplam bu veri türü için en büyük değeri aşarsa, işlev bir hata döndürür. + +Sadece sayılar için çalışır. + +## sumMap (anahtar, değer) {#agg_functions-summap} + +Toplam thelar ‘value’ belirtilen tuş accordinglara göre dizi ‘key’ dizi. +Eleman sayısı ‘key’ ve ‘value’ toplam her satır için aynı olmalıdır. +Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. + +Örnek: + +``` sql +CREATE TABLE sum_map( + date Date, + timeslot DateTime, + statusMap Nested( + status UInt16, + requests UInt64 + ) +) ENGINE = Log; +INSERT INTO sum_map VALUES + ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), + ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), + ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), + ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); +SELECT + timeslot, + sumMap(statusMap.status, statusMap.requests) +FROM sum_map +GROUP BY timeslot +``` + +``` text +┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┐ +│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ +│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ +└─────────────────────┴──────────────────────────────────────────────┘ +``` + +## skewPop {#skewpop} + +Hesaplar [çarpıklık](https://en.wikipedia.org/wiki/Skewness) bir sıra. + +``` sql +skewPop(expr) +``` + +**Parametre** + +`expr` — [İfade](../syntax.md#syntax-expressions) bir numara döndürüyor. + +**Döndürülen değer** + +The skewness of the given distribution. Type — [Float64](../../sql_reference/data_types/float.md) + +**Örnek** + +``` sql +SELECT skewPop(value) FROM series_with_value_column +``` + +## skewSamp {#skewsamp} + +Hesaplar [örnek çarpıklık](https://en.wikipedia.org/wiki/Skewness) bir sıra. + +Bir rassal değişkenin çarpıklığının tarafsız bir tahminini temsil eder. + +``` sql +skewSamp(expr) +``` + +**Parametre** + +`expr` — [İfade](../syntax.md#syntax-expressions) bir numara döndürüyor. + +**Döndürülen değer** + +The skewness of the given distribution. Type — [Float64](../../sql_reference/data_types/float.md). Eğer `n <= 1` (`n` örnek boyutudur), daha sonra işlev döner `nan`. + +**Örnek** + +``` sql +SELECT skewSamp(value) FROM series_with_value_column +``` + +## kurtPop {#kurtpop} + +Hesaplar [kurtosis](https://en.wikipedia.org/wiki/Kurtosis) bir sıra. + +``` sql +kurtPop(expr) +``` + +**Parametre** + +`expr` — [İfade](../syntax.md#syntax-expressions) bir numara döndürüyor. + +**Döndürülen değer** + +The kurtosis of the given distribution. Type — [Float64](../../sql_reference/data_types/float.md) + +**Örnek** + +``` sql +SELECT kurtPop(value) FROM series_with_value_column +``` + +## kurtSamp {#kurtsamp} + +Hesaplar [örnek kurtoz](https://en.wikipedia.org/wiki/Kurtosis) bir sıra. + +Eğer geçen değerleri örnek oluşturur, eğer bir rassal değişken kurtosis tarafsız bir tahmini temsil eder. + +``` sql +kurtSamp(expr) +``` + +**Parametre** + +`expr` — [İfade](../syntax.md#syntax-expressions) bir numara döndürüyor. + +**Döndürülen değer** + +The kurtosis of the given distribution. Type — [Float64](../../sql_reference/data_types/float.md). Eğer `n <= 1` (`n` örnek bir boyutudur), daha sonra işlev döner `nan`. + +**Örnek** + +``` sql +SELECT kurtSamp(value) FROM series_with_value_column +``` + +## timeSeriesGroupSum(uıd, zaman damgası, değer) {#agg-function-timeseriesgroupsum} + +`timeSeriesGroupSum` örnek zaman damgası değil hizalama farklı zaman serileri toplayabilir. +İki örnek zaman damgası arasında doğrusal enterpolasyon kullanacak ve daha sonra zaman serilerini birlikte toplayacaktır. + +- `uid` zaman serisi benzersiz kimliği mi, `UInt64`. +- `timestamp` milisaniye veya mikrosaniye desteklemek için Int64 türüdür. +- `value` metr .iktir. + +İşlev, tuples dizisini döndürür `(timestamp, aggregated_value)` çiftliler. + +Bu işlevi kullanmadan önce emin olun `timestamp` artan düzende. + +Örnek: + +``` text +┌─uid─┬─timestamp─┬─value─┐ +│ 1 │ 2 │ 0.2 │ +│ 1 │ 7 │ 0.7 │ +│ 1 │ 12 │ 1.2 │ +│ 1 │ 17 │ 1.7 │ +│ 1 │ 25 │ 2.5 │ +│ 2 │ 3 │ 0.6 │ +│ 2 │ 8 │ 1.6 │ +│ 2 │ 12 │ 2.4 │ +│ 2 │ 18 │ 3.6 │ +│ 2 │ 24 │ 4.8 │ +└─────┴───────────┴───────┘ +``` + +``` sql +CREATE TABLE time_series( + uid UInt64, + timestamp Int64, + value Float64 +) ENGINE = Memory; +INSERT INTO time_series VALUES + (1,2,0.2),(1,7,0.7),(1,12,1.2),(1,17,1.7),(1,25,2.5), + (2,3,0.6),(2,8,1.6),(2,12,2.4),(2,18,3.6),(2,24,4.8); + +SELECT timeSeriesGroupSum(uid, timestamp, value) +FROM ( + SELECT * FROM time_series order by timestamp ASC +); +``` + +Ve sonuç olacak: + +``` text +[(2,0.2),(3,0.9),(7,2.1),(8,2.4),(12,3.6),(17,5.1),(18,5.4),(24,7.2),(25,2.5)] +``` + +## timeSeriesGroupRateSum(uıd, ts, val) {#agg-function-timeseriesgroupratesum} + +Benzer şekilde timeSeriesGroupRateSum, timeSeriesGroupRateSum zaman serisi ve daha sonra toplam oranları birlikte oranını hesaplar. +Ayrıca, bu işlevi kullanmadan önce zaman damgası yükseliş sırasına göre olmalıdır. + +Bu işlevi kullanın, yukarıdaki sonuç olacaktır: + +``` text +[(2,0),(3,0.1),(7,0.3),(8,0.3),(12,0.3),(17,0.3),(18,0.3),(24,0.3),(25,0.1)] +``` + +## avg (x) {#agg_function-avg} + +Ortalama hesaplar. +Sadece sayılar için çalışır. +Sonuç Her zaman Float64. + +## avgWeighted {#avgweighted} + +Hesaplar [ağırlıklı aritmetik ortalama](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean). + +**Sözdizimi** + +``` sql +avgWeighted(x, weight) +``` + +**Parametre** + +- `x` — Values. [Tamsayı](../data_types/int_uint.md) veya [kayan nokta](../data_types/float.md). +- `weight` — Weights of the values. [Tamsayı](../data_types/int_uint.md) veya [kayan nokta](../data_types/float.md). + +Türü `x` ve `weight` aynı olmalıdır. + +**Döndürülen değer** + +- Ağırlıklı ortalama. +- `NaN`. Tüm ağırlıklar 0'a eşitse. + +Tür: [Float64](../data_types/float.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT avgWeighted(x, w) +FROM values('x Int8, w Int8', (4, 1), (1, 0), (10, 2)) +``` + +Sonuç: + +``` text +┌─avgWeighted(x, weight)─┐ +│ 8 │ +└────────────────────────┘ +``` + +## uniq {#agg_function-uniq} + +Bağımsız değişken farklı değerlerin yaklaşık sayısını hesaplar. + +``` sql +uniq(x[, ...]) +``` + +**Parametre** + +Fonksiyon değişken sayıda parametre alır. Parametreler olabilir `Tuple`, `Array`, `Date`, `DateTime`, `String` veya sayısal türleri. + +**Döndürülen değer** + +- A [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip numarası. + +**Uygulama Detayları** + +İşlev: + +- Toplamdaki tüm parametreler için bir karma hesaplar, daha sonra hesaplamalarda kullanır. + +- Bir adaptif örnekleme algoritması kullanır. Hesaplama durumu için işlev, 65536'ya kadar öğe karma değerlerinin bir örneğini kullanır. + + This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions. + +- Sonucu deterministically sağlar (sorgu işleme sırasına bağlı değildir). + +Bu işlevi hemen hemen tüm senaryolarda kullanmanızı öneririz. + +**Ayrıca Bakınız** + +- [uniqCombined](#agg_function-uniqcombined) +- [uniqCombined64](#agg_function-uniqcombined64) +- [uniqHLL12](#agg_function-uniqhll12) +- [uniqExact](#agg_function-uniqexact) + +## uniqCombined {#agg_function-uniqcombined} + +Farklı bağımsız değişken değerlerinin yaklaşık sayısını hesaplar. + +``` sql +uniqCombined(HLL_precision)(x[, ...]) +``` + +Bu `uniqCombined` fonksiyon, farklı değerlerin sayısını hesaplamak için iyi bir seçimdir. + +**Parametre** + +Fonksiyon değişken sayıda parametre alır. Parametreler olabilir `Tuple`, `Array`, `Date`, `DateTime`, `String` veya sayısal türleri. + +`HLL_precision` hücre sayısının baz-2 logaritmasıdır. [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). İsteğe bağlı olarak işlevi kullanabilirsiniz `uniqCombined(x[, ...])`. İçin varsayılan değer `HLL_precision` etkin bir şekilde 96 KiB alan olan 17'dir (2^17 hücre, her biri 6 bit). + +**Döndürülen değer** + +- Numara [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip numarası. + +**Uygulama Detayları** + +İşlev: + +- Bir karma hesaplar (64-bit karma için `String` ve 32-bit aksi halde) agregadaki tüm parametreler için, hesaplamalarda kullanır. + +- Bir hata düzeltme tablosu ile dizi, karma tablo ve HyperLogLog: üç algoritmaları bir arada kullanır. + + For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. + +- Sonucu deterministically sağlar (sorgu işleme sırasına bağlı değildir). + +!!! note "Not" + Olmayan için 32-bit karma kullandığından-`String` tipi, sonuç cardinalities önemli ölçüde daha büyük için çok yüksek hata olacak `UINT_MAX` (birkaç on milyarlarca farklı değerden sonra hata hızla artacaktır), bu durumda kullanmanız gerekir [uniqCombined64](#agg_function-uniqcombined64) + +İle karşılaştırıldığında [uniq](#agg_function-uniq) fonksiyonu, `uniqCombined`: + +- Birkaç kez daha az bellek tüketir. +- Birkaç kat daha yüksek doğrulukla hesaplar. +- Genellikle biraz daha düşük performansa sahiptir. Bazı senaryolarda, `uniqCombined` daha iyi performans gösterebilir `uniq` örneğin, ağ üzerinden çok sayıda toplama durumu ileten dağıtılmış sorgularla. + +**Ayrıca Bakınız** + +- [uniq](#agg_function-uniq) +- [uniqCombined64](#agg_function-uniqcombined64) +- [uniqHLL12](#agg_function-uniqhll12) +- [uniqExact](#agg_function-uniqexact) + +## uniqCombined64 {#agg_function-uniqcombined64} + +Aynı olarak [uniqCombined](#agg_function-uniqcombined), ancak tüm veri türleri için 64 bit karma kullanır. + +## uniqHLL12 {#agg_function-uniqhll12} + +Farklı argüman değerlerinin yaklaşık sayısını hesaplar [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algoritma. + +``` sql +uniqHLL12(x[, ...]) +``` + +**Parametre** + +Fonksiyon değişken sayıda parametre alır. Parametreler olabilir `Tuple`, `Array`, `Date`, `DateTime`, `String` veya sayısal türleri. + +**Döndürülen değer** + +- A [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip numarası. + +**Uygulama Detayları** + +İşlev: + +- Toplamdaki tüm parametreler için bir karma hesaplar, daha sonra hesaplamalarda kullanır. + +- Farklı bağımsız değişken değerlerinin sayısını yaklaştırmak için HyperLogLog algoritmasını kullanır. + + 212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). + +- Belirli sonucu sağlar (sorgu işleme sırasına bağlı değildir). + +Bu işlevi kullanmanızı önermiyoruz. Çoğu durumda, kullan [uniq](#agg_function-uniq) veya [uniqCombined](#agg_function-uniqcombined) işlev. + +**Ayrıca Bakınız** + +- [uniq](#agg_function-uniq) +- [uniqCombined](#agg_function-uniqcombined) +- [uniqExact](#agg_function-uniqexact) + +## uniqExact {#agg_function-uniqexact} + +Farklı bağımsız değişken değerlerinin tam sayısını hesaplar. + +``` sql +uniqExact(x[, ...]) +``` + +Kullan... `uniqExact` kesinlikle kesin bir sonuca ihtiyacınız varsa işlev. Aksi takdirde kullanın [uniq](#agg_function-uniq) işlev. + +Bu `uniqExact` fonksiyonu daha fazla bellek kullanır `uniq`, çünkü farklı değerlerin sayısı arttıkça devletin büyüklüğü sınırsız büyümeye sahiptir. + +**Parametre** + +Fonksiyon değişken sayıda parametre alır. Parametreler olabilir `Tuple`, `Array`, `Date`, `DateTime`, `String` veya sayısal türleri. + +**Ayrıca Bakınız** + +- [uniq](#agg_function-uniq) +- [uniqCombined](#agg_function-uniqcombined) +- [uniqHLL12](#agg_function-uniqhll12) + +## groupArray (x), groupArray (max\_size)(x) {#agg_function-grouparray} + +Bağımsız değişken değerleri dizisi oluşturur. +Değerler diziye herhangi bir (belirsiz) sırayla eklenebilir. + +İkinci versiyonu (ile `max_size` parametre), elde edilen dizinin boyutunu sınırlar `max_size` öğeler. +Mesela, `groupArray (1) (x)` eşdeğ toer equivalentdir `[any (x)]`. + +Bazı durumlarda, hala yürütme sırasına güvenebilirsiniz. Bu, aşağıdaki durumlar için geçerlidir `SELECT` kullanan bir alt sorgudan gelir `ORDER BY`. + +## grouparrayınsertat (değer, konum) {#grouparrayinsertatvalue-position} + +Belirtilen konumda diziye bir değer ekler. + +!!! note "Not" + Bu işlev, SQL dizileri için geleneksel tek tabanlı konumların aksine sıfır tabanlı konumlar kullanır. + +Accepts the value and position as input. If several values ​​are inserted into the same position, any of them might end up in the resulting array (the first one will be used in the case of single-threaded execution). If no value is inserted into a position, the position is assigned the default value. + +İsteğe bağlı parametreler: + +- Boş pozisyonlarda ikame için varsayılan değer. +- Elde edilen dizinin uzunluğu. Bu, tüm toplama anahtarları için aynı boyuttaki dizileri almanızı sağlar. Bu parametreyi kullanırken, varsayılan değer belirtilmelidir. + +## groupArrayMovingSum {#agg_function-grouparraymovingsum} + +Giriş değerlerinin hareketli toplamını hesaplar. + +``` sql +groupArrayMovingSum(numbers_for_summing) +groupArrayMovingSum(window_size)(numbers_for_summing) +``` + +İşlev, pencere boyutunu bir parametre olarak alabilir. Belirtilmemiş bırakılırsa, işlev, sütundaki satır sayısına eşit pencere boyutunu alır. + +**Parametre** + +- `numbers_for_summing` — [İfade](../syntax.md#syntax-expressions) sayısal veri türü değeri ile sonuçlanır. +- `window_size` — Size of the calculation window. + +**Döndürülen değerler** + +- Giriş verileri ile aynı boyut ve türdeki dizi. + +**Örnek** + +Örnek tablo: + +``` sql +CREATE TABLE t +( + `int` UInt8, + `float` Float32, + `dec` Decimal32(2) +) +ENGINE = TinyLog +``` + +``` text +┌─int─┬─float─┬──dec─┐ +│ 1 │ 1.1 │ 1.10 │ +│ 2 │ 2.2 │ 2.20 │ +│ 4 │ 4.4 │ 4.40 │ +│ 7 │ 7.77 │ 7.77 │ +└─────┴───────┴──────┘ +``` + +Sorgu: + +``` sql +SELECT + groupArrayMovingSum(int) AS I, + groupArrayMovingSum(float) AS F, + groupArrayMovingSum(dec) AS D +FROM t +``` + +``` text +┌─I──────────┬─F───────────────────────────────┬─D──────────────────────┐ +│ [1,3,7,14] │ [1.1,3.3000002,7.7000003,15.47] │ [1.10,3.30,7.70,15.47] │ +└────────────┴─────────────────────────────────┴────────────────────────┘ +``` + +``` sql +SELECT + groupArrayMovingSum(2)(int) AS I, + groupArrayMovingSum(2)(float) AS F, + groupArrayMovingSum(2)(dec) AS D +FROM t +``` + +``` text +┌─I──────────┬─F───────────────────────────────┬─D──────────────────────┐ +│ [1,3,6,11] │ [1.1,3.3000002,6.6000004,12.17] │ [1.10,3.30,6.60,12.17] │ +└────────────┴─────────────────────────────────┴────────────────────────┘ +``` + +## groupArrayMovingAvg {#agg_function-grouparraymovingavg} + +Giriş değerlerinin hareketli ortalamasını hesaplar. + +``` sql +groupArrayMovingAvg(numbers_for_summing) +groupArrayMovingAvg(window_size)(numbers_for_summing) +``` + +İşlev, pencere boyutunu bir parametre olarak alabilir. Belirtilmemiş bırakılırsa, işlev, sütundaki satır sayısına eşit pencere boyutunu alır. + +**Parametre** + +- `numbers_for_summing` — [İfade](../syntax.md#syntax-expressions) sayısal veri türü değeri ile sonuçlanır. +- `window_size` — Size of the calculation window. + +**Döndürülen değerler** + +- Giriş verileri ile aynı boyut ve türdeki dizi. + +İşlev kullanır [sıfıra doğru yuvarlama](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). Sonuç veri türü için önemsiz ondalık basamaklar keser. + +**Örnek** + +Örnek tablo `b`: + +``` sql +CREATE TABLE t +( + `int` UInt8, + `float` Float32, + `dec` Decimal32(2) +) +ENGINE = TinyLog +``` + +``` text +┌─int─┬─float─┬──dec─┐ +│ 1 │ 1.1 │ 1.10 │ +│ 2 │ 2.2 │ 2.20 │ +│ 4 │ 4.4 │ 4.40 │ +│ 7 │ 7.77 │ 7.77 │ +└─────┴───────┴──────┘ +``` + +Sorgu: + +``` sql +SELECT + groupArrayMovingAvg(int) AS I, + groupArrayMovingAvg(float) AS F, + groupArrayMovingAvg(dec) AS D +FROM t +``` + +``` text +┌─I─────────┬─F───────────────────────────────────┬─D─────────────────────┐ +│ [0,0,1,3] │ [0.275,0.82500005,1.9250001,3.8675] │ [0.27,0.82,1.92,3.86] │ +└───────────┴─────────────────────────────────────┴───────────────────────┘ +``` + +``` sql +SELECT + groupArrayMovingAvg(2)(int) AS I, + groupArrayMovingAvg(2)(float) AS F, + groupArrayMovingAvg(2)(dec) AS D +FROM t +``` + +``` text +┌─I─────────┬─F────────────────────────────────┬─D─────────────────────┐ +│ [0,1,3,5] │ [0.55,1.6500001,3.3000002,6.085] │ [0.55,1.65,3.30,6.08] │ +└───────────┴──────────────────────────────────┴───────────────────────┘ +``` + +## groupUniqArray (x), groupUniqArray (max\_size)(x) {#groupuniqarrayx-groupuniqarraymax-sizex} + +Farklı bağımsız değişken değerlerinden bir dizi oluşturur. Bellek tüketimi için aynıdır `uniqExact` işlev. + +İkinci versiyonu (ile `max_size` parametre), elde edilen dizinin boyutunu sınırlar `max_size` öğeler. +Mesela, `groupUniqArray(1)(x)` eşdeğ toer equivalentdir `[any(x)]`. + +## quantile {#quantile} + +Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) sayısal veri dizisinin. + +Bu işlev geçerlidir [rezerv reservoiruar örnek samplinglemesi](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192'ye kadar bir rezervuar boyutu ve örnekleme için rastgele sayı üreteci ile. Sonuç deterministik değildir. Tam bir miktar elde etmek için [quantileExact](#quantileexact) işlev. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantile(level)(expr) +``` + +Takma ad: `median`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). +- `expr` — Expression over the column values resulting in numeric [veri türleri](../../sql_reference/data_types/index.md#data_types), [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). + +**Döndürülen değer** + +- Belirtilen seviyenin yaklaşık miktarı. + +Tür: + +- [Float64](../../sql_reference/data_types/float.md) sayısal veri türü girişi için. +- [Tarihli](../../sql_reference/data_types/date.md) giriş değerleri varsa `Date` tür. +- [DateTime](../../sql_reference/data_types/datetime.md) giriş değerleri varsa `DateTime` tür. + +**Örnek** + +Giriş tablosu: + +``` text +┌─val─┐ +│ 1 │ +│ 1 │ +│ 2 │ +│ 3 │ +└─────┘ +``` + +Sorgu: + +``` sql +SELECT quantile(val) FROM t +``` + +Sonuç: + +``` text +┌─quantile(val)─┐ +│ 1.5 │ +└───────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## quantileDeterministic {#quantiledeterministic} + +Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) sayısal veri dizisinin. + +Bu işlev geçerlidir [rezerv reservoiruar örnek samplinglemesi](https://en.wikipedia.org/wiki/Reservoir_sampling) 8192'ye kadar bir rezervuar boyutu ve örnekleme deterministik algoritması ile. Sonuç deterministiktir. Tam bir miktar elde etmek için [quantileExact](#quantileexact) işlev. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantileDeterministic(level)(expr, determinator) +``` + +Takma ad: `medianDeterministic`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). +- `expr` — Expression over the column values resulting in numeric [veri türleri](../../sql_reference/data_types/index.md#data_types), [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). +- `determinator` — Number whose hash is used instead of a random number generator in the reservoir sampling algorithm to make the result of sampling deterministic. As a determinator you can use any deterministic positive number, for example, a user id or an event id. If the same determinator value occures too often, the function works incorrectly. + +**Döndürülen değer** + +- Belirtilen seviyenin yaklaşık miktarı. + +Tür: + +- [Float64](../../sql_reference/data_types/float.md) sayısal veri türü girişi için. +- [Tarihli](../../sql_reference/data_types/date.md) giriş değerleri varsa `Date` tür. +- [DateTime](../../sql_reference/data_types/datetime.md) giriş değerleri varsa `DateTime` tür. + +**Örnek** + +Giriş tablosu: + +``` text +┌─val─┐ +│ 1 │ +│ 1 │ +│ 2 │ +│ 3 │ +└─────┘ +``` + +Sorgu: + +``` sql +SELECT quantileDeterministic(val, 1) FROM t +``` + +Sonuç: + +``` text +┌─quantileDeterministic(val, 1)─┐ +│ 1.5 │ +└───────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## quantileExact {#quantileexact} + +Tam olarak hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) sayısal veri dizisinin. + +To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` bellek, nerede `n` geçirilen değerler say .ısıdır. Bununla birlikte, az sayıda değer için, işlev çok etkilidir. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantileExact(level)(expr) +``` + +Takma ad: `medianExact`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). +- `expr` — Expression over the column values resulting in numeric [veri türleri](../../sql_reference/data_types/index.md#data_types), [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). + +**Döndürülen değer** + +- Belirtilen seviyenin miktarı. + +Tür: + +- [Float64](../../sql_reference/data_types/float.md) sayısal veri türü girişi için. +- [Tarihli](../../sql_reference/data_types/date.md) giriş değerleri varsa `Date` tür. +- [DateTime](../../sql_reference/data_types/datetime.md) giriş değerleri varsa `DateTime` tür. + +**Örnek** + +Sorgu: + +``` sql +SELECT quantileExact(number) FROM numbers(10) +``` + +Sonuç: + +``` text +┌─quantileExact(number)─┐ +│ 5 │ +└───────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## quantilexactweighted {#quantileexactweighted} + +Tam olarak hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) her elemanın ağırlığını dikkate alarak sayısal bir veri dizisinin. + +To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Each value is counted with its weight, as if it is present `weight` times. A hash table is used in the algorithm. Because of this, if the passed values ​​are frequently repeated, the function consumes less RAM than [quantileExact](#quantileexact). Bunun yerine bu işlevi kullanabilirsiniz `quantileExact` ve 1 ağırlığını belirtin. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantileExactWeighted(level)(expr, weight) +``` + +Takma ad: `medianExactWeighted`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). +- `expr` — Expression over the column values resulting in numeric [veri türleri](../../sql_reference/data_types/index.md#data_types), [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). +- `weight` — Column with weights of sequence members. Weight is a number of value occurrences. + +**Döndürülen değer** + +- Belirtilen seviyenin miktarı. + +Tür: + +- [Float64](../../sql_reference/data_types/float.md) sayısal veri türü girişi için. +- [Tarihli](../../sql_reference/data_types/date.md) giriş değerleri varsa `Date` tür. +- [DateTime](../../sql_reference/data_types/datetime.md) giriş değerleri varsa `DateTime` tür. + +**Örnek** + +Giriş tablosu: + +``` text +┌─n─┬─val─┐ +│ 0 │ 3 │ +│ 1 │ 2 │ +│ 2 │ 1 │ +│ 5 │ 4 │ +└───┴─────┘ +``` + +Sorgu: + +``` sql +SELECT quantileExactWeighted(n, val) FROM t +``` + +Sonuç: + +``` text +┌─quantileExactWeighted(n, val)─┐ +│ 1 │ +└───────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## quantileTiming {#quantiletiming} + +Belirlenen hassas hesaplar ile [quantile](https://en.wikipedia.org/wiki/Quantile) sayısal veri dizisinin. + +Sonuç deterministiktir (sorgu işleme sırasına bağlı değildir). Fonksiyon yükleme web sayfaları kez veya arka uç yanıt süreleri gibi dağılımları tanımlamak dizileri ile çalışmak için optimize edilmiştir. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantileTiming(level)(expr) +``` + +Takma ad: `medianTiming`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). + +- `expr` — [İfade](../syntax.md#syntax-expressions) bir sütun değerleri üzerinde dönen bir [Yüzdürmek\*](../../sql_reference/data_types/float.md)- tip numarası. + + - If negative values are passed to the function, the behavior is undefined. + - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. + +**Doğruluk** + +Hesaplama doğru ise: + +- Toplam değer sayısı 5670'i geçmez. +- Toplam değer sayısı 5670'i aşıyor, ancak sayfa yükleme süresi 1024 ms'den az. + +Aksi takdirde, hesaplamanın sonucu 16 MS'nin en yakın katlarına yuvarlanır. + +!!! note "Not" + Sayfa yükleme süresi nicelerini hesaplamak için, bu işlev daha etkili ve doğrudur [quantile](#quantile). + +**Döndürülen değer** + +- Belirtilen seviyenin miktarı. + +Tür: `Float32`. + +!!! note "Not" + İşlev valuese hiçbir değer geçir (ilmem (işse (kullanırken `quantileTimingIf`), [Nine](../../sql_reference/data_types/float.md#data_type-float-nan-inf) döndürülür. Bunun amacı, bu vakaları sıfır ile sonuçlanan vakalardan ayırmaktır. Görmek [ORDER BY FLA BYGE](../statements/select.md#select-order-by) sıralama ile ilgili notlar için `NaN` değerler. + +**Örnek** + +Giriş tablosu: + +``` text +┌─response_time─┐ +│ 72 │ +│ 112 │ +│ 126 │ +│ 145 │ +│ 104 │ +│ 242 │ +│ 313 │ +│ 168 │ +│ 108 │ +└───────────────┘ +``` + +Sorgu: + +``` sql +SELECT quantileTiming(response_time) FROM t +``` + +Sonuç: + +``` text +┌─quantileTiming(response_time)─┐ +│ 126 │ +└───────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## niceletimingweighted {#quantiletimingweighted} + +Belirlenen hassas hesaplar ile [quantile](https://en.wikipedia.org/wiki/Quantile) her sıra üyesi ağırlığına göre sayısal veri dizisi. + +Sonuç deterministiktir (sorgu işleme sırasına bağlı değildir). Fonksiyon yükleme web sayfaları kez veya arka uç yanıt süreleri gibi dağılımları tanımlamak dizileri ile çalışmak için optimize edilmiştir. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantileTimingWeighted(level)(expr, weight) +``` + +Takma ad: `medianTimingWeighted`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). + +- `expr` — [İfade](../syntax.md#syntax-expressions) bir sütun değerleri üzerinde dönen bir [Yüzdürmek\*](../../sql_reference/data_types/float.md)- tip numarası. + + - If negative values are passed to the function, the behavior is undefined. + - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. + +- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. + +**Doğruluk** + +Hesaplama doğru ise: + +- Toplam değer sayısı 5670'i geçmez. +- Toplam değer sayısı 5670'i aşıyor, ancak sayfa yükleme süresi 1024 ms'den az. + +Aksi takdirde, hesaplamanın sonucu 16 MS'nin en yakın katlarına yuvarlanır. + +!!! note "Not" + Sayfa yükleme süresi nicelerini hesaplamak için, bu işlev daha etkili ve doğrudur [quantile](#quantile). + +**Döndürülen değer** + +- Belirtilen seviyenin miktarı. + +Tür: `Float32`. + +!!! note "Not" + İşlev valuese hiçbir değer geçir (ilmem (işse (kullanırken `quantileTimingIf`), [Nine](../../sql_reference/data_types/float.md#data_type-float-nan-inf) döndürülür. Bunun amacı, bu vakaları sıfır ile sonuçlanan vakalardan ayırmaktır. Görmek [ORDER BY FLA BYGE](../statements/select.md#select-order-by) sıralama ile ilgili notlar için `NaN` değerler. + +**Örnek** + +Giriş tablosu: + +``` text +┌─response_time─┬─weight─┐ +│ 68 │ 1 │ +│ 104 │ 2 │ +│ 112 │ 3 │ +│ 126 │ 2 │ +│ 138 │ 1 │ +│ 162 │ 1 │ +└───────────────┴────────┘ +``` + +Sorgu: + +``` sql +SELECT quantileTimingWeighted(response_time, weight) FROM t +``` + +Sonuç: + +``` text +┌─quantileTimingWeighted(response_time, weight)─┐ +│ 112 │ +└───────────────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## quantileTDigest {#quantiletdigest} + +Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) kullanarak sayısal veri diz ofisinin [t-dig -est](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algoritma. + +Maksimum hata %1'dir. Bellek tüketimi `log(n)`, nere `n` değer say isısıdır. Sonuç, sorguyu çalıştırma sırasına bağlıdır ve nondeterministic. + +Fonksiyonun performansı, performanstan daha düşüktür [quantile](#quantile) veya [quantileTiming](#quantiletiming). Durum boyutunun hassasiyete oranı açısından, bu işlev çok daha iyidir `quantile`. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantileTDigest(level)(expr) +``` + +Takma ad: `medianTDigest`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). +- `expr` — Expression over the column values resulting in numeric [veri türleri](../../sql_reference/data_types/index.md#data_types), [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). + +**Döndürülen değer** + +- Belirtilen seviyenin yaklaşık miktarı. + +Tür: + +- [Float64](../../sql_reference/data_types/float.md) sayısal veri türü girişi için. +- [Tarihli](../../sql_reference/data_types/date.md) giriş değerleri varsa `Date` tür. +- [DateTime](../../sql_reference/data_types/datetime.md) giriş değerleri varsa `DateTime` tür. + +**Örnek** + +Sorgu: + +``` sql +SELECT quantileTDigest(number) FROM numbers(10) +``` + +Sonuç: + +``` text +┌─quantileTDigest(number)─┐ +│ 4.5 │ +└─────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## quantileTDigestWeighted {#quantiletdigestweighted} + +Yaklaşık hesaplar [quantile](https://en.wikipedia.org/wiki/Quantile) kullanarak sayısal veri diz ofisinin [t-dig -est](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algoritma. İşlev, her sıra üyesinin ağırlığını dikkate alır. Maksimum hata %1'dir. Bellek tüketimi `log(n)`, nere `n` değer say isısıdır. + +Fonksiyonun performansı, performanstan daha düşüktür [quantile](#quantile) veya [quantileTiming](#quantiletiming). Durum boyutunun hassasiyete oranı açısından, bu işlev çok daha iyidir `quantile`. + +Sonuç, sorguyu çalıştırma sırasına bağlıdır ve nondeterministic. + +Çoklu kullanırken `quantile*` bir sorguda farklı düzeylerde işlevler, iç durumları birleştirilmez (diğer bir deyişle, sorgu olabilir daha az verimli çalışır). Bu durumda, kullan [quantiles](#quantiles) işlev. + +**Sözdizimi** + +``` sql +quantileTDigest(level)(expr) +``` + +Takma ad: `medianTDigest`. + +**Parametre** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` aralığında değer `[0.01, 0.99]`. Varsayılan değer: 0.5. Yanında `level=0.5` fonksiyon hesaplar [medyan](https://en.wikipedia.org/wiki/Median). +- `expr` — Expression over the column values resulting in numeric [veri türleri](../../sql_reference/data_types/index.md#data_types), [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). +- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. + +**Döndürülen değer** + +- Belirtilen seviyenin yaklaşık miktarı. + +Tür: + +- [Float64](../../sql_reference/data_types/float.md) sayısal veri türü girişi için. +- [Tarihli](../../sql_reference/data_types/date.md) giriş değerleri varsa `Date` tür. +- [DateTime](../../sql_reference/data_types/datetime.md) giriş değerleri varsa `DateTime` tür. + +**Örnek** + +Sorgu: + +``` sql +SELECT quantileTDigestWeighted(number, 1) FROM numbers(10) +``` + +Sonuç: + +``` text +┌─quantileTDigestWeighted(number, 1)─┐ +│ 4.5 │ +└────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [medyan](#median) +- [quantiles](#quantiles) + +## medyan {#median} + +Bu `median*` fonksiyonlar karşılık gelen takma adlardır `quantile*` işlevler. Sayısal bir veri örneğinin medyanını hesaplarlar. + +İşlevler: + +- `median` — Alias for [quantile](#quantile). +- `medianDeterministic` — Alias for [quantileDeterministic](#quantiledeterministic). +- `medianExact` — Alias for [quantileExact](#quantileexact). +- `medianExactWeighted` — Alias for [quantilexactweighted](#quantileexactweighted). +- `medianTiming` — Alias for [quantileTiming](#quantiletiming). +- `medianTimingWeighted` — Alias for [niceletimingweighted](#quantiletimingweighted). +- `medianTDigest` — Alias for [quantileTDigest](#quantiletdigest). +- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](#quantiletdigestweighted). + +**Örnek** + +Giriş tablosu: + +``` text +┌─val─┐ +│ 1 │ +│ 1 │ +│ 2 │ +│ 3 │ +└─────┘ +``` + +Sorgu: + +``` sql +SELECT medianDeterministic(val, 1) FROM t +``` + +Sonuç: + +``` text +┌─medianDeterministic(val, 1)─┐ +│ 1.5 │ +└─────────────────────────────┘ +``` + +## quantiles(level1, level2, …)(x) {#quantiles} + +Tüm quantile fonksiyonları da karşılık gelen quantile fonksiyonlarına sahiptir: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. Bu işlevler, listelenen seviyelerin tüm nicelerini tek geçişte hesaplar ve elde edilen değerlerin bir dizisini döndürür. + +## varSamp (x) {#varsampx} + +Miktarı hesaplar `Σ((x - x̅)^2) / (n - 1)`, nere `n` örneklem büyüklüğü ve `x̅`ortalama değer isidir `x`. + +Bir rassal değişkenin varyansının tarafsız bir tahminini temsil eder, eğer geçirilen değerler numunesini oluşturursa. + +Dönüşler `Float64`. Ne zaman `n <= 1`, dönüşler `+∞`. + +## varPop (x) {#varpopx} + +Miktarı hesaplar `Σ((x - x̅)^2) / n`, nere `n` örneklem büyüklüğü ve `x̅`ortalama değer isidir `x`. + +Başka bir deyişle, bir dizi değer için dağılım. Dönüşler `Float64`. + +## stddevSamp(x) {#stddevsampx} + +Sonuç kareköküne eşittir `varSamp(x)`. + +## stddevPop(x) {#stddevpopx} + +Sonuç kareköküne eşittir `varPop(x)`. + +## topK (N) (x) {#topknx} + +Belirtilen sütundaki yaklaşık en sık değerleri bir dizi döndürür. Elde edilen dizi, değerlerin yaklaşık frekansının azalan sırasına göre sıralanır (değerlerin kendileri tarafından değil). + +Uygular [Filtrelenmiş Yerden Tasarruf](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) TopK analiz etmek için algoritma, azaltmak ve birleştirmek algoritması dayalı [Paralel Alan Tasarrufu](https://arxiv.org/pdf/1401.0702.pdf). + +``` sql +topK(N)(column) +``` + +Bu işlev garantili bir sonuç sağlamaz. Bazı durumlarda, hatalar oluşabilir ve en sık kullanılan değerler olmayan sık değerler döndürebilir. + +Biz kullanmanızı öneririz `N < 10` değer; performans büyük ile azalır `N` değerler. Maksimum değeri `N = 65536`. + +**Parametre** + +- ‘N’ dönmek için Öğe sayısıdır. + +Parametre atlanırsa, varsayılan değer 10 kullanılır. + +**Değişkenler** + +- ' x ' – The value to calculate frequency. + +**Örnek** + +Tak thee the [OnTime](../../getting_started/example_datasets/ontime.md) veri kümesi ve üç en sık oluşan değerleri seçin `AirlineID` sütun. + +``` sql +SELECT topK(3)(AirlineID) AS res +FROM ontime +``` + +``` text +┌─res─────────────────┐ +│ [19393,19790,19805] │ +└─────────────────────┘ +``` + +## topKWeighted {#topkweighted} + +Benzer `topK` ancak tamsayı türünde bir ek argüman alır - `weight`. Her değer muhasebeleştirilir `weight` frekans hesaplaması için zamanlar. + +**Sözdizimi** + +``` sql +topKWeighted(N)(x, weight) +``` + +**Parametre** + +- `N` — The number of elements to return. + +**Değişkenler** + +- `x` – The value. +- `weight` — The weight. [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değer** + +Maksimum yaklaşık ağırlık toplamına sahip değerlerin bir dizisini döndürür. + +**Örnek** + +Sorgu: + +``` sql +SELECT topKWeighted(10)(number, number) FROM numbers(1000) +``` + +Sonuç: + +``` text +┌─topKWeighted(10)(number, number)──────────┐ +│ [999,998,997,996,995,994,993,992,991,990] │ +└───────────────────────────────────────────┘ +``` + +## covarSamp(x, y) {#covarsampx-y} + +Değerini hesaplar `Σ((x - x̅)(y - y̅)) / (n - 1)`. + +Float64 Döndürür. Ne zaman `n <= 1`, returns +∞. + +## covarPop (x, y) {#covarpopx-y} + +Değerini hesaplar `Σ((x - x̅)(y - y̅)) / n`. + +## corr(x, y) {#corrx-y} + +Pearson korelasyon katsayısını hesaplar: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. + +## categoricalınformationvalue {#categoricalinformationvalue} + +Değerini hesaplar `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` her kategori için. + +``` sql +categoricalInformationValue(category1, category2, ..., tag) +``` + +Sonuç, ayrık (kategorik) bir özelliğin nasıl olduğunu gösterir `[category1, category2, ...]` değerini öngör aen bir öğrenme modeline katkıda `tag`. + +## simpleLinearRegression {#simplelinearregression} + +Basit (tek boyutlu) doğrusal regresyon gerçekleştirir. + +``` sql +simpleLinearRegression(x, y) +``` + +Parametre: + +- `x` — Column with dependent variable values. +- `y` — Column with explanatory variable values. + +Döndürülen değerler: + +Devamlılar `(a, b)` ortaya çıkan hat linetın `y = a*x + b`. + +**Örnekler** + +``` sql +SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [0, 1, 2, 3]) +``` + +``` text +┌─arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [0, 1, 2, 3])─┐ +│ (1,0) │ +└───────────────────────────────────────────────────────────────────┘ +``` + +``` sql +SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6]) +``` + +``` text +┌─arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6])─┐ +│ (1,3) │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## stokastiklinearregression {#agg_functions-stochasticlinearregression} + +Bu fonksiyon stokastik doğrusal regresyon uygular. Öğrenme oranı, L2 regularization katsayısı, mini-batch boyutu için özel parametreleri destekler ve ağırlıkları güncellemek için birkaç yöntem vardır ([Adem](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) (varsayılan olarak kullanılır), [basit SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [İvme](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)). + +### Parametre {#agg_functions-stochasticlinearregression-parameters} + +4 özelleştirilebilir parametre vardır. Onlar sırayla işleve geçirilir, ancak dört varsayılan değerleri kullanılacak geçmek gerek yoktur, ancak iyi bir model bazı parametre ayarlama gerekli. + +``` text +stochasticLinearRegression(1.0, 1.0, 10, 'SGD') +``` + +1. `learning rate` Gradyan iniş adımı gerçekleştirildiğinde adım uzunluğundaki katsayıdır. Çok büyük öğrenme oranı, modelin sonsuz ağırlıklarına neden olabilir. Default is `0.00001`. +2. `l2 regularization coefficient` hangi overfitting önlemek için yardımcı olabilir. Default is `0.1`. +3. `mini-batch size` gradyanların hesaplanacağı ve Gradyan inişinin bir adımını gerçekleştirmek için toplanacağı öğelerin sayısını ayarlar. Saf stokastik iniş bir eleman kullanır, ancak küçük partilere(yaklaşık 10 eleman) sahip olmak degrade adımları daha kararlı hale getirir. Default is `15`. +4. `method for updating weights` onlar : `Adam` (varsayılan olarak), `SGD`, `Momentum`, `Nesterov`. `Momentum` ve `Nesterov` biraz daha fazla hesaplama ve bellek gerektirir, ancak stokastik Gradyan yöntemlerinin yakınsama hızı ve kararlılığı açısından yararlı olurlar. + +### Kullanma {#agg_functions-stochasticlinearregression-usage} + +`stochasticLinearRegression` iki adımda kullanılır: modelin takılması ve yeni verilerin tahmin edilmesi. Modeli sığdırmak ve daha sonra kullanım için durumunu kaydetmek için kullandığımız `-State` temel olarak durumu kurtaran birleştirici (model ağırlıkları, vb.). +Fonksiyonu kullan wedığımızı tahmin etmek [evalMLMethod](../functions/machine_learning_functions.md#machine_learning_methods-evalmlmethod), bir argüman olarak bir durumu yanı sıra tahmin etmek için özellikler alır. + + + +**1.** Uydurma + +Böyle bir sorgu kullanılabilir. + +``` sql +CREATE TABLE IF NOT EXISTS train_data +( + param1 Float64, + param2 Float64, + target Float64 +) ENGINE = Memory; + +CREATE TABLE your_model ENGINE = Memory AS SELECT +stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) +AS state FROM train_data; +``` + +Burada ayrıca veri eklememiz gerekiyor `train_data` Tablo. Parametrelerin sayısı sabit değildir, sadece argümanların sayısına bağlıdır, `linearRegressionState`. Hepsi sayısal değerler olmalıdır. +Hedef değere sahip sütunun(tahmin etmeyi öğrenmek istediğimiz) ilk argüman olarak eklendiğini unutmayın. + +**2.** Öngören + +Bir durumu tabloya kaydettikten sonra, tahmin için birden çok kez kullanabilir, hatta diğer durumlarla birleşebilir ve yeni daha iyi modeller oluşturabiliriz. + +``` sql +WITH (SELECT state FROM your_model) AS model SELECT +evalMLMethod(model, param1, param2) FROM test_data +``` + +Sorgu, tahmin edilen değerlerin bir sütununu döndürür. Not ilk argüman `evalMLMethod` oluyor `AggregateFunctionState` nesne, sonraki özelliklerin sütunlarıdır. + +`test_data` bir tablo gibi mi `train_data` ancak hedef değer içermeyebilir. + +### Not {#agg_functions-stochasticlinearregression-notes} + +1. İki modeli birleştirmek için Kullanıcı böyle bir sorgu oluşturabilir: + `sql SELECT state1 + state2 FROM your_models` + nerede `your_models` tablo her iki modeli de içerir. Bu sorgu yeni dönecektir `AggregateFunctionState` nesne. + +2. Kullanıcı, modeli kaydetmeden oluşturulan modelin ağırlıklarını kendi amaçları için alabilir `-State` birleştirici kullanılır. + `sql SELECT stochasticLinearRegression(0.01)(target, param1, param2) FROM train_data` + Bu sorgu modele uyacak ve ağırlıklarını geri getirecektir-ilk önce modelin parametrelerine karşılık gelen ağırlıklar, sonuncusu önyargıdır. Yani yukarıdaki örnekte sorgu 3 değer içeren bir sütun döndürecektir. + +**Ayrıca Bakınız** + +- [stochasticLogisticRegression](#agg_functions-stochasticlogisticregression) +- [Doğrusal ve lojistik regresyonlar arasındaki fark](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) + +## stochasticLogisticRegression {#agg_functions-stochasticlogisticregression} + +Bu işlev stokastik lojistik regresyon uygular. İkili sınıflandırma problemi için kullanılabilir, stochasticLinearRegression ile aynı özel parametreleri destekler ve aynı şekilde çalışır. + +### Parametre {#agg_functions-stochasticlogisticregression-parameters} + +Parametreler tam olarak stochasticLinearRegression ile aynıdır: +`learning rate`, `l2 regularization coefficient`, `mini-batch size`, `method for updating weights`. +Daha fazla bilgi için bkz. [parametre](#agg_functions-stochasticlinearregression-parameters). + +``` text +stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') +``` + +1. Uydurma + + + + See the `Fitting` section in the [stochasticLinearRegression](#stochasticlinearregression-usage-fitting) description. + + Predicted labels have to be in \[-1, 1\]. + +1. Öngören + + + + Using saved state we can predict probability of object having label `1`. + + ``` sql + WITH (SELECT state FROM your_model) AS model SELECT + evalMLMethod(model, param1, param2) FROM test_data + ``` + + The query will return a column of probabilities. Note that first argument of `evalMLMethod` is `AggregateFunctionState` object, next are columns of features. + + We can also set a bound of probability, which assigns elements to different labels. + + ``` sql + SELECT ans < 1.1 AND ans > 0.5 FROM + (WITH (SELECT state FROM your_model) AS model SELECT + evalMLMethod(model, param1, param2) AS ans FROM test_data) + ``` + + Then the result will be labels. + + `test_data` is a table like `train_data` but may not contain target value. + +**Ayrıca Bakınız** + +- [stokastiklinearregression](#agg_functions-stochasticlinearregression) +- [Doğrusal ve lojistik regresyonlar arasındaki fark.](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) + +## groupBitmapAnd {#groupbitmapand} + +Bu VE bir bitmap sütun, Uınt64 tür iade önem, hesaplamaları suffix ekleme -Devlet, sonra iade [bitmap nesnesi](../../sql_reference/functions/bitmap_functions.md). + +``` sql +groupBitmapAnd(expr) +``` + +**Parametre** + +`expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` tür. + +**Dönüş değeri** + +Bu değer `UInt64` tür. + +**Örnek** + +``` sql +DROP TABLE IF EXISTS bitmap_column_expr_test2; +CREATE TABLE bitmap_column_expr_test2 +( + tag_id String, + z AggregateFunction(groupBitmap, UInt32) +) +ENGINE = MergeTree +ORDER BY tag_id; + +INSERT INTO bitmap_column_expr_test2 VALUES ('tag1', bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32)))); +INSERT INTO bitmap_column_expr_test2 VALUES ('tag2', bitmapBuild(cast([6,7,8,9,10,11,12,13,14,15] as Array(UInt32)))); +INSERT INTO bitmap_column_expr_test2 VALUES ('tag3', bitmapBuild(cast([2,4,6,8,10,12] as Array(UInt32)))); + +SELECT groupBitmapAnd(z) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); +┌─groupBitmapAnd(z)─┐ +│ 3 │ +└───────────────────┘ + +SELECT arraySort(bitmapToArray(groupBitmapAndState(z))) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); +┌─arraySort(bitmapToArray(groupBitmapAndState(z)))─┐ +│ [6,8,10] │ +└──────────────────────────────────────────────────┘ +``` + +## groupBitmapOr {#groupbitmapor} + +YA da bir bitmap sütun, Uınt64 tür iade önem, hesaplamaları suffix ekleme -Devlet, sonra iade [bitmap nesnesi](../../sql_reference/functions/bitmap_functions.md). Bu eşdeğerdir `groupBitmapMerge`. + +``` sql +groupBitmapOr(expr) +``` + +**Parametre** + +`expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` tür. + +**Dönüş değeri** + +Bu değer `UInt64` tür. + +**Örnek** + +``` sql +DROP TABLE IF EXISTS bitmap_column_expr_test2; +CREATE TABLE bitmap_column_expr_test2 +( + tag_id String, + z AggregateFunction(groupBitmap, UInt32) +) +ENGINE = MergeTree +ORDER BY tag_id; + +INSERT INTO bitmap_column_expr_test2 VALUES ('tag1', bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32)))); +INSERT INTO bitmap_column_expr_test2 VALUES ('tag2', bitmapBuild(cast([6,7,8,9,10,11,12,13,14,15] as Array(UInt32)))); +INSERT INTO bitmap_column_expr_test2 VALUES ('tag3', bitmapBuild(cast([2,4,6,8,10,12] as Array(UInt32)))); + +SELECT groupBitmapOr(z) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); +┌─groupBitmapOr(z)─┐ +│ 15 │ +└──────────────────┘ + +SELECT arraySort(bitmapToArray(groupBitmapOrState(z))) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); +┌─arraySort(bitmapToArray(groupBitmapOrState(z)))─┐ +│ [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] │ +└─────────────────────────────────────────────────┘ +``` + +## groupBitmapXor {#groupbitmapxor} + +Bir bitmap sütun, Uınt64 tür iade önem hesaplamaları XOR, suffix ekleme -Devlet, sonra iade [bitmap nesnesi](../../sql_reference/functions/bitmap_functions.md). + +``` sql +groupBitmapOr(expr) +``` + +**Parametre** + +`expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` tür. + +**Dönüş değeri** + +Bu değer `UInt64` tür. + +**Örnek** + +``` sql +DROP TABLE IF EXISTS bitmap_column_expr_test2; +CREATE TABLE bitmap_column_expr_test2 +( + tag_id String, + z AggregateFunction(groupBitmap, UInt32) +) +ENGINE = MergeTree +ORDER BY tag_id; + +INSERT INTO bitmap_column_expr_test2 VALUES ('tag1', bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32)))); +INSERT INTO bitmap_column_expr_test2 VALUES ('tag2', bitmapBuild(cast([6,7,8,9,10,11,12,13,14,15] as Array(UInt32)))); +INSERT INTO bitmap_column_expr_test2 VALUES ('tag3', bitmapBuild(cast([2,4,6,8,10,12] as Array(UInt32)))); + +SELECT groupBitmapXor(z) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); +┌─groupBitmapXor(z)─┐ +│ 10 │ +└───────────────────┘ + +SELECT arraySort(bitmapToArray(groupBitmapXorState(z))) FROM bitmap_column_expr_test2 WHERE like(tag_id, 'tag%'); +┌─arraySort(bitmapToArray(groupBitmapXorState(z)))─┐ +│ [1,3,5,6,8,10,11,13,14,15] │ +└──────────────────────────────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/agg_functions/reference/) diff --git a/docs/tr/sql_reference/data_types/aggregatefunction.md b/docs/tr/sql_reference/data_types/aggregatefunction.md new file mode 100644 index 00000000000..622e02d9674 --- /dev/null +++ b/docs/tr/sql_reference/data_types/aggregatefunction.md @@ -0,0 +1,70 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 52 +toc_title: AggregateFunction (ad, types_of_arguments...) +--- + +# AggregateFunction(name, types\_of\_arguments…) {#data-type-aggregatefunction} + +Aggregate functions can have an implementation-defined intermediate state that can be serialized to an AggregateFunction(…) data type and stored in a table, usually, by means of [materyalize bir görünüm](../../sql_reference/statements/select.md#create-view). Bir toplama işlevi durumu üretmek için ortak yolu ile toplama işlevi çağırarak olduğunu `-State` sonek. Gelecekte toplanmanın nihai sonucunu elde etmek için, aynı toplama işlevini `-Merge`sonek. + +`AggregateFunction` — parametric data type. + +**Parametre** + +- Toplama işlevinin adı. + + If the function is parametric, specify its parameters too. + +- Toplama işlevi bağımsız değişkenleri türleri. + +**Örnek** + +``` sql +CREATE TABLE t +( + column1 AggregateFunction(uniq, UInt64), + column2 AggregateFunction(anyIf, String, UInt8), + column3 AggregateFunction(quantiles(0.5, 0.9), UInt64) +) ENGINE = ... +``` + +[uniq](../../sql_reference/aggregate_functions/reference.md#agg_function-uniq), anyİf ([herhangi](../../sql_reference/aggregate_functions/reference.md#agg_function-any)+[Eğer](../../sql_reference/aggregate_functions/combinators.md#agg-functions-combinator-if)) ve [quantiles](../../sql_reference/aggregate_functions/reference.md) ClickHouse desteklenen toplam işlevleri vardır. + +## Kullanma {#usage} + +### Veri Ekleme {#data-insertion} + +Veri eklemek için şunları kullanın `INSERT SELECT` agr aggregateega ile `-State`- işlevler. + +**Fonksiyon örnekleri** + +``` sql +uniqState(UserID) +quantilesState(0.5, 0.9)(SendTiming) +``` + +Karşılık gelen fonksiyonların aksine `uniq` ve `quantiles`, `-State`- fonksiyonlar son değer yerine durumu döndürür. Başka bir deyişle, bir değer döndürür `AggregateFunction` tür. + +Sonuç inlarında `SELECT` sorgu, değerleri `AggregateFunction` türü, Tüm ClickHouse çıktı biçimleri için uygulamaya özgü ikili gösterime sahiptir. Örneğin, veri dökümü, `TabSeparated` ile format `SELECT` sorgu, daha sonra bu dökümü kullanarak geri yüklenebilir `INSERT` sorgu. + +### Veri Seçimi {#data-selection} + +Veri seçerken `AggregatingMergeTree` tablo kullanın `GROUP BY` yan tümce ve veri eklerken aynı toplama işlevleri, ancak kullanarak `-Merge`sonek. + +Bir toplama fonksiyonu ile `-Merge` sonek, bir dizi durum alır, bunları birleştirir ve tam veri toplama sonucunu döndürür. + +Örneğin, aşağıdaki iki sorgu aynı sonucu döndürür: + +``` sql +SELECT uniq(UserID) FROM table + +SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP BY RegionID) +``` + +## Kullanım Örneği {#usage-example} + +Görmek [AggregatingMergeTree](../../engines/table_engines/mergetree_family/aggregatingmergetree.md) motor açıklaması. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nested_data_structures/aggregatefunction/) diff --git a/docs/tr/sql_reference/data_types/array.md b/docs/tr/sql_reference/data_types/array.md new file mode 100644 index 00000000000..1db16ebd1fc --- /dev/null +++ b/docs/tr/sql_reference/data_types/array.md @@ -0,0 +1,77 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 51 +toc_title: Dizi(T) +--- + +# Dizi(t) {#data-type-array} + +Bir dizi `T`- tip öğeleri. `T` herhangi bir veri türü, bir dizi dahil edilebilir. + +## Bir dizi oluşturma {#creating-an-array} + +Bir dizi oluşturmak için bir işlev kullanabilirsiniz: + +``` sql +array(T) +``` + +Köşeli parantez de kullanabilirsiniz. + +``` sql +[] +``` + +Bir dizi oluşturma örneği: + +``` sql +SELECT array(1, 2) AS x, toTypeName(x) +``` + +``` text +┌─x─────┬─toTypeName(array(1, 2))─┐ +│ [1,2] │ Array(UInt8) │ +└───────┴─────────────────────────┘ +``` + +``` sql +SELECT [1, 2] AS x, toTypeName(x) +``` + +``` text +┌─x─────┬─toTypeName([1, 2])─┐ +│ [1,2] │ Array(UInt8) │ +└───────┴────────────────────┘ +``` + +## Veri Türleri İle Çalışma {#working-with-data-types} + +Anında bir dizi oluştururken, ClickHouse bağımsız değişken türünü otomatik olarak listelenen tüm bağımsız değişkenleri depolayabilen en dar veri türü olarak tanımlar. Eğer herhangi bir [Nullable](nullable.md#data_type-nullable) veya edebi [NULL](../../sql_reference/syntax.md#null-literal) değerler, bir dizi öğesinin türü de olur [Nullable](nullable.md). + +ClickHouse veri türünü belirleyemedi, bir özel durum oluşturur. Örneğin, aynı anda dizeler ve sayılarla bir dizi oluşturmaya çalışırken bu olur (`SELECT array(1, 'a')`). + +Otomatik veri türü algılama örnekleri: + +``` sql +SELECT array(1, 2, NULL) AS x, toTypeName(x) +``` + +``` text +┌─x──────────┬─toTypeName(array(1, 2, NULL))─┐ +│ [1,2,NULL] │ Array(Nullable(UInt8)) │ +└────────────┴───────────────────────────────┘ +``` + +Uyumsuz veri türleri dizisi oluşturmaya çalışırsanız, ClickHouse bir özel durum atar: + +``` sql +SELECT array(1, 'a') +``` + +``` text +Received exception from server (version 1.1.54388): +Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not. +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/array/) diff --git a/docs/tr/sql_reference/data_types/boolean.md b/docs/tr/sql_reference/data_types/boolean.md new file mode 100644 index 00000000000..e0641847b98 --- /dev/null +++ b/docs/tr/sql_reference/data_types/boolean.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 43 +toc_title: Boeanoleanean +--- + +# Boole Değerleri {#boolean-values} + +Boole değerleri için ayrı bir tür yoktur. 0 veya 1 değerleriyle sınırlı Uİnt8 türünü kullanın. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/boolean/) diff --git a/docs/tr/sql_reference/data_types/date.md b/docs/tr/sql_reference/data_types/date.md new file mode 100644 index 00000000000..fd47a49313c --- /dev/null +++ b/docs/tr/sql_reference/data_types/date.md @@ -0,0 +1,15 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 47 +toc_title: Tarihli +--- + +# Tarihli {#date} + +Tarihli. 1970-01-01 (imzasız) gün sayısı olarak iki bayt olarak saklanır. Unix döneminin başlangıcından hemen sonra, derleme aşamasında bir sabit tarafından tanımlanan üst eşiğe kadar değerlerin depolanmasına izin verir (şu anda, bu 2106 yılına kadar, ancak tam olarak desteklenen son yıl 2105'tir). +Minimum değer 0000-00-00 olarak çıktıdır. + +Tarih değeri saat dilimi olmadan depolanır. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/date/) diff --git a/docs/tr/sql_reference/data_types/datetime.md b/docs/tr/sql_reference/data_types/datetime.md new file mode 100644 index 00000000000..bc5feeed3b5 --- /dev/null +++ b/docs/tr/sql_reference/data_types/datetime.md @@ -0,0 +1,129 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 48 +toc_title: DateTime +--- + +# Datetime {#data_type-datetime} + +Bir takvim tarih ve bir günün bir saat olarak ifade edilebilir, zaman içinde bir anlık saklamak için izin verir. + +Sözdizimi: + +``` sql +DateTime([timezone]) +``` + +Desteklenen değerler aralığı: \[1970-01-01 00:00:00, 2105-12-31 23:59:59\]. + +Çözünürlük: 1 saniye. + +## Kullanım Açıklamaları {#usage-remarks} + +Zaman içindeki nokta bir [Unix zaman damgası](https://en.wikipedia.org/wiki/Unix_time), ne olursa olsun saat dilimi veya gün ışığından yararlanma saati. Ayrıca, `DateTime` tür, tüm sütun için aynı olan saat dilimini depolayabilir, bu da `DateTime` tür değerleri metin biçiminde görüntülenir ve dizeler olarak belirtilen değerlerin nasıl ayrıştırılır (‘2020-01-01 05:00:01’). Saat dilimi tablo (veya resultset) satırlarında depolanır, ancak sütun meta verileri depolanır. +Desteklenen saat dilimlerinin bir listesi şu adreste bulunabilir: [IANA Saat Dilimi veritabanı](https://www.iana.org/time-zones). +Bu `tzdata` paket, içeren [IANA Saat Dilimi veritabanı](https://www.iana.org/time-zones), sisteme Kurul .malıdır. Kullan... `timedatectl list-timezones` yerel bir sistem tarafından bilinen zaman dilimlerini listelemek için komut. + +İçin bir saat dilimi açıkça ayarlayabilirsiniz `DateTime`- bir tablo oluştururken sütunları yazın. Saat dilimi ayarlanmamışsa, ClickHouse değerini kullanır [saat dilimi](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-timezone) sunucu ayarlarında veya ClickHouse sunucusunun başlatıldığı anda işletim sistemi ayarlarında parametre. + +Bu [clickhouse-müşteri](../../interfaces/cli.md) veri türünü başlatırken bir saat dilimi açıkça ayarlanmamışsa, sunucu saat dilimini varsayılan olarak uygular. İstemci saat dilimini kullanmak için `clickhouse-client` ile... `--use_client_time_zone` parametre. + +ClickHouse çıkış değerleri `YYYY-MM-DD hh:mm:ss` varsayılan olarak metin biçimi. Çıkış ile değiştirebilirsiniz [formatDateTime](../../sql_reference/functions/date_time_functions.md#formatdatetime) işlev. + +Clickhouse'a veri eklerken, Tarih ve saat dizelerinin farklı biçimlerini kullanabilirsiniz. [date\_time\_input\_format](../../operations/settings/settings.md#settings-date_time_input_format) ayar. + +## Örnekler {#examples} + +**1.** Bir tablo ile bir tablo oluşturma `DateTime`- sütun yazın ve içine veri ekleme: + +``` sql +CREATE TABLE dt +( + `timestamp` DateTime('Europe/Moscow'), + `event_id` UInt8 +) +ENGINE = TinyLog; +``` + +``` sql +INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2); +``` + +``` sql +SELECT * FROM dt; +``` + +``` text +┌───────────timestamp─┬─event_id─┐ +│ 2019-01-01 03:00:00 │ 1 │ +│ 2019-01-01 00:00:00 │ 2 │ +└─────────────────────┴──────────┘ +``` + +- Bir tamsayı olarak datetime eklerken, Unıx Zaman Damgası (UTC) olarak kabul edilir. `1546300800` temsil etmek `'2019-01-01 00:00:00'` UTC. Ancak, `timestamp` sütun vardır `Europe/Moscow` (UTC+3) belirtilen zaman dilimi, dize olarak çıkış yaparken değer olarak gösterilecektir `'2019-01-01 03:00:00'` +- Dize değerini datetime olarak eklerken, sütun saat diliminde olduğu kabul edilir. `'2019-01-01 00:00:00'` will gibi muamele `Europe/Moscow` saat dilimi ve farklı kaydedildi `1546290000`. + +**2.** Üzerinde filtreleme `DateTime` değerler + +``` sql +SELECT * FROM dt WHERE timestamp = toDateTime('2019-01-01 00:00:00', 'Europe/Moscow') +``` + +``` text +┌───────────timestamp─┬─event_id─┐ +│ 2019-01-01 00:00:00 │ 2 │ +└─────────────────────┴──────────┘ +``` + +`DateTime` sütun değerleri, bir dize değeri kullanılarak filtrelenebilir `WHERE` yüklem. Dönüştürül willecektir `DateTime` otomatik olarak: + +``` sql +SELECT * FROM dt WHERE timestamp = '2019-01-01 00:00:00' +``` + +``` text +┌───────────timestamp─┬─event_id─┐ +│ 2019-01-01 03:00:00 │ 1 │ +└─────────────────────┴──────────┘ +``` + +**3.** Bir saat dilimi almak `DateTime`- type Col columnum columnn: + +``` sql +SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x +``` + +``` text +┌──────────────column─┬─x─────────────────────────┐ +│ 2019-10-16 04:12:04 │ DateTime('Europe/Moscow') │ +└─────────────────────┴───────────────────────────┘ +``` + +**4.** Zaman dilimi dönüştürme + +``` sql +SELECT +toDateTime(timestamp, 'Europe/London') as lon_time, +toDateTime(timestamp, 'Europe/Moscow') as mos_time +FROM dt +``` + +``` text +┌───────────lon_time──┬────────────mos_time─┐ +│ 2019-01-01 00:00:00 │ 2019-01-01 03:00:00 │ +│ 2018-12-31 21:00:00 │ 2019-01-01 00:00:00 │ +└─────────────────────┴─────────────────────┘ +``` + +## Ayrıca Bakınız {#see-also} + +- [Tip dönüştürme fonksiyonları](../../sql_reference/functions/type_conversion_functions.md) +- [Tarih ve saatlerle çalışmak için işlevler](../../sql_reference/functions/date_time_functions.md) +- [Dizilerle çalışmak için işlevler](../../sql_reference/functions/array_functions.md) +- [Bu `date_time_input_format` ayar](../../operations/settings/settings.md#settings-date_time_input_format) +- [Bu `timezone` sunucu yapılandırma parametresi](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-timezone) +- [Tarih ve saatlerle çalışmak için operatörler](../../sql_reference/operators.md#operators-datetime) +- [Bu `Date` veri türü](date.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/datetime/) diff --git a/docs/tr/sql_reference/data_types/datetime64.md b/docs/tr/sql_reference/data_types/datetime64.md new file mode 100644 index 00000000000..82839e174da --- /dev/null +++ b/docs/tr/sql_reference/data_types/datetime64.md @@ -0,0 +1,104 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 49 +toc_title: DateTime64 +--- + +# Datetime64 {#data_type-datetime64} + +Tanımlanmış alt saniye hassasiyetle, bir takvim tarihi ve bir günün saati olarak ifade edilebilir, zaman içinde bir anlık saklamak için izin verir + +Kene boyutu (hassas): 10-hassaslık ikincilikler + +Sözdizimi: + +``` sql +DateTime64(precision, [timezone]) +``` + +DAHİLİ olarak, verileri bir dizi olarak saklar ‘ticks’ epoch başlangıçtan beri (1970-01-01 00:00:00 UTC) Int64 olarak. Kene çözünürlüğü hassasiyet parametresi tarafından belirlenir. Ayrıca, `DateTime64` tür, tüm sütun için aynı olan saat dilimini depolayabilir, bu da `DateTime64` tür değerleri metin biçiminde görüntülenir ve dizeler olarak belirtilen değerlerin nasıl ayrıştırılır (‘2020-01-01 05:00:01.000’). Saat dilimi tablo (veya resultset) satırlarında depolanır, ancak sütun meta verileri depolanır. Ayrıntıları görün [DateTime](datetime.md). + +## Örnekler {#examples} + +**1.** İle bir tablo oluşturma `DateTime64`- sütun yazın ve içine veri ekleme: + +``` sql +CREATE TABLE dt +( + `timestamp` DateTime64(3, 'Europe/Moscow'), + `event_id` UInt8 +) +ENGINE = TinyLog +``` + +``` sql +INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2) +``` + +``` sql +SELECT * FROM dt +``` + +``` text +┌───────────────timestamp─┬─event_id─┐ +│ 2019-01-01 03:00:00.000 │ 1 │ +│ 2019-01-01 00:00:00.000 │ 2 │ +└─────────────────────────┴──────────┘ +``` + +- Bir tamsayı olarak datetime eklerken, uygun şekilde ölçeklendirilmiş bir Unıx Zaman Damgası (UTC) olarak kabul edilir. `1546300800000` (hassas 3 ile) temsil eder `'2019-01-01 00:00:00'` UTC. Ancak, `timestamp` sütun vardır `Europe/Moscow` (UTC+3) belirtilen zaman dilimi, bir dize olarak çıkış yaparken değer olarak gösterilir `'2019-01-01 03:00:00'` +- Dize değerini datetime olarak eklerken, sütun saat diliminde olduğu kabul edilir. `'2019-01-01 00:00:00'` will gibi muamele `Europe/Moscow` saat dilimi ve olarak saklanır `1546290000000`. + +**2.** Üzerinde filtreleme `DateTime64` değerler + +``` sql +SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow') +``` + +``` text +┌───────────────timestamp─┬─event_id─┐ +│ 2019-01-01 00:00:00.000 │ 2 │ +└─────────────────────────┴──────────┘ +``` + +Aksine `DateTime`, `DateTime64` değerler dönüştürülmez `String` otomatik olarak + +**3.** Bir saat dilimi almak `DateTime64`- tip değeri: + +``` sql +SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x +``` + +``` text +┌──────────────────column─┬─x──────────────────────────────┐ +│ 2019-10-16 04:12:04.000 │ DateTime64(3, 'Europe/Moscow') │ +└─────────────────────────┴────────────────────────────────┘ +``` + +**4.** Zaman dilimi dönüştürme + +``` sql +SELECT +toDateTime64(timestamp, 3, 'Europe/London') as lon_time, +toDateTime64(timestamp, 3, 'Europe/Moscow') as mos_time +FROM dt +``` + +``` text +┌───────────────lon_time──┬────────────────mos_time─┐ +│ 2019-01-01 00:00:00.000 │ 2019-01-01 03:00:00.000 │ +│ 2018-12-31 21:00:00.000 │ 2019-01-01 00:00:00.000 │ +└─────────────────────────┴─────────────────────────┘ +``` + +## Ayrıca Bakınız {#see-also} + +- [Tip dönüştürme fonksiyonları](../../sql_reference/functions/type_conversion_functions.md) +- [Tarih ve saatlerle çalışmak için işlevler](../../sql_reference/functions/date_time_functions.md) +- [Dizilerle çalışmak için işlevler](../../sql_reference/functions/array_functions.md) +- [Bu `date_time_input_format` ayar](../../operations/settings/settings.md#settings-date_time_input_format) +- [Bu `timezone` sunucu yapılandırma parametresi](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-timezone) +- [Tarih ve saatlerle çalışmak için operatörler](../../sql_reference/operators.md#operators-datetime) +- [`Date` veri türü](date.md) +- [`DateTime` veri türü](datetime.md) diff --git a/docs/tr/sql_reference/data_types/decimal.md b/docs/tr/sql_reference/data_types/decimal.md new file mode 100644 index 00000000000..6ec15a52e12 --- /dev/null +++ b/docs/tr/sql_reference/data_types/decimal.md @@ -0,0 +1,109 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 42 +toc_title: "Ondal\u0131k" +--- + +# Ondalık(P, S), Decimal32 (S), Decimal64( S), Decimal128 (S) {#decimalp-s-decimal32s-decimal64s-decimal128s} + +Ekleme, çıkarma ve çarpma işlemleri sırasında hassasiyeti koruyan imzalı sabit noktalı sayılar. Bölünme için en az önemli basamak atılır (yuvarlatılmamış). + +## Parametre {#parameters} + +- P-hassas. Geçerli Aralık: \[1: 38 \]. Kaç ondalık basamak sayısı (kesir dahil) olabilir belirler. +- S-scale. Geçerli Aralık: \[0: P\]. Kaç ondalık basamak kesir olabilir belirler. + +P parametre değerine bağlı olarak ondalık (P, S) bir eşanlamlıdır: +- P \[ 1 : 9\] - Decimal32(S) için) +- P \[ 10 : 18\] - Decimal64(ler) için) +- P \[ 19 : 38\] - Decimal128(ler) için) + +## Ondalık değer aralıkları {#decimal-value-ranges} + +- Decimal32(S) - ( -1 \* 10^(9 - S), 1 \* 10^(9-S) ) +- Decimal64(S) - ( -1 \* 10^(18 - S), 1 \* 10^(18-S) ) +- Decimal128(S) - ( -1 \* 10^(38 - S), 1 \* 10^(38-S) ) + +Örneğin, Decimal32 (4) -99999.9999 99999.9999 0.0001 adım ile sayılar içerebilir. + +## İç temsil {#internal-representation} + +Dahili veri, ilgili bit genişliğine sahip normal imzalı tamsayılar olarak temsil edilir. Bellekte saklanabilen gerçek değer aralıkları, yukarıda belirtilenden biraz daha büyüktür ve yalnızca bir dizeden dönüştürmede kontrol edilir. + +Modern CPU 128-bit tamsayıları doğal olarak desteklemediğinden, Decimal128 üzerindeki işlemler öykünülür. Bu Decimal128 nedeniyle Decimal32/Decimal64'ten önemli ölçüde daha yavaş çalışır. + +## İşlemler ve sonuç türü {#operations-and-result-type} + +Ondalık sonuçtaki ikili işlemler daha geniş sonuç türünde (herhangi bir bağımsız değişken sırası ile) sonuçlanır. + +- Decimal64(S1) Decimal32 (S2) - \> Decimal64 (S) +- Decimal128(S1) Decimal32 (S2) - \> Decimal128(S) +- Decimal128(S1) Decimal64 (S2) - \> Decimal128(S) + +Ölçek kuralları: + +- ekleme, çıkarma: s = max (S1, S2). +- multuply: S = S1 + S2. +- böl: S = S1. + +Ondalık ve tamsayılar arasındaki benzer işlemler için sonuç, bir bağımsız değişkenle aynı boyutta ondalık olur. + +Ondalık ve Float32 / Float64 arasındaki işlemler tanımlanmamıştır. Bunlara ihtiyacınız varsa, todecimal32, toDecimal64, toDecimal128 veya toFloat32, toFloat64 builtins kullanarak bağımsız değişkenlerden birini açıkça yayınlayabilirsiniz. Sonucun hassasiyeti kaybedeceğini ve tür dönüşümünün hesaplamalı olarak pahalı bir işlem olduğunu unutmayın. + +Float64 (örneğin, var veya stddev) ondalık dönüş sonucu bazı işlevler. Ara hesaplamalar hala Float64 ve aynı değerlere sahip ondalık girişler arasında farklı sonuçlara yol açabilecek ondalık olarak gerçekleştirilebilir. + +## Taşma kontrolleri {#overflow-checks} + +Ondalık hesaplamalar sırasında tamsayı taşmaları gerçekleşebilir. Bir kesirdeki aşırı rakamlar atılır (yuvarlatılmamış). Tamsayı bölümünde aşırı basamak bir istisna yol açacaktır. + +``` sql +SELECT toDecimal32(2, 4) AS x, x / 3 +``` + +``` text +┌──────x─┬─divide(toDecimal32(2, 4), 3)─┐ +│ 2.0000 │ 0.6666 │ +└────────┴──────────────────────────────┘ +``` + +``` sql +SELECT toDecimal32(4.2, 8) AS x, x * x +``` + +``` text +DB::Exception: Scale is out of bounds. +``` + +``` sql +SELECT toDecimal32(4.2, 8) AS x, 6 * x +``` + +``` text +DB::Exception: Decimal math overflow. +``` + +Taşma kontrolleri operasyonların yavaşlamasına neden olur. Taşmaların mümkün olmadığı biliniyorsa, kontrolleri kullanarak devre dışı bırakmak mantıklıdır `decimal_check_overflow` ayar. Kontroller devre dışı bırakıldığında ve taşma gerçekleştiğinde, sonuç yanlış olacaktır: + +``` sql +SET decimal_check_overflow = 0; +SELECT toDecimal32(4.2, 8) AS x, 6 * x +``` + +``` text +┌──────────x─┬─multiply(6, toDecimal32(4.2, 8))─┐ +│ 4.20000000 │ -17.74967296 │ +└────────────┴──────────────────────────────────┘ +``` + +Taşma kontrolleri sadece aritmetik işlemlerde değil, değer karşılaştırmasında da gerçekleşir: + +``` sql +SELECT toDecimal32(1, 8) < 100 +``` + +``` text +DB::Exception: Can't compare. +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/decimal/) diff --git a/docs/tr/sql_reference/data_types/domains/index.md b/docs/tr/sql_reference/data_types/domains/index.md new file mode 100644 index 00000000000..e2632761647 --- /dev/null +++ b/docs/tr/sql_reference/data_types/domains/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Etkiler +toc_priority: 56 +--- + + diff --git a/docs/tr/sql_reference/data_types/domains/ipv4.md b/docs/tr/sql_reference/data_types/domains/ipv4.md new file mode 100644 index 00000000000..51611ad97de --- /dev/null +++ b/docs/tr/sql_reference/data_types/domains/ipv4.md @@ -0,0 +1,84 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 59 +toc_title: Ipv44 +--- + +## Ipv44 {#ipv4} + +`IPv4` dayalı bir doma aindir `UInt32` tip ve IPv4 değerlerini depolamak için yazılan bir yedek olarak hizmet eder. İnsan dostu giriş-çıkış biçimi ve muayene ile ilgili sütun tipi bilgileri ile kompakt depolama sağlar. + +### Temel Kullanım {#basic-usage} + +``` sql +CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY url; + +DESCRIBE TABLE hits; +``` + +``` text +┌─name─┬─type───┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┐ +│ url │ String │ │ │ │ │ +│ from │ IPv4 │ │ │ │ │ +└──────┴────────┴──────────────┴────────────────────┴─────────┴──────────────────┘ +``` + +Veya IPv4 etki alanını anahtar olarak kullanabilirsiniz: + +``` sql +CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; +``` + +`IPv4` etki alanı IPv4 dizeleri olarak özel giriş biçimini destekler: + +``` sql +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); + +SELECT * FROM hits; +``` + +``` text +┌─url────────────────────────────────┬───────────from─┐ +│ https://clickhouse.tech/docs/en/ │ 116.106.34.242 │ +│ https://wikipedia.org │ 116.253.40.133 │ +│ https://clickhouse.tech │ 183.247.232.58 │ +└────────────────────────────────────┴────────────────┘ +``` + +Değerler kompakt ikili formda saklanır: + +``` sql +SELECT toTypeName(from), hex(from) FROM hits LIMIT 1; +``` + +``` text +┌─toTypeName(from)─┬─hex(from)─┐ +│ IPv4 │ B7F7E83A │ +└──────────────────┴───────────┘ +``` + +Etki alanı değerleri örtülü olarak dışındaki türlere dönüştürülemez `UInt32`. +Dönüştürmek istiyorsanız `IPv4` bir dizeye değer, bunu açıkça yapmak zorundasınız `IPv4NumToString()` işlev: + +``` sql +SELECT toTypeName(s), IPv4NumToString(from) as s FROM hits LIMIT 1; +``` + + ┌─toTypeName(IPv4NumToString(from))─┬─s──────────────┐ + │ String │ 183.247.232.58 │ + └───────────────────────────────────┴────────────────┘ + +Ya da bir döküm `UInt32` değer: + +``` sql +SELECT toTypeName(i), CAST(from as UInt32) as i FROM hits LIMIT 1; +``` + +``` text +┌─toTypeName(CAST(from, 'UInt32'))─┬──────────i─┐ +│ UInt32 │ 3086477370 │ +└──────────────────────────────────┴────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/domains/ipv4) diff --git a/docs/tr/sql_reference/data_types/domains/ipv6.md b/docs/tr/sql_reference/data_types/domains/ipv6.md new file mode 100644 index 00000000000..71c9db90f9f --- /dev/null +++ b/docs/tr/sql_reference/data_types/domains/ipv6.md @@ -0,0 +1,86 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 60 +toc_title: IPv6 +--- + +## IPv6 {#ipv6} + +`IPv6` dayalı bir doma aindir `FixedString(16)` tip ve IPv6 değerlerini depolamak için yazılan bir yedek olarak hizmet eder. İnsan dostu giriş-çıkış biçimi ve muayene ile ilgili sütun tipi bilgileri ile kompakt depolama sağlar. + +### Temel Kullanım {#basic-usage} + +``` sql +CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY url; + +DESCRIBE TABLE hits; +``` + +``` text +┌─name─┬─type───┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┐ +│ url │ String │ │ │ │ │ +│ from │ IPv6 │ │ │ │ │ +└──────┴────────┴──────────────┴────────────────────┴─────────┴──────────────────┘ +``` + +Veya kullanabilirsiniz `IPv6` anahtar olarak etki alanı: + +``` sql +CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; +``` + +`IPv6` etki alanı IPv6 dizeleri olarak özel girişi destekler: + +``` sql +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); + +SELECT * FROM hits; +``` + +``` text +┌─url────────────────────────────────┬─from──────────────────────────┐ +│ https://clickhouse.tech │ 2001:44c8:129:2632:33:0:252:2 │ +│ https://clickhouse.tech/docs/en/ │ 2a02:e980:1e::1 │ +│ https://wikipedia.org │ 2a02:aa08:e000:3100::2 │ +└────────────────────────────────────┴───────────────────────────────┘ +``` + +Değerler kompakt ikili formda saklanır: + +``` sql +SELECT toTypeName(from), hex(from) FROM hits LIMIT 1; +``` + +``` text +┌─toTypeName(from)─┬─hex(from)────────────────────────┐ +│ IPv6 │ 200144C8012926320033000002520002 │ +└──────────────────┴──────────────────────────────────┘ +``` + +Etki alanı değerleri örtülü olarak dışındaki türlere dönüştürülemez `FixedString(16)`. +Dönüştürmek istiyorsanız `IPv6` bir dizeye değer, bunu açıkça yapmak zorundasınız `IPv6NumToString()` işlev: + +``` sql +SELECT toTypeName(s), IPv6NumToString(from) as s FROM hits LIMIT 1; +``` + +``` text +┌─toTypeName(IPv6NumToString(from))─┬─s─────────────────────────────┐ +│ String │ 2001:44c8:129:2632:33:0:252:2 │ +└───────────────────────────────────┴───────────────────────────────┘ +``` + +Ya da bir döküm `FixedString(16)` değer: + +``` sql +SELECT toTypeName(i), CAST(from as FixedString(16)) as i FROM hits LIMIT 1; +``` + +``` text +┌─toTypeName(CAST(from, 'FixedString(16)'))─┬─i───────┐ +│ FixedString(16) │ ��� │ +└───────────────────────────────────────────┴─────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/domains/ipv6) diff --git a/docs/tr/sql_reference/data_types/domains/overview.md b/docs/tr/sql_reference/data_types/domains/overview.md new file mode 100644 index 00000000000..97e3ef5018a --- /dev/null +++ b/docs/tr/sql_reference/data_types/domains/overview.md @@ -0,0 +1,32 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 58 +toc_title: "Genel bak\u0131\u015F" +--- + +# Etkiler {#domains} + +Etki alanları, varolan temel türün üstüne bazı ek özellikler ekleyen, ancak temel veri türünün kablolu ve disk üstü biçimini sağlam bırakan özel amaçlı türlerdir. Şu anda, ClickHouse kullanıcı tanımlı etki alanlarını desteklemiyor. + +Örneğin, ilgili taban türünün kullanılabileceği her yerde etki alanlarını kullanabilirsiniz: + +- Etki alanı türünde bir sütun oluşturma +- Alan sütunundan/alanına değerleri okuma / yazma +- Bir temel türü bir dizin olarak kullanılabilir, bir dizin olarak kullanın +- Etki alanı sütun değerleri ile çağrı fonksiyonları + +### Alanların ekstra özellikleri {#extra-features-of-domains} + +- Açık sütun türü adı `SHOW CREATE TABLE` veya `DESCRIBE TABLE` +- İle insan dostu format inputtan giriş `INSERT INTO domain_table(domain_column) VALUES(...)` +- İçin insan dostu forma outputta çıktı `SELECT domain_column FROM domain_table` +- Harici bir kaynaktan insan dostu biçimde veri yükleme: `INSERT INTO domain_table FORMAT CSV ...` + +### Sınırlamalar {#limitations} + +- Temel türün dizin sütununu etki alanı türüne dönüştürülemiyor `ALTER TABLE`. +- Başka bir sütun veya tablodan veri eklerken dize değerlerini dolaylı olarak etki alanı değerlerine dönüştüremez. +- Etki alanı, depolanan değerler üzerinde hiçbir kısıtlama ekler. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/domains/overview) diff --git a/docs/tr/sql_reference/data_types/enum.md b/docs/tr/sql_reference/data_types/enum.md new file mode 100644 index 00000000000..65c687835dc --- /dev/null +++ b/docs/tr/sql_reference/data_types/enum.md @@ -0,0 +1,132 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 50 +toc_title: Enum +--- + +# Enum {#enum} + +Adlandırılmış değerlerden oluşan numaralandırılmış tür. + +Adlandırılmış değerler olarak bildirilmelidir `'string' = integer` çiftliler. ClickHouse yalnızca sayıları saklar, ancak adları aracılığıyla değerlerle işlemleri destekler. + +ClickHouse destekler: + +- 8-bit `Enum`. En fazla 256 değerleri numaralandırılmış içerebilir `[-128, 127]` Aralık. +- 16-bit `Enum`. En fazla 65536 değerleri numaralandırılmış içerebilir `[-32768, 32767]` Aralık. + +ClickHouse otomatik olarak türünü seçer `Enum` veri eklendiğinde. Ayrıca kullanabilirsiniz `Enum8` veya `Enum16` türleri depolama boyutunda emin olmak için. + +## Kullanım Örnekleri {#usage-examples} + +Burada bir tablo oluşturuyoruz `Enum8('hello' = 1, 'world' = 2)` type Col columnum columnn: + +``` sql +CREATE TABLE t_enum +( + x Enum('hello' = 1, 'world' = 2) +) +ENGINE = TinyLog +``` + +Sütun `x` yalnızca tür tanımında listelenen değerleri depolayabilir: `'hello'` veya `'world'`. Başka bir değer kaydetmeye çalışırsanız, ClickHouse bir özel durum yükseltir. Bunun için 8-bit boyutu `Enum` otomatik olarak seçilir. + +``` sql +INSERT INTO t_enum VALUES ('hello'), ('world'), ('hello') +``` + +``` text +Ok. +``` + +``` sql +INSERT INTO t_enum values('a') +``` + +``` text +Exception on client: +Code: 49. DB::Exception: Unknown element 'a' for type Enum('hello' = 1, 'world' = 2) +``` + +Tablodan veri sorguladığınızda, ClickHouse dize değerleri `Enum`. + +``` sql +SELECT * FROM t_enum +``` + +``` text +┌─x─────┐ +│ hello │ +│ world │ +│ hello │ +└───────┘ +``` + +Satırların sayısal eşdeğerlerini görmeniz gerekiyorsa, `Enum` tamsayı türüne değer. + +``` sql +SELECT CAST(x, 'Int8') FROM t_enum +``` + +``` text +┌─CAST(x, 'Int8')─┐ +│ 1 │ +│ 2 │ +│ 1 │ +└─────────────────┘ +``` + +Bir sorguda bir Enum değeri oluşturmak için, ayrıca kullanmanız gerekir `CAST`. + +``` sql +SELECT toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)')) +``` + +``` text +┌─toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)'))─┐ +│ Enum8('a' = 1, 'b' = 2) │ +└─────────────────────────────────────────────────────┘ +``` + +## Genel Kurallar ve kullanım {#general-rules-and-usage} + +Değerlerin her birine aralıkta bir sayı atanır `-128 ... 127` için `Enum8` veya aralık inta `-32768 ... 32767` için `Enum16`. Tüm dizeler ve sayılar farklı olmalıdır. Boş bir dize izin verilir. Bu tür belirtilmişse (bir tablo tanımında), sayılar rasgele bir sırada olabilir. Ancak, sipariş önemli değil. + +Ne dize ne de sayısal değer bir `Enum` olabilir [NULL](../../sql_reference/syntax.md). + +Bir `Enum` içerdiği olabilir [Nullable](nullable.md) tür. Yani sorguyu kullanarak bir tablo oluşturursanız + +``` sql +CREATE TABLE t_enum_nullable +( + x Nullable( Enum8('hello' = 1, 'world' = 2) ) +) +ENGINE = TinyLog +``` + +bu mağaza değil sadece `'hello'` ve `'world'`, ama `NULL`, yanında. + +``` sql +INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) +``` + +RAM, bir `Enum` sütun aynı şekilde saklanır `Int8` veya `Int16` karşılık gelen sayısal değerlerin. + +Metin formunda okurken, ClickHouse değeri bir dize olarak ayrıştırır ve karşılık gelen dizeyi Enum değerleri kümesinden arar. Bulunmazsa, bir istisna atılır. Metin biçiminde okurken, dize okunur ve karşılık gelen sayısal değer aranır. Bulunmazsa bir istisna atılır. +Metin formunda yazarken, değeri karşılık gelen dize olarak yazar. Sütun verileri çöp içeriyorsa (geçerli kümeden olmayan sayılar), bir özel durum atılır. İkili formda okurken ve yazarken, Int8 ve Int16 veri türleri ile aynı şekilde çalışır. +Örtülü varsayılan değer, en düşük sayıya sahip değerdir. + +Sırasında `ORDER BY`, `GROUP BY`, `IN`, `DISTINCT` ve böylece, Enumlar karşılık gelen sayılarla aynı şekilde davranır. Örneğin, sipariş onları sayısal olarak sıralar. Eşitlik ve karşılaştırma işleçleri, alttaki sayısal değerler üzerinde yaptıkları gibi Enumlarda aynı şekilde çalışır. + +Enum değerleri sayılarla karşılaştırılamaz. Enums sabit bir dize ile karşılaştırılabilir. Karşılaştırılan dize Enum için geçerli bir değer değilse, bir özel durum atılır. IN operatörü, sol taraftaki Enum ve sağ taraftaki bir dizi dizeyle desteklenir. Dizeler, karşılık gelen Enumun değerleridir. + +Most numeric and string operations are not defined for Enum values, e.g. adding a number to an Enum or concatenating a string to an Enum. +Ancak, Enum doğal bir `toString` dize değerini döndüren işlev. + +Enum değerleri de kullanarak sayısal türlere dönüştürülebilir `toT` fonksiyon, burada t sayısal bir türdür. T enum'un temel sayısal türüne karşılık geldiğinde, bu dönüşüm sıfır maliyetlidir. +Enum türü, yalnızca değer kümesi değiştirilirse, alter kullanılarak maliyet olmadan değiştirilebilir. Her iki ekleme ve Alter kullanarak Enum üyeleri kaldırmak mümkündür (kaldırma yalnızca kaldırılan değer tabloda hiç kullanılmadıysa güvenlidir). Bir koruma olarak, önceden tanımlanmış bir Enum üyesinin sayısal değerini değiştirmek bir istisna atar. + +ALTER kullanarak, bir Enum8 için bir Enum16 veya tam tersi, Int8 için Int16 değiştirme gibi değiştirmek mümkündür. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/enum/) diff --git a/docs/tr/sql_reference/data_types/fixedstring.md b/docs/tr/sql_reference/data_types/fixedstring.md new file mode 100644 index 00000000000..bf716f4d591 --- /dev/null +++ b/docs/tr/sql_reference/data_types/fixedstring.md @@ -0,0 +1,63 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 45 +toc_title: FixedString(N) +--- + +# Fixedstring {#fixedstring} + +Sabit uzunlukta bir dize `N` bayt (ne karakter ne de kod noktaları). + +Bir sütun bildirmek için `FixedString` yazın, aşağıdaki sözdizimini kullanın: + +``` sql + FixedString(N) +``` + +Nerede `N` doğal bir sayıdır. + +Bu `FixedString` veri tam olarak uzunluğa sahip olduğunda tür etkilidir `N` baytlar. Diğer tüm durumlarda, verimliliği düşürmesi muhtemeldir. + +Verimli bir şekilde depolan theabilen değerlere örnekler `FixedString`- yazılan sütunlar: + +- IP adreslerinin ikili gösterimi (`FixedString(16)` IPv6 için). +- Language codes (ru\_RU, en\_US … ). +- Currency codes (USD, RUB … ). +- Karma ikili gösterimi (`FixedString(16)` MD5 için, `FixedString(32)` SHA256 için). + +UUID değerlerini depolamak için [UUID](uuid.md) veri türü. + +Verileri eklerken, ClickHouse: + +- Dize daha az içeriyorsa, boş bayt ile bir dize tamamlar `N` baytlar. +- Atar `Too large value for FixedString(N)` dize birden fazla içeriyorsa, özel durum `N` baytlar. + +Verileri seçerken, ClickHouse dize sonunda boş bayt kaldırmaz. Eğer kullanıyorsanız `WHERE` yan tümcesi, null bayt el ile eşleştirmek için eklemelisiniz `FixedString` değer. Kullanımı için aşağıdaki örnek, nasıl gösterir `WHERE` fık withra ile `FixedString`. + +Aşağıdaki tabloyu tek ile düşünelim `FixedString(2)` sütun: + +``` text +┌─name──┐ +│ b │ +└───────┘ +``` + +Sorgu `SELECT * FROM FixedStringTable WHERE a = 'b'` sonuç olarak herhangi bir veri döndürmez. Filtre desenini boş baytlarla tamamlamalıyız. + +``` sql +SELECT * FROM FixedStringTable +WHERE a = 'b\0' +``` + +``` text +┌─a─┐ +│ b │ +└───┘ +``` + +Bu davranış için MySQL farklıdır `CHAR` tür (burada dizeler boşluklarla doldurulur ve boşluklar çıktı için kaldırılır). + +Not uzunluğu `FixedString(N)` değer sabittir. Bu [uzunluk](../../sql_reference/functions/array_functions.md#array_functions-length) fonksiyon döndürür `N` hatta eğer `FixedString(N)` değer yalnızca boş baytlarla doldurulur, ancak [boş](../../sql_reference/functions/string_functions.md#empty) fonksiyon döndürür `1` bu durumda. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/fixedstring/) diff --git a/docs/tr/sql_reference/data_types/float.md b/docs/tr/sql_reference/data_types/float.md new file mode 100644 index 00000000000..0b5bd76be86 --- /dev/null +++ b/docs/tr/sql_reference/data_types/float.md @@ -0,0 +1,87 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 41 +toc_title: Float32, Float64 +--- + +# Float32, Float64 {#float32-float64} + +[Kayan nokta numaraları](https://en.wikipedia.org/wiki/IEEE_754). + +Türleri C türlerine eşdeğerdir: + +- `Float32` - `float` +- `Float64` - `double` + +Verileri mümkün olduğunda tamsayı biçiminde saklamanızı öneririz. Örneğin, sabit hassas sayıları parasal tutarlar veya sayfa yükleme süreleri gibi milisaniye cinsinden tamsayı değerlerine dönüştürün. + +## Kayan noktalı sayıları kullanma {#using-floating-point-numbers} + +- Kayan noktalı sayılarla yapılan hesaplamalar yuvarlama hatası oluşturabilir. + + + +``` sql +SELECT 1 - 0.9 +``` + +``` text +┌───────minus(1, 0.9)─┐ +│ 0.09999999999999998 │ +└─────────────────────┘ +``` + +- Hesaplamanın sonucu hesaplama yöntemine (bilgisayar sisteminin işlemci tipi ve mimarisi) bağlıdır. +- Kayan nokta hesaplamaları, sonsuzluk gibi sayılarla sonuçlanabilir (`Inf`) ve “not-a-number” (`NaN`). Hesaplamaların sonuçlarını işlerken bu dikkate alınmalıdır. +- Kayan noktalı sayıları metinden ayrıştırırken, sonuç en yakın makine tarafından temsil edilebilir sayı olmayabilir. + +## N andan ve In andf {#data_type-float-nan-inf} + +Standart SQL aksine, ClickHouse kayan noktalı sayılar aşağıdaki kategorileri destekler: + +- `Inf` – Infinity. + + + +``` sql +SELECT 0.5 / 0 +``` + +``` text +┌─divide(0.5, 0)─┐ +│ inf │ +└────────────────┘ +``` + +- `-Inf` – Negative infinity. + + + +``` sql +SELECT -0.5 / 0 +``` + +``` text +┌─divide(-0.5, 0)─┐ +│ -inf │ +└─────────────────┘ +``` + +- `NaN` – Not a number. + + + +``` sql +SELECT 0 / 0 +``` + +``` text +┌─divide(0, 0)─┐ +│ nan │ +└──────────────┘ +``` + + See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select.md). + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/float/) diff --git a/docs/tr/sql_reference/data_types/index.md b/docs/tr/sql_reference/data_types/index.md new file mode 100644 index 00000000000..c2fdda0c8e8 --- /dev/null +++ b/docs/tr/sql_reference/data_types/index.md @@ -0,0 +1,15 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Veri T\xFCrleri" +toc_priority: 37 +toc_title: "Giri\u015F" +--- + +# Veri Türleri {#data_types} + +ClickHouse tablo hücrelerinde veri çeşitli saklayabilirsiniz. + +Bu bölümde desteklenen veri türleri ve varsa bunları kullanmak ve/veya uygulamak için özel hususlar açıklanmaktadır. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/) diff --git a/docs/tr/sql_reference/data_types/int_uint.md b/docs/tr/sql_reference/data_types/int_uint.md new file mode 100644 index 00000000000..2c7b47eb4c6 --- /dev/null +++ b/docs/tr/sql_reference/data_types/int_uint.md @@ -0,0 +1,26 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 40 +toc_title: "U\u0130nt8, U\u0130nt16, U\u0130nt32, Uint64, Int8, Int16, Int32, Int64" +--- + +# Uİnt8, Uİnt16, Uİnt32, Uint64, Int8, Int16, Int32, Int64 {#uint8-uint16-uint32-uint64-int8-int16-int32-int64} + +Sabit uzunlukta tamsayılar, veya bir işareti olmadan. + +## İnt Aralıkları {#int-ranges} + +- Int8- \[-128: 127\] +- Int16 - \[-32768: 32767\] +- Int32 - \[-2147483648: 2147483647\] +- Int64 - \[-9223372036854775808: 9223372036854775807\] + +## Uint Aralıkları {#uint-ranges} + +- Uİnt8- \[0: 255\] +- Uİnt16- \[0: 65535\] +- Uİnt32- \[0: 4294967295\] +- Uİnt64 - \[0: 18446744073709551615\] + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/int_uint/) diff --git a/docs/tr/sql_reference/data_types/nested_data_structures/index.md b/docs/tr/sql_reference/data_types/nested_data_structures/index.md new file mode 100644 index 00000000000..53a998fa8a8 --- /dev/null +++ b/docs/tr/sql_reference/data_types/nested_data_structures/index.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "\u0130\xE7 \u0130\xE7e Veri Yap\u0131lar\u0131" +toc_hidden: true +toc_priority: 54 +toc_title: "gizlenmi\u015F" +--- + +# İç İçe Veri Yapıları {#nested-data-structures} + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nested_data_structures/) diff --git a/docs/tr/sql_reference/data_types/nested_data_structures/nested.md b/docs/tr/sql_reference/data_types/nested_data_structures/nested.md new file mode 100644 index 00000000000..8cd2dc3a558 --- /dev/null +++ b/docs/tr/sql_reference/data_types/nested_data_structures/nested.md @@ -0,0 +1,106 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 57 +toc_title: "\u0130\xE7 \u0130\xE7e (Name1 Type1, Name2 Type2,...)" +--- + +# Nested(name1 Type1, Name2 Type2, …) {#nestedname1-type1-name2-type2} + +A nested data structure is like a table inside a cell. The parameters of a nested data structure – the column names and types – are specified the same way as in a [CREATE TABLE](../../../sql_reference/statements/create.md) sorgu. Her tablo satırı, iç içe geçmiş veri yapısındaki herhangi bir sayıda satıra karşılık gelebilir. + +Örnek: + +``` sql +CREATE TABLE test.visits +( + CounterID UInt32, + StartDate Date, + Sign Int8, + IsNew UInt8, + VisitID UInt64, + UserID UInt64, + ... + Goals Nested + ( + ID UInt32, + Serial UInt32, + EventTime DateTime, + Price Int64, + OrderID String, + CurrencyID UInt32 + ), + ... +) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192, Sign) +``` + +Bu örnek bildirir `Goals` dönüşümlerle ilgili verileri içeren iç içe veri yapısı (ulaşılan hedefler). Her satır içinde ‘visits’ tablo sıfır veya dönüşüm herhangi bir sayıda karşılık gelebilir. + +Sadece tek bir yuvalama seviyesi desteklenir. Diziler içeren iç içe geçmiş yapıların sütunları çok boyutlu dizilere eşdeğerdir, bu nedenle sınırlı desteğe sahiptirler (bu sütunları MergeTree altyapısı ile tablolarda depolamak için destek yoktur). + +Çoğu durumda, iç içe geçmiş bir veri yapısıyla çalışırken, sütunları bir nokta ile ayrılmış sütun adlarıyla belirtilir. Bu sütunlar eşleşen türleri bir dizi oluşturur. Tek bir iç içe geçmiş veri yapısının tüm sütun dizileri aynı uzunluğa sahiptir. + +Örnek: + +``` sql +SELECT + Goals.ID, + Goals.EventTime +FROM test.visits +WHERE CounterID = 101500 AND length(Goals.ID) < 5 +LIMIT 10 +``` + +``` text +┌─Goals.ID───────────────────────┬─Goals.EventTime───────────────────────────────────────────────────────────────────────────┐ +│ [1073752,591325,591325] │ ['2014-03-17 16:38:10','2014-03-17 16:38:48','2014-03-17 16:42:27'] │ +│ [1073752] │ ['2014-03-17 00:28:25'] │ +│ [1073752] │ ['2014-03-17 10:46:20'] │ +│ [1073752,591325,591325,591325] │ ['2014-03-17 13:59:20','2014-03-17 22:17:55','2014-03-17 22:18:07','2014-03-17 22:18:51'] │ +│ [] │ [] │ +│ [1073752,591325,591325] │ ['2014-03-17 11:37:06','2014-03-17 14:07:47','2014-03-17 14:36:21'] │ +│ [] │ [] │ +│ [] │ [] │ +│ [591325,1073752] │ ['2014-03-17 00:46:05','2014-03-17 00:46:05'] │ +│ [1073752,591325,591325,591325] │ ['2014-03-17 13:28:33','2014-03-17 13:30:26','2014-03-17 18:51:21','2014-03-17 18:51:45'] │ +└────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +İç içe geçmiş bir veri yapısını aynı uzunlukta birden çok sütun dizisi kümesi olarak düşünmek en kolay yoldur. + +Bir SELECT sorgusunun tek tek sütunlar yerine tüm iç içe geçmiş veri yapısının adını belirtebileceği tek yer array JOIN yan tümcesi. Daha fazla bilgi için, bkz. “ARRAY JOIN clause”. Örnek: + +``` sql +SELECT + Goal.ID, + Goal.EventTime +FROM test.visits +ARRAY JOIN Goals AS Goal +WHERE CounterID = 101500 AND length(Goals.ID) < 5 +LIMIT 10 +``` + +``` text +┌─Goal.ID─┬──────Goal.EventTime─┐ +│ 1073752 │ 2014-03-17 16:38:10 │ +│ 591325 │ 2014-03-17 16:38:48 │ +│ 591325 │ 2014-03-17 16:42:27 │ +│ 1073752 │ 2014-03-17 00:28:25 │ +│ 1073752 │ 2014-03-17 10:46:20 │ +│ 1073752 │ 2014-03-17 13:59:20 │ +│ 591325 │ 2014-03-17 22:17:55 │ +│ 591325 │ 2014-03-17 22:18:07 │ +│ 591325 │ 2014-03-17 22:18:51 │ +│ 1073752 │ 2014-03-17 11:37:06 │ +└─────────┴─────────────────────┘ +``` + +İç içe geçmiş veri yapısının tamamı için SELECT gerçekleştiremezsiniz. Yalnızca bir parçası olan tek tek sütunları açıkça listeleyebilirsiniz. + +Bir INSERT sorgusu için, iç içe geçmiş bir veri yapısının tüm bileşen sütun dizilerini ayrı ayrı (tek tek sütun dizileri gibi) iletmelisiniz. Ekleme sırasında, sistem aynı uzunluğa sahip olduklarını kontrol eder. + +Bir tanımlama sorgusu için, iç içe geçmiş bir veri yapısındaki sütunlar aynı şekilde ayrı olarak listelenir. + +İç içe geçmiş bir veri yapısındaki öğeler için ALTER sorgusu sınırlamaları vardır. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nested_data_structures/nested/) diff --git a/docs/tr/sql_reference/data_types/nullable.md b/docs/tr/sql_reference/data_types/nullable.md new file mode 100644 index 00000000000..1950e2b1b77 --- /dev/null +++ b/docs/tr/sql_reference/data_types/nullable.md @@ -0,0 +1,46 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 54 +toc_title: Nullable +--- + +# Nullable (typename) {#data_type-nullable} + +Özel işaretleyici saklamak için izin verir ([NULL](../../sql_reference/syntax.md)) bu ifade eder “missing value” tarafından izin verilen normal değerlerin yanında `TypeName`. Örneğin, bir `Nullable(Int8)` tipi sütun saklayabilirsiniz `Int8` değerleri yazın ve değeri olmayan satırlar depolayacaktır `NULL`. + +İçin... `TypeName`, bileşik veri türlerini kullanamazsınız [Dizi](array.md) ve [Demet](tuple.md). Bileşik veri türleri şunları içerebilir `Nullable` gibi tür değerleri `Array(Nullable(Int8))`. + +A `Nullable` tür alanı tablo dizinlerine dahil edilemez. + +`NULL` herhangi biri için varsayılan değer mi `Nullable` ClickHouse sunucu yapılandırmasında aksi belirtilmediği sürece yazın. + +## Depolama Özellikleri {#storage-features} + +İçermek `Nullable` bir tablo sütunundaki değerleri yazın, ClickHouse ile ayrı bir dosya kullanır `NULL` değerleri ile normal dosyaya ek olarak Maskeler. Maskeli girişleri ClickHouse ayırt izin dosyası `NULL` ve her tablo satırı için karşılık gelen veri türünün varsayılan değeri. Ek bir dosya nedeniyle, `Nullable` sütun, benzer bir normal olana kıyasla ek depolama alanı tüketir. + +!!! info "Not" + Kullanım `Nullable` neredeyse her zaman performansı olumsuz etkiler, veritabanlarınızı tasarlarken bunu aklınızda bulundurun. + +## Kullanım Örneği {#usage-example} + +``` sql +CREATE TABLE t_null(x Int8, y Nullable(Int8)) ENGINE TinyLog +``` + +``` sql +INSERT INTO t_null VALUES (1, NULL), (2, 3) +``` + +``` sql +SELECT x + y FROM t_null +``` + +``` text +┌─plus(x, y)─┐ +│ ᴺᵁᴸᴸ │ +│ 5 │ +└────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/nullable/) diff --git a/docs/tr/sql_reference/data_types/special_data_types/expression.md b/docs/tr/sql_reference/data_types/special_data_types/expression.md new file mode 100644 index 00000000000..a098b0c6365 --- /dev/null +++ b/docs/tr/sql_reference/data_types/special_data_types/expression.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 58 +toc_title: "\u0130fade" +--- + +# İfade {#expression} + +İfadeler, lambda'ları yüksek mertebeden işlevlerde temsil etmek için kullanılır. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/expression/) diff --git a/docs/tr/sql_reference/data_types/special_data_types/index.md b/docs/tr/sql_reference/data_types/special_data_types/index.md new file mode 100644 index 00000000000..700bf57784b --- /dev/null +++ b/docs/tr/sql_reference/data_types/special_data_types/index.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "\xD6zel Veri T\xFCrleri" +toc_hidden: true +toc_priority: 55 +toc_title: "gizlenmi\u015F" +--- + +# Özel Veri Türleri {#special-data-types} + +Özel veri türü değerleri Tablo veya çıktı sorgu sonuçlarında kaydetmek için seri hale getirilemez, ancak sorgu yürütme sırasında bir ara sonuç olarak kullanılabilir. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/) diff --git a/docs/tr/sql_reference/data_types/special_data_types/interval.md b/docs/tr/sql_reference/data_types/special_data_types/interval.md new file mode 100644 index 00000000000..e95f5a98cfd --- /dev/null +++ b/docs/tr/sql_reference/data_types/special_data_types/interval.md @@ -0,0 +1,85 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 61 +toc_title: "Aral\u0131kl\u0131" +--- + +# Aralıklı {#data-type-interval} + +Zaman ve Tarih aralıklarını temsil eden veri türleri ailesi. Ortaya çıkan türleri [INTERVAL](../../../sql_reference/operators.md#operator-interval) operatör. + +!!! warning "Uyarıcı" + `Interval` veri türü değerleri tablolarda saklanamaz. + +Yapılı: + +- İmzasız bir tamsayı değeri olarak zaman aralığı. +- Bir aralık türü. + +Desteklenen Aralık türleri: + +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `QUARTER` +- `YEAR` + +Her Aralık türü için ayrı bir veri türü vardır. Örneğin, `DAY` Aralık karşılık gelir `IntervalDay` veri türü: + +``` sql +SELECT toTypeName(INTERVAL 4 DAY) +``` + +``` text +┌─toTypeName(toIntervalDay(4))─┐ +│ IntervalDay │ +└──────────────────────────────┘ +``` + +## Kullanım Açıklamaları {#data-type-interval-usage-remarks} + +Kullanabilirsiniz `Interval`- aritmetik işlemlerde değerler yazın [Tarihli](../../../sql_reference/data_types/date.md) ve [DateTime](../../../sql_reference/data_types/datetime.md)- tip değerleri. Örneğin, geçerli saate 4 gün ekleyebilirsiniz: + +``` sql +SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY +``` + +``` text +┌───current_date_time─┬─plus(now(), toIntervalDay(4))─┐ +│ 2019-10-23 10:58:45 │ 2019-10-27 10:58:45 │ +└─────────────────────┴───────────────────────────────┘ +``` + +Farklı tiplere sahip aralıklar birleştirilemez. Gibi aralıklarla kullanamazsınız `4 DAY 1 HOUR`. Aralıkların, örneğin aralığın en küçük birimine eşit veya daha küçük olan birimlerdeki aralıkları belirtin `1 day and an hour` aralık olarak ifade edilebilir `25 HOUR` veya `90000 SECOND`. + +İle aritmetik işlemler yapamazsınız `Interval`- değerleri yazın, ancak farklı türde aralıklar ekleyebilirsiniz. `Date` veya `DateTime` veri türleri. Mesela: + +``` sql +SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR +``` + +``` text +┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ +│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +``` + +Aşağıdaki sorgu bir özel duruma neden olur: + +``` sql +select now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR) +``` + +``` text +Received exception from server (version 19.14.1): +Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argument types for function plus: if one argument is Interval, then another must be Date or DateTime.. +``` + +## Ayrıca Bakınız {#see-also} + +- [INTERVAL](../../../sql_reference/operators.md#operator-interval) operatör +- [toİnterval](../../../sql_reference/functions/type_conversion_functions.md#function-tointerval) tip dönüştürme işlevleri diff --git a/docs/tr/sql_reference/data_types/special_data_types/nothing.md b/docs/tr/sql_reference/data_types/special_data_types/nothing.md new file mode 100644 index 00000000000..046c36ab047 --- /dev/null +++ b/docs/tr/sql_reference/data_types/special_data_types/nothing.md @@ -0,0 +1,26 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 60 +toc_title: "Hi\xE7bir \u015Fey" +--- + +# Hiçbir şey {#nothing} + +Bu veri türünün tek amacı, bir değerin beklenmediği durumları temsil etmektir. Yani bir oluşturamazsınız `Nothing` type value. + +Örneğin, literal [NULL](../../../sql_reference/syntax.md#null-literal) türü vardır `Nullable(Nothing)`. Daha fazla görmek [Nullable](../../../sql_reference/data_types/nullable.md). + +Bu `Nothing` tür boş dizileri belirtmek için de kullanılabilir: + +``` sql +SELECT toTypeName(array()) +``` + +``` text +┌─toTypeName(array())─┐ +│ Array(Nothing) │ +└─────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/nothing/) diff --git a/docs/tr/sql_reference/data_types/special_data_types/set.md b/docs/tr/sql_reference/data_types/special_data_types/set.md new file mode 100644 index 00000000000..f12fac0e56f --- /dev/null +++ b/docs/tr/sql_reference/data_types/special_data_types/set.md @@ -0,0 +1,12 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 59 +toc_title: Koymak +--- + +# Koymak {#set} + +Sağ yarısı için kullanılan bir [IN](../../../sql_reference/statements/select.md#select-in-operators) ifade. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/special_data_types/set/) diff --git a/docs/tr/sql_reference/data_types/string.md b/docs/tr/sql_reference/data_types/string.md new file mode 100644 index 00000000000..0b7c5918dc7 --- /dev/null +++ b/docs/tr/sql_reference/data_types/string.md @@ -0,0 +1,20 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 44 +toc_title: Dize +--- + +# Dize {#string} + +Keyfi uzunlukta dizeler. Uzunluk sınırlı değildir. Değer, boş bayt da dahil olmak üzere rasgele bir bayt kümesi içerebilir. +Dize türü türleri değiştirir VARCHAR, BLOB, CLOB, ve diğerleri diğer DBMSs. + +## Kodlamalar {#encodings} + +ClickHouse kodlama kavramına sahip değildir. Dizeler, depolanan ve olduğu gibi çıkan rasgele bir bayt kümesi içerebilir. +Metinleri saklamanız gerekiyorsa, UTF-8 kodlamasını kullanmanızı öneririz. En azından, terminaliniz UTF-8 kullanıyorsa (önerildiği gibi), dönüşüm yapmadan değerlerinizi okuyabilir ve yazabilirsiniz. +Benzer şekilde, dizelerle çalışmak için belirli işlevler, dizenin UTF-8 kodlu bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışan ayrı varyasyonlara sahiptir. +Örneğin, ‘length’ işlev, bayt cinsinden dize uzunluğunu hesaplar; ‘lengthUTF8’ işlev, değerin UTF-8 kodlanmış olduğunu varsayarak Unicode kod noktalarındaki dize uzunluğunu hesaplar. + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/string/) diff --git a/docs/tr/sql_reference/data_types/tuple.md b/docs/tr/sql_reference/data_types/tuple.md new file mode 100644 index 00000000000..64f2b2b2aec --- /dev/null +++ b/docs/tr/sql_reference/data_types/tuple.md @@ -0,0 +1,52 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 53 +toc_title: Tuple (T1, T2, ...) +--- + +# Tuple(t1, T2, …) {#tuplet1-t2} + +Elemanlarının bir demet, her bir birey olması [tür](index.md#data_types). + +Tuples geçici sütun gruplama için kullanılır. Sütunlar, bir sorguda bir In ifadesi kullanıldığında ve lambda işlevlerinin belirli resmi parametrelerini belirtmek için gruplandırılabilir. Daha fazla bilgi için bölümlere bakın [Operatör İNLERDE](../../sql_reference/statements/select.md) ve [Yüksek mertebeden fonksiyonlar](../../sql_reference/functions/higher_order_functions.md). + +Tuples bir sorgunun sonucu olabilir. Bu durumda, json dışındaki metin formatları için değerler köşeli parantez içinde virgülle ayrılır. JSON formatlarında, tuples diziler olarak çıktılanır (köşeli parantez içinde). + +## Bir Tuple oluşturma {#creating-a-tuple} + +Bir tuple oluşturmak için bir işlev kullanabilirsiniz: + +``` sql +tuple(T1, T2, ...) +``` + +Bir tuple oluşturma örneği: + +``` sql +SELECT tuple(1,'a') AS x, toTypeName(x) +``` + +``` text +┌─x───────┬─toTypeName(tuple(1, 'a'))─┐ +│ (1,'a') │ Tuple(UInt8, String) │ +└─────────┴───────────────────────────┘ +``` + +## Veri Türleri İle Çalışma {#working-with-data-types} + +Anında bir tuple oluştururken, ClickHouse her bağımsız değişkenin türünü bağımsız değişken değerini depolayabilen türlerin en azı olarak otomatik olarak algılar. Argüman ise [NULL](../../sql_reference/syntax.md#null-literal), tuple elemanının türü [Nullable](nullable.md). + +Otomatik veri türü algılama örneği: + +``` sql +SELECT tuple(1, NULL) AS x, toTypeName(x) +``` + +``` text +┌─x────────┬─toTypeName(tuple(1, NULL))──────┐ +│ (1,NULL) │ Tuple(UInt8, Nullable(Nothing)) │ +└──────────┴─────────────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/tuple/) diff --git a/docs/tr/sql_reference/data_types/uuid.md b/docs/tr/sql_reference/data_types/uuid.md new file mode 100644 index 00000000000..9fb502227be --- /dev/null +++ b/docs/tr/sql_reference/data_types/uuid.md @@ -0,0 +1,77 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 46 +toc_title: UUID +--- + +# UUID {#uuid-data-type} + +Evrensel olarak benzersiz bir tanımlayıcı (UUID), kayıtları tanımlamak için kullanılan 16 baytlık bir sayıdır. UUID hakkında ayrıntılı bilgi için bkz [Vikipedi](https://en.wikipedia.org/wiki/Universally_unique_identifier). + +UUID türü değeri örneği aşağıda temsil edilmektedir: + +``` text +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +``` + +Yeni bir kayıt eklerken UUID sütun değerini belirtmezseniz, UUID değeri sıfır ile doldurulur: + +``` text +00000000-0000-0000-0000-000000000000 +``` + +## Nasıl Oluşturulur {#how-to-generate} + +UUID değerini oluşturmak için ClickHouse, [generateuuıdv4](../../sql_reference/functions/uuid_functions.md) işlev. + +## Kullanım Örneği {#usage-example} + +**Örnek 1** + +Bu örnek, UUID türü sütunuyla bir tablo oluşturma ve tabloya bir değer ekleme gösterir. + +``` sql +CREATE TABLE t_uuid (x UUID, y String) ENGINE=TinyLog +``` + +``` sql +INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1' +``` + +``` sql +SELECT * FROM t_uuid +``` + +``` text +┌────────────────────────────────────x─┬─y─────────┐ +│ 417ddc5d-e556-4d27-95dd-a34d84e46a50 │ Example 1 │ +└──────────────────────────────────────┴───────────┘ +``` + +**Örnek 2** + +Bu örnekte, yeni bir kayıt eklerken UUID sütun değeri belirtilmedi. + +``` sql +INSERT INTO t_uuid (y) VALUES ('Example 2') +``` + +``` sql +SELECT * FROM t_uuid +``` + +``` text +┌────────────────────────────────────x─┬─y─────────┐ +│ 417ddc5d-e556-4d27-95dd-a34d84e46a50 │ Example 1 │ +│ 00000000-0000-0000-0000-000000000000 │ Example 2 │ +└──────────────────────────────────────┴───────────┘ +``` + +## Kısıtlama {#restrictions} + +UUID veri türü sadece hangi fonksiyonları destekler [Dize](string.md) veri türü de destekler (örneğin, [dakika](../../sql_reference/aggregate_functions/reference.md#agg_function-min), [maksimum](../../sql_reference/aggregate_functions/reference.md#agg_function-max), ve [sayma](../../sql_reference/aggregate_functions/reference.md#agg_function-count)). + +UUID veri türü aritmetik işlemler tarafından desteklenmez (örneğin, [abs](../../sql_reference/functions/arithmetic_functions.md#arithm_func-abs)) veya toplama fonksiyonları gibi [toplam](../../sql_reference/aggregate_functions/reference.md#agg_function-sum) ve [avg](../../sql_reference/aggregate_functions/reference.md#agg_function-avg). + +[Orijinal makale](https://clickhouse.tech/docs/en/data_types/uuid/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts.md new file mode 100644 index 00000000000..ac85ccd1724 --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts.md @@ -0,0 +1,56 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 39 +toc_title: "Genel A\xE7\u0131klama" +--- + +# Dış Söz Dictionarieslükler {#dicts-external-dicts} + +Çeşitli veri kaynaklarından kendi sözlükleri ekleyebilirsiniz. Bir sözlük için veri kaynağı, yerel bir metin veya yürütülebilir dosya, bir HTTP(s) kaynağı veya başka bir DBMS olabilir. Daha fazla bilgi için, bkz. “[Dış sözlükler için kaynaklar](external_dicts_dict_sources.md)”. + +ClickHouse: + +- Sözlükleri RAM'de tamamen veya kısmen saklar. +- Sözlükleri periyodik olarak günceller ve eksik değerleri dinamik olarak yükler. Başka bir deyişle, sözlükler dinamik olarak yüklenebilir. +- Xml dosyaları ile harici sözlükler oluşturmak için izin verir veya [DDL sorguları](../../statements/create.md#create-dictionary-query). + +Dış sözlüklerin yapılandırması bir veya daha fazla xml dosyasında bulunabilir. Yapılandırma yolu belirtilen [dictionaries\_config](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-dictionaries_config) parametre. + +Sözlükler sunucu başlangıçta veya ilk kullanımda, bağlı olarak yüklenebilir [dictionaries\_lazy\_load](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load) ayar. + +Sözlük yapılandırma dosyası aşağıdaki biçime sahiptir: + +``` xml + + An optional element with any content. Ignored by the ClickHouse server. + + + /etc/metrika.xml + + + + + + + + +``` + +Yapabilirsin [yapılandırmak](external_dicts_dict.md) aynı dosyada sözlükler herhangi bir sayıda. + +[Sözlükler için DDL sorguları](../../statements/create.md#create-dictionary-query) sunucu yapılandırmasında herhangi bir ek kayıt gerektirmez. Tablolar veya görünümler gibi birinci sınıf varlıklar olarak sözlüklerle çalışmaya izin verirler. + +!!! attention "Dikkat" + Küçük bir sözlük için değerleri, bir `SELECT` sorgu (bkz. [dönüştürmek](../../../sql_reference/functions/other_functions.md) işlev). Bu işlevsellik harici sözlüklerle ilgili değildir. + +## Ayrıca Bakınız {#ext-dicts-see-also} + +- [Harici bir sözlük yapılandırma](external_dicts_dict.md) +- [Sözlükleri bellekte saklama](external_dicts_dict_layout.md) +- [Sözlük Güncellemeleri](external_dicts_dict_lifetime.md) +- [Dış Sözlüklerin kaynakları](external_dicts_dict_sources.md) +- [Sözlük anahtarı ve alanları](external_dicts_dict_structure.md) +- [Harici Sözlüklerle çalışmak için işlevler](../../../sql_reference/functions/ext_dict_functions.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md new file mode 100644 index 00000000000..b931143903e --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md @@ -0,0 +1,53 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 40 +toc_title: "Harici bir s\xF6zl\xFCk yap\u0131land\u0131rma" +--- + +# Harici bir sözlük yapılandırma {#dicts-external-dicts-dict} + +Sözlük xml dosyası kullanılarak yapılandırılmışsa, sözlük yapılandırması aşağıdaki yapıya sahiptir: + +``` xml + + dict_name + + + + + + + + + + + + + + + + + +``` + +İlgili [DDL-sorgu](../../statements/create.md#create-dictionary-query) aşağıdaki yapıya sahiptir: + +``` sql +CREATE DICTIONARY dict_name +( + ... -- attributes +) +PRIMARY KEY ... -- complex or single key configuration +SOURCE(...) -- Source configuration +LAYOUT(...) -- Memory layout configuration +LIFETIME(...) -- Lifetime of dictionary in memory +``` + +- `name` – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`. +- [kaynaklı](external_dicts_dict_sources.md) — Source of the dictionary. +- [düzen](external_dicts_dict_layout.md) — Dictionary layout in memory. +- [yapılı](external_dicts_dict_structure.md) — Structure of the dictionary . A key and attributes that can be retrieved by this key. +- [ömür](external_dicts_dict_lifetime.md) — Frequency of dictionary updates. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md new file mode 100644 index 00000000000..6945173c529 --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md @@ -0,0 +1,70 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 45 +toc_title: "Hiyerar\u015Fik s\xF6zl\xFCkler" +--- + +# Hiyerarşik Sözlükler {#hierarchical-dictionaries} + +ClickHouse bir hiyerarşik sözlükler destekler [sayısal tuş](external_dicts_dict_structure.md#ext_dict-numeric-key). + +Aşağıdaki hiyerarşik yapıya bakın: + +``` text +0 (Common parent) +│ +├── 1 (Russia) +│ │ +│ └── 2 (Moscow) +│ │ +│ └── 3 (Center) +│ +└── 4 (Great Britain) + │ + └── 5 (London) +``` + +Bu hiyerarşi aşağıdaki sözlük tablosu olarak ifade edilebilir. + +| region\_id | parent\_region | region\_name | +|------------|----------------|--------------| +| 1 | 0 | Rusya | +| 2 | 1 | Moskova | +| 3 | 2 | Merkezli | +| 4 | 0 | İngiltere | +| 5 | 4 | Londra | + +Bu tablo bir sütun içerir `parent_region` bu öğe için en yakın ebeveynin anahtarını içerir. + +ClickHouse destekler [hiyerarşik](external_dicts_dict_structure.md#hierarchical-dict-attr) için mülkiyet [dış sözlük](index.md) öznitelik. Bu özellik, yukarıda açıklanana benzer hiyerarşik sözlüğü yapılandırmanıza izin verir. + +Bu [dictGetHierarchy](../../../sql_reference/functions/ext_dict_functions.md#dictgethierarchy) fonksiyonu bir elemanın üst zincir almak için izin verir. + +Örneğimiz için, sözlüğün yapısı aşağıdaki gibi olabilir: + +``` xml + + + + region_id + + + + parent_region + UInt64 + 0 + true + + + + region_name + String + + + + + +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_hierarchical/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md new file mode 100644 index 00000000000..174d6830839 --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md @@ -0,0 +1,373 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 41 +toc_title: "S\xF6zl\xFCkleri bellekte saklama" +--- + +# Sözlükleri Bellekte Saklama {#dicts-external-dicts-dict-layout} + +Sözlükleri bellekte saklamanın çeşitli yolları vardır. + +Biz tavsiye [düzlük](#flat), [karıştırıyordu](#dicts-external_dicts_dict_layout-hashed) ve [complex\_key\_hashed](#complex-key-hashed). hangi optimum işleme hızı sağlamak. + +Önbelleğe alma, potansiyel olarak düşük performans ve en uygun parametreleri seçmede zorluklar nedeniyle önerilmez. Bölümünde devamını oku “[önbellek](#cache)”. + +Sözlük performansını artırmanın birkaç yolu vardır: + +- Sonra sözlük ile çalışmak için işlevi çağırın `GROUP BY`. +- Mark enjektif olarak ayıklamak için nitelikler. Farklı öznitelik değerleri farklı anahtarlara karşılık geliyorsa, bir öznitelik ınjective olarak adlandırılır. Yani ne zaman `GROUP BY` anahtar tarafından bir öznitelik değeri getiren bir işlev kullanır, bu işlev otomatik olarak dışarı alınır `GROUP BY`. + +ClickHouse, sözlüklerdeki hatalar için bir istisna oluşturur. Hata örnekleri: + +- Erişilen sözlük yüklenemedi. +- Bir sorgulama hatası `cached` sözlük. + +Sen dış sözlükler ve durumları listesini görüntüleyebilirsiniz `system.dictionaries` Tablo. + +Yapılandırma şöyle görünüyor: + +``` xml + + + ... + + + + + + ... + + +``` + +İlgili [DDL-sorgu](../../statements/create.md#create-dictionary-query): + +``` sql +CREATE DICTIONARY (...) +... +LAYOUT(LAYOUT_TYPE(param value)) -- layout settings +... +``` + +## Sözlükleri Bellekte Saklamanın Yolları {#ways-to-store-dictionaries-in-memory} + +- [düzlük](#flat) +- [karıştırıyordu](#dicts-external_dicts_dict_layout-hashed) +- [sparse\_hashed](#dicts-external_dicts_dict_layout-sparse_hashed) +- [önbellek](#cache) +- [range\_hashed](#range-hashed) +- [complex\_key\_hashed](#complex-key-hashed) +- [complex\_key\_cache](#complex-key-cache) +- [ıp\_trie](#ip-trie) + +### düzlük {#flat} + +Sözlük tamamen düz diziler şeklinde bellekte saklanır. Sözlük ne kadar bellek kullanıyor? Miktar, en büyük anahtarın boyutuyla orantılıdır (kullanılan alanda). + +Sözlük anahtarı vardır `UInt64` yazın ve değeri 500.000 ile sınırlıdır. Sözlük oluştururken daha büyük bir anahtar bulunursa, ClickHouse bir özel durum atar ve sözlüğü oluşturmaz. + +Her türlü kaynak desteklenmektedir. Güncellerken, veriler (bir dosyadan veya bir tablodan) bütünüyle okunur. + +Bu yöntem, sözlüğü saklamak için mevcut tüm yöntemler arasında en iyi performansı sağlar. + +Yapılandırma örneği: + +``` xml + + + +``` + +veya + +``` sql +LAYOUT(FLAT()) +``` + +### karıştırıyordu {#dicts-external_dicts_dict_layout-hashed} + +Sözlük tamamen bir karma tablo şeklinde bellekte saklanır. Sözlük, uygulamada herhangi bir tanımlayıcıya sahip herhangi bir sayıda öğe içerebilir, anahtar sayısı on milyonlarca öğeye ulaşabilir. + +Her türlü kaynak desteklenmektedir. Güncellerken, veriler (bir dosyadan veya bir tablodan) bütünüyle okunur. + +Yapılandırma örneği: + +``` xml + + + +``` + +veya + +``` sql +LAYOUT(HASHED()) +``` + +### sparse\_hashed {#dicts-external_dicts_dict_layout-sparse_hashed} + +Benzer `hashed`, ancak daha fazla CPU kullanımı lehine daha az bellek kullanır. + +Yapılandırma örneği: + +``` xml + + + +``` + +``` sql +LAYOUT(SPARSE_HASHED()) +``` + +### complex\_key\_hashed {#complex-key-hashed} + +Bu tür depolama kompozit ile kullanım içindir [anahtarlar](external_dicts_dict_structure.md). Benzer `hashed`. + +Yapılandırma örneği: + +``` xml + + + +``` + +``` sql +LAYOUT(COMPLEX_KEY_HASHED()) +``` + +### range\_hashed {#range-hashed} + +Sözlük, sıralı bir aralık dizisi ve bunlara karşılık gelen değerleri olan bir karma tablo şeklinde bellekte saklanır. + +Bu depolama yöntemi, hashed ile aynı şekilde çalışır ve anahtara ek olarak tarih/saat (rasgele sayısal tür) aralıklarının kullanılmasına izin verir. + +Örnek: tablo, her reklamveren için biçimdeki indirimleri içerir: + +``` text ++---------|-------------|-------------|------+ +| advertiser id | discount start date | discount end date | amount | ++===============+=====================+===================+========+ +| 123 | 2015-01-01 | 2015-01-15 | 0.15 | ++---------|-------------|-------------|------+ +| 123 | 2015-01-16 | 2015-01-31 | 0.25 | ++---------|-------------|-------------|------+ +| 456 | 2015-01-01 | 2015-01-15 | 0.05 | ++---------|-------------|-------------|------+ +``` + +Tarih aralıkları için bir örnek kullanmak için, `range_min` ve `range_max` element inler [yapılı](external_dicts_dict_structure.md). Bu elemanlar elemanları içermelidir `name` ve`type` (eğer `type` belirtilmemişse, varsayılan tür kullanılır-Tarih). `type` herhangi bir sayısal tür olabilir (Date / DateTime / Uint64 / Int32 / others). + +Örnek: + +``` xml + + + Id + + + first + Date + + + last + Date + + ... +``` + +veya + +``` sql +CREATE DICTIONARY somedict ( + id UInt64, + first Date, + last Date +) +PRIMARY KEY id +LAYOUT(RANGE_HASHED()) +RANGE(MIN first MAX last) +``` + +Bu sözlüklerle çalışmak için, `dictGetT` bir aralığın seçildiği işlev: + +``` sql +dictGetT('dict_name', 'attr_name', id, date) +``` + +Bu işlev belirtilen değerin değerini döndürür `id`s ve geçirilen tarihi içeren tarih aralığı. + +Algoritmanın detayları: + +- Eğer... `id` not fo orund veya a range is not fo aund for the `id`, sözlük için varsayılan değeri döndürür. +- Çakışan aralıklar varsa, herhangi birini kullanabilirsiniz. +- Aralık sınırlayıcı ise `NULL` veya geçersiz bir tarih (örneğin 1900-01-01 veya 2039-01-01), Aralık açık bırakılır. Aralık her iki tarafta da açık olabilir. + +Yapılandırma örneği: + +``` xml + + + + ... + + + + + + + + Abcdef + + + StartTimeStamp + UInt64 + + + EndTimeStamp + UInt64 + + + XXXType + String + + + + + + +``` + +veya + +``` sql +CREATE DICTIONARY somedict( + Abcdef UInt64, + StartTimeStamp UInt64, + EndTimeStamp UInt64, + XXXType String DEFAULT '' +) +PRIMARY KEY Abcdef +RANGE(MIN StartTimeStamp MAX EndTimeStamp) +``` + +### önbellek {#cache} + +Sözlük, sabit sayıda hücre içeren bir önbellekte saklanır. Bu hücreler sık kullanılan elementleri içerir. + +Bir sözlük ararken, önce önbellek aranır. Her veri bloğu için, önbellekte bulunmayan veya güncel olmayan tüm anahtarlar, kaynak kullanılarak istenir `SELECT attrs... FROM db.table WHERE id IN (k1, k2, ...)`. Alınan veriler daha sonra önbelleğe yazılır. + +Önbellek sözlükleri için, sona erme [ömür](external_dicts_dict_lifetime.md) önbellekteki verilerin ayarlanabilir. Eğer daha fazla zaman `lifetime` bir hücrede veri yüklenmesinden bu yana geçti, hücrenin değeri kullanılmaz ve bir dahaki sefere kullanılması gerektiğinde yeniden istenir. +Bu, sözlükleri saklamanın tüm yollarından en az etkilidir. Önbelleğin hızı, doğru ayarlara ve kullanım senaryosuna bağlıdır. Bir önbellek türü sözlüğü, yalnızca isabet oranları yeterince yüksek olduğunda (önerilen %99 ve daha yüksek) iyi performans gösterir. Sen ortalama isabet oranı görebilirsiniz `system.dictionaries` Tablo. + +Önbellek performansını artırmak için bir alt sorgu ile kullanın `LIMIT`, ve harici sözlük ile işlevini çağırın. + +Destek [kaynaklılar](external_dicts_dict_sources.md): MySQL, ClickHouse, yürütülebilir, HTTP. + +Ayarlar örneği: + +``` xml + + + + 1000000000 + + +``` + +veya + +``` sql +LAYOUT(CACHE(SIZE_IN_CELLS 1000000000)) +``` + +Yeterince büyük bir önbellek boyutu ayarlayın. Sen hücre sayısını seçmek için deneme gerekir: + +1. Bazı değer ayarlayın. +2. Önbellek tamamen doluncaya kadar sorguları çalıştırın. +3. Kullanarak bellek tüketimini değerlendirmek `system.dictionaries` Tablo. +4. Gerekli bellek tüketimine ulaşılana kadar hücre sayısını artırın veya azaltın. + +!!! warning "Uyarıcı" + Rasgele okuma ile sorguları işlemek için yavaş olduğundan, ClickHouse kaynak olarak kullanmayın. + +### complex\_key\_cache {#complex-key-cache} + +Bu tür depolama kompozit ile kullanım içindir [anahtarlar](external_dicts_dict_structure.md). Benzer `cache`. + +### ıp\_trie {#ip-trie} + +Bu tür depolama, ağ öneklerini (IP adresleri) asn gibi meta verilere eşlemek içindir. + +Örnek: tablo, ağ önekleri ve bunlara karşılık gelen sayı ve ülke kodu içerir: + +``` text + +-----------|-----|------+ + | prefix | asn | cca2 | + +=================+=======+========+ + | 202.79.32.0/20 | 17501 | NP | + +-----------|-----|------+ + | 2620:0:870::/48 | 3856 | US | + +-----------|-----|------+ + | 2a02:6b8:1::/48 | 13238 | RU | + +-----------|-----|------+ + | 2001:db8::/32 | 65536 | ZZ | + +-----------|-----|------+ +``` + +Bu tür bir düzen kullanırken, yapının bileşik bir anahtarı olmalıdır. + +Örnek: + +``` xml + + + + prefix + String + + + + asn + UInt32 + + + + cca2 + String + ?? + + ... +``` + +veya + +``` sql +CREATE DICTIONARY somedict ( + prefix String, + asn UInt32, + cca2 String DEFAULT '??' +) +PRIMARY KEY prefix +``` + +Anahtarın izin verilen bir IP öneki içeren yalnızca bir dize türü özniteliği olması gerekir. Diğer türler henüz desteklenmiyor. + +Sorgular için aynı işlevleri kullanmanız gerekir (`dictGetT` bir tuple ile) kompozit tuşları ile sözlükler gelince: + +``` sql +dictGetT('dict_name', 'attr_name', tuple(ip)) +``` + +İşlev ya alır `UInt32` IPv4 için veya `FixedString(16)` IPv6 için: + +``` sql +dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) +``` + +Diğer türler henüz desteklenmiyor. İşlev, bu IP adresine karşılık gelen önek için özniteliği döndürür. Örtüşen önekler varsa, en spesifik olanı döndürülür. + +Veri bir saklanan `trie`. Tamamen RAM'e uyması gerekir. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_layout/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md new file mode 100644 index 00000000000..d630e9e0128 --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md @@ -0,0 +1,91 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 42 +toc_title: "S\xF6zl\xFCk G\xFCncellemeleri" +--- + +# Sözlük Güncellemeleri {#dictionary-updates} + +ClickHouse sözlükleri periyodik olarak günceller. Tam olarak karşıdan yüklenen sözlükler için Güncelleştirme aralığı ve önbelleğe alınmış sözlükler için geçersiz kılma aralığı `` saniyeler içinde etiketleyin. + +Sözlük güncelleştirmeleri (ilk kullanım için yükleme dışında) sorguları engellemez. Güncellemeler sırasında, bir sözlüğün eski sürümü kullanılır. Güncelleştirme sırasında bir hata oluşursa, hata sunucu günlüğüne yazılır ve sorgular sözlüklerin eski sürümünü kullanmaya devam eder. + +Ayarlar örneği: + +``` xml + + ... + 300 + ... + +``` + +``` sql +CREATE DICTIONARY (...) +... +LIFETIME(300) +... +``` + +Ayar `0` (`LIFETIME(0)`) söz dictionarieslük .lerin güncel updatinglenmesini engeller. + +Yükseltmeler için bir zaman aralığı ayarlayabilirsiniz ve ClickHouse bu aralıkta eşit rastgele bir zaman seçecektir. Bu, çok sayıda sunucuda yükseltme yaparken yükü sözlük kaynağına dağıtmak için gereklidir. + +Ayarlar örneği: + +``` xml + + ... + + 300 + 360 + + ... + +``` + +veya + +``` sql +LIFETIME(MIN 300 MAX 360) +``` + +Eğer `0` ve `0`, ClickHouse sözlüğü zaman aşımı ile yeniden yüklemez. +Bu durumda, Sözlük yapılandırma dosyası değiştirilmişse veya ClickHouse sözlüğü daha önce yeniden yükleyebilir. `SYSTEM RELOAD DICTIONARY` komut yürütüldü. + +Sözlükleri yükseltirken, ClickHouse sunucusu türüne bağlı olarak farklı mantık uygular [kaynaklı](external_dicts_dict_sources.md): + +Sözlükleri yükseltirken, ClickHouse sunucusu türüne bağlı olarak farklı mantık uygular [kaynaklı](external_dicts_dict_sources.md): + +- Bir metin dosyası için değişiklik zamanını kontrol eder. Zaman önceden kaydedilmiş zaman farklıysa, sözlük güncelleştirilir. +- Myısam tabloları için, değişiklik zamanı bir `SHOW TABLE STATUS` sorgu. +- Diğer kaynaklardan gelen sözlükler varsayılan olarak her zaman güncellenir. + +MySQL (InnoDB), ODBC ve ClickHouse kaynakları için, sözlükleri her seferinde değil, gerçekten değiştiyse güncelleyecek bir sorgu ayarlayabilirsiniz. Bunu yapmak için şu adımları izleyin: + +- Sözlük tablosu, kaynak verileri güncelleştirildiğinde her zaman değişen bir alana sahip olmalıdır. +- Kaynak ayarları, değişen alanı alan bir sorgu belirtmeniz gerekir. ClickHouse sunucu sorgu sonucu bir satır olarak yorumlar ve bu satır önceki durumuna göre değişmişse, sözlük güncelleştirilir. Sorguda belirtme `` için ayar fieldlardaki alan [kaynaklı](external_dicts_dict_sources.md). + +Ayarlar örneği: + +``` xml + + ... + + ... + SELECT update_time FROM dictionary_source where id = 1 + + ... + +``` + +veya + +``` sql +... +SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source where id = 1')) +... +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_lifetime/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md new file mode 100644 index 00000000000..d2ea01c54ce --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md @@ -0,0 +1,608 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 43 +toc_title: "D\u0131\u015F S\xF6zl\xFCklerin kaynaklar\u0131" +--- + +# Dış Sözlüklerin Kaynakları {#dicts-external-dicts-dict-sources} + +Harici bir sözlük birçok farklı kaynaktan bağlanabilir. + +Sözlük xml dosyası kullanılarak yapılandırılmışsa, yapılandırma şöyle görünür: + +``` xml + + + ... + + + + + + ... + + ... + +``` + +Durumunda [DDL-sorgu](../../statements/create.md#create-dictionary-query), eşit yapılandırma gibi görünüyor olacak: + +``` sql +CREATE DICTIONARY dict_name (...) +... +SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration +... +``` + +Kaynak yapılandırılmış `source` bölme. + +Kaynak türleri (`source_type`): + +- [Yerel dosya](#dicts-external_dicts_dict_sources-local_file) +- [Yürütülebilir dosya](#dicts-external_dicts_dict_sources-executable) +- [HTTP (s)](#dicts-external_dicts_dict_sources-http) +- DBMS + - [ODBC](#dicts-external_dicts_dict_sources-odbc) + - [MySQL](#dicts-external_dicts_dict_sources-mysql) + - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) + - [MongoDB](#dicts-external_dicts_dict_sources-mongodb) + - [Redis](#dicts-external_dicts_dict_sources-redis) + +## Yerel Dosya {#dicts-external_dicts_dict_sources-local_file} + +Ayarlar örneği: + +``` xml + + + /opt/dictionaries/os.tsv + TabSeparated + + +``` + +veya + +``` sql +SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) +``` + +Ayar alanları: + +- `path` – The absolute path to the file. +- `format` – The file format. All the formats described in “[Biçimliler](../../../interfaces/formats.md#formats)” desteklenir. + +## Yürütülebilir Dosya {#dicts-external_dicts_dict_sources-executable} + +Yürütülebilir dosyalarla çalışmak Aşağıdakilere bağlıdır [sözlük bellekte nasıl saklanır](external_dicts_dict_layout.md). Sözlük kullanılarak saklan theıyorsa `cache` ve `complex_key_cache` ClickHouse, yürütülebilir dosyanın STDIN'SİNE bir istek göndererek gerekli anahtarları ister. Aksi takdirde, clickhouse yürütülebilir dosyayı başlatır ve çıktısını sözlük verileri olarak değerlendirir. + +Ayarlar örneği: + +``` xml + + + cat /opt/dictionaries/os.tsv + TabSeparated + + +``` + +veya + +``` sql +SOURCE(EXECUTABLE(command 'cat /opt/dictionaries/os.tsv' format 'TabSeparated')) +``` + +Ayar alanları: + +- `command` – The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). +- `format` – The file format. All the formats described in “[Biçimliler](../../../interfaces/formats.md#formats)” desteklenir. + +## Http (s) {#dicts-external_dicts_dict_sources-http} + +Bir HTTP (s) sunucusuyla çalışmak Aşağıdakilere bağlıdır [sözlük bellekte nasıl saklanır](external_dicts_dict_layout.md). Sözlük kullanılarak saklan theıyorsa `cache` ve `complex_key_cache`, ClickHouse aracılığıyla bir istek göndererek gerekli anahtarları ister `POST` yöntem. + +Ayarlar örneği: + +``` xml + + + http://[::1]/os.tsv + TabSeparated + + user + password + + +
+ API-KEY + key +
+ + + +``` + +veya + +``` sql +SOURCE(HTTP( + url 'http://[::1]/os.tsv' + format 'TabSeparated' + credentials(user 'user' password 'password') + headers(header(name 'API-KEY' value 'key')) +)) +``` + +Clickhouse'un bir HTTPS kaynağına erişebilmesi için şunları yapmanız gerekir [openssl'yi yapılandırma](../../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-openssl) sunucu yapılandırmasında. + +Ayar alanları: + +- `url` – The source URL. +- `format` – The file format. All the formats described in “[Biçimliler](../../../interfaces/formats.md#formats)” desteklenir. +- `credentials` – Basic HTTP authentication. Optional parameter. + - `user` – Username required for the authentication. + - `password` – Password required for the authentication. +- `headers` – All custom HTTP headers entries used for the HTTP request. Optional parameter. + - `header` – Single HTTP header entry. + - `name` – Identifiant name used for the header send on the request. + - `value` – Value set for a specific identifiant name. + +## ODBC {#dicts-external_dicts_dict_sources-odbc} + +ODBC sürücüsü olan herhangi bir veritabanını bağlamak için bu yöntemi kullanabilirsiniz. + +Ayarlar örneği: + +``` xml + + + DatabaseName + ShemaName.TableName
+ DSN=some_parameters + SQL_QUERY + + +``` + +veya + +``` sql +SOURCE(ODBC( + db 'DatabaseName' + table 'SchemaName.TableName' + connection_string 'DSN=some_parameters' + invalidate_query 'SQL_QUERY' +)) +``` + +Ayar alanları: + +- `db` – Name of the database. Omit it if the database name is set in the `` parametre. +- `table` – Name of the table and schema if exists. +- `connection_string` – Connection string. +- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Sözlükleri güncelleme](external_dicts_dict_lifetime.md). + +ClickHouse, ODBC sürücüsünden alıntı sembolleri alır ve sorgulardaki tüm ayarları sürücüye aktarır, bu nedenle tablo adını veritabanındaki tablo adı durumuna göre ayarlamak gerekir. + +Oracle kullanırken kodlamalarla ilgili bir sorununuz varsa, ilgili [FAQ](../../../faq/general.md#oracle-odbc-encodings) makale. + +### ODBC Sözlük işlevselliği bilinen güvenlik açığı {#known-vulnerability-of-the-odbc-dictionary-functionality} + +!!! attention "Dikkat" + ODBC sürücüsü bağlantı parametresi aracılığıyla veritabanına bağlanırken `Servername` yerine. Bu durumda değerler `USERNAME` ve `PASSWORD` itibaren `odbc.ini` uzak sunucuya gönderilir ve tehlikeye girebilir. + +**Güvensiz kullanım örneği** + +PostgreSQL için unixodbc'yi yapılandıralım. İçeriği `/etc/odbc.ini`: + +``` text +[gregtest] +Driver = /usr/lib/psqlodbca.so +Servername = localhost +PORT = 5432 +DATABASE = test_db +#OPTION = 3 +USERNAME = test +PASSWORD = test +``` + +Daha sonra aşağıdaki gibi bir sorgu yaparsanız + +``` sql +SELECT * FROM odbc('DSN=gregtest;Servername=some-server.com', 'test_db'); +``` + +ODBC sürücüsü değerleri gönderir `USERNAME` ve `PASSWORD` itibaren `odbc.ini` -e doğru `some-server.com`. + +### PostgreSQL Bağlanma Örneği {#example-of-connecting-postgresql} + +UB .untu OS. + +PostgreSQL için UNİXODBC ve ODBC sürücüsünü yükleme: + +``` bash +$ sudo apt-get install -y unixodbc odbcinst odbc-postgresql +``` + +Yapılandırma `/etc/odbc.ini` (veya `~/.odbc.ini`): + +``` text + [DEFAULT] + Driver = myconnection + + [myconnection] + Description = PostgreSQL connection to my_db + Driver = PostgreSQL Unicode + Database = my_db + Servername = 127.0.0.1 + UserName = username + Password = password + Port = 5432 + Protocol = 9.3 + ReadOnly = No + RowVersioning = No + ShowSystemTables = No + ConnSettings = +``` + +Clickhouse'da sözlük yapılandırması: + +``` xml + + + table_name + + + + + DSN=myconnection + postgresql_table
+ + + + 300 + 360 + + + + + + + id + + + some_column + UInt64 + 0 + + + + +``` + +veya + +``` sql +CREATE DICTIONARY table_name ( + id UInt64, + some_column UInt64 DEFAULT 0 +) +PRIMARY KEY id +SOURCE(ODBC(connection_string 'DSN=myconnection' table 'postgresql_table')) +LAYOUT(HASHED()) +LIFETIME(MIN 300 MAX 360) +``` + +Düzenlemeniz gerekebilir `odbc.ini` sürücü ile kitaplığın tam yolunu belirtmek için `DRIVER=/usr/local/lib/psqlodbcw.so`. + +### MS SQL Server bağlanma örneği {#example-of-connecting-ms-sql-server} + +UB .untu OS. + +Sürücüyü yükleme: : + +``` bash +$ sudo apt-get install tdsodbc freetds-bin sqsh +``` + +Sürücüyü yapılandırma: + +``` bash + $ cat /etc/freetds/freetds.conf + ... + + [MSSQL] + host = 192.168.56.101 + port = 1433 + tds version = 7.0 + client charset = UTF-8 + + $ cat /etc/odbcinst.ini + ... + + [FreeTDS] + Description = FreeTDS + Driver = /usr/lib/x86_64-linux-gnu/odbc/libtdsodbc.so + Setup = /usr/lib/x86_64-linux-gnu/odbc/libtdsS.so + FileUsage = 1 + UsageCount = 5 + + $ cat ~/.odbc.ini + ... + + [MSSQL] + Description = FreeTDS + Driver = FreeTDS + Servername = MSSQL + Database = test + UID = test + PWD = test + Port = 1433 +``` + +Clickhouse'da sözlüğü yapılandırma: + +``` xml + + + test + + + dict
+ DSN=MSSQL;UID=test;PWD=test + + + + + 300 + 360 + + + + + + + + + k + + + s + String + + + + + +``` + +veya + +``` sql +CREATE DICTIONARY test ( + k UInt64, + s String DEFAULT '' +) +PRIMARY KEY k +SOURCE(ODBC(table 'dict' connection_string 'DSN=MSSQL;UID=test;PWD=test')) +LAYOUT(FLAT()) +LIFETIME(MIN 300 MAX 360) +``` + +## DBMS {#dbms} + +### Mysql {#dicts-external_dicts_dict_sources-mysql} + +Ayarlar örneği: + +``` xml + + + 3306 + clickhouse + qwerty + + example01-1 + 1 + + + example01-2 + 1 + + db_name + table_name
+ id=10 + SQL_QUERY + + +``` + +veya + +``` sql +SOURCE(MYSQL( + port 3306 + user 'clickhouse' + password 'qwerty' + replica(host 'example01-1' priority 1) + replica(host 'example01-2' priority 1) + db 'db_name' + table 'table_name' + where 'id=10' + invalidate_query 'SQL_QUERY' +)) +``` + +Ayar alanları: + +- `port` – The port on the MySQL server. You can specify it for all replicas, or for each one individually (inside ``). + +- `user` – Name of the MySQL user. You can specify it for all replicas, or for each one individually (inside ``). + +- `password` – Password of the MySQL user. You can specify it for all replicas, or for each one individually (inside ``). + +- `replica` – Section of replica configurations. There can be multiple sections. + + - `replica/host` – The MySQL host. + - `replica/priority` – The replica priority. When attempting to connect, ClickHouse traverses the replicas in order of priority. The lower the number, the higher the priority. + +- `db` – Name of the database. + +- `table` – Name of the table. + +- `where` – The selection criteria. The syntax for conditions is the same as for `WHERE` MySQL, örneğin, `id > 10 AND id < 20`. İsteğe bağlı parametre. + +- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Sözlükleri güncelleme](external_dicts_dict_lifetime.md). + +MySQL yuva üzerinden yerel bir ana bilgisayara bağlanabilir. Bunu yapmak için, ayarlayın `host` ve `socket`. + +Ayarlar örneği: + +``` xml + + + localhost + /path/to/socket/file.sock + clickhouse + qwerty + db_name + table_name
+ id=10 + SQL_QUERY + + +``` + +veya + +``` sql +SOURCE(MYSQL( + host 'localhost' + socket '/path/to/socket/file.sock' + user 'clickhouse' + password 'qwerty' + db 'db_name' + table 'table_name' + where 'id=10' + invalidate_query 'SQL_QUERY' +)) +``` + +### ClickHouse {#dicts-external_dicts_dict_sources-clickhouse} + +Ayarlar örneği: + +``` xml + + + example01-01-1 + 9000 + default + + default + ids
+ id=10 + + +``` + +veya + +``` sql +SOURCE(CLICKHOUSE( + host 'example01-01-1' + port 9000 + user 'default' + password '' + db 'default' + table 'ids' + where 'id=10' +)) +``` + +Ayar alanları: + +- `host` – The ClickHouse host. If it is a local host, the query is processed without any network activity. To improve fault tolerance, you can create a [Dağılı](../../../engines/table_engines/special/distributed.md) tablo ve sonraki yapılandırmalarda girin. +- `port` – The port on the ClickHouse server. +- `user` – Name of the ClickHouse user. +- `password` – Password of the ClickHouse user. +- `db` – Name of the database. +- `table` – Name of the table. +- `where` – The selection criteria. May be omitted. +- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Sözlükleri güncelleme](external_dicts_dict_lifetime.md). + +### Mongodb {#dicts-external_dicts_dict_sources-mongodb} + +Ayarlar örneği: + +``` xml + + + localhost + 27017 + + + test + dictionary_source + + +``` + +veya + +``` sql +SOURCE(MONGO( + host 'localhost' + port 27017 + user '' + password '' + db 'test' + collection 'dictionary_source' +)) +``` + +Ayar alanları: + +- `host` – The MongoDB host. +- `port` – The port on the MongoDB server. +- `user` – Name of the MongoDB user. +- `password` – Password of the MongoDB user. +- `db` – Name of the database. +- `collection` – Name of the collection. + +### Redis {#dicts-external_dicts_dict_sources-redis} + +Ayarlar örneği: + +``` xml + + + localhost + 6379 + simple + 0 + + +``` + +veya + +``` sql +SOURCE(REDIS( + host 'localhost' + port 6379 + storage_type 'simple' + db_index 0 +)) +``` + +Ayar alanları: + +- `host` – The Redis host. +- `port` – The port on the Redis server. +- `storage_type` – The structure of internal Redis storage using for work with keys. `simple` basit kaynaklar ve karma tek anahtar kaynaklar içindir, `hash_map` iki anahtarlı karma kaynaklar içindir. Ranged kaynakları ve karmaşık anahtarlı önbellek kaynakları desteklenmez. İhmal edilebilir, varsayılan değer `simple`. +- `db_index` – The specific numeric index of Redis logical database. May be omitted, default value is 0. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_sources/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md new file mode 100644 index 00000000000..bd404e3b822 --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md @@ -0,0 +1,175 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 44 +toc_title: "S\xF6zl\xFCk anahtar\u0131 ve alanlar\u0131" +--- + +# Sözlük anahtarı ve alanları {#dictionary-key-and-fields} + +Bu `` yan tümcesi sözlük anahtarı ve sorgular için kullanılabilir alanları açıklar. + +XML açıklaması: + +``` xml + + + + Id + + + + + + + ... + + + +``` + +Nitelikler elemanlarda açıklanmıştır: + +- `` — [Anahtar sütun](external_dicts_dict_structure.md#ext_dict_structure-key). +- `` — [Veri sütunu](external_dicts_dict_structure.md#ext_dict_structure-attributes). Birden fazla sayıda özellik olabilir. + +DDL sorgusu: + +``` sql +CREATE DICTIONARY dict_name ( + Id UInt64, + -- attributes +) +PRIMARY KEY Id +... +``` + +Öznitelikler sorgu gövdesinde açıklanmıştır: + +- `PRIMARY KEY` — [Anahtar sütun](external_dicts_dict_structure.md#ext_dict_structure-key) +- `AttrName AttrType` — [Veri sütunu](external_dicts_dict_structure.md#ext_dict_structure-attributes). Birden fazla sayıda özellik olabilir. + +## Anahtar {#ext_dict_structure-key} + +ClickHouse aşağıdaki anahtar türlerini destekler: + +- Sayısal tuş. `UInt64`. Tanımlanan `` etiket veya kullanma `PRIMARY KEY` kelime. +- Kompozit anahtar. Farklı türde değerler kümesi. Etiket definedinde tanımlı `` veya `PRIMARY KEY` kelime. + +Bir xml yapısı şunları içerebilir `` veya ``. DDL sorgusu tek içermelidir `PRIMARY KEY`. + +!!! warning "Uyarıcı" + Anahtarı bir öznitelik olarak tanımlamamalısınız. + +### Sayısal Tuş {#ext_dict-numeric-key} + +Tür: `UInt64`. + +Yapılandırma örneği: + +``` xml + + Id + +``` + +Yapılandırma alanları: + +- `name` – The name of the column with keys. + +DDL sorgusu için: + +``` sql +CREATE DICTIONARY ( + Id UInt64, + ... +) +PRIMARY KEY Id +... +``` + +- `PRIMARY KEY` – The name of the column with keys. + +### Kompozit Anahtar {#composite-key} + +Anahtar bir olabilir `tuple` her türlü alandan. Bu [düzen](external_dicts_dict_layout.md) bu durumda olmalıdır `complex_key_hashed` veya `complex_key_cache`. + +!!! tip "Uç" + Bileşik bir anahtar tek bir elemandan oluşabilir. Bu, örneğin bir dizeyi anahtar olarak kullanmayı mümkün kılar. + +Anahtar yapısı eleman ayarlanır ``. Anahtar alanlar sözlük ile aynı biçimde belirtilir [öznitelik](external_dicts_dict_structure.md). Örnek: + +``` xml + + + + field1 + String + + + field2 + UInt32 + + ... + +... +``` + +veya + +``` sql +CREATE DICTIONARY ( + field1 String, + field2 String + ... +) +PRIMARY KEY field1, field2 +... +``` + +Bir sorgu için `dictGet*` fonksiyon, bir tuple anahtar olarak geçirilir. Örnek: `dictGetString('dict_name', 'attr_name', tuple('string for field1', num_for_field2))`. + +## Öznitelik {#ext_dict_structure-attributes} + +Yapılandırma örneği: + +``` xml + + ... + + Name + ClickHouseDataType + + rand64() + true + true + true + + +``` + +veya + +``` sql +CREATE DICTIONARY somename ( + Name ClickHouseDataType DEFAULT '' EXPRESSION rand64() HIERARCHICAL INJECTIVE IS_OBJECT_ID +) +``` + +Yapılandırma alanları: + +| Etiket | Açıklama | Gerekli | +|------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| `name` | Sütun adı. | Evet | +| `type` | ClickHouse veri türü.
ClickHouse, sözlükten belirtilen veri türüne değer atmaya çalışır. Örneğin, MySQL için alan olabilir `TEXT`, `VARCHAR`, veya `BLOB` MySQL kaynak tablosunda, ancak şu şekilde yüklenebilir `String` Clickhouse'da.
[Nullable](../../../sql_reference/data_types/nullable.md) desteklenmiyor. | Evet | +| `null_value` | Varolan olmayan bir öğe için varsayılan değer.
Örnekte, boş bir dizedir. Kullanamazsınız `NULL` bu alanda. | Evet | +| `expression` | [İfade](../../syntax.md#syntax-expressions) bu ClickHouse değeri yürütür.
İfade, uzak SQL veritabanında bir sütun adı olabilir. Bu nedenle, uzak sütun için bir diğer ad oluşturmak için kullanabilirsiniz.

Varsayılan değer: ifade yok. | Hayır | +| `hierarchical` | Eğer `true`, öznitelik, geçerli anahtar için bir üst anahtarın değerini içerir. Görmek [Hiyerarşik Sözlükler](external_dicts_dict_hierarchical.md).

Varsayılan değer: `false`. | Hayır | +| `injective` | Olup olmadığını gösteren bayrak `id -> attribute` ima isge is [enjektif](https://en.wikipedia.org/wiki/Injective_function).
Eğer `true`, ClickHouse sonra otomatik olarak yerleştirebilirsiniz `GROUP BY` fık .ra ile ilgili istek dictionariesleriniz Genellikle bu tür taleplerin miktarını önemli ölçüde azaltır.

Varsayılan değer: `false`. | Hayır | +| `is_object_id` | Bir MongoDB belgesi için sorgunun yürütülüp yürütülmediğini gösteren bayrak `ObjectID`.

Varsayılan değer: `false`. | Hayır | + +## Ayrıca Bakınız {#see-also} + +- [Harici sözlüklerle çalışmak için işlevler](../../../sql_reference/functions/ext_dict_functions.md). + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_structure/) diff --git a/docs/tr/sql_reference/dictionaries/external_dictionaries/index.md b/docs/tr/sql_reference/dictionaries/external_dictionaries/index.md new file mode 100644 index 00000000000..d5eaa1e55b6 --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/external_dictionaries/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "D\u0131\u015F S\xF6z Dictionariesl\xFCkler" +toc_priority: 37 +--- + + diff --git a/docs/tr/sql_reference/dictionaries/index.md b/docs/tr/sql_reference/dictionaries/index.md new file mode 100644 index 00000000000..a8a6c3605b4 --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/index.md @@ -0,0 +1,22 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "S\xF6zl\xFCkler" +toc_priority: 35 +toc_title: "Giri\u015F" +--- + +# Sözlükler {#dictionaries} + +Bir sözlük bir haritalama (`key -> attributes`) bu referans listeleri çeşitli türleri için uygundur. + +ClickHouse, sorgularda kullanılabilecek sözlüklerle çalışmak için özel işlevleri destekler. Sözlükleri işlevlerle kullanmak daha kolay ve daha verimlidir. `JOIN` referans tabloları ile. + +[NULL](../syntax.md#null) değerler sözlükte saklanamaz. + +ClickHouse destekler: + +- [Dahili sözlükler](internal_dicts.md#internal_dicts) bir özel ile [fonksiyonlar kümesi](../../sql_reference/functions/ym_dict_functions.md). +- [Eklenti (harici) söz dictionarieslükler](external_dictionaries/external_dicts.md) ile bir [fonksiyonlar net](../../sql_reference/functions/ext_dict_functions.md). + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/) diff --git a/docs/tr/sql_reference/dictionaries/internal_dicts.md b/docs/tr/sql_reference/dictionaries/internal_dicts.md new file mode 100644 index 00000000000..b7a1d46207c --- /dev/null +++ b/docs/tr/sql_reference/dictionaries/internal_dicts.md @@ -0,0 +1,55 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 39 +toc_title: "\u0130\xE7 S\xF6z Dictionariesl\xFCkler" +--- + +# İç söz dictionarieslükler {#internal_dicts} + +ClickHouse, bir geobase ile çalışmak için yerleşik bir özellik içerir. + +Bu size sağlar: + +- Adını istediğiniz dilde almak için bölgenin kimliğini kullanın. +- Bir şehir, bölge, federal bölge, ülke veya kıtanın kimliğini almak için bölgenin kimliğini kullanın. +- Bir bölgenin başka bir bölgenin parçası olup olmadığını kontrol edin. +- Ana bölgeler zinciri alın. + +Tüm fonksiyonları destek “translocality,” aynı anda bölge mülkiyeti farklı bakış açıları kullanma yeteneği. Daha fazla bilgi için bölüme bakın “Functions for working with Yandex.Metrica dictionaries”. + +İç sözlükler varsayılan pakette devre dışı bırakılır. +Bunları etkinleştirmek için, parametreleri uncomment `path_to_regions_hierarchy_file` ve `path_to_regions_names_files` sunucu yapılandırma dosyasında. + +Geobase metin dosyalarından yüklenir. + +Place the `regions_hierarchy*.txt` dosyaları içine `path_to_regions_hierarchy_file` dizin. Bu yapılandırma parametresi, `regions_hierarchy.txt` dosya (varsayılan bölgesel hiyerarşi) ve diğer dosyalar (`regions_hierarchy_ua.txt`) aynı dizinde bulunmalıdır. + +Koy... `regions_names_*.txt` dosyalar içinde `path_to_regions_names_files` dizin. + +Bu dosyaları kendiniz de oluşturabilirsiniz. Dosya biçimi aşağıdaki gibidir: + +`regions_hierarchy*.txt`: TabSeparated (başlık yok), sütunlar: + +- bölge kimliği (`UInt32`) +- üst bölge kimliği (`UInt32`) +- bölge tipi (`UInt8`): 1 - kıta, 3-ülke, 4-federal bölge, 5-bölge, 6-şehir; diğer türlerin değerleri yoktur +- nüfuslu (`UInt32`) — optional column + +`regions_names_*.txt`: TabSeparated (başlık yok), sütunlar: + +- bölge kimliği (`UInt32`) +- bölge adı (`String`) — Can't contain tabs or line feeds, even escaped ones. + +RAM'de depolamak için düz bir dizi kullanılır. Bu nedenle, IDs bir milyondan fazla olmamalıdır. + +Sözlükler sunucuyu yeniden başlatmadan güncellenebilir. Ancak, kullanılabilir sözlükler kümesi güncelleştirilmez. +Güncellemeler için dosya değiştirme süreleri kontrol edilir. Bir dosya değiştiyse, sözlük güncelleştirilir. +Değişiklikleri kontrol etmek için Aralık `builtin_dictionaries_reload_interval` parametre. +Sözlük güncelleştirmeleri (ilk kullanımda yükleme dışında) sorguları engellemez. Güncelleştirmeler sırasında, sorgular sözlüklerin eski sürümlerini kullanır. Güncelleştirme sırasında bir hata oluşursa, hata sunucu günlüğüne yazılır ve sorgular sözlüklerin eski sürümünü kullanmaya devam eder. + +Sözlükleri geobase ile periyodik olarak güncellemenizi öneririz. Bir güncelleme sırasında yeni dosyalar oluşturun ve bunları ayrı bir konuma yazın. Her şey hazır olduğunda, bunları sunucu tarafından kullanılan dosyalara yeniden adlandırın. + +OS tanımlayıcıları ve Yandex ile çalışmak için işlevler de vardır.Metrica arama motorları, ancak kullanılmamalıdır. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/dicts/internal_dicts/) diff --git a/docs/tr/sql_reference/functions/arithmetic_functions.md b/docs/tr/sql_reference/functions/arithmetic_functions.md new file mode 100644 index 00000000000..594e9bd4699 --- /dev/null +++ b/docs/tr/sql_reference/functions/arithmetic_functions.md @@ -0,0 +1,87 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 35 +toc_title: Aritmetik +--- + +# Aritmetik fonksiyonlar {#arithmetic-functions} + +Tüm aritmetik işlevler için, sonuç türü, böyle bir tür varsa, sonucun sığdığı en küçük sayı türü olarak hesaplanır. Minimum, bit sayısına, imzalanıp imzalanmadığına ve yüzüp yüzmediğine bağlı olarak aynı anda alınır. Yeterli bit yoksa, en yüksek bit türü alınır. + +Örnek: + +``` sql +SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 + 0 + 0) +``` + +``` text +┌─toTypeName(0)─┬─toTypeName(plus(0, 0))─┬─toTypeName(plus(plus(0, 0), 0))─┬─toTypeName(plus(plus(plus(0, 0), 0), 0))─┐ +│ UInt8 │ UInt16 │ UInt32 │ UInt64 │ +└───────────────┴────────────────────────┴─────────────────────────────────┴──────────────────────────────────────────┘ +``` + +Aritmetik fonksiyonlar, uint8, Uİnt16, Uİnt32, Uint64, Int8, Int16, Int32, Int64, Float32 veya Float64 türlerinden herhangi bir çift için çalışır. + +Taşma, C++ile aynı şekilde üretilir. + +## artı (a, b), A + B operatörü {#plusa-b-a-b-operator} + +Sayıların toplamını hesaplar. +Ayrıca bir tarih veya tarih ve Saat ile tamsayı numaraları ekleyebilirsiniz. Bir tarih durumunda, bir tamsayı eklemek, karşılık gelen gün sayısını eklemek anlamına gelir. Zamanla bir tarih için, karşılık gelen saniye sayısını eklemek anlamına gelir. + +## eksi (a, b), A - B operatörü {#minusa-b-a-b-operator} + +Farkı hesaplar. Sonuç her zaman imzalanır. + +You can also calculate integer numbers from a date or date with time. The idea is the same – see above for ‘plus’. + +## çarp operatorma (a, b), A \* B operatörü {#multiplya-b-a-b-operator} + +Sayıların ürününü hesaplar. + +## böl (a, b), A / B operatörü {#dividea-b-a-b-operator} + +Sayıların bölümünü hesaplar. Sonuç türü her zaman bir kayan nokta türüdür. +Tam sayı bölümü değildir. Tamsayı bölümü için, ‘intDiv’ işlev. +Sıfıra bölerek zaman olsun ‘inf’, ‘-inf’, veya ‘nan’. + +## ıntdiv(a, b) {#intdiva-b} + +Sayıların bölümünü hesaplar. Tamsayılara bölünür, yuvarlanır (mutlak değere göre). +Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. + +## ıntdivorzero(a, b) {#intdivorzeroa-b} + +Farklıdır ‘intDiv’ bu, sıfıra bölünürken veya en az bir negatif sayıyı eksi bir ile bölerek sıfır döndürür. + +## modulo (a, b), A % B operatörü {#moduloa-b-a-b-operator} + +Bölünmeden sonra kalan hesaplar. +Bağımsız değişkenler kayan nokta sayılarıysa, ondalık bölümü bırakarak tamsayılara önceden dönüştürülürler. +Kalan C++ile aynı anlamda alınır. Kesik bölme negatif sayılar için kullanılır. +Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. + +## moduloOrZero(a, b) {#moduloorzeroa-b} + +Farklıdır ‘modulo’ bölen sıfır olduğunda sıfır döndürür. + +## negate (a), - bir operatör {#negatea-a-operator} + +Ters işareti ile bir sayı hesaplar. Sonuç her zaman imzalanır. + +## abs (a) {#arithm_func-abs} + +\(A\) sayısının mutlak değerini hesaplar. Yani, \< 0 ise,- A döndürür. imzasız türler için hiçbir şey yapmaz. İmzalı tamsayı türleri için imzasız bir sayı döndürür. + +## gcd (a, b) {#gcda-b} + +Sayıların en büyük ortak böleni döndürür. +Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. + +## lcm(a, b) {#lcma-b} + +Sayıların en az ortak katını döndürür. +Sıfıra bölünürken veya en az negatif sayıyı eksi bir ile bölürken bir istisna atılır. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/arithmetic_functions/) diff --git a/docs/tr/sql_reference/functions/array_functions.md b/docs/tr/sql_reference/functions/array_functions.md new file mode 100644 index 00000000000..9e3221435ed --- /dev/null +++ b/docs/tr/sql_reference/functions/array_functions.md @@ -0,0 +1,1061 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 46 +toc_title: "Dizilerle \xE7al\u0131\u015Fma" +--- + +# Dizilerle çalışmak için işlevler {#functions-for-working-with-arrays} + +## boş {#function-empty} + +Boş bir dizi için 1 veya boş olmayan bir dizi için 0 döndürür. +Sonuç türü Uint8'dir. +İşlev ayrıca dizeler için de çalışır. + +## notEmpty {#function-notempty} + +Boş bir dizi için 0 veya boş olmayan bir dizi için 1 döndürür. +Sonuç türü Uint8'dir. +İşlev ayrıca dizeler için de çalışır. + +## uzunluk {#array_functions-length} + +Dizideki öğe sayısını döndürür. +Sonuç türü Uint64'tür. +İşlev ayrıca dizeler için de çalışır. + +## emptyArrayUİnt8, emptyArrayUİnt16, emptyArrayUİnt32, emptyArrayUİnt64 {#emptyarrayuint8-emptyarrayuint16-emptyarrayuint32-emptyarrayuint64} + +## emptyArrayİnt8, emptyArrayİnt16, emptyArrayİnt32, emptyArrayİnt64 {#emptyarrayint8-emptyarrayint16-emptyarrayint32-emptyarrayint64} + +## emptyArrayFloat32, emptyArrayFloat64 {#emptyarrayfloat32-emptyarrayfloat64} + +## emptyArrayDate, emptyArrayDateTime {#emptyarraydate-emptyarraydatetime} + +## emptyArrayString {#emptyarraystring} + +Sıfır bağımsız değişkeni kabul eder ve uygun türde boş bir dizi döndürür. + +## emptyArrayToSingle {#emptyarraytosingle} + +Boş bir dizi kabul eder ve varsayılan değere eşit bir tek öğe dizisi döndürür. + +## Aralık (bitiş), Aralık(başlangıç, bitiş \[, adım\]) {#rangeend-rangestart-end-step} + +1 Adım-başından sonuna kadar sayıların bir dizi döndürür. +Eğer argüman `start` belirtilmemiş, varsayılan olarak 0. +Eğer argüman `step` belirtilmemiş, varsayılan olarak 1. +Neredeyse pythonic gibi davranışlar `range`. Ancak fark, tüm argümanların tipinin olması gerektiğidir `UInt` şiir. +Bir veri bloğunda toplam uzunluğu 100.000.000'den fazla öğe olan diziler oluşturulursa, bir istisna atılır. + +## array(x1, …), operator \[x1, …\] {#arrayx1-operator-x1} + +İşlev bağımsız değişkenlerinden bir dizi oluşturur. +Bağımsız değişkenler sabit olmalı ve en küçük ortak türe sahip türlere sahip olmalıdır. En az bir argüman geçirilmelidir, çünkü aksi takdirde hangi tür dizinin oluşturulacağı açık değildir. Yani, boş bir dizi oluşturmak için bu işlevi kullanamazsınız (bunu yapmak için, ‘emptyArray\*’ fonksiyon yukarıda açıklandığı. +Ret anur anns an ‘Array(T)’ sonucu yazın, nerede ‘T’ geçirilen bağımsız değişkenlerin en küçük ortak türüdür. + +## arrayConcat {#arrayconcat} + +Argüman olarak geçirilen dizileri birleştirir. + +``` sql +arrayConcat(arrays) +``` + +**Parametre** + +- `arrays` – Arbitrary number of arguments of [Dizi](../../sql_reference/data_types/array.md) tür. + **Örnek** + + + +``` sql +SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res +``` + +``` text +┌─res───────────┐ +│ [1,2,3,4,5,6] │ +└───────────────┘ +``` + +## arrayElement (arr, n), operatör arr\[n\] {#arrayelementarr-n-operator-arrn} + +İnd theex ile eleman alın `n` diz theiden `arr`. `n` herhangi bir tamsayı türü olmalıdır. +Bir dizideki dizinler birinden başlar. +Negatif dizinler desteklenir. Bu durumda, sondan numaralandırılmış ilgili elemanı seçer. Mesela, `arr[-1]` dizideki son öğedir. + +Dizin bir dizinin sınırlarının dışına düşerse, bazı varsayılan değer döndürür (sayılar için 0, dizeler için boş bir dize vb.).), sabit olmayan bir dizi ve sabit bir dizin 0 olan durum hariç (bu durumda bir hata olacaktır `Array indices are 1-based`). + +## has (arr, elem) {#hasarr-elem} + +Olup olmadığını denetler ‘arr’ dizi var ‘elem’ öğe. +Öğe dizide değilse 0 veya varsa 1 değerini döndürür. + +`NULL` değer olarak iş islenir. + +``` sql +SELECT has([1, 2, NULL], NULL) +``` + +``` text +┌─has([1, 2, NULL], NULL)─┐ +│ 1 │ +└─────────────────────────┘ +``` + +## hasAll {#hasall} + +Bir dizi başka bir alt kümesi olup olmadığını denetler. + +``` sql +hasAll(set, subset) +``` + +**Parametre** + +- `set` – Array of any type with a set of elements. +- `subset` – Array of any type with elements that should be tested to be a subset of `set`. + +**Dönüş değerleri** + +- `1`, eğer `set` tüm öğeleri içerir `subset`. +- `0`, başka. + +**Tuhaf özellikler** + +- Boş bir dizi, herhangi bir dizinin bir alt kümesidir. +- `Null` bir değer olarak işlenir. +- Her iki dizideki değerlerin sırası önemli değil. + +**Örnekler** + +`SELECT hasAll([], [])` döner 1. + +`SELECT hasAll([1, Null], [Null])` döner 1. + +`SELECT hasAll([1.0, 2, 3, 4], [1, 3])` döner 1. + +`SELECT hasAll(['a', 'b'], ['a'])` döner 1. + +`SELECT hasAll([1], ['a'])` 0 döndürür. + +`SELECT hasAll([[1, 2], [3, 4]], [[1, 2], [3, 5]])` 0 döndürür. + +## hasAny {#hasany} + +İki dizinin bazı öğelerle kesiştiği olup olmadığını kontrol eder. + +``` sql +hasAny(array1, array2) +``` + +**Parametre** + +- `array1` – Array of any type with a set of elements. +- `array2` – Array of any type with a set of elements. + +**Dönüş değerleri** + +- `1`, eğer `array1` ve `array2` en azından benzer bir öğeye sahip olun. +- `0`, başka. + +**Tuhaf özellikler** + +- `Null` bir değer olarak işlenir. +- Her iki dizideki değerlerin sırası önemli değil. + +**Örnekler** + +`SELECT hasAny([1], [])` dönüşler `0`. + +`SELECT hasAny([Null], [Null, 1])` dönüşler `1`. + +`SELECT hasAny([-128, 1., 512], [1])` dönüşler `1`. + +`SELECT hasAny([[1, 2], [3, 4]], ['a', 'c'])` dönüşler `0`. + +`SELECT hasAll([[1, 2], [3, 4]], [[1, 2], [1, 2]])` dönüşler `1`. + +## ındexof(arr, x) {#indexofarr-x} + +İlk dizini döndürür ‘x’ dizide ise öğe (1'den başlayarak) veya değilse 0. + +Örnek: + +``` sql +SELECT indexOf([1, 3, NULL, NULL], NULL) +``` + +``` text +┌─indexOf([1, 3, NULL, NULL], NULL)─┐ +│ 3 │ +└───────────────────────────────────┘ +``` + +Elem setents set to `NULL` normal değerler olarak ele alınır. + +## countEqual(arr, x) {#countequalarr-x} + +X eşit dizideki öğelerin sayısını döndürür. Arraycount eşdeğer (elem - \> elem = x, arr). + +`NULL` öğeler ayrı değerler olarak işlenir. + +Örnek: + +``` sql +SELECT countEqual([1, 2, NULL, NULL], NULL) +``` + +``` text +┌─countEqual([1, 2, NULL, NULL], NULL)─┐ +│ 2 │ +└──────────────────────────────────────┘ +``` + +## arrayEnumerate(arr) {#array_functions-arrayenumerate} + +Returns the array \[1, 2, 3, …, length (arr) \] + +Bu işlev normalde ARRAY JOIN ile kullanılır. ARRAY JOİN uyguladıktan sonra her dizi için sadece bir kez bir şey saymaya izin verir. Örnek: + +``` sql +SELECT + count() AS Reaches, + countIf(num = 1) AS Hits +FROM test.hits +ARRAY JOIN + GoalsReached, + arrayEnumerate(GoalsReached) AS num +WHERE CounterID = 160656 +LIMIT 10 +``` + +``` text +┌─Reaches─┬──Hits─┐ +│ 95606 │ 31406 │ +└─────────┴───────┘ +``` + +Bu örnekte, Reaches dönüşümlerin sayısıdır (ARRAY JOİN uygulandıktan sonra alınan dizeler) ve İsabetler sayfa görüntüleme sayısıdır (ARRAY JOİN önce dizeler). Bu özel durumda, aynı sonucu daha kolay bir şekilde alabilirsiniz: + +``` sql +SELECT + sum(length(GoalsReached)) AS Reaches, + count() AS Hits +FROM test.hits +WHERE (CounterID = 160656) AND notEmpty(GoalsReached) +``` + +``` text +┌─Reaches─┬──Hits─┐ +│ 95606 │ 31406 │ +└─────────┴───────┘ +``` + +Bu fonksiyon aynı zamanda yüksek mertebeden fonksiyonlarda da kullanılabilir. Örneğin, bir koşulla eşleşen öğeler için dizi dizinleri almak için kullanabilirsiniz. + +## arrayEnumerateUniq(arr, …) {#arrayenumerateuniqarr} + +Kaynak diziyle aynı boyutta bir dizi döndürür ve her öğe için aynı değere sahip öğeler arasında konumunun ne olduğunu gösterir. +Örneğin: arrayEnumerateUniq(\[10, 20, 10, 30\]) = \[1, 1, 2, 1\]. + +Bu işlev, dizi birleştirme ve dizi öğelerinin toplanmasını kullanırken kullanışlıdır. +Örnek: + +``` sql +SELECT + Goals.ID AS GoalID, + sum(Sign) AS Reaches, + sumIf(Sign, num = 1) AS Visits +FROM test.visits +ARRAY JOIN + Goals, + arrayEnumerateUniq(Goals.ID) AS num +WHERE CounterID = 160656 +GROUP BY GoalID +ORDER BY Reaches DESC +LIMIT 10 +``` + +``` text +┌──GoalID─┬─Reaches─┬─Visits─┐ +│ 53225 │ 3214 │ 1097 │ +│ 2825062 │ 3188 │ 1097 │ +│ 56600 │ 2803 │ 488 │ +│ 1989037 │ 2401 │ 365 │ +│ 2830064 │ 2396 │ 910 │ +│ 1113562 │ 2372 │ 373 │ +│ 3270895 │ 2262 │ 812 │ +│ 1084657 │ 2262 │ 345 │ +│ 56599 │ 2260 │ 799 │ +│ 3271094 │ 2256 │ 812 │ +└─────────┴─────────┴────────┘ +``` + +Bu örnekte, her hedef kimliğinin dönüşüm sayısı (hedefler iç içe geçmiş veri yapısındaki her öğe, bir dönüşüm olarak adlandırdığımız ulaşılan bir hedeftir) ve oturum sayısı Hesaplaması vardır. ARRAY JOİN olmadan, oturum sayısını sum(Sign) olarak sayardık. Ancak bu özel durumda, satırlar iç içe geçmiş hedefler yapısıyla çarpıldı, bu nedenle her oturumu bir kez saymak için arrayenumerateuniq değerine bir koşul uyguluyoruz(Goals.ID) fonksiyonu. + +Arrayenumerateuniq işlevi, bağımsız değişkenlerle aynı boyutta birden çok dizi alabilir. Bu durumda, tüm dizilerde aynı konumlardaki elemanların tuplesleri için benzersizlik düşünülür. + +``` sql +SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res +``` + +``` text +┌─res───────────┐ +│ [1,2,1,1,2,1] │ +└───────────────┘ +``` + +Bu, iç içe geçmiş bir veri yapısı ve bu yapıdaki birden çok öğe arasında daha fazla toplama ile dizi birleşimini kullanırken gereklidir. + +## arrayPopBack {#arraypopback} + +Son öğeyi diziden kaldırır. + +``` sql +arrayPopBack(array) +``` + +**Parametre** + +- `array` – Array. + +**Örnek** + +``` sql +SELECT arrayPopBack([1, 2, 3]) AS res +``` + +``` text +┌─res───┐ +│ [1,2] │ +└───────┘ +``` + +## arrayPopFront {#arraypopfront} + +İlk öğeyi diziden kaldırır. + +``` sql +arrayPopFront(array) +``` + +**Parametre** + +- `array` – Array. + +**Örnek** + +``` sql +SELECT arrayPopFront([1, 2, 3]) AS res +``` + +``` text +┌─res───┐ +│ [2,3] │ +└───────┘ +``` + +## arrayPushBack {#arraypushback} + +Dizinin sonuna bir öğe ekler. + +``` sql +arrayPushBack(array, single_value) +``` + +**Parametre** + +- `array` – Array. +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` dizinin veri türü için yazın. Clickhouse'daki veri türleri hakkında daha fazla bilgi için bkz. “[Veri türleri](../../sql_reference/data_types/index.md#data_types)”. Olabilir `NULL`. Fonksiyon bir ekler `NULL` bir dizi için öğe ve dizi öğeleri türü dönüştürür `Nullable`. + +**Örnek** + +``` sql +SELECT arrayPushBack(['a'], 'b') AS res +``` + +``` text +┌─res───────┐ +│ ['a','b'] │ +└───────────┘ +``` + +## arrayPushFront {#arraypushfront} + +Dizinin başına bir öğe ekler. + +``` sql +arrayPushFront(array, single_value) +``` + +**Parametre** + +- `array` – Array. +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` dizinin veri türü için yazın. Clickhouse'daki veri türleri hakkında daha fazla bilgi için bkz. “[Veri türleri](../../sql_reference/data_types/index.md#data_types)”. Olabilir `NULL`. Fonksiyon bir ekler `NULL` bir dizi için öğe ve dizi öğeleri türü dönüştürür `Nullable`. + +**Örnek** + +``` sql +SELECT arrayPushFront(['b'], 'a') AS res +``` + +``` text +┌─res───────┐ +│ ['a','b'] │ +└───────────┘ +``` + +## arrayResize {#arrayresize} + +Dizinin uzunluğunu değiştirir. + +``` sql +arrayResize(array, size[, extender]) +``` + +**Parametre:** + +- `array` — Array. +- `size` — Required length of the array. + - Eğer `size` dizinin orijinal boyutundan daha az, dizi sağdan kesilir. +- Eğer `size` dizinin başlangıç boyutundan daha büyük, dizi sağa uzatılır `extender` dizi öğelerinin veri türü için değerler veya varsayılan değerler. +- `extender` — Value for extending an array. Can be `NULL`. + +**Döndürülen değer:** + +Bir dizi uzunluk `size`. + +**Arama örnekleri** + +``` sql +SELECT arrayResize([1], 3) +``` + +``` text +┌─arrayResize([1], 3)─┐ +│ [1,0,0] │ +└─────────────────────┘ +``` + +``` sql +SELECT arrayResize([1], 3, NULL) +``` + +``` text +┌─arrayResize([1], 3, NULL)─┐ +│ [1,NULL,NULL] │ +└───────────────────────────┘ +``` + +## arraySlice {#arrayslice} + +Dizinin bir dilimini döndürür. + +``` sql +arraySlice(array, offset[, length]) +``` + +**Parametre** + +- `array` – Array of data. +- `offset` – Indent from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. +- `length` - Gerekli dilimin uzunluğu. Negatif bir değer belirtirseniz, işlev açık bir dilim döndürür `[offset, array_length - length)`. Değeri atlarsanız, işlev dilimi döndürür `[offset, the_end_of_array]`. + +**Örnek** + +``` sql +SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res +``` + +``` text +┌─res────────┐ +│ [2,NULL,4] │ +└────────────┘ +``` + +Ar arrayray elem toents set to `NULL` normal değerler olarak ele alınır. + +## arraySort(\[func,\] arr, …) {#array_functions-sort} + +Elemanları sıralar `arr` artan düzende dizi. Eğer... `func` fonksiyonu belirtilir, sıralama düzeni sonucu belirlenir `func` fonksiyon dizinin elemanlarına uygulanır. Eğer `func` birden fazla argüman kabul eder, `arraySort` fonksiyon argümanları birkaç diziler geçirilir `func` karşılık gelir. Ayrıntılı örnekler sonunda gösterilir `arraySort` açıklama. + +Tamsayı değerleri sıralama örneği: + +``` sql +SELECT arraySort([1, 3, 3, 0]); +``` + +``` text +┌─arraySort([1, 3, 3, 0])─┐ +│ [0,1,3,3] │ +└─────────────────────────┘ +``` + +Dize değerleri sıralama örneği: + +``` sql +SELECT arraySort(['hello', 'world', '!']); +``` + +``` text +┌─arraySort(['hello', 'world', '!'])─┐ +│ ['!','hello','world'] │ +└────────────────────────────────────┘ +``` + +Aşağıdaki sıralama sırasını göz önünde bulundurun `NULL`, `NaN` ve `Inf` değerler: + +``` sql +SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]); +``` + +``` text +┌─arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf])─┐ +│ [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL] │ +└───────────────────────────────────────────────────────────┘ +``` + +- `-Inf` değerler dizide ilk sırada yer alır. +- `NULL` değerler dizideki son değerlerdir. +- `NaN` değerler hemen önce `NULL`. +- `Inf` değerler hemen önce `NaN`. + +Not thate that `arraySort` is a [yüksek sipariş fonksiyonu](higher_order_functions.md). Bir lambda işlevini ilk argüman olarak iletebilirsiniz. Bu durumda, sıralama sırası, dizinin elemanlarına uygulanan lambda işlevinin sonucu ile belirlenir. + +Aşağıdaki örneği ele alalım: + +``` sql +SELECT arraySort((x) -> -x, [1, 2, 3]) as res; +``` + +``` text +┌─res─────┐ +│ [3,2,1] │ +└─────────┘ +``` + +For each element of the source array, the lambda function returns the sorting key, that is, \[1 –\> -1, 2 –\> -2, 3 –\> -3\]. Since the `arraySort` fonksiyon tuşları artan sırayla sıralar, sonuç \[3, 2, 1\]. Böylece, `(x) –> -x` lambda fonksiyonu setleri [azalan düzen](#array_functions-reverse-sort) bir sıralama içinde. + +Lambda işlevi birden çok bağımsız değişken kabul edebilir. Bu durumda, geçmek gerekir `arraySort` işlev lambda işlevinin argümanlarının karşılık geleceği aynı uzunlukta birkaç dizi. Elde edilen dizi ilk giriş dizisinden elemanlardan oluşacaktır; bir sonraki giriş dizilerinden elemanlar sıralama anahtarlarını belirtir. Mesela: + +``` sql +SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; +``` + +``` text +┌─res────────────────┐ +│ ['world', 'hello'] │ +└────────────────────┘ +``` + +Burada, ikinci dizide (\[2, 1\]) geçirilen öğeler, kaynak diziden karşılık gelen öğe için bir sıralama anahtarı tanımlar (\[‘hello’, ‘world’\]), bu, \[‘hello’ –\> 2, ‘world’ –\> 1\]. Since the lambda function doesn't use `x`, kaynak dizinin gerçek değerleri sonuçtaki sırayı etkilemez. Böyle, ‘hello’ sonuçtaki ikinci eleman olacak ve ‘world’ ilk olacak. + +Diğer örnekler aşağıda gösterilmiştir. + +``` sql +SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res; +``` + +``` text +┌─res─────┐ +│ [2,1,0] │ +└─────────┘ +``` + +``` sql +SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res; +``` + +``` text +┌─res─────┐ +│ [2,1,0] │ +└─────────┘ +``` + +!!! note "Not" + Sıralama verimliliğini artırmak için, [Schwartzian dönüşümü](https://en.wikipedia.org/wiki/Schwartzian_transform) kullanılır. + +## arrayReverseSort(\[func,\] arr, …) {#array_functions-reverse-sort} + +Elemanları sıralar `arr` azalan sırayla dizi. Eğer... `func` fonksiyon belirtilir, `arr` sonucuna göre sıra islanır. `func` işlev dizinin öğelerine uygulanır ve sonra sıralanmış dizi tersine çevrilir. Eğer `func` birden fazla argüman kabul eder, `arrayReverseSort` fonksiyon argümanları birkaç diziler geçirilir `func` karşılık gelir. Ayrıntılı örnekler sonunda gösterilir `arrayReverseSort` açıklama. + +Tamsayı değerleri sıralama örneği: + +``` sql +SELECT arrayReverseSort([1, 3, 3, 0]); +``` + +``` text +┌─arrayReverseSort([1, 3, 3, 0])─┐ +│ [3,3,1,0] │ +└────────────────────────────────┘ +``` + +Dize değerleri sıralama örneği: + +``` sql +SELECT arrayReverseSort(['hello', 'world', '!']); +``` + +``` text +┌─arrayReverseSort(['hello', 'world', '!'])─┐ +│ ['world','hello','!'] │ +└───────────────────────────────────────────┘ +``` + +Aşağıdaki sıralama sırasını göz önünde bulundurun `NULL`, `NaN` ve `Inf` değerler: + +``` sql +SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res; +``` + +``` text +┌─res───────────────────────────────────┐ +│ [inf,3,2,1,-4,-inf,nan,nan,NULL,NULL] │ +└───────────────────────────────────────┘ +``` + +- `Inf` değerler dizide ilk sırada yer alır. +- `NULL` değerler dizideki son değerlerdir. +- `NaN` değerler hemen önce `NULL`. +- `-Inf` değerler hemen önce `NaN`. + +Not `arrayReverseSort` is a [yüksek sipariş fonksiyonu](higher_order_functions.md). Bir lambda işlevini ilk argüman olarak iletebilirsiniz. Örnek aşağıda gösterilmiştir. + +``` sql +SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res; +``` + +``` text +┌─res─────┐ +│ [1,2,3] │ +└─────────┘ +``` + +Dizi aşağıdaki şekilde sıralanır: + +1. İlk başta, kaynak dizi (\[1, 2, 3\]), dizinin elemanlarına uygulanan lambda işlevinin sonucuna göre sıralanır. Sonuç bir dizidir \[3, 2, 1\]. +2. Önceki adımda elde edilen dizi tersine çevrilir. Yani, nihai sonuç \[1, 2, 3\]. + +Lambda işlevi birden çok bağımsız değişken kabul edebilir. Bu durumda, geçmek gerekir `arrayReverseSort` işlev lambda işlevinin argümanlarının karşılık geleceği aynı uzunlukta birkaç dizi. Elde edilen dizi ilk giriş dizisinden elemanlardan oluşacaktır; bir sonraki giriş dizilerinden elemanlar sıralama anahtarlarını belirtir. Mesela: + +``` sql +SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; +``` + +``` text +┌─res───────────────┐ +│ ['hello','world'] │ +└───────────────────┘ +``` + +Bu örnekte, dizi aşağıdaki şekilde sıralanır: + +1. İlk başta, kaynak dizi (\[‘hello’, ‘world’\]) dizilerin elemanlarına uygulanan lambda işlevinin sonucuna göre sıralanır. İkinci dizide geçirilen öğeler (\[2, 1\]), kaynak diziden karşılık gelen öğeler için sıralama anahtarlarını tanımlar. Sonuç bir dizidir \[‘world’, ‘hello’\]. +2. Önceki adımda sıralanmış dizi tersine çevrilir. Yani, nihai sonuç \[‘hello’, ‘world’\]. + +Diğer örnekler aşağıda gösterilmiştir. + +``` sql +SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res; +``` + +``` text +┌─res─────┐ +│ [5,3,4] │ +└─────────┘ +``` + +``` sql +SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res; +``` + +``` text +┌─res─────┐ +│ [4,3,5] │ +└─────────┘ +``` + +## arrayUniq(arr, …) {#arrayuniqarr} + +Bir bağımsız değişken geçirilirse, dizideki farklı öğelerin sayısını sayar. +Birden çok bağımsız değişken geçirilirse, birden çok dizideki karşılık gelen konumlardaki farklı öğe kümelerinin sayısını sayar. + +Bir dizideki benzersiz öğelerin bir listesini almak istiyorsanız, arrayreduce kullanabilirsiniz(‘groupUniqArray’, arr). + +## arrayJoin(arr) {#array-functions-join} + +Özel bir işlev. Bölümüne bakınız [“ArrayJoin function”](array_join.md#functions_arrayjoin). + +## arrayDifference {#arraydifference} + +Bitişik dizi öğeleri arasındaki farkı hesaplar. İlk öğenin 0 olacağı bir dizi döndürür, ikincisi arasındaki farktır `a[1] - a[0]`, etc. The type of elements in the resulting array is determined by the type inference rules for subtraction (e.g. `UInt8` - `UInt8` = `Int16`). + +**Sözdizimi** + +``` sql +arrayDifference(array) +``` + +**Parametre** + +- `array` – [Dizi](https://clickhouse.yandex/docs/en/data_types/array/). + +**Döndürülen değerler** + +Bitişik öğeler arasındaki farklar dizisini döndürür. + +Tür: [Uİnt\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#uint-ranges), [Tamsayı\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#int-ranges), [Yüzdürmek\*](https://clickhouse.yandex/docs/en/data_types/float/). + +**Örnek** + +Sorgu: + +``` sql +SELECT arrayDifference([1, 2, 3, 4]) +``` + +Sonuç: + +``` text +┌─arrayDifference([1, 2, 3, 4])─┐ +│ [0,1,1,1] │ +└───────────────────────────────┘ +``` + +Sonuç türü Int64 nedeniyle taşma örneği: + +Sorgu: + +``` sql +SELECT arrayDifference([0, 10000000000000000000]) +``` + +Sonuç: + +``` text +┌─arrayDifference([0, 10000000000000000000])─┐ +│ [0,-8446744073709551616] │ +└────────────────────────────────────────────┘ +``` + +## arrayDistinct {#arraydistinct} + +Bir dizi alır, yalnızca farklı öğeleri içeren bir dizi döndürür. + +**Sözdizimi** + +``` sql +arrayDistinct(array) +``` + +**Parametre** + +- `array` – [Dizi](https://clickhouse.yandex/docs/en/data_types/array/). + +**Döndürülen değerler** + +Farklı öğeleri içeren bir dizi döndürür. + +**Örnek** + +Sorgu: + +``` sql +SELECT arrayDistinct([1, 2, 2, 3, 1]) +``` + +Sonuç: + +``` text +┌─arrayDistinct([1, 2, 2, 3, 1])─┐ +│ [1,2,3] │ +└────────────────────────────────┘ +``` + +## arrayEnumerateDense(arr) {#array_functions-arrayenumeratedense} + +Kaynak diziyle aynı boyutta bir dizi döndürür ve her öğenin kaynak dizide ilk olarak nerede göründüğünü gösterir. + +Örnek: + +``` sql +SELECT arrayEnumerateDense([10, 20, 10, 30]) +``` + +``` text +┌─arrayEnumerateDense([10, 20, 10, 30])─┐ +│ [1,2,1,3] │ +└───────────────────────────────────────┘ +``` + +## arrayıntersect(arr) {#array-functions-arrayintersect} + +Birden çok dizi alır, tüm kaynak dizilerde bulunan öğeleri içeren bir dizi döndürür. Elde edilen dizideki öğeler sırası ilk dizideki ile aynıdır. + +Örnek: + +``` sql +SELECT + arrayIntersect([1, 2], [1, 3], [2, 3]) AS no_intersect, + arrayIntersect([1, 2], [1, 3], [1, 4]) AS intersect +``` + +``` text +┌─no_intersect─┬─intersect─┐ +│ [] │ [1] │ +└──────────────┴───────────┘ +``` + +## arrayReduce {#arrayreduce} + +Dizi öğelerine bir toplama işlevi uygular ve sonucunu döndürür. Toplama işlevinin adı, tek tırnak içinde bir dize olarak geçirilir `'max'`, `'sum'`. Parametrik toplama işlevleri kullanıldığında, parametre parantez içinde işlev adından sonra gösterilir `'uniqUpTo(6)'`. + +**Sözdizimi** + +``` sql +arrayReduce(agg_func, arr1, arr2, ..., arrN) +``` + +**Parametre** + +- `agg_func` — The name of an aggregate function which should be a constant [dize](../../sql_reference/data_types/string.md). +- `arr` — Any number of [dizi](../../sql_reference/data_types/array.md) sütunları toplama işlevinin parametreleri olarak yazın. + +**Döndürülen değer** + +**Örnek** + +``` sql +SELECT arrayReduce('max', [1, 2, 3]) +``` + +``` text +┌─arrayReduce('max', [1, 2, 3])─┐ +│ 3 │ +└───────────────────────────────┘ +``` + +Bir toplama işlevi birden çok bağımsız değişken alırsa, bu işlev aynı boyuttaki birden çok diziye uygulanmalıdır. + +``` sql +SELECT arrayReduce('maxIf', [3, 5], [1, 0]) +``` + +``` text +┌─arrayReduce('maxIf', [3, 5], [1, 0])─┐ +│ 3 │ +└──────────────────────────────────────┘ +``` + +Parametrik toplama fonksiyonu ile örnek: + +``` sql +SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +``` + +``` text +┌─arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])─┐ +│ 4 │ +└─────────────────────────────────────────────────────────────┘ +``` + +## arrayReduceİnRanges {#arrayreduceinranges} + +Belirli aralıklardaki dizi öğelerine bir toplama işlevi uygular ve her aralığa karşılık gelen sonucu içeren bir dizi döndürür. Fonksiyon aynı sonucu birden fazla olarak döndürür `arrayReduce(agg_func, arraySlice(arr1, index, length), ...)`. + +**Sözdizimi** + +``` sql +arrayReduceInRanges(agg_func, ranges, arr1, arr2, ..., arrN) +``` + +**Parametre** + +- `agg_func` — The name of an aggregate function which should be a constant [dize](../../sql_reference/data_types/string.md). +- `ranges` — The ranges to aggretate which should be an [dizi](../../sql_reference/data_types/array.md) -den [Demetler](../../sql_reference/data_types/tuple.md) indeks ve her aralığın uzunluğunu içeren. +- `arr` — Any number of [dizi](../../sql_reference/data_types/array.md) sütunları toplama işlevinin parametreleri olarak yazın. + +**Döndürülen değer** + +**Örnek** + +``` sql +SELECT arrayReduceInRanges( + 'sum', + [(1, 5), (2, 3), (3, 4), (4, 4)], + [1000000, 200000, 30000, 4000, 500, 60, 7] +) AS res +``` + +``` text +┌─res─────────────────────────┐ +│ [1234500,234000,34560,4567] │ +└─────────────────────────────┘ +``` + +## arrayReverse(arr) {#arrayreverse} + +Öğeleri ters sırada içeren orijinal diziyle aynı boyutta bir dizi döndürür. + +Örnek: + +``` sql +SELECT arrayReverse([1, 2, 3]) +``` + +``` text +┌─arrayReverse([1, 2, 3])─┐ +│ [3,2,1] │ +└─────────────────────────┘ +``` + +## ters (arr) {#array-functions-reverse} + +Eşanlamlı [“arrayReverse”](#array_functions-arrayreverse) + +## arrayFlatten {#arrayflatten} + +Bir dizi diziyi düz bir diziye dönüştürür. + +İşlev: + +- İç içe geçmiş dizilerin herhangi bir derinliği için geçerlidir. +- Zaten düz olan dizileri değiştirmez. + +Düzleştirilmiş dizi, tüm kaynak dizilerdeki tüm öğeleri içerir. + +**Sözdizimi** + +``` sql +flatten(array_of_arrays) +``` + +Takma ad: `flatten`. + +**Parametre** + +- `array_of_arrays` — [Dizi](../../sql_reference/data_types/array.md) dizilerin. Mesela, `[[1,2,3], [4,5]]`. + +**Örnekler** + +``` sql +SELECT flatten([[[1]], [[2], [3]]]) +``` + +``` text +┌─flatten(array(array([1]), array([2], [3])))─┐ +│ [1,2,3] │ +└─────────────────────────────────────────────┘ +``` + +## arrayCompact {#arraycompact} + +Ardışık yinelenen öğeleri bir diziden kaldırır. Sonuç değerlerinin sırası, kaynak dizideki sıraya göre belirlenir. + +**Sözdizimi** + +``` sql +arrayCompact(arr) +``` + +**Parametre** + +`arr` — The [dizi](../../sql_reference/data_types/array.md) incelemek. + +**Döndürülen değer** + +Yinelenen olmadan dizi. + +Tür: `Array`. + +**Örnek** + +Sorgu: + +``` sql +SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) +``` + +Sonuç: + +``` text +┌─arrayCompact([1, 1, nan, nan, 2, 3, 3, 3])─┐ +│ [1,nan,nan,2,3] │ +└────────────────────────────────────────────┘ +``` + +## arrayZip {#arrayzip} + +Birden çok diziyi tek bir dizide birleştirir. Elde edilen dizi, listelenen bağımsız değişken sırasına göre gruplandırılmış kaynak dizilerin karşılık gelen öğelerini içerir. + +**Sözdizimi** + +``` sql +arrayZip(arr1, arr2, ..., arrN) +``` + +**Parametre** + +- `arrN` — [Dizi](../data_types/array.md). + +İşlev, farklı türde herhangi bir dizi alabilir. Tüm giriş dizileri eşit boyutta olmalıdır. + +**Döndürülen değer** + +- Gruplandırılmış kaynak dizilerden öğelerle dizi [Demetler](../data_types/tuple.md). Veri türleri tuple giriş dizileri türleri ile aynıdır ve diziler geçirilir aynı sırada. + +Tür: [Dizi](../data_types/array.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]) +``` + +Sonuç: + +``` text +┌─arrayZip(['a', 'b', 'c'], [5, 2, 1])─┐ +│ [('a',5),('b',2),('c',1)] │ +└──────────────────────────────────────┘ +``` + +## arrayAUC {#arrayauc} + +Auc'yi hesaplayın (makine öğreniminde bir kavram olan eğrinin altındaki alan, daha fazla ayrıntıya bakın: https://en.wikipedia.org/wiki/Receiver\_operating\_characteristic\#Area\_under\_the\_curve). + +**Sözdizimi** + +``` sql +arrayAUC(arr_scores, arr_labels) +``` + +**Parametre** +- `arr_scores` — scores prediction model gives. +- `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negtive sample. + +**Döndürülen değer** +Float64 türü ile AUC değerini döndürür. + +**Örnek** +Sorgu: + +``` sql +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +``` + +Sonuç: + +``` text +┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ +│ 0.75 │ +└────────────────────────────────────────---──┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/array_functions/) diff --git a/docs/tr/sql_reference/functions/array_join.md b/docs/tr/sql_reference/functions/array_join.md new file mode 100644 index 00000000000..f59ed95805b --- /dev/null +++ b/docs/tr/sql_reference/functions/array_join.md @@ -0,0 +1,37 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 61 +toc_title: arrayJoin +--- + +# arrayjoin işlevi {#functions_arrayjoin} + +Bu çok sıradışı bir işlevdir. + +Normal işlevler bir dizi satırı değiştirmez, ancak her satırdaki değerleri değiştirir (harita). +Toplama işlevleri bir dizi satırı sıkıştırır (katlayın veya azaltın). +Bu ‘arrayJoin’ işlev her satırı alır ve bir dizi satır oluşturur (açılır). + +Bu işlev bir diziyi bağımsız değişken olarak alır ve kaynak satırı dizideki öğe sayısı için birden çok satıra yayar. +Sütunlardaki tüm değerler, bu işlevin uygulandığı sütundaki değerler dışında kopyalanır; karşılık gelen dizi değeri ile değiştirilir. + +Bir sorgu birden çok kullanabilirsiniz `arrayJoin` işlevler. Bu durumda, dönüşüm birden çok kez gerçekleştirilir. + +Daha geniş olanaklar sağlayan SELECT sorgusunda dizi birleştirme sözdizimini not alın. + +Örnek: + +``` sql +SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src +``` + +``` text +┌─dst─┬─\'Hello\'─┬─src─────┐ +│ 1 │ Hello │ [1,2,3] │ +│ 2 │ Hello │ [1,2,3] │ +│ 3 │ Hello │ [1,2,3] │ +└─────┴───────────┴─────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/array_join/) diff --git a/docs/tr/sql_reference/functions/bit_functions.md b/docs/tr/sql_reference/functions/bit_functions.md new file mode 100644 index 00000000000..1e8d9761d5d --- /dev/null +++ b/docs/tr/sql_reference/functions/bit_functions.md @@ -0,0 +1,255 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 48 +toc_title: Bitlik +--- + +# Bit fonksiyonları {#bit-functions} + +Bit işlevleri, uint8, Uİnt16, Uİnt32, Uint64, Int8, Int16, Int32, Int64, Float32 veya Float64 türlerinden herhangi bir çift için çalışır. + +Sonuç türü, bağımsız değişkenlerinin maksimum bitlerine eşit bit içeren bir tamsayıdır. Bağımsız değişkenlerden en az biri imzalanırsa, sonuç imzalı bir sayıdır. Bir bağımsız değişken bir kayan noktalı sayı ise, Int64 için cast. + +## bıtor(a, b) {#bitanda-b} + +## bitOr(a, b) {#bitora-b} + +## bitXor(a, b) {#bitxora-b} + +## bitNot (a) {#bitnota} + +## bitShiftLeft(a, b) {#bitshiftlefta-b} + +## bitShiftRight(a, b) {#bitshiftrighta-b} + +## bitRotateLeft(a, b) {#bitrotatelefta-b} + +## bitRotateRight(a, b) {#bitrotaterighta-b} + +## bitTest {#bittest} + +Herhangi bir tamsayı alır ve dönüştürür [ikili form](https://en.wikipedia.org/wiki/Binary_number), belirtilen konumda bir bit değerini döndürür. Geri sayım sağdan sola 0 başlar. + +**Sözdizimi** + +``` sql +SELECT bitTest(number, index) +``` + +**Parametre** + +- `number` – integer number. +- `index` – position of bit. + +**Döndürülen değerler** + +Belirtilen konumda bit değeri döndürür. + +Tür: `UInt8`. + +**Örnek** + +Örneğin, taban-2 (ikili) sayı sistemindeki 43 sayısı 101011'dir. + +Sorgu: + +``` sql +SELECT bitTest(43, 1) +``` + +Sonuç: + +``` text +┌─bitTest(43, 1)─┐ +│ 1 │ +└────────────────┘ +``` + +Başka bir örnek: + +Sorgu: + +``` sql +SELECT bitTest(43, 2) +``` + +Sonuç: + +``` text +┌─bitTest(43, 2)─┐ +│ 0 │ +└────────────────┘ +``` + +## bitTestAll {#bittestall} + +Sonucu döndürür [mantıksal conjuction](https://en.wikipedia.org/wiki/Logical_conjunction) Verilen pozisyonlarda tüm bitlerin (ve operatörü). Geri sayım sağdan sola 0 başlar. + +Bitsel işlemler için conjuction: + +0 AND 0 = 0 + +0 AND 1 = 0 + +1 AND 0 = 0 + +1 AND 1 = 1 + +**Sözdizimi** + +``` sql +SELECT bitTestAll(number, index1, index2, index3, index4, ...) +``` + +**Parametre** + +- `number` – integer number. +- `index1`, `index2`, `index3`, `index4` – positions of bit. For example, for set of positions (`index1`, `index2`, `index3`, `index4`) doğru ise ve sadece tüm pozisyon trueları doğru ise (`index1` ⋀ `index2`, ⋀ `index3` ⋀ `index4`). + +**Döndürülen değerler** + +Mantıksal conjuction sonucunu döndürür. + +Tür: `UInt8`. + +**Örnek** + +Örneğin, taban-2 (ikili) sayı sistemindeki 43 sayısı 101011'dir. + +Sorgu: + +``` sql +SELECT bitTestAll(43, 0, 1, 3, 5) +``` + +Sonuç: + +``` text +┌─bitTestAll(43, 0, 1, 3, 5)─┐ +│ 1 │ +└────────────────────────────┘ +``` + +Başka bir örnek: + +Sorgu: + +``` sql +SELECT bitTestAll(43, 0, 1, 3, 5, 2) +``` + +Sonuç: + +``` text +┌─bitTestAll(43, 0, 1, 3, 5, 2)─┐ +│ 0 │ +└───────────────────────────────┘ +``` + +## bitTestAny {#bittestany} + +Sonucu döndürür [mantıksal ayrılma](https://en.wikipedia.org/wiki/Logical_disjunction) Verilen konumlardaki tüm bitlerin (veya operatör). Geri sayım sağdan sola 0 başlar. + +Bitsel işlemler için ayrılma: + +0 OR 0 = 0 + +0 OR 1 = 1 + +1 OR 0 = 1 + +1 OR 1 = 1 + +**Sözdizimi** + +``` sql +SELECT bitTestAny(number, index1, index2, index3, index4, ...) +``` + +**Parametre** + +- `number` – integer number. +- `index1`, `index2`, `index3`, `index4` – positions of bit. + +**Döndürülen değerler** + +Mantıksal disjuction sonucunu döndürür. + +Tür: `UInt8`. + +**Örnek** + +Örneğin, taban-2 (ikili) sayı sistemindeki 43 sayısı 101011'dir. + +Sorgu: + +``` sql +SELECT bitTestAny(43, 0, 2) +``` + +Sonuç: + +``` text +┌─bitTestAny(43, 0, 2)─┐ +│ 1 │ +└──────────────────────┘ +``` + +Başka bir örnek: + +Sorgu: + +``` sql +SELECT bitTestAny(43, 4, 2) +``` + +Sonuç: + +``` text +┌─bitTestAny(43, 4, 2)─┐ +│ 0 │ +└──────────────────────┘ +``` + +## bitCount {#bitcount} + +Bir sayının ikili gösteriminde birine ayarlanmış bit sayısını hesaplar. + +**Sözdizimi** + +``` sql +bitCount(x) +``` + +**Parametre** + +- `x` — [Tamsayı](../../sql_reference/data_types/int_uint.md) veya [kayan nokta](../../sql_reference/data_types/float.md) numara. İşlev, bellekteki değer gösterimini kullanır. Kayan noktalı sayıları desteklemeye izin verir. + +**Döndürülen değer** + +- Giriş numarasında birine ayarlanmış bit sayısı. + +İşlev, giriş değerini daha büyük bir türe dönüştürmez ([işaret uzantısı](https://en.wikipedia.org/wiki/Sign_extension)). Bu yüzden, örneğin , `bitCount(toUInt8(-1)) = 8`. + +Tür: `UInt8`. + +**Örnek** + +Örneğin 333 sayısını alın. İkili gösterimi: 0000000101001101. + +Sorgu: + +``` sql +SELECT bitCount(333) +``` + +Sonuç: + +``` text +┌─bitCount(333)─┐ +│ 5 │ +└───────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/bit_functions/) diff --git a/docs/tr/sql_reference/functions/bitmap_functions.md b/docs/tr/sql_reference/functions/bitmap_functions.md new file mode 100644 index 00000000000..1b4fb53e0e3 --- /dev/null +++ b/docs/tr/sql_reference/functions/bitmap_functions.md @@ -0,0 +1,496 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 49 +toc_title: "E\u015Flem" +--- + +# Bitmap işlevleri {#bitmap-functions} + +Bitmap işlevleri iki bit eşlemler nesne değeri hesaplama için çalışmak, yeni bitmap veya kardinality formül hesaplama, and, or, xor ve not, vb gibi kullanırken döndürmektir. + +Bitmap nesnesi için 2 çeşit inşaat yöntemi vardır. Biri-State ile toplama işlevi groupBitmap tarafından inşa edilecek, diğeri Array nesnesi tarafından inşa edilecek. Ayrıca bitmap nesnesini dizi nesnesine dönüştürmektir. + +Roaringbitmap, Bitmap nesnelerinin gerçek depolanması sırasında bir veri yapısına sarılır. Önemlilik 32'den küçük veya eşit olduğunda, Set objet kullanır. Kardinality 32'den büyük olduğunda, roaringbitmap nesnesi kullanır. Bu nedenle düşük kardinalite kümesinin depolanması daha hızlıdır. + +RoaringBitmap hakkında daha fazla bilgi için bkz: [CRoaring](https://github.com/RoaringBitmap/CRoaring). + +## bitmapBuild {#bitmap_functions-bitmapbuild} + +İmzasız tamsayı dizisinden bir bit eşlem oluşturun. + +``` sql +bitmapBuild(array) +``` + +**Parametre** + +- `array` – unsigned integer array. + +**Örnek** + +``` sql +SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res) +``` + +``` text +┌─res─┬─toTypeName(bitmapBuild([1, 2, 3, 4, 5]))─────┐ +│  │ AggregateFunction(groupBitmap, UInt8) │ +└─────┴──────────────────────────────────────────────┘ +``` + +## bitmapToArray {#bitmaptoarray} + +Bitmap'i tamsayı dizisine dönüştürün. + +``` sql +bitmapToArray(bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res +``` + +``` text +┌─res─────────┐ +│ [1,2,3,4,5] │ +└─────────────┘ +``` + +## bitmapsubsetınrange {#bitmap-functions-bitmapsubsetinrange} + +Belirtilen aralıktaki alt kümesi döndürür (range\_end içermez). + +``` sql +bitmapSubsetInRange(bitmap, range_start, range_end) +``` + +**Parametre** + +- `bitmap` – [Bitmap nesnesi](#bitmap_functions-bitmapbuild). +- `range_start` – range start point. Type: [Uİnt32](../../sql_reference/data_types/int_uint.md). +- `range_end` – range end point(excluded). Type: [Uİnt32](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +``` sql +SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res +``` + +``` text +┌─res───────────────┐ +│ [30,31,32,33,100] │ +└───────────────────┘ +``` + +## bitmapSubsetLimit {#bitmapsubsetlimit} + +Arasında alınan n öğeleri ile bitmap bir alt kümesi oluşturur `range_start` ve `cardinality_limit`. + +**Sözdizimi** + +``` sql +bitmapSubsetLimit(bitmap, range_start, cardinality_limit) +``` + +**Parametre** + +- `bitmap` – [Bitmap nesnesi](#bitmap_functions-bitmapbuild). +- `range_start` – The subset starting point. Type: [Uİnt32](../../sql_reference/data_types/int_uint.md). +- `cardinality_limit` – The subset cardinality upper limit. Type: [Uİnt32](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değer** + +Alt. + +Tür: `Bitmap object`. + +**Örnek** + +Sorgu: + +``` sql +SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res +``` + +Sonuç: + +``` text +┌─res───────────────────────┐ +│ [30,31,32,33,100,200,500] │ +└───────────────────────────┘ +``` + +## bitmapContains {#bitmap_functions-bitmapcontains} + +Bit eşlem bir öğe içerip içermediğini denetler. + +``` sql +bitmapContains(haystack, needle) +``` + +**Parametre** + +- `haystack` – [Bitmap nesnesi](#bitmap_functions-bitmapbuild), fonksiyon arar nerede. +- `needle` – Value that the function searches. Type: [Uİnt32](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- 0 — If `haystack` içermez `needle`. +- 1 — If `haystack` içeriyor `needle`. + +Tür: `UInt8`. + +**Örnek** + +``` sql +SELECT bitmapContains(bitmapBuild([1,5,7,9]), toUInt32(9)) AS res +``` + +``` text +┌─res─┐ +│ 1 │ +└─────┘ +``` + +## bitmapHasAny {#bitmaphasany} + +İki bit eşlemin bazı öğelerle kesiştiği olup olmadığını kontrol eder. + +``` sql +bitmapHasAny(bitmap1, bitmap2) +``` + +Eğer eminseniz `bitmap2` kesinlikle bir öğe içerir, kullanmayı düşünün [bitmapContains](#bitmap_functions-bitmapcontains) işlev. Daha verimli çalışır. + +**Parametre** + +- `bitmap*` – bitmap object. + +**Dönüş değerleri** + +- `1`, eğer `bitmap1` ve `bitmap2` en azından benzer bir öğeye sahip olun. +- `0`, başka. + +**Örnek** + +``` sql +SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res +``` + +``` text +┌─res─┐ +│ 1 │ +└─────┘ +``` + +## bitmapHasAll {#bitmaphasall} + +Benzer `hasAll(array, array)` ilk bit eşlem, ikincisinin tüm öğelerini içeriyorsa, 1 değerini döndürür, aksi halde 0. +İkinci bağımsız değişken boş bir bit eşlem ise, 1 döndürür. + +``` sql +bitmapHasAll(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res +``` + +``` text +┌─res─┐ +│ 0 │ +└─────┘ +``` + +## bitmapCardinality {#bitmapcardinality} + +Retrun bit eşlem kardinalite türü Uİnt64. + +``` sql +bitmapCardinality(bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res +``` + +``` text +┌─res─┐ +│ 5 │ +└─────┘ +``` + +## bitmapMin {#bitmapmin} + +Kümedeki uint64 türünün en küçük değerini yeniden çalıştırın, küme boşsa UİNT32\_MAX. + + bitmapMin(bitmap) + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapMin(bitmapBuild([1, 2, 3, 4, 5])) AS res +``` + + ┌─res─┐ + │ 1 │ + └─────┘ + +## bitmapMax {#bitmapmax} + +Küme boşsa, kümedeki uint64 türünün en büyük değerini 0 olarak yeniden çalıştırın. + + bitmapMax(bitmap) + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapMax(bitmapBuild([1, 2, 3, 4, 5])) AS res +``` + + ┌─res─┐ + │ 5 │ + └─────┘ + +## bitmapTransform {#bitmaptransform} + +Bitmap'teki bir değer dizisini başka bir değer dizisine dönüştürün, sonuç yeni bir bitmap'tir. + + bitmapTransform(bitmap, from_array, to_array) + +**Parametre** + +- `bitmap` – bitmap object. +- `from_array` – UInt32 array. For idx in range \[0, from\_array.size()), if bitmap contains from\_array\[idx\], then replace it with to\_array\[idx\]. Note that the result depends on array ordering if there are common elements between from\_array and to\_array. +- `to_array` – UInt32 array, its size shall be the same to from\_array. + +**Örnek** + +``` sql +SELECT bitmapToArray(bitmapTransform(bitmapBuild([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), cast([5,999,2] as Array(UInt32)), cast([2,888,20] as Array(UInt32)))) AS res +``` + + ┌─res───────────────────┐ + │ [1,3,4,6,7,8,9,10,20] │ + └───────────────────────┘ + +## bitmapAnd {#bitmapand} + +İki bitmap ve hesaplama, sonuç yeni bir bitmap'tir. + +``` sql +bitmapAnd(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +``` + +``` text +┌─res─┐ +│ [3] │ +└─────┘ +``` + +## bitmapOr {#bitmapor} + +İki bitmap veya hesaplama, sonuç yeni bir bitmap'tir. + +``` sql +bitmapOr(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +``` + +``` text +┌─res─────────┐ +│ [1,2,3,4,5] │ +└─────────────┘ +``` + +## bitmapXor {#bitmapxor} + +İki bitmap XOR hesaplama, sonuç yeni bir bitmap. + +``` sql +bitmapXor(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +``` + +``` text +┌─res───────┐ +│ [1,2,4,5] │ +└───────────┘ +``` + +## bitmapAndnot {#bitmapandnot} + +İki bit eşlem andnot hesaplama, sonuç yeni bir bit eşlem. + +``` sql +bitmapAndnot(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +``` + +``` text +┌─res───┐ +│ [1,2] │ +└───────┘ +``` + +## bitmapAndCardinality {#bitmapandcardinality} + +İki bitmap ve hesaplama, uint64 türünün kardinalliğini döndürür. + +``` sql +bitmapAndCardinality(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; +``` + +``` text +┌─res─┐ +│ 1 │ +└─────┘ +``` + +## bitmapOrCardinality {#bitmaporcardinality} + +İki bitmap veya hesaplama, uint64 türünün kardinalliğini döndürür. + +``` sql +bitmapOrCardinality(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; +``` + +``` text +┌─res─┐ +│ 5 │ +└─────┘ +``` + +## bitmapXorCardinality {#bitmapxorcardinality} + +İki bitmap XOR hesaplama, uint64 türünün kardinalliğini döndürür. + +``` sql +bitmapXorCardinality(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; +``` + +``` text +┌─res─┐ +│ 4 │ +└─────┘ +``` + +## bitmapAndnotCardinality {#bitmapandnotcardinality} + +İki bitmap andnot hesaplama, uint64 türünün kardinalliğini döndürür. + +``` sql +bitmapAndnotCardinality(bitmap,bitmap) +``` + +**Parametre** + +- `bitmap` – bitmap object. + +**Örnek** + +``` sql +SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; +``` + +``` text +┌─res─┐ +│ 2 │ +└─────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/bitmap_functions/) diff --git a/docs/tr/sql_reference/functions/comparison_functions.md b/docs/tr/sql_reference/functions/comparison_functions.md new file mode 100644 index 00000000000..d041459bb05 --- /dev/null +++ b/docs/tr/sql_reference/functions/comparison_functions.md @@ -0,0 +1,37 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 36 +toc_title: "Kar\u015F\u0131la\u015Ft\u0131rma" +--- + +# Karşılaştırma fonksiyonları {#comparison-functions} + +Karşılaştırma işlevleri her zaman 0 veya 1 (Uint8) döndürür. + +Aşağıdaki türler karşılaştırılabilir: + +- şiir +- dizeler ve sabit dizeler +- tarihliler +- tarihleri ile saatleri + +her grup içinde, ancak farklı gruplar arasında değil. + +Örneğin, bir tarihi bir dizeyle karşılaştıramazsınız. Bir tarih dizesi dönüştürmek, ya da tam tersi bir işlev kullanmak zorunda. + +Dizeler baytlarla karşılaştırılır. Daha kısa bir dize, onunla başlayan ve en az bir karakter daha içeren tüm dizelerden daha küçüktür. + +## eşittir, a = b VE a = = B operatörü {#function-equals} + +## notEquals, a ! operatör = b VE A \<\> b {#function-notequals} + +## l ,ess, \< operat \ operatör {#function-greater} + +## lessOrEquals, \< = operatör {#function-lessorequals} + +## greaterOrEquals, \> = operatör {#function-greaterorequals} + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/comparison_functions/) diff --git a/docs/tr/sql_reference/functions/conditional_functions.md b/docs/tr/sql_reference/functions/conditional_functions.md new file mode 100644 index 00000000000..b2e918666a0 --- /dev/null +++ b/docs/tr/sql_reference/functions/conditional_functions.md @@ -0,0 +1,207 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 43 +toc_title: "Ko\u015Fullu " +--- + +# Koşullu fonksiyonlar {#conditional-functions} + +## eğer {#if} + +Koşullu dallanmayı kontrol eder. Çoğu sistemin aksine, ClickHouse her zaman her iki ifadeyi de değerlendirir `then` ve `else`. + +**Sözdizimi** + +``` sql +SELECT if(cond, then, else) +``` + +Eğer durum `cond` sıfır olmayan bir değere değerlendirir, ifadenin sonucunu döndürür `then` ve ifad andenin sonucu `else` varsa, atlanır. Eğer... `cond` sıfır veya `NULL` fakat daha sonra sonucu `then` ifade atlanır ve sonucu `else` Ifade, varsa, döndürülür. + +**Parametre** + +- `cond` – The condition for evaluation that can be zero or not. The type is UInt8, Nullable(UInt8) or NULL. +- `then` - Koşul karşılanırsa dönmek için ifade. +- `else` - Koşul karşılanmazsa dönmek için ifade. + +**Döndürülen değerler** + +İşlev yürütür `then` ve `else` ifadeler ve koşulun olup olmadığına bağlı olarak sonucunu döndürür `cond` sıfır ya da değil. + +**Örnek** + +Sorgu: + +``` sql +SELECT if(1, plus(2, 2), plus(2, 6)) +``` + +Sonuç: + +``` text +┌─plus(2, 2)─┐ +│ 4 │ +└────────────┘ +``` + +Sorgu: + +``` sql +SELECT if(0, plus(2, 2), plus(2, 6)) +``` + +Sonuç: + +``` text +┌─plus(2, 6)─┐ +│ 8 │ +└────────────┘ +``` + +- `then` ve `else` en düşük ortak türe sahip olmalıdır. + +**Örnek:** + +Bunu al `LEFT_RIGHT` Tablo: + +``` sql +SELECT * +FROM LEFT_RIGHT + +┌─left─┬─right─┐ +│ ᴺᵁᴸᴸ │ 4 │ +│ 1 │ 3 │ +│ 2 │ 2 │ +│ 3 │ 1 │ +│ 4 │ ᴺᵁᴸᴸ │ +└──────┴───────┘ +``` + +Aşağıdaki sorgu karşılaştırır `left` ve `right` değerler: + +``` sql +SELECT + left, + right, + if(left < right, 'left is smaller than right', 'right is greater or equal than left') AS is_smaller +FROM LEFT_RIGHT +WHERE isNotNull(left) AND isNotNull(right) + +┌─left─┬─right─┬─is_smaller──────────────────────────┐ +│ 1 │ 3 │ left is smaller than right │ +│ 2 │ 2 │ right is greater or equal than left │ +│ 3 │ 1 │ right is greater or equal than left │ +└──────┴───────┴─────────────────────────────────────┘ +``` + +Not: `NULL` bu örnekte değerler kullanılmaz, kontrol edin [Koşullardaki boş değerler](#null-values-in-conditionals) bölme. + +## Üçlü Operatör {#ternary-operator} + +Aynı gibi çalışıyor. `if` işlev. + +Sözdizimi: `cond ? then : else` + +Dönüşler `then` eğer... `cond` true (sıfırdan büyük) olarak değerlendirir, aksi takdirde döndürür `else`. + +- `cond` türü olmalıdır `UInt8`, ve `then` ve `else` en düşük ortak türe sahip olmalıdır. + +- `then` ve `else` olabilir `NULL` + +**Ayrıca bakınız** + +- [ifNotFinite](other_functions.md#ifnotfinite). + +## multiİf {#multiif} + +Yaz allowsmanızı sağlar [CASE](../operators.md#operator_case) operatör sorguda daha kompakt. + +Sözdizimi: `multiIf(cond_1, then_1, cond_2, then_2, ..., else)` + +**Parametre:** + +- `cond_N` — The condition for the function to return `then_N`. +- `then_N` — The result of the function when executed. +- `else` — The result of the function if none of the conditions is met. + +İşlev kabul eder `2N+1` parametre. + +**Döndürülen değerler** + +İşlev, değerlerden birini döndürür `then_N` veya `else` bu koşullara bağlı olarak `cond_N`. + +**Örnek** + +Yine kullanarak `LEFT_RIGHT` Tablo. + +``` sql +SELECT + left, + right, + multiIf(left < right, 'left is smaller', left > right, 'left is greater', left = right, 'Both equal', 'Null value') AS result +FROM LEFT_RIGHT + +┌─left─┬─right─┬─result──────────┐ +│ ᴺᵁᴸᴸ │ 4 │ Null value │ +│ 1 │ 3 │ left is smaller │ +│ 2 │ 2 │ Both equal │ +│ 3 │ 1 │ left is greater │ +│ 4 │ ᴺᵁᴸᴸ │ Null value │ +└──────┴───────┴─────────────────┘ +``` + +## Koşullu sonuçları doğrudan kullanma {#using-conditional-results-directly} + +Koşullar her zaman sonuç `0`, `1` veya `NULL`. Böylece koşullu sonuçları doğrudan bu şekilde kullanabilirsiniz: + +``` sql +SELECT left < right AS is_small +FROM LEFT_RIGHT + +┌─is_small─┐ +│ ᴺᵁᴸᴸ │ +│ 1 │ +│ 0 │ +│ 0 │ +│ ᴺᵁᴸᴸ │ +└──────────┘ +``` + +## Koşullardaki boş değerler {#null-values-in-conditionals} + +Ne zaman `NULL` değerler koşullarla ilgilidir, sonuç da olacaktır `NULL`. + +``` sql +SELECT + NULL < 1, + 2 < NULL, + NULL < NULL, + NULL = NULL + +┌─less(NULL, 1)─┬─less(2, NULL)─┬─less(NULL, NULL)─┬─equals(NULL, NULL)─┐ +│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ +└───────────────┴───────────────┴──────────────────┴────────────────────┘ +``` + +Bu nedenle, sorgularınızı türleri dikkatli bir şekilde oluşturmalısınız `Nullable`. + +Aşağıdaki örnek, eşittir koşulu eklemek başarısız tarafından bu gösterir `multiIf`. + +``` sql +SELECT + left, + right, + multiIf(left < right, 'left is smaller', left > right, 'right is smaller', 'Both equal') AS faulty_result +FROM LEFT_RIGHT + +┌─left─┬─right─┬─faulty_result────┐ +│ ᴺᵁᴸᴸ │ 4 │ Both equal │ +│ 1 │ 3 │ left is smaller │ +│ 2 │ 2 │ Both equal │ +│ 3 │ 1 │ right is smaller │ +│ 4 │ ᴺᵁᴸᴸ │ Both equal │ +└──────┴───────┴──────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/conditional_functions/) diff --git a/docs/tr/sql_reference/functions/date_time_functions.md b/docs/tr/sql_reference/functions/date_time_functions.md new file mode 100644 index 00000000000..fdcbacc7749 --- /dev/null +++ b/docs/tr/sql_reference/functions/date_time_functions.md @@ -0,0 +1,450 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 39 +toc_title: "Tarih ve Saatlerle \xE7al\u0131\u015Fma" +--- + +# Tarih ve saatlerle çalışmak için işlevler {#functions-for-working-with-dates-and-times} + +Saat dilimleri için destek + +Saat dilimi için mantıksal kullanımı olan tarih ve Saat ile çalışmak için tüm işlevler, ikinci bir isteğe bağlı saat dilimi bağımsız değişkeni kabul edebilir. Örnek: Asya / Yekaterinburg. Bu durumda, yerel (varsayılan) yerine belirtilen saat dilimini kullanırlar. + +``` sql +SELECT + toDateTime('2016-06-15 23:00:00') AS time, + toDate(time) AS date_local, + toDate(time, 'Asia/Yekaterinburg') AS date_yekat, + toString(time, 'US/Samoa') AS time_samoa +``` + +``` text +┌────────────────time─┬─date_local─┬─date_yekat─┬─time_samoa──────────┐ +│ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-16 │ 2016-06-15 09:00:00 │ +└─────────────────────┴────────────┴────────────┴─────────────────────┘ +``` + +UTC'DEN saat sayısına göre farklı olan yalnızca saat dilimleri desteklenir. + +## toTimeZone {#totimezone} + +Saat veya tarih ve saati belirtilen saat dilimine dönüştürün. + +## toYear {#toyear} + +Bir tarihi veya tarihi zamanla yıl numarasını (AD) içeren bir Uınt16 numarasına dönüştürür. + +## toQuarter {#toquarter} + +Bir tarihi veya tarihi zaman ile çeyrek sayısını içeren bir Uİnt8 numarasına dönüştürür. + +## toMonth {#tomonth} + +Bir tarih veya tarih ile saati, ay numarasını (1-12) içeren bir Uİnt8 numarasına dönüştürür. + +## bugünyıl {#todayofyear} + +Bir tarih veya tarih ile saat, yılın gün sayısını (1-366) içeren bir Uınt16 numarasına dönüştürür. + +## bugünay {#todayofmonth} + +Bir tarih veya tarih ile saat, Ayın gün sayısını (1-31) içeren bir Uınt8 numarasına dönüştürür. + +## bugünhafta {#todayofweek} + +Bir tarih veya tarih ile saat, haftanın gününün sayısını içeren bir Uınt8 numarasına dönüştürür (Pazartesi 1 ve pazar 7'dir). + +## toHour {#tohour} + +Saatli bir tarihi, 24 saatlik süre (0-23) saat sayısını içeren bir Uınt8 numarasına dönüştürür. +This function assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always true – even in Moscow the clocks were twice changed at a different time). + +## toMinute {#tominute} + +Saatli bir tarihi, saatin dakika sayısını (0-59) içeren bir Uınt8 numarasına dönüştürür. + +## toSecond {#tosecond} + +Dakika (0-59) ikinci sayısını içeren bir uınt8 numarasına zaman ile bir tarih dönüştürür. +Sıçrama saniye hesaba değildir. + +## toUnixTimestamp {#to-unix-timestamp} + +DateTime argümanı için: değeri dahili sayısal gösterimine dönüştürür (Unıx Zaman Damgası). +String argümanı için: datetime'ı dizeden saat dilimine göre ayrıştırın (isteğe bağlı ikinci argüman, sunucu zaman dilimi varsayılan olarak kullanılır) ve karşılık gelen unıx zaman damgasını döndürür. +Tarih argümanı için: davranış belirtilmemiş. + +**Sözdizimi** + +``` sql +toUnixTimestamp(datetime) +toUnixTimestamp(str, [timezone]) +``` + +**Döndürülen değer** + +- Unix zaman damgasını döndürür. + +Tür: `UInt32`. + +**Örnek** + +Sorgu: + +``` sql +SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp +``` + +Sonuç: + +``` text +┌─unix_timestamp─┐ +│ 1509836867 │ +└────────────────┘ +``` + +## toStartOfYear {#tostartofyear} + +Yılın ilk gününe kadar bir tarih veya tarih aşağı yuvarlar. +Tarihi döndürür. + +## toStartOfİSOYear {#tostartofisoyear} + +ISO yılın ilk gününe kadar bir tarih veya tarih aşağı yuvarlar. +Tarihi döndürür. + +## toStartOfQuarter {#tostartofquarter} + +Çeyrek ilk güne kadar bir tarih veya tarih aşağı yuvarlar. +Çeyreğin ilk günü 1 Ocak, 1 Nisan, 1 Temmuz veya 1 ekim'dir. +Tarihi döndürür. + +## toStartOfMonth {#tostartofmonth} + +Ayın ilk gününe kadar bir tarih veya tarih aşağı yuvarlar. +Tarihi döndürür. + +!!! attention "Dikkat" + Yanlış tarihleri ayrıştırma davranışı uygulamaya özeldir. ClickHouse sıfır tarihi döndürebilir, bir istisna atabilir veya yapabilir “natural” taşmak. + +## toMonday {#tomonday} + +En yakın Pazartesi günü bir tarih veya tarih aşağı yuvarlar. +Tarihi döndürür. + +## toStartOfWeek(t \[, mod\]) {#tostartofweektmode} + +Modu ile en yakın pazar veya Pazartesi zaman bir tarih veya tarih aşağı yuvarlar. +Tarihi döndürür. +Mod bağımsız değişkeni, toWeek () için mod bağımsız değişkeni gibi çalışır. Tek bağımsız değişken sözdizimi için 0 mod değeri kullanılır. + +## toStartOfDay {#tostartofday} + +Günün başlangıcına kadar bir tarih aşağı yuvarlar. + +## toStartOfHour {#tostartofhour} + +Saat başlangıcına kadar bir tarih aşağı yuvarlar. + +## toStartOfMinute {#tostartofminute} + +Dakikanın başlangıcına kadar bir tarih aşağı yuvarlar. + +## toStartOfFiveMinute {#tostartoffiveminute} + +Beş dakikalık aralığın başlangıcına kadar bir tarih aşağı yuvarlar. + +## toStartOfTenMinutes {#tostartoftenminutes} + +On dakikalık aralığın başlangıcına kadar bir tarih aşağı yuvarlar. + +## toStartOfFifteenMinutes {#tostartoffifteenminutes} + +On beş dakikalık aralığın başlangıcına kadar tarih aşağı yuvarlar. + +## toStartOfİnterval (time\_or\_data, Aralık x birimi \[, time\_zone\]) {#tostartofintervaltime-or-data-interval-x-unit-time-zone} + +Bu, diğer işlevlerin bir genellemesidir `toStartOf*`. Mesela, +`toStartOfInterval(t, INTERVAL 1 year)` aynı döndürür `toStartOfYear(t)`, +`toStartOfInterval(t, INTERVAL 1 month)` aynı döndürür `toStartOfMonth(t)`, +`toStartOfInterval(t, INTERVAL 1 day)` aynı döndürür `toStartOfDay(t)`, +`toStartOfInterval(t, INTERVAL 15 minute)` aynı döndürür `toStartOfFifteenMinutes(t)` vb. + +## toTime {#totime} + +Belirli bir sabit tarihe zaman ile bir tarih dönüştürür, zaman korurken. + +## toRelativeYearNum {#torelativeyearnum} + +Geçmişte belirli bir sabit noktadan başlayarak, yıl sayısına saat veya tarih ile bir tarih dönüştürür. + +## toRelativeQuarterNum {#torelativequarternum} + +Geçmişte belirli bir sabit noktadan başlayarak, çeyrek sayısına saat veya tarih ile bir tarih dönüştürür. + +## toRelativeMonthNum {#torelativemonthnum} + +Geçmişte belirli bir sabit noktadan başlayarak, Ayın sayısına saat veya tarih ile bir tarih dönüştürür. + +## toRelativeWeekNum {#torelativeweeknum} + +Geçmişte belirli bir sabit noktadan başlayarak, haftanın sayısına saat veya tarih ile bir tarih dönüştürür. + +## toRelativeDayNum {#torelativedaynum} + +Geçmişte belirli bir sabit noktadan başlayarak, günün sayısına saat veya tarih ile bir tarih dönüştürür. + +## toRelativeHourNum {#torelativehournum} + +Geçmişte belirli bir sabit noktadan başlayarak, saat veya tarih ile bir tarih saat sayısına dönüştürür. + +## toRelativeMinuteNum {#torelativeminutenum} + +Geçmişte belirli bir sabit noktadan başlayarak, dakika sayısına saat veya tarih ile bir tarih dönüştürür. + +## toRelativeSecondNum {#torelativesecondnum} + +Geçmişte belirli bir sabit noktadan başlayarak, ikinci sayısına saat veya tarih ile bir tarih dönüştürür. + +## toİSOYear {#toisoyear} + +ISO yıl numarasını içeren bir uınt16 numarasına bir tarih veya tarih zaman dönüştürür. + +## toİSOWeek {#toisoweek} + +ISO hafta numarasını içeren bir uınt8 numarasına bir tarih veya tarih zaman dönüştürür. + +## toWeek (tarih \[, mod\]) {#toweekdatemode} + +Bu işlev, date veya datetime için hafta numarasını döndürür. ToWeek () ' in iki bağımsız değişkenli formu, haftanın pazar veya Pazartesi günü başlayıp başlamadığını ve dönüş değerinin 0 ile 53 arasında mı yoksa 1 ile 53 arasında mı olması gerektiğini belirlemenizi sağlar. Mod bağımsız değişkeni atlanırsa, varsayılan mod 0'dır. +`toISOWeek()`eşdeğer bir uyumluluk işlevidir `toWeek(date,3)`. +Aşağıdaki tabloda mod bağımsız değişkeni nasıl çalıştığını açıklar. + +| Modu | Haftanın ilk günü | Aralık | Week 1 is the first week … | +|------|-------------------|--------|----------------------------------| +| 0 | Pazar | 0-53 | bu yıl bir pazar günü ile | +| 1 | Pazartesi | 0-53 | bu yıl 4 veya daha fazla gün ile | +| 2 | Pazar | 1-53 | bu yıl bir pazar günü ile | +| 3 | Pazartesi | 1-53 | bu yıl 4 veya daha fazla gün ile | +| 4 | Pazar | 0-53 | bu yıl 4 veya daha fazla gün ile | +| 5 | Pazartesi | 0-53 | bu yıl bir Pazartesi ile | +| 6 | Pazar | 1-53 | bu yıl 4 veya daha fazla gün ile | +| 7 | Pazartesi | 1-53 | bu yıl bir Pazartesi ile | +| 8 | Pazar | 1-53 | 1 Ocak içerir | +| 9 | Pazartesi | 1-53 | 1 Ocak içerir | + +Bir anlamı olan mod değerleri için “with 4 or more days this year,” haftalar ISO 8601: 1988'e göre numaralandırılmıştır: + +- 1 Ocak içeren haftanın yeni yılda 4 veya daha fazla günü varsa, 1. haftadır. + +- Aksi takdirde, bir önceki yılın son haftasıdır ve bir sonraki hafta 1. haftadır. + +Bir anlamı olan mod değerleri için “contains January 1”, 1 Ocak haftanın 1.haft .asıdır. Haftanın yeni yılda kaç gün içerdiği önemli değil, sadece bir gün içerse bile. + +``` sql +toWeek(date, [, mode][, Timezone]) +``` + +**Parametre** + +- `date` – Date or DateTime. +- `mode` – Optional parameter, Range of values is \[0,9\], default is 0. +- `Timezone` – Optional parameter, it behaves like any other conversion function. + +**Örnek** + +``` sql +SELECT toDate('2016-12-27') AS date, toWeek(date) AS week0, toWeek(date,1) AS week1, toWeek(date,9) AS week9; +``` + +``` text +┌───────date─┬─week0─┬─week1─┬─week9─┐ +│ 2016-12-27 │ 52 │ 52 │ 1 │ +└────────────┴───────┴───────┴───────┘ +``` + +## toYearWeek (tarih \[, mod\]) {#toyearweekdatemode} + +Bir tarih için yıl ve hafta döndürür. Sonuçtaki yıl, yılın ilk ve son haftası için tarih argümanındaki yıldan farklı olabilir. + +Mod bağımsız değişkeni, toWeek () için mod bağımsız değişkeni gibi çalışır. Tek bağımsız değişken sözdizimi için 0 mod değeri kullanılır. + +`toISOYear()`eşdeğer bir uyumluluk işlevidir `intDiv(toYearWeek(date,3),100)`. + +**Örnek** + +``` sql +SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9; +``` + +``` text +┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┐ +│ 2016-12-27 │ 201652 │ 201652 │ 201701 │ +└────────────┴───────────┴───────────┴───────────┘ +``` + +## şimdi {#now} + +Sıfır bağımsız değişkeni kabul eder ve geçerli saati istek yürütme anlarından birinde döndürür. +Bu işlev, isteğin tamamlanması uzun zaman alsa bile bir sabit döndürür. + +## bugünkü {#today} + +Sıfır bağımsız değişkeni kabul eder ve geçerli tarihi, istek yürütme anlarından birinde döndürür. +Olarak aynı ‘toDate(now())’. + +## dün {#yesterday} + +Sıfır bağımsız değişkeni kabul eder ve istek yürütme anlarından birinde dünün tarihini döndürür. +Olarak aynı ‘today() - 1’. + +## zaman dilimi {#timeslot} + +Yarım saat için zaman yuvarlar. +Bu fonksiyon (kayıt olmak için özeldir.Metrica, yarım saat, bir izleme etiketi, tek bir kullanıcının ardışık sayfa görüntülemelerini, zaman içinde bu miktardan kesinlikle daha fazla farklılık gösteriyorsa, bir oturumu iki oturuma bölmek için minimum zaman miktarıdır. Bu, ilgili oturumda bulunan sayfa görüntülemelerini aramak için tuples (etiket kimliği, kullanıcı kimliği ve zaman dilimi) kullanılabileceği anlamına gelir. + +## toYYYYMM {#toyyyymm} + +Bir tarih veya tarih ile saat, yıl ve ay numarasını (YYYY \* 100 + MM) içeren bir Uınt32 numarasına dönüştürür. + +## toYYYYMMDD {#toyyyymmdd} + +Bir tarih veya tarih ile saat, yıl ve ay numarasını içeren bir Uınt32 numarasına dönüştürür (YYYY \* 10000 + MM \* 100 + DD). + +## toYYYYMMDDhhmmss {#toyyyymmddhhmmss} + +Bir tarihi veya tarihi, yıl ve ay numarasını içeren bir Uınt64 numarasına dönüştürür (YYYY \* 1000000 + MM \* 1000000 + DD \* 1000000 + hh \* 10000 + mm \* 100 + ss). + +## addYears, addMonths, addWeeks, addDays, addHours, addMinutes, addSeconds, addQuarters {#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters} + +İşlev, bir tarih/DateTime aralığına bir tarih/DateTime ekler ve ardından Tarih/Datetime'ı döndürür. Mesela: + +``` sql +WITH + toDate('2018-01-01') AS date, + toDateTime('2018-01-01 00:00:00') AS date_time +SELECT + addYears(date, 1) AS add_years_with_date, + addYears(date_time, 1) AS add_years_with_date_time +``` + +``` text +┌─add_years_with_date─┬─add_years_with_date_time─┐ +│ 2019-01-01 │ 2019-01-01 00:00:00 │ +└─────────────────────┴──────────────────────────┘ +``` + +## subtractYears, subtractMonths, subtractWeeks, subtractDays, subtractHours, subtractMinutes, subtractSeconds, subtractQuarters {#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters} + +Fonksiyon bir tarih/DateTime aralığını bir tarih/DateTime olarak çıkarır ve ardından Tarih/Datetime'ı döndürür. Mesela: + +``` sql +WITH + toDate('2019-01-01') AS date, + toDateTime('2019-01-01 00:00:00') AS date_time +SELECT + subtractYears(date, 1) AS subtract_years_with_date, + subtractYears(date_time, 1) AS subtract_years_with_date_time +``` + +``` text +┌─subtract_years_with_date─┬─subtract_years_with_date_time─┐ +│ 2018-01-01 │ 2018-01-01 00:00:00 │ +└──────────────────────────┴───────────────────────────────┘ +``` + +## dateDiff {#datediff} + +İki Date veya DateTime değerleri arasındaki farkı döndürür. + +**Sözdizimi** + +``` sql +dateDiff('unit', startdate, enddate, [timezone]) +``` + +**Parametre** + +- `unit` — Time unit, in which the returned value is expressed. [Dize](../syntax.md#syntax-string-literal). + + Supported values: + + | unit | + | ---- | + |second | + |minute | + |hour | + |day | + |week | + |month | + |quarter | + |year | + +- `startdate` — The first time value to compare. [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). + +- `enddate` — The second time value to compare. [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). + +- `timezone` — Optional parameter. If specified, it is applied to both `startdate` ve `enddate`. Belirtilmemişse, saat dilimleri `startdate` ve `enddate` kullanılır. Aynı değilse, sonuç belirtilmemiştir. + +**Döndürülen değer** + +Arasındaki fark `startdate` ve `enddate` ifade edilen `unit`. + +Tür: `int`. + +**Örnek** + +Sorgu: + +``` sql +SELECT dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00')); +``` + +Sonuç: + +``` text +┌─dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00'))─┐ +│ 25 │ +└────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## (StartTime, Süresi,\[, Boyutu zaman yuvasının\]) {#timeslotsstarttime-duration-size} + +Başlayan bir zaman aralığı için ‘StartTime’ ve devam etmek için ‘Duration’ saniye, bu aralıktan aşağı yuvarlanan noktalardan oluşan zaman içinde bir dizi moment döndürür ‘Size’ saniyeler içinde. ‘Size’ isteğe bağlı bir parametredir: varsayılan olarak 1800 olarak ayarlanmış bir sabit Uİnt32. +Mesela, `timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. +Bu, ilgili oturumda sayfa görüntülemelerini aramak için gereklidir. + +## formatDateTime (saat, Biçim \[, Saat Dilimi\]) {#formatdatetime} + +Function formats a Time according given Format string. N.B.: Format is a constant expression, e.g. you can not have multiple formats for single result column. + +Biçim için desteklenen değiştiriciler: +(“Example” sütun, zaman için biçimlendirme sonucunu gösterir `2018-01-02 22:33:44`) + +| Değiştirici | Açıklama | Örnek | +|-------------|----------------------------------------------------------|------------| +| %C | yıl 100'e bölünür ve tamsayıya kesilir (00-99) | 20 | +| %d | Ayın günü, sıfır yastıklı (01-31) | 02 | +| %D | Kısa MM/DD/YY tarih, eşdeğer %m / %d / % y | 01/02/18 | +| %e | Ayın günü, boşluk dolgulu (1-31) | 2 | +| %F | kısa YYYY-AA-DD tarih, eşdeğer %Y-%m - %d | 2018-01-02 | +| %H | 24 saat formatında saat (00-23) | 22 | +| %I | 12h formatında saat (01-12) | 10 | +| %j | yılın günü (001-366) | 002 | +| %metre | ondalık sayı olarak ay (01-12) | 01 | +| %M | dakika (00-59) | 33 | +| %ve | new-line char (ac (ter (") | | +| %p | AM veya PM atama | PM | +| %R | 24-hour HH: MM Zaman, eşdeğer %H:%M | 22:33 | +| %S | ikinci (00-59) | 44 | +| %t | yatay-sekme karakteri (') | | +| %T | ISO 8601 saat biçimi (HH:MM:SS), eşdeğer %H:%M: % S | 22:33:44 | +| %u | ISO 8601 hafta içi sayı olarak Pazartesi olarak 1 (1-7) | 2 | +| %V | ISO 8601 hafta numarası (01-53) | 01 | +| %g | Pazar günü 0 (0-6) olarak ondalık sayı olarak hafta içi) | 2 | +| %y | Yıl, son iki basamak (00-99) | 18 | +| %Y | Yıllık | 2018 | +| %% | im | % | + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/date_time_functions/) diff --git a/docs/tr/sql_reference/functions/encoding_functions.md b/docs/tr/sql_reference/functions/encoding_functions.md new file mode 100644 index 00000000000..49ad1467957 --- /dev/null +++ b/docs/tr/sql_reference/functions/encoding_functions.md @@ -0,0 +1,175 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 52 +toc_title: "Kodlam\u0131\u015F" +--- + +# Kodlama fonksiyonları {#encoding-functions} + +## kömürleşmek {#char} + +Geçirilen bağımsız değişkenlerin sayısı olarak uzunluğu olan dizeyi döndürür ve her bayt karşılık gelen bağımsız değişken değerine sahiptir. Sayısal türlerin birden çok bağımsız değişkeni kabul eder. Bağımsız değişken değeri uint8 veri türü aralığının dışındaysa, Olası yuvarlama ve taşma ile Uint8'e dönüştürülür. + +**Sözdizimi** + +``` sql +char(number_1, [number_2, ..., number_n]); +``` + +**Parametre** + +- `number_1, number_2, ..., number_n` — Numerical arguments interpreted as integers. Types: [Tamsayı](../../sql_reference/data_types/int_uint.md), [Yüzdürmek](../../sql_reference/data_types/float.md). + +**Döndürülen değer** + +- verilen bayt dizisi. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello +``` + +Sonuç: + +``` text +┌─hello─┐ +│ hello │ +└───────┘ +``` + +Karşılık gelen baytları geçirerek bir rasgele kodlama dizesi oluşturabilirsiniz. İşte UTF-8 için örnek: + +Sorgu: + +``` sql +SELECT char(0xD0, 0xBF, 0xD1, 0x80, 0xD0, 0xB8, 0xD0, 0xB2, 0xD0, 0xB5, 0xD1, 0x82) AS hello; +``` + +Sonuç: + +``` text +┌─hello──┐ +│ привет │ +└────────┘ +``` + +Sorgu: + +``` sql +SELECT char(0xE4, 0xBD, 0xA0, 0xE5, 0xA5, 0xBD) AS hello; +``` + +Sonuç: + +``` text +┌─hello─┐ +│ 你好 │ +└───────┘ +``` + +## büyü {#hex} + +Bağımsız değişkenin onaltılık gösterimini içeren bir dize döndürür. + +**Sözdizimi** + +``` sql +hex(arg) +``` + +İşlev büyük harfler kullanıyor `A-F` ve herhangi bir önek kullanmamak (gibi `0x`) veya sonekler (gibi `h`). + +Tamsayı argümanları için, onaltılık basamak yazdırır (“nibbles”) en önemliden en önemlisine (big endian veya “human readable” sipariş). En önemli sıfır olmayan baytla başlar (önde gelen sıfır bayt atlanır), ancak önde gelen basamak sıfır olsa bile her baytın her iki basamağını da yazdırır. + +Örnek: + +**Örnek** + +Sorgu: + +``` sql +SELECT hex(1); +``` + +Sonuç: + +``` text +01 +``` + +Tip değerleri `Date` ve `DateTime` karşılık gelen tamsayılar olarak biçimlendirilir (tarih için çağdan bu yana geçen gün sayısı ve datetime için Unix zaman damgasının değeri). + +İçin `String` ve `FixedString`, tüm bayt sadece iki onaltılık sayı olarak kodlanır. Sıfır bayt ihmal edilmez. + +Kayan nokta ve ondalık türlerinin değerleri, bellekteki gösterimi olarak kodlanır. Küçük endian mimarisini desteklediğimiz için, bunlar küçük endian'da kodlanmıştır. Sıfır önde gelen / sondaki bayt ihmal edilmez. + +**Parametre** + +- `arg` — A value to convert to hexadecimal. Types: [Dize](../../sql_reference/data_types/string.md), [Uİnt](../../sql_reference/data_types/int_uint.md), [Yüzdürmek](../../sql_reference/data_types/float.md), [Ondalık](../../sql_reference/data_types/decimal.md), [Tarihli](../../sql_reference/data_types/date.md) veya [DateTime](../../sql_reference/data_types/datetime.md). + +**Döndürülen değer** + +- Bağımsız değişken onaltılık gösterimi ile bir dize. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT hex(toFloat32(number)) as hex_presentation FROM numbers(15, 2); +``` + +Sonuç: + +``` text +┌─hex_presentation─┐ +│ 00007041 │ +│ 00008041 │ +└──────────────────┘ +``` + +Sorgu: + +``` sql +SELECT hex(toFloat64(number)) as hex_presentation FROM numbers(15, 2); +``` + +Sonuç: + +``` text +┌─hex_presentation─┐ +│ 0000000000002E40 │ +│ 0000000000003040 │ +└──────────────────┘ +``` + +## unhex (str) {#unhexstr} + +Onaltılık basamak herhangi bir sayıda içeren bir dize kabul eder ve karşılık gelen bayt içeren bir dize döndürür. Hem büyük hem de küçük harfleri destekler a-F. onaltılık basamak sayısı bile olmak zorunda değildir. Tek ise, son rakam 00-0F baytın en az önemli yarısı olarak yorumlanır. Bağımsız değişken dizesi onaltılık basamaklardan başka bir şey içeriyorsa, uygulama tanımlı bazı sonuçlar döndürülür (bir özel durum atılmaz). +Sonucu bir sayıya dönüştürmek istiyorsanız, ‘reverse’ ve ‘reinterpretAsType’ işlevler. + +## UUİDStringToNum (str) {#uuidstringtonumstr} + +Biçiminde 36 karakter içeren bir dize kabul eder `123e4567-e89b-12d3-a456-426655440000` ve bir fixedstring(16) bayt kümesi olarak döndürür. + +## UUİDNumToString (str) {#uuidnumtostringstr} + +FixedString(16) değerini kabul eder. Metin biçiminde 36 karakter içeren bir dize döndürür. + +## bitmaskToList (num) {#bitmasktolistnum} + +Bir tamsayı kabul eder. Özetlendiğinde kaynak sayısını toplayan iki güç listesini içeren bir dize döndürür. Artan düzende metin biçiminde boşluk bırakmadan virgülle ayrılırlar. + +## bitmaskToArray (num) {#bitmasktoarraynum} + +Bir tamsayı kabul eder. Özetlendiğinde kaynak sayısını toplayan iki güç listesini içeren bir uint64 sayı dizisi döndürür. Dizideki sayılar artan düzendedir. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/encoding_functions/) diff --git a/docs/tr/sql_reference/functions/ext_dict_functions.md b/docs/tr/sql_reference/functions/ext_dict_functions.md new file mode 100644 index 00000000000..564ec9d5df0 --- /dev/null +++ b/docs/tr/sql_reference/functions/ext_dict_functions.md @@ -0,0 +1,205 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 58 +toc_title: "Harici S\xF6zl\xFCklerle \xE7al\u0131\u015Fma" +--- + +# Harici Sözlüklerle Çalışmak İçin İşlevler {#ext_dict_functions} + +Dış sözlükleri bağlama ve yapılandırma hakkında bilgi için bkz. [Dış söz dictionarieslükler](../../sql_reference/dictionaries/external_dictionaries/external_dicts.md). + +## dictGet {#dictget} + +Harici bir sözlükten bir değer alır. + +``` sql +dictGet('dict_name', 'attr_name', id_expr) +dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) +``` + +**Parametre** + +- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). +- `attr_name` — Name of the column of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). +- `id_expr` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql_reference/data_types/int_uint.md) veya [Demet](../../sql_reference/data_types/tuple.md)- sözlük yapılandırmasına bağlı olarak değer yazın. +- `default_value_expr` — Value returned if the dictionary doesn't contain a row with the `id_expr` anahtar. [İfade](../syntax.md#syntax-expressions) veri türü için yapılandırılmış değeri döndürme `attr_name` öznitelik. + +**Döndürülen değer** + +- ClickHouse özniteliği başarıyla ayrıştırırsa [öznitelik veri türü](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes), fonksiyonlar karşılık gelen sözlük özniteliğinin değerini döndürür `id_expr`. + +- Anahtar yoksa, karşılık gelen `id_expr`, söz thelükte, sonra: + + - `dictGet` returns the content of the `` element specified for the attribute in the dictionary configuration. + - `dictGetOrDefault` returns the value passed as the `default_value_expr` parameter. + +Clickhouse, özniteliğin değerini ayrıştıramazsa veya değer öznitelik veri türüyle eşleşmiyorsa bir özel durum atar. + +**Örnek** + +Metin dosyası oluşturma `ext-dict-text.csv` aşağıdakileri içeren: + +``` text +1,1 +2,2 +``` + +İlk sütun `id` ikinci sütun `c1`. + +Dış sözlüğü yapılandırma: + +``` xml + + + ext-dict-test + + + /path-to/ext-dict-test.csv + CSV + + + + + + + + id + + + c1 + UInt32 + + + + 0 + + +``` + +Sorguyu gerçekleştir: + +``` sql +SELECT + dictGetOrDefault('ext-dict-test', 'c1', number + 1, toUInt32(number * 10)) AS val, + toTypeName(val) AS type +FROM system.numbers +LIMIT 3 +``` + +``` text +┌─val─┬─type───┐ +│ 1 │ UInt32 │ +│ 2 │ UInt32 │ +│ 20 │ UInt32 │ +└─────┴────────┘ +``` + +**Ayrıca Bakınız** + +- [Dış Söz Dictionarieslükler](../../sql_reference/dictionaries/external_dictionaries/external_dicts.md) + +## dictHas {#dicthas} + +Bir anahtar sözlükte mevcut olup olmadığını denetler. + +``` sql +dictHas('dict_name', id_expr) +``` + +**Parametre** + +- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). +- `id_expr` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip değeri. + +**Döndürülen değer** + +- 0, anahtar yoksa. +- 1, bir anahtar varsa. + +Tür: `UInt8`. + +## dictGetHierarchy {#dictgethierarchy} + +Bir anahtarın tüm ebeveynlerini içeren bir dizi oluşturur. [hiyerarş dictionaryik sözlük](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md). + +**Sözdizimi** + +``` sql +dictGetHierarchy('dict_name', key) +``` + +**Parametre** + +- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). +- `key` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip değeri. + +**Döndürülen değer** + +- Anahtar için ebeveynler. + +Tür: [Dizi (Uİnt64)](../../sql_reference/data_types/array.md). + +## dictİsİn {#dictisin} + +Sözlükteki tüm hiyerarşik zincir boyunca bir anahtarın atasını kontrol eder. + +``` sql +dictIsIn('dict_name', child_id_expr, ancestor_id_expr) +``` + +**Parametre** + +- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). +- `child_id_expr` — Key to be checked. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip değeri. +- `ancestor_id_expr` — Alleged ancestor of the `child_id_expr` anahtar. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip değeri. + +**Döndürülen değer** + +- 0, eğer `child_id_expr` bir çocuk değil mi `ancestor_id_expr`. +- 1, Eğer `child_id_expr` bir çocuk `ancestor_id_expr` veya eğer `child_id_expr` is an `ancestor_id_expr`. + +Tür: `UInt8`. + +## Diğer Fonksiyonlar {#ext_dict_functions-other} + +ClickHouse sözlük yapılandırma ne olursa olsun belirli bir veri türü için sözlük öznitelik değerlerini dönüştürmek özel işlevleri destekler. + +İşlevler: + +- `dictGetInt8`, `dictGetInt16`, `dictGetInt32`, `dictGetInt64` +- `dictGetUInt8`, `dictGetUInt16`, `dictGetUInt32`, `dictGetUInt64` +- `dictGetFloat32`, `dictGetFloat64` +- `dictGetDate` +- `dictGetDateTime` +- `dictGetUUID` +- `dictGetString` + +Tüm bu işlevler `OrDefault` değişiklik. Mesela, `dictGetDateOrDefault`. + +Sözdizimi: + +``` sql +dictGet[Type]('dict_name', 'attr_name', id_expr) +dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) +``` + +**Parametre** + +- `dict_name` — Name of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). +- `attr_name` — Name of the column of the dictionary. [String lit literal](../syntax.md#syntax-string-literal). +- `id_expr` — Key value. [İfade](../syntax.md#syntax-expressions) dönen bir [Uİnt64](../../sql_reference/data_types/int_uint.md)- tip değeri. +- `default_value_expr` — Value which is returned if the dictionary doesn't contain a row with the `id_expr` anahtar. [İfade](../syntax.md#syntax-expressions) veri türü için yapılandırılmış bir değer döndürme `attr_name` öznitelik. + +**Döndürülen değer** + +- ClickHouse özniteliği başarıyla ayrıştırırsa [öznitelik veri türü](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md#ext_dict_structure-attributes), fonksiyonlar karşılık gelen sözlük özniteliğinin değerini döndürür `id_expr`. + +- İsten nomiyorsa `id_expr` söz thelükte o zaman: + + - `dictGet[Type]` returns the content of the `` element specified for the attribute in the dictionary configuration. + - `dictGet[Type]OrDefault` returns the value passed as the `default_value_expr` parameter. + +Clickhouse, özniteliğin değerini ayrıştıramazsa veya değer öznitelik veri türüyle eşleşmiyorsa bir özel durum atar. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/ext_dict_functions/) diff --git a/docs/tr/sql_reference/functions/functions_for_nulls.md b/docs/tr/sql_reference/functions/functions_for_nulls.md new file mode 100644 index 00000000000..59567067c42 --- /dev/null +++ b/docs/tr/sql_reference/functions/functions_for_nulls.md @@ -0,0 +1,312 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 63 +toc_title: "Null arg\xFCmanlarla \xE7al\u0131\u015Fma" +--- + +# Null agregalarla çalışmak için işlevler {#functions-for-working-with-nullable-aggregates} + +## isNull {#isnull} + +Bağımsız değişken olup olmadığını denetler [NULL](../syntax.md#null). + +``` sql +isNull(x) +``` + +**Parametre** + +- `x` — A value with a non-compound data type. + +**Döndürülen değer** + +- `1` eğer `x` oluyor `NULL`. +- `0` eğer `x` değildir `NULL`. + +**Örnek** + +Giriş tablosu + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Sorgu + +``` sql +SELECT x FROM t_null WHERE isNull(y) +``` + +``` text +┌─x─┐ +│ 1 │ +└───┘ +``` + +## isNotNull {#isnotnull} + +Bağımsız değişken olup olmadığını denetler [NULL](../syntax.md#null). + +``` sql +isNotNull(x) +``` + +**Parametre:** + +- `x` — A value with a non-compound data type. + +**Döndürülen değer** + +- `0` eğer `x` oluyor `NULL`. +- `1` eğer `x` değildir `NULL`. + +**Örnek** + +Giriş tablosu + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Sorgu + +``` sql +SELECT x FROM t_null WHERE isNotNull(y) +``` + +``` text +┌─x─┐ +│ 2 │ +└───┘ +``` + +## birleşmek {#coalesce} + +Olup olmadığını soldan sağa denetler `NULL` argümanlar geçti ve ilk olmayan döndürür-`NULL` değişken. + +``` sql +coalesce(x,...) +``` + +**Parametre:** + +- Bileşik olmayan tipte herhangi bir sayıda parametre. Tüm parametreler veri türüne göre uyumlu olmalıdır. + +**Döndürülen değerler** + +- İlk sigara-`NULL` değişken. +- `NULL`, eğer tüm argümanlar `NULL`. + +**Örnek** + +Bir müşteriyle iletişim kurmak için birden çok yol belirtebilecek kişilerin listesini düşünün. + +``` text +┌─name─────┬─mail─┬─phone─────┬──icq─┐ +│ client 1 │ ᴺᵁᴸᴸ │ 123-45-67 │ 123 │ +│ client 2 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ +└──────────┴──────┴───────────┴──────┘ +``` + +Bu `mail` ve `phone` alanlar String tip ofindedir, ancak `icq` Fi fieldeld is `UInt32`, bu yüzden dönüştürülmesi gerekiyor `String`. + +Müşteri için ilk kullanılabilir iletişim yöntemini kişi listesinden alın: + +``` sql +SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook +``` + +``` text +┌─name─────┬─coalesce(mail, phone, CAST(icq, 'Nullable(String)'))─┐ +│ client 1 │ 123-45-67 │ +│ client 2 │ ᴺᵁᴸᴸ │ +└──────────┴──────────────────────────────────────────────────────┘ +``` + +## ifNull {#ifnull} + +Ana bağımsız değişken ise alternatif bir değer döndürür `NULL`. + +``` sql +ifNull(x,alt) +``` + +**Parametre:** + +- `x` — The value to check for `NULL`. +- `alt` — The value that the function returns if `x` oluyor `NULL`. + +**Döndürülen değerler** + +- Değer `x`, eğer `x` değildir `NULL`. +- Değer `alt`, eğer `x` oluyor `NULL`. + +**Örnek** + +``` sql +SELECT ifNull('a', 'b') +``` + +``` text +┌─ifNull('a', 'b')─┐ +│ a │ +└──────────────────┘ +``` + +``` sql +SELECT ifNull(NULL, 'b') +``` + +``` text +┌─ifNull(NULL, 'b')─┐ +│ b │ +└───────────────────┘ +``` + +## nullİf {#nullif} + +Dönüşler `NULL` argümanlar eşitse. + +``` sql +nullIf(x, y) +``` + +**Parametre:** + +`x`, `y` — Values for comparison. They must be compatible types, or ClickHouse will generate an exception. + +**Döndürülen değerler** + +- `NULL`, argümanlar eşitse. +- Bu `x` bağımsız değişkenler eşit değilse, değer. + +**Örnek** + +``` sql +SELECT nullIf(1, 1) +``` + +``` text +┌─nullIf(1, 1)─┐ +│ ᴺᵁᴸᴸ │ +└──────────────┘ +``` + +``` sql +SELECT nullIf(1, 2) +``` + +``` text +┌─nullIf(1, 2)─┐ +│ 1 │ +└──────────────┘ +``` + +## assumeNotNull {#assumenotnull} + +Bir tür değeri ile sonuçlanır [Nullable](../../sql_reference/data_types/nullable.md) bir sigara için- `Nullable` eğer değer değil `NULL`. + +``` sql +assumeNotNull(x) +``` + +**Parametre:** + +- `x` — The original value. + +**Döndürülen değerler** + +- Olmayan orijinal değeri-`Nullable` tipi, değilse `NULL`. +- Olmayan için varsayılan değer-`Nullable` özgün değer ise yazın `NULL`. + +**Örnek** + +Düşünün `t_null` Tablo. + +``` sql +SHOW CREATE TABLE t_null +``` + +``` text +┌─statement─────────────────────────────────────────────────────────────────┐ +│ CREATE TABLE default.t_null ( x Int8, y Nullable(Int8)) ENGINE = TinyLog │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Uygula `assumeNotNull` fonksiyonu için `y` sütun. + +``` sql +SELECT assumeNotNull(y) FROM t_null +``` + +``` text +┌─assumeNotNull(y)─┐ +│ 0 │ +│ 3 │ +└──────────────────┘ +``` + +``` sql +SELECT toTypeName(assumeNotNull(y)) FROM t_null +``` + +``` text +┌─toTypeName(assumeNotNull(y))─┐ +│ Int8 │ +│ Int8 │ +└──────────────────────────────┘ +``` + +## toNullable {#tonullable} + +Bağımsız değişken türünü dönüştürür `Nullable`. + +``` sql +toNullable(x) +``` + +**Parametre:** + +- `x` — The value of any non-compound type. + +**Döndürülen değer** + +- Bir ile giriş değeri `Nullable` tür. + +**Örnek** + +``` sql +SELECT toTypeName(10) +``` + +``` text +┌─toTypeName(10)─┐ +│ UInt8 │ +└────────────────┘ +``` + +``` sql +SELECT toTypeName(toNullable(10)) +``` + +``` text +┌─toTypeName(toNullable(10))─┐ +│ Nullable(UInt8) │ +└────────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/functions_for_nulls/) diff --git a/docs/tr/sql_reference/functions/geo.md b/docs/tr/sql_reference/functions/geo.md new file mode 100644 index 00000000000..353418c7b97 --- /dev/null +++ b/docs/tr/sql_reference/functions/geo.md @@ -0,0 +1,510 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 62 +toc_title: "Co\u011Frafi koordinatlar ile \xE7al\u0131\u015Fma" +--- + +# Coğrafi Koordinatlarla çalışmak için fonksiyonlar {#functions-for-working-with-geographical-coordinates} + +## greatCircleDistance {#greatcircledistance} + +Dünya yüzeyindeki iki nokta arasındaki mesafeyi kullanarak hesaplayın [büyük daire formülü](https://en.wikipedia.org/wiki/Great-circle_distance). + +``` sql +greatCircleDistance(lon1Deg, lat1Deg, lon2Deg, lat2Deg) +``` + +**Giriş parametreleri** + +- `lon1Deg` — Longitude of the first point in degrees. Range: `[-180°, 180°]`. +- `lat1Deg` — Latitude of the first point in degrees. Range: `[-90°, 90°]`. +- `lon2Deg` — Longitude of the second point in degrees. Range: `[-180°, 180°]`. +- `lat2Deg` — Latitude of the second point in degrees. Range: `[-90°, 90°]`. + +Pozitif değerler Kuzey enlemi ve Doğu boylamına karşılık gelir ve negatif değerler Güney enlemi ve Batı boylamına karşılık gelir. + +**Döndürülen değer** + +Dünya yüzeyindeki iki nokta arasındaki mesafe, metre cinsinden. + +Girdi parametre değerleri aralığın dışına düştüğünde bir özel durum oluşturur. + +**Örnek** + +``` sql +SELECT greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673) +``` + +``` text +┌─greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673)─┐ +│ 14132374.194975413 │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## pointİnEllipses {#pointinellipses} + +Noktanın elipslerden en az birine ait olup olmadığını kontrol eder. +Koordinatlar kartezyen koordinat sisteminde geometriktir. + +``` sql +pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) +``` + +**Giriş parametreleri** + +- `x, y` — Coordinates of a point on the plane. +- `xᵢ, yᵢ` — Coordinates of the center of the `i`-inci üç nokta. +- `aᵢ, bᵢ` — Axes of the `i`- x, y koordinatları birimlerinde üç nokta. + +Giriş parametreleri olmalıdır `2+4⋅n`, nere `n` elips sayısıdır. + +**Döndürülen değerler** + +`1` nokta elipslerden en az birinin içindeyse; `0`hayır değil. + +**Örnek** + +``` sql +SELECT pointInEllipses(10., 10., 10., 9.1, 1., 0.9999) +``` + +``` text +┌─pointInEllipses(10., 10., 10., 9.1, 1., 0.9999)─┐ +│ 1 │ +└─────────────────────────────────────────────────┘ +``` + +## pointİnPolygon {#pointinpolygon} + +Noktanın düzlemdeki poligona ait olup olmadığını kontrol eder. + +``` sql +pointInPolygon((x, y), [(a, b), (c, d) ...], ...) +``` + +**Giriş değerleri** + +- `(x, y)` — Coordinates of a point on the plane. Data type — [Demet](../../sql_reference/data_types/tuple.md) — A tuple of two numbers. +- `[(a, b), (c, d) ...]` — Polygon vertices. Data type — [Dizi](../../sql_reference/data_types/array.md). Her köşe bir çift koordinat ile temsil edilir `(a, b)`. Köşeler saat yönünde veya saat yönünün tersine sırayla belirtilmelidir. Minimum köşe sayısı 3'tür. Çokgen sabit olmalıdır. +- Fonksiyon ayrıca delikli çokgenleri de destekler (bölümleri keser). Bu durumda, işlevin ek argümanlarını kullanarak kesilen bölümleri tanımlayan çokgenler ekleyin. İşlev, basit olmayan bağlı çokgenleri desteklemez. + +**Döndürülen değerler** + +`1` nokta çokgenin içinde ise, `0` hayır değil. +Nokta çokgen sınırında ise, işlev 0 veya 1 döndürebilir. + +**Örnek** + +``` sql +SELECT pointInPolygon((3., 3.), [(6, 0), (8, 4), (5, 8), (0, 2)]) AS res +``` + +``` text +┌─res─┐ +│ 1 │ +└─────┘ +``` + +## geohashEncode {#geohashencode} + +Enlem ve boylamı bir geohash-string olarak kodlar, Lütfen bakınız (http://geohash.org/, https://en.wikipedia.org/wiki/Geohash). + +``` sql +geohashEncode(longitude, latitude, [precision]) +``` + +**Giriş değerleri** + +- boylam-kodlamak istediğiniz koordinatın boylam kısmı. Aralık floatingta yüz floatingen`[-180°, 180°]` +- latitude-kodlamak istediğiniz koordinatın enlem kısmı. Aralık floatingta yüz floatingen `[-90°, 90°]` +- hassas-isteğe bağlı, elde edilen kodlanmış dizenin uzunluğu, varsayılan olarak `12`. Aralıktaki tamsayı `[1, 12]`. Herhangi bir değer daha az `1` veya daha büyük `12` sessizce dönüştürülür `12`. + +**Döndürülen değerler** + +- alfanümerik `String` kodlanmış koordinat (base32-kodlama alfabesinin değiştirilmiş versiyonu kullanılır). + +**Örnek** + +``` sql +SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res +``` + +``` text +┌─res──────────┐ +│ ezs42d000000 │ +└──────────────┘ +``` + +## geohashDecode {#geohashdecode} + +Herhangi bir geohash kodlu dizeyi boylam ve enlem olarak çözer. + +**Giriş değerleri** + +- kodlanmış dize-geohash kodlanmış dize. + +**Döndürülen değerler** + +- (boylam, enlem) - 2-tuple `Float64` boylam ve enlem değerleri. + +**Örnek** + +``` sql +SELECT geohashDecode('ezs42') AS res +``` + +``` text +┌─res─────────────────────────────┐ +│ (-5.60302734375,42.60498046875) │ +└─────────────────────────────────┘ +``` + +## geoToH3 {#geotoh3} + +Dönüşler [H3](https://uber.github.io/h3/#/documentation/overview/introduction) nokta Endeksi `(lon, lat)` belirtilen çözünürlük ile. + +[H3](https://uber.github.io/h3/#/documentation/overview/introduction) Dünya yüzeyinin altıgen fayanslara bile bölündüğü coğrafi bir indeksleme sistemidir. Bu sistem hiyerarşiktir, örn. üst seviyedeki her altıgen yedi hatta daha küçük olanlara bölünebilir. + +Bu endeks öncelikle kovalama yerleri ve diğer coğrafi manipülasyonlar için kullanılır. + +**Sözdizimi** + +``` sql +geoToH3(lon, lat, resolution) +``` + +**Parametre** + +- `lon` — Longitude. Type: [Float64](../../sql_reference/data_types/float.md). +- `lat` — Latitude. Type: [Float64](../../sql_reference/data_types/float.md). +- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- Altıgen dizin numarası. +- Hata durumunda 0. + +Tür: `UInt64`. + +**Örnek** + +Sorgu: + +``` sql +SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index +``` + +Sonuç: + +``` text +┌────────────h3Index─┐ +│ 644325524701193974 │ +└────────────────────┘ +``` + +## geohashesİnBox {#geohashesinbox} + +Verilen kutunun içine giren ve verilen kutunun sınırlarını kesişen, temel olarak diziye düzleştirilmiş bir 2D ızgarası olan bir dizi geohash kodlu dizeler dizisi döndürür. + +**Giriş değerleri** + +- longitude\_min - min boylam, aralıkta kayan değer `[-180°, 180°]` +- latitude\_min - min enlem, aralıkta kayan değer `[-90°, 90°]` +- longitude\_max-maksimum boylam, aralıkta kayan değer `[-180°, 180°]` +- latitude\_max-maksimum enlem, aralıkta kayan değer `[-90°, 90°]` +- hassas-geohash hassas, `UInt8` Aralık inta `[1, 12]` + +Lütfen tüm koordinat parametrelerinin aynı tipte olması gerektiğini unutmayın: `Float32` veya `Float64`. + +**Döndürülen değerler** + +- verilen alanı kapsayan geohash kutularının hassas uzun dizeleri dizisi, öğelerin sırasına güvenmemelisiniz. +- \[\]- eğer boş dizi *dakika* değerleri *enlem* ve *Boylam* karşılık gelenden daha az değil *maksimum* değerler. + +Ortaya çıkan dizi 10'000' 000 ürün uzunluğundaysa, işlevin bir istisna atacağını lütfen unutmayın. + +**Örnek** + +``` sql +SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos +``` + +``` text +┌─thasos──────────────────────────────────────┐ +│ ['sx1q','sx1r','sx32','sx1w','sx1x','sx38'] │ +└─────────────────────────────────────────────┘ +``` + +## h3GetBaseCell {#h3getbasecell} + +Dizin temel hücre numarasını döndürür. + +**Sözdizimi** + +``` sql +h3GetBaseCell(index) +``` + +**Parametre** + +- `index` — Hexagon index number. Type: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- Altıgen baz hücre numarası. Tür: [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT h3GetBaseCell(612916788725809151) as basecell +``` + +Sonuç: + +``` text +┌─basecell─┐ +│ 12 │ +└──────────┘ +``` + +## h3HexAreaM2 {#h3hexaream2} + +Verilen çözünürlükte metrekare ortalama altıgen alan. + +**Sözdizimi** + +``` sql +h3HexAreaM2(resolution) +``` + +**Parametre** + +- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- Area in m². Type: [Float64](../../sql_reference/data_types/float.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT h3HexAreaM2(13) as area +``` + +Sonuç: + +``` text +┌─area─┐ +│ 43.9 │ +└──────┘ +``` + +## h3İndexesAreNeighbors {#h3indexesareneighbors} + +Sağlanan H3indexlerin komşu olup olmadığını döndürür. + +**Sözdizimi** + +``` sql +h3IndexesAreNeighbors(index1, index2) +``` + +**Parametre** + +- `index1` — Hexagon index number. Type: [Uİnt64](../../sql_reference/data_types/int_uint.md). +- `index2` — Hexagon index number. Type: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- Dönüşler `1` dizinler komşu ise, `0` başka. Tür: [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT h3IndexesAreNeighbors(617420388351344639, 617420388352655359) AS n +``` + +Sonuç: + +``` text +┌─n─┐ +│ 1 │ +└───┘ +``` + +## h3ToChildren {#h3tochildren} + +Verilen dizinin alt dizinlerini içeren bir dizi döndürür. + +**Sözdizimi** + +``` sql +h3ToChildren(index, resolution) +``` + +**Parametre** + +- `index` — Hexagon index number. Type: [Uİnt64](../../sql_reference/data_types/int_uint.md). +- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- Alt H3 dizinleri ile dizi. Dizi türü: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT h3ToChildren(599405990164561919, 6) AS children +``` + +Sonuç: + +``` text +┌─children───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ [603909588852408319,603909588986626047,603909589120843775,603909589255061503,603909589389279231,603909589523496959,603909589657714687] │ +└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## h3ToParent {#h3toparent} + +Verilen dizini içeren üst (kaba) dizini döndürür. + +**Sözdizimi** + +``` sql +h3ToParent(index, resolution) +``` + +**Parametre** + +- `index` — Hexagon index number. Type: [Uİnt64](../../sql_reference/data_types/int_uint.md). +- `resolution` — Index resolution. Range: `[0, 15]`. Tür: [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- Ana H3 Endeksi. Tür: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT h3ToParent(599405990164561919, 3) as parent +``` + +Sonuç: + +``` text +┌─────────────parent─┐ +│ 590398848891879423 │ +└────────────────────┘ +``` + +## h3ToString {#h3tostring} + +Dizinin H3ındex gösterimini dize gösterimine dönüştürür. + +``` sql +h3ToString(index) +``` + +**Parametre** + +- `index` — Hexagon index number. Type: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- H3 dizininin dize gösterimi. Tür: [Dize](../../sql_reference/data_types/string.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT h3ToString(617420388352917503) as h3_string +``` + +Sonuç: + +``` text +┌─h3_string───────┐ +│ 89184926cdbffff │ +└─────────────────┘ +``` + +## stringToH3 {#stringtoh3} + +Dize gösterimini H3ındex (Uİnt64) gösterimine dönüştürür. + +``` sql +stringToH3(index_str) +``` + +**Parametre** + +- `index_str` — String representation of the H3 index. Type: [Dize](../../sql_reference/data_types/string.md). + +**Döndürülen değerler** + +- Altıgen dizin numarası. Hata 0 döndürür. Tür: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT stringToH3('89184926cc3ffff') as index +``` + +Sonuç: + +``` text +┌──────────────index─┐ +│ 617420388351344639 │ +└────────────────────┘ +``` + +## h3GetResolution {#h3getresolution} + +Dizin çözünürlüğünü döndürür. + +**Sözdizimi** + +``` sql +h3GetResolution(index) +``` + +**Parametre** + +- `index` — Hexagon index number. Type: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değerler** + +- Dizin çözünürlüğü. Aralık: `[0, 15]`. Tür: [Uİnt8](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT h3GetResolution(617420388352917503) as res +``` + +Sonuç: + +``` text +┌─res─┐ +│ 9 │ +└─────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/geo/) diff --git a/docs/tr/sql_reference/functions/hash_functions.md b/docs/tr/sql_reference/functions/hash_functions.md new file mode 100644 index 00000000000..fbb21597943 --- /dev/null +++ b/docs/tr/sql_reference/functions/hash_functions.md @@ -0,0 +1,446 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 50 +toc_title: Karma +--- + +# Karma fonksiyonlar {#hash-functions} + +Hash fonksiyonları elementlerin deterministik sözde rastgele karıştırma için kullanılabilir. + +## halfMD5 {#hash-functions-halfmd5} + +[Yorumluyor](../../sql_reference/functions/type_conversion_functions.md#type_conversion_functions-reinterpretAsString) tüm giriş parametrelerini dizeler olarak hesaplar ve [MD5](https://en.wikipedia.org/wiki/MD5) her biri için karma değeri. Sonra karmaları birleştirir, elde edilen dizenin karmasının ilk 8 baytını alır ve bunları şöyle yorumlar `UInt64` büyük endian bayt sırasına göre. + +``` sql +halfMD5(par1, ...) +``` + +İşlev nispeten yavaştır (işlemci çekirdeği başına saniyede 5 milyon kısa dizge). +Kullanmayı düşünün [sifash64](#hash_functions-siphash64) bunun yerine işlev. + +**Parametre** + +Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql_reference/data_types/index.md). + +**Döndürülen Değer** + +A [Uİnt64](../../sql_reference/data_types/int_uint.md) veri türü karma değeri. + +**Örnek** + +``` sql +SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type +``` + +``` text +┌────────halfMD5hash─┬─type───┐ +│ 186182704141653334 │ UInt64 │ +└────────────────────┴────────┘ +``` + +## MD5 {#hash_functions-md5} + +MD5 bir dizeden hesaplar ve elde edilen bayt kümesini FixedString(16) olarak döndürür. +Özellikle MD5'E ihtiyacınız yoksa, ancak iyi bir şifreleme 128 bit karmasına ihtiyacınız varsa, ‘sipHash128’ bunun yerine işlev. +Md5sum yardımcı programı tarafından çıktı ile aynı sonucu elde etmek istiyorsanız, lower(hex(MD5(s))) kullanın. + +## sifash64 {#hash_functions-siphash64} + +64-bit üretir [Sifash](https://131002.net/siphash/) karma değeri. + +``` sql +sipHash64(par1,...) +``` + +Bu bir şifreleme karma işlevidir. En az üç kat daha hızlı çalışır [MD5](#hash_functions-md5) işlev. + +İşlev [yorumluyor](../../sql_reference/functions/type_conversion_functions.md#type_conversion_functions-reinterpretAsString) tüm giriş parametreleri dizeleri olarak ve bunların her biri için karma değerini hesaplar. Sonra aşağıdaki algoritma ile karmaları birleştirir: + +1. Tüm giriş parametrelerini karma yaptıktan sonra, işlev karma dizisini alır. +2. Fonksiyon birinci ve ikinci öğeleri alır ve bunların dizisi için bir karma hesaplar. +3. Daha sonra işlev, önceki adımda hesaplanan karma değeri ve ilk karma dizinin üçüncü öğesini alır ve bunların dizisi için bir karma hesaplar. +4. Önceki adım, ilk karma dizinin kalan tüm öğeleri için tekrarlanır. + +**Parametre** + +Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql_reference/data_types/index.md). + +**Döndürülen Değer** + +A [Uİnt64](../../sql_reference/data_types/int_uint.md) veri türü karma değeri. + +**Örnek** + +``` sql +SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type +``` + +``` text +┌──────────────SipHash─┬─type───┐ +│ 13726873534472839665 │ UInt64 │ +└──────────────────────┴────────┘ +``` + +## sifash128 {#hash_functions-siphash128} + +Bir dizeden Sifash hesaplar. +Bir dize türü bağımsız değişkeni kabul eder. Fixedstring(16) Döndürür. +Sifash64'ten farklıdır, çünkü son xor katlama durumu sadece 128 bit'e kadar yapılır. + +## cityHash64 {#cityhash64} + +64-bit üretir [CityHash](https://github.com/google/cityhash) karma değeri. + +``` sql +cityHash64(par1,...) +``` + +Bu hızlı olmayan şifreleme karma işlevidir. Dize parametreleri için CityHash algoritmasını ve diğer veri türleriyle parametreler için uygulamaya özgü hızlı kriptografik olmayan karma işlevini kullanır. İşlev, nihai sonuçları almak için CityHash birleştiricisini kullanır. + +**Parametre** + +Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql_reference/data_types/index.md). + +**Döndürülen Değer** + +A [Uİnt64](../../sql_reference/data_types/int_uint.md) veri türü karma değeri. + +**Örnekler** + +Çağrı örneği: + +``` sql +SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type +``` + +``` text +┌─────────────CityHash─┬─type───┐ +│ 12072650598913549138 │ UInt64 │ +└──────────────────────┴────────┘ +``` + +Aşağıdaki örnek, tüm tablonun sağlama toplamının satır sırasına kadar doğrulukla nasıl hesaplanacağını gösterir: + +``` sql +SELECT groupBitXor(cityHash64(*)) FROM table +``` + +## intHash32 {#inthash32} + +Herhangi bir tamsayı türünden 32 bit karma kodu hesaplar. +Bu, sayılar için ortalama kalitenin nispeten hızlı bir kriptografik olmayan karma işlevidir. + +## intHash64 {#inthash64} + +Herhangi bir tamsayı türünden 64 bit karma kodu hesaplar. +Inthash32'den daha hızlı çalışır. Ortalama kalite. + +## SHA1 {#sha1} + +## SHA224 {#sha224} + +## SHA256 {#sha256} + +Bir dizeden SHA-1, SHA-224 veya SHA-256 hesaplar ve elde edilen bayt kümesini FixedString(20), FixedString(28) veya FixedString(32) olarak döndürür. +İşlev oldukça yavaş çalışır (SHA-1, işlemci çekirdeği başına saniyede yaklaşık 5 milyon kısa dizgiyi işler, SHA-224 ve SHA-256 ise yaklaşık 2.2 milyon işlem yapar). +Bu işlevi yalnızca belirli bir karma işleve ihtiyacınız olduğunda ve bunu seçemediğinizde kullanmanızı öneririz. +Bu gibi durumlarda bile, SELECTS'TE uygulamak yerine, tabloya eklerken işlev çevrimdışı ve ön hesaplama değerlerini uygulamanızı öneririz. + +## URLHash(url \[, N\]) {#urlhashurl-n} + +Bir tür normalleştirme kullanarak bir URL'den elde edilen bir dize için hızlı, iyi kalitede olmayan şifreleme karma işlevi. +`URLHash(s)` – Calculates a hash from a string without one of the trailing symbols `/`,`?` veya `#` sonunda, varsa. +`URLHash(s, N)` – Calculates a hash from a string up to the N level in the URL hierarchy, without one of the trailing symbols `/`,`?` veya `#` sonunda, varsa. +Düzeyleri URLHierarchy aynıdır. Bu fonksiyon (kayıt olmak için özeldir.Metrica. + +## farmHash64 {#farmhash64} + +64-bit üretir [FarmHash](https://github.com/google/farmhash) karma değeri. + +``` sql +farmHash64(par1, ...) +``` + +Fonksiyonu kullanır `Hash64` tüm yöntem [mevcut yöntemler](https://github.com/google/farmhash/blob/master/src/farmhash.h). + +**Parametre** + +Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql_reference/data_types/index.md). + +**Döndürülen Değer** + +A [Uİnt64](../../sql_reference/data_types/int_uint.md) veri türü karma değeri. + +**Örnek** + +``` sql +SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type +``` + +``` text +┌─────────────FarmHash─┬─type───┐ +│ 17790458267262532859 │ UInt64 │ +└──────────────────────┴────────┘ +``` + +## javaHash {#hash_functions-javahash} + +Hesaplıyor [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) bir ipten. Bu karma işlevi ne hızlı ne de iyi bir kaliteye sahip değildir. Bunu kullanmanın tek nedeni, bu algoritmanın zaten başka bir sistemde kullanılmasıdır ve tam olarak aynı sonucu hesaplamanız gerekir. + +**Sözdizimi** + +``` sql +SELECT javaHash(''); +``` + +**Döndürülen değer** + +A `Int32` veri türü karma değeri. + +**Örnek** + +Sorgu: + +``` sql +SELECT javaHash('Hello, world!'); +``` + +Sonuç: + +``` text +┌─javaHash('Hello, world!')─┐ +│ -1880044555 │ +└───────────────────────────┘ +``` + +## javaHashUTF16LE {#javahashutf16le} + +Hesaplıyor [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) bir dizeden, UTF-16LE kodlamasında bir dizeyi temsil eden bayt içerdiğini varsayarak. + +**Sözdizimi** + +``` sql +javaHashUTF16LE(stringUtf16le) +``` + +**Parametre** + +- `stringUtf16le` — a string in UTF-16LE encoding. + +**Döndürülen değer** + +A `Int32` veri türü karma değeri. + +**Örnek** + +UTF-16LE kodlanmış dize ile doğru sorgu. + +Sorgu: + +``` sql +SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')) +``` + +Sonuç: + +``` text +┌─javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le'))─┐ +│ 3556498 │ +└──────────────────────────────────────────────────────────────┘ +``` + +## hiveHash {#hash-functions-hivehash} + +Hesaplıyor `HiveHash` bir ipten. + +``` sql +SELECT hiveHash(''); +``` + +Bu sadece [JavaHash](#hash_functions-javahash) sıfırlanmış işaret biti ile. Bu işlev kullanılır [Apache Kov Hanı](https://en.wikipedia.org/wiki/Apache_Hive) 3.0 öncesi sürümler için. Bu karma işlevi ne hızlı ne de iyi bir kaliteye sahip değildir. Bunu kullanmanın tek nedeni, bu algoritmanın zaten başka bir sistemde kullanılmasıdır ve tam olarak aynı sonucu hesaplamanız gerekir. + +**Döndürülen değer** + +A `Int32` veri türü karma değeri. + +Tür: `hiveHash`. + +**Örnek** + +Sorgu: + +``` sql +SELECT hiveHash('Hello, world!'); +``` + +Sonuç: + +``` text +┌─hiveHash('Hello, world!')─┐ +│ 267439093 │ +└───────────────────────────┘ +``` + +## metroHash64 {#metrohash64} + +64-bit üretir [MetroHash](http://www.jandrewrogers.com/2015/05/27/metrohash/) karma değeri. + +``` sql +metroHash64(par1, ...) +``` + +**Parametre** + +Fonksiyon, değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql_reference/data_types/index.md). + +**Döndürülen Değer** + +A [Uİnt64](../../sql_reference/data_types/int_uint.md) veri türü karma değeri. + +**Örnek** + +``` sql +SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type +``` + +``` text +┌────────────MetroHash─┬─type───┐ +│ 14235658766382344533 │ UInt64 │ +└──────────────────────┴────────┘ +``` + +## jumpConsistentHash {#jumpconsistenthash} + +Bir Uint64 Formu jumpconsistenthash hesaplar. +İki bağımsız değişkeni kabul eder: bir uint64 tipi anahtar ve kova sayısı. Int32 Döndürür. +Daha fazla bilgi için bağlantıya bakın: [JumpConsistentHash](https://arxiv.org/pdf/1406.2294.pdf) + +## murmurHash2\_32, murmurHash2\_64 {#murmurhash2-32-murmurhash2-64} + +Üreten bir [MurmurHash2](https://github.com/aappleby/smhasher) karma değeri. + +``` sql +murmurHash2_32(par1, ...) +murmurHash2_64(par1, ...) +``` + +**Parametre** + +Her iki işlev de değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql_reference/data_types/index.md). + +**Döndürülen Değer** + +- Bu `murmurHash2_32` fonksiyon hash değerini döndürür [Uİnt32](../../sql_reference/data_types/int_uint.md) veri türü. +- Bu `murmurHash2_64` fonksiyon hash değerini döndürür [Uİnt64](../../sql_reference/data_types/int_uint.md) veri türü. + +**Örnek** + +``` sql +SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type +``` + +``` text +┌──────────MurmurHash2─┬─type───┐ +│ 11832096901709403633 │ UInt64 │ +└──────────────────────┴────────┘ +``` + +## murmurHash3\_32, murmurHash3\_64 {#murmurhash3-32-murmurhash3-64} + +Üreten bir [MurmurHash3](https://github.com/aappleby/smhasher) karma değeri. + +``` sql +murmurHash3_32(par1, ...) +murmurHash3_64(par1, ...) +``` + +**Parametre** + +Her iki işlev de değişken sayıda giriş parametresi alır. Parametreler herhangi biri olabilir [desteklenen veri türleri](../../sql_reference/data_types/index.md). + +**Döndürülen Değer** + +- Bu `murmurHash3_32` fonksiyon bir [Uİnt32](../../sql_reference/data_types/int_uint.md) veri türü karma değeri. +- Bu `murmurHash3_64` fonksiyon bir [Uİnt64](../../sql_reference/data_types/int_uint.md) veri türü karma değeri. + +**Örnek** + +``` sql +SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type +``` + +``` text +┌─MurmurHash3─┬─type───┐ +│ 2152717 │ UInt32 │ +└─────────────┴────────┘ +``` + +## murmurHash3\_128 {#murmurhash3-128} + +128-bit üretir [MurmurHash3](https://github.com/aappleby/smhasher) karma değeri. + +``` sql +murmurHash3_128( expr ) +``` + +**Parametre** + +- `expr` — [İfadeler](../syntax.md#syntax-expressions) dönen bir [Dize](../../sql_reference/data_types/string.md)- tip değeri. + +**Döndürülen Değer** + +A [FixedString (16)](../../sql_reference/data_types/fixedstring.md) veri türü karma değeri. + +**Örnek** + +``` sql +SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type +``` + +``` text +┌─MurmurHash3──────┬─type────────────┐ +│ 6�1�4"S5KT�~~q │ FixedString(16) │ +└──────────────────┴─────────────────┘ +``` + +## xxHash32, xxHash64 {#hash-functions-xxhash32} + +Hesaplıyor `xxHash` bir ipten. İki tat, 32 ve 64 bit olarak önerilmiştir. + +``` sql +SELECT xxHash32(''); + +OR + +SELECT xxHash64(''); +``` + +**Döndürülen değer** + +A `Uint32` veya `Uint64` veri türü karma değeri. + +Tür: `xxHash`. + +**Örnek** + +Sorgu: + +``` sql +SELECT xxHash32('Hello, world!'); +``` + +Sonuç: + +``` text +┌─xxHash32('Hello, world!')─┐ +│ 834093149 │ +└───────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [xxHash](http://cyan4973.github.io/xxHash/). + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/hash_functions/) diff --git a/docs/tr/sql_reference/functions/higher_order_functions.md b/docs/tr/sql_reference/functions/higher_order_functions.md new file mode 100644 index 00000000000..e5faadc689a --- /dev/null +++ b/docs/tr/sql_reference/functions/higher_order_functions.md @@ -0,0 +1,264 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 57 +toc_title: "Y\xFCksek Sipari\u015F" +--- + +# Yüksek mertebeden fonksiyonlar {#higher-order-functions} + +## `->` operatör, lambda (params, expr) fonksiyonu {#operator-lambdaparams-expr-function} + +Allows describing a lambda function for passing to a higher-order function. The left side of the arrow has a formal parameter, which is any ID, or multiple formal parameters – any IDs in a tuple. The right side of the arrow has an expression that can use these formal parameters, as well as any table columns. + +Örnekler: `x -> 2 * x, str -> str != Referer.` + +Daha yüksek mertebeden işlevler yalnızca Lambda işlevlerini işlevsel argümanları olarak kabul edebilir. + +Birden çok bağımsız değişkeni kabul eden bir lambda işlevi, daha yüksek mertebeden bir işleve geçirilebilir. Bu durumda, yüksek mertebeden işlev, bu bağımsız değişkenlerin karşılık geleceği aynı uzunlukta birkaç diziden geçirilir. + +Gibi bazı işlevler için [arrayCount](#higher_order_functions-array-count) veya [arraySum](#higher_order_functions-array-count), ilk argüman (lambda işlevi) ihmal edilebilir. Bu durumda, aynı eşleme varsayılır. + +Aşağıdaki işlevler için bir lambda işlevi ihmal edilemez: + +- [arrayMap](#higher_order_functions-array-map) +- [arrayFilter](#higher_order_functions-array-filter) +- [arrayFill](#higher_order_functions-array-fill) +- [arrayReverseFill](#higher_order_functions-array-reverse-fill) +- [arraySplit](#higher_order_functions-array-split) +- [arrayReverseSplit](#higher_order_functions-array-reverse-split) +- [arrayFirst](#higher_order_functions-array-first) +- [arrayFirstİndex](#higher_order_functions-array-first-index) + +### arrayMap(func, arr1, …) {#higher_order_functions-array-map} + +Özgün uygulamadan elde edilen bir dizi döndürür `func` fonksiyon inunda her ele elementmana `arr` dizi. + +Örnekler: + +``` sql +SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res; +``` + +``` text +┌─res─────┐ +│ [3,4,5] │ +└─────────┘ +``` + +Aşağıdaki örnek, farklı dizilerden bir öğe kümesinin nasıl oluşturulacağını gösterir: + +``` sql +SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res +``` + +``` text +┌─res─────────────────┐ +│ [(1,4),(2,5),(3,6)] │ +└─────────────────────┘ +``` + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayMap` işlev. + +### arrayFilter(func, arr1, …) {#higher_order_functions-array-filter} + +Yalnızca öğeleri içeren bir dizi döndürür `arr1` hangi için `func` 0'dan başka bir şey döndürür. + +Örnekler: + +``` sql +SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res +``` + +``` text +┌─res───────────┐ +│ ['abc World'] │ +└───────────────┘ +``` + +``` sql +SELECT + arrayFilter( + (i, x) -> x LIKE '%World%', + arrayEnumerate(arr), + ['Hello', 'abc World'] AS arr) + AS res +``` + +``` text +┌─res─┐ +│ [2] │ +└─────┘ +``` + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFilter` işlev. + +### arrayFill(func, arr1, …) {#higher_order_functions-array-fill} + +Tarama yoluyla `arr1` ilk öğeden son öğeye ve değiştir `arr1[i]` tarafından `arr1[i - 1]` eğer `func` 0 döndürür. İlk eleman `arr1` değiştir notilm .eyecektir. + +Örnekler: + +``` sql +SELECT arrayFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res +``` + +``` text +┌─res──────────────────────────────┐ +│ [1,1,3,11,12,12,12,5,6,14,14,14] │ +└──────────────────────────────────┘ +``` + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFill` işlev. + +### arrayReverseFill(func, arr1, …) {#higher_order_functions-array-reverse-fill} + +Tarama yoluyla `arr1` son öğeden ilk öğeye ve değiştir `arr1[i]` tarafından `arr1[i + 1]` eğer `func` 0 döndürür. The La lastst element of `arr1` değiştir notilm .eyecektir. + +Örnekler: + +``` sql +SELECT arrayReverseFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res +``` + +``` text +┌─res────────────────────────────────┐ +│ [1,3,3,11,12,5,5,5,6,14,NULL,NULL] │ +└────────────────────────────────────┘ +``` + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayReverseFill` işlev. + +### arraySplit(func, arr1, …) {#higher_order_functions-array-split} + +Bölme `arr1` birden fazla diziye. Ne zaman `func` 0'dan başka bir şey döndürür, dizi öğenin sol tarafında bölünecektir. Dizi ilk öğeden önce bölünmez. + +Örnekler: + +``` sql +SELECT arraySplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res +``` + +``` text +┌─res─────────────┐ +│ [[1,2,3],[4,5]] │ +└─────────────────┘ +``` + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arraySplit` işlev. + +### arrayReverseSplit(func, arr1, …) {#higher_order_functions-array-reverse-split} + +Bölme `arr1` birden fazla diziye. Ne zaman `func` 0'dan başka bir şey döndürür, dizi öğenin sağ tarafında bölünecektir. Dizi son öğeden sonra bölünmez. + +Örnekler: + +``` sql +SELECT arrayReverseSplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res +``` + +``` text +┌─res───────────────┐ +│ [[1],[2,3,4],[5]] │ +└───────────────────┘ +``` + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arraySplit` işlev. + +### arrayCount(\[func,\] arr1, …) {#higher_order_functions-array-count} + +Func 0'dan başka bir şey döndüren arr dizisindeki öğelerin sayısını döndürür. Eğer ‘func’ belirtilmemişse, dizideki sıfır olmayan öğelerin sayısını döndürür. + +### arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1} + +İçinde en az bir öğe varsa 1 değerini döndürür ‘arr’ hangi için ‘func’ 0'dan başka bir şey döndürür. Aksi takdirde, 0 döndürür. + +### arrayAll(\[func,\] arr1, …) {#arrayallfunc-arr1} + +Döner 1 Eğer ‘func’ içindeki tüm öğeler için 0'dan başka bir şey döndürür ‘arr’. Aksi takdirde, 0 döndürür. + +### arraySum(\[func,\] arr1, …) {#higher-order-functions-array-sum} + +Toplamını döndürür ‘func’ değerler. İşlev atlanırsa, sadece dizi öğelerinin toplamını döndürür. + +### arrayFirst(func, arr1, …) {#higher_order_functions-array-first} + +İlk öğeyi döndürür ‘arr1’ dizi hangi ‘func’ 0'dan başka bir şey döndürür. + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFirst` işlev. + +### arrayFirstIndex(func, arr1, …) {#higher_order_functions-array-first-index} + +İlk öğenin dizinini döndürür ‘arr1’ dizi hangi ‘func’ 0'dan başka bir şey döndürür. + +İlk argümanın (lambda işlevi) atlanamayacağını unutmayın. `arrayFirstIndex` işlev. + +### arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1} + +Kaynak dizideki öğelerin kısmi toplamlarının bir dizisini döndürür (çalışan bir toplam). Eğer... `func` işlev belirtilir, daha sonra dizi öğelerinin değerleri toplanmadan önce bu işlev tarafından dönüştürülür. + +Örnek: + +``` sql +SELECT arrayCumSum([1, 1, 1, 1]) AS res +``` + +``` text +┌─res──────────┐ +│ [1, 2, 3, 4] │ +└──────────────┘ +``` + +### arrayCumSumNonNegative(arr) {#arraycumsumnonnegativearr} + +Aynı olarak `arrayCumSum`, kaynak dizideki öğelerin kısmi toplamlarının bir dizisini döndürür (çalışan bir toplam). Farklı `arrayCumSum`, daha sonra döndürülen değer sıfırdan küçük bir değer içerdiğinde, değer sıfır ile değiştirilir ve sonraki hesaplama sıfır parametrelerle gerçekleştirilir. Mesela: + +``` sql +SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res +``` + +``` text +┌─res───────┐ +│ [1,2,0,1] │ +└───────────┘ +``` + +### arraySort(\[func,\] arr1, …) {#arraysortfunc-arr1} + +Öğeleri sıralama sonucu bir dizi döndürür `arr1` artan düzende. Eğer... `func` fonksiyon belirtilir, sıralama sırası fonksiyonun sonucu ile belirlenir `func` dizi elemanlarına uygulanır (diziler) + +Bu [Schwartzian dönüşümü](https://en.wikipedia.org/wiki/Schwartzian_transform) sıralama verimliliğini artırmak için kullanılır. + +Örnek: + +``` sql +SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]); +``` + +``` text +┌─res────────────────┐ +│ ['world', 'hello'] │ +└────────────────────┘ +``` + +Hakkında daha fazla bilgi için `arraySort` yöntem, görmek [Dizilerle çalışmak için işlevler](array_functions.md#array_functions-sort) bölme. + +### arrayReverseSort(\[func,\] arr1, …) {#arrayreversesortfunc-arr1} + +Öğeleri sıralama sonucu bir dizi döndürür `arr1` azalan sırada. Eğer... `func` fonksiyon belirtilir, sıralama sırası fonksiyonun sonucu ile belirlenir `func` dizi (diziler) elemanlarına uygulanır. + +Örnek: + +``` sql +SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; +``` + +``` text +┌─res───────────────┐ +│ ['hello','world'] │ +└───────────────────┘ +``` + +Hakkında daha fazla bilgi için `arrayReverseSort` yöntem, görmek [Dizilerle çalışmak için işlevler](array_functions.md#array_functions-reverse-sort) bölme. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/higher_order_functions/) diff --git a/docs/tr/sql_reference/functions/in_functions.md b/docs/tr/sql_reference/functions/in_functions.md new file mode 100644 index 00000000000..481c1b61802 --- /dev/null +++ b/docs/tr/sql_reference/functions/in_functions.md @@ -0,0 +1,26 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 60 +toc_title: "In \u0130\u015Flet theic implementingisinin uygulanmas\u0131" +--- + +# In operatörünü uygulamak için işlevler {#functions-for-implementing-the-in-operator} + +## içinde, notİn, globalİn, globalNotİn {#in-functions} + +Bölümüne bakınız [Operatör İNLERDE](../statements/select.md#select-in-operators). + +## tuple(x, y, …), operator (x, y, …) {#tuplex-y-operator-x-y} + +Birden çok sütun gruplama sağlayan bir işlev. +For columns with the types T1, T2, …, it returns a Tuple(T1, T2, …) type tuple containing these columns. There is no cost to execute the function. +Tuples normalde bir argüman için Ara değerler olarak kullanılır operatörler, veya lambda fonksiyonlarının resmi parametrelerin bir listesini oluşturmak için. Tuples bir masaya yazılamaz. + +## tupleElement (tuple, n), operatör x. N {#tupleelementtuple-n-operator-x-n} + +Bir tuple bir sütun alma sağlayan bir işlev. +‘N’ 1'den başlayarak sütun dizinidir. N sabit olmalıdır. ‘N’ bir sabit olması gerekir. ‘N’ tuple boyutundan daha büyük olmayan katı bir pozitif tamsayı olmalıdır. +İşlevi yürütmek için hiçbir maliyet yoktur. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/in_functions/) diff --git a/docs/tr/sql_reference/functions/index.md b/docs/tr/sql_reference/functions/index.md new file mode 100644 index 00000000000..01961c69526 --- /dev/null +++ b/docs/tr/sql_reference/functions/index.md @@ -0,0 +1,74 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "\u0130\u015Flevler" +toc_priority: 32 +toc_title: "Giri\u015F" +--- + +# İşlevler {#functions} + +En az\* iki tür fonksiyon vardır-düzenli Fonksiyonlar (sadece denir “functions”) and aggregate functions. These are completely different concepts. Regular functions work as if they are applied to each row separately (for each row, the result of the function doesn't depend on the other rows). Aggregate functions accumulate a set of values from various rows (i.e. they depend on the entire set of rows). + +Bu bölümde düzenli işlevleri tartışıyoruz. Toplama işlevleri için bölüme bakın “Aggregate functions”. + +\* - Üçüncü bir işlev türü vardır ‘arrayJoin’ fonksiyon aittir; tablo fonksiyonları da ayrı ayrı belirtilebilir.\* + +## Güçlü yazarak {#strong-typing} + +Standart SQL aksine, ClickHouse güçlü yazarak vardır. Başka bir deyişle, türler arasında örtük dönüşümler yapmaz. Her işlev belirli bir tür kümesi için çalışır. Bu, bazen tür dönüştürme işlevlerini kullanmanız gerektiği anlamına gelir. + +## Ortak subexpression eliminasyonu {#common-subexpression-elimination} + +Aynı AST (aynı kayıt veya sözdizimsel ayrıştırma aynı sonucu) olan bir sorgudaki tüm ifadeler aynı değerlere sahip olarak kabul edilir. Bu tür ifadeler bir kez birleştirilir ve yürütülür. Aynı alt sorgular da bu şekilde elimine edilir. + +## Sonuç türleri {#types-of-results} + +Tüm işlevler sonuç olarak tek bir dönüş döndürür (birkaç değer değil, sıfır değer değil). Sonuç türü genellikle değerlerle değil, yalnızca bağımsız değişken türleriyle tanımlanır. Özel durumlar tupleElement işlevi (a.n işleci) ve tofixedstring işlevidir. + +## Devamlılar {#constants} + +Basitlik için, bazı işlevler yalnızca bazı argümanlar için sabitlerle çalışabilir. Örneğin, LİKE operatörünün doğru argümanı sabit olmalıdır. +Hemen hemen tüm işlevler sabit argümanlar için bir sabit döndürür. İstisna, rasgele sayılar üreten işlevlerdir. +Bu ‘now’ işlev, farklı zamanlarda çalıştırılan sorgular için farklı değerler döndürür, ancak sonuç sabit olarak kabul edilir, çünkü sabitlik yalnızca tek bir sorguda önemlidir. +Sabit bir ifade de sabit olarak kabul edilir (örneğin, LİKE operatörünün sağ yarısı birden fazla sabitten oluşturulabilir). + +Fonksiyonlar sabit ve sabit olmayan argümanlar için farklı şekillerde uygulanabilir (farklı kod yürütülür). Ancak, bir sabit ve yalnızca aynı değeri içeren gerçek bir sütun için sonuçlar birbiriyle eşleşmelidir. + +## NULL işleme {#null-processing} + +Fonksiyonlar aşağıdaki davranışlara sahiptir: + +- İşlevin argümanlarından en az biri ise `NULL`, fonksiyon sonucu da `NULL`. +- Her işlevin açıklamasında ayrı ayrı belirtilen özel davranış. ClickHouse kaynak kodunda, bu işlevler `UseDefaultImplementationForNulls=false`. + +## Süreklilik {#constancy} + +Functions can't change the values of their arguments – any changes are returned as the result. Thus, the result of calculating separate functions does not depend on the order in which the functions are written in the query. + +## Hata işleme {#error-handling} + +Veriler geçersizse bazı işlevler bir istisna oluşturabilir. Bu durumda, sorgu iptal edilir ve bir hata metni istemciye döndürülür. Dağıtılmış işlem için sunuculardan birinde bir özel durum oluştuğunda, diğer sunucular da sorguyu iptal etmeye çalışır. + +## Argüman ifadelerinin değerlendirilmesi {#evaluation-of-argument-expressions} + +Hemen hemen tüm programlama dillerinde, argümanlardan biri belirli operatörler için değerlendirilmeyebilir. Bu genellikle operatörler `&&`, `||`, ve `?:`. +Ancak Clickhouse'da, fonksiyonların (operatörler) argümanları her zaman değerlendirilir. Bunun nedeni, sütunların tüm bölümlerinin her satırı ayrı ayrı hesaplamak yerine bir kerede değerlendirilmesidir. + +## Dağıtılmış sorgu işleme işlevleri gerçekleştirme {#performing-functions-for-distributed-query-processing} + +Dağıtılmış sorgu işleme için, sorgu işlemenin mümkün olduğu kadar çok aşaması uzak sunucularda gerçekleştirilir ve aşamaların geri kalanı (Ara sonuçları ve bundan sonra her şeyi birleştirme) istek sahibi sunucuda gerçekleştirilir. + +Bu, işlevlerin farklı sunucularda gerçekleştirilebileceği anlamına gelir. +Örneğin, sorguda `SELECT f(sum(g(x))) FROM distributed_table GROUP BY h(y),` + +- eğer bir `distributed_table` en az iki parçaya sahiptir, fonksiyonlar ‘g’ ve ‘h’ uzak sunucularda gerçekleştirilir ve işlev ‘f’ ıstekçi sunucuda gerçekleştirilir. +- eğer bir `distributed_table` sadece bir parça var, tüm ‘f’, ‘g’, ve ‘h’ fonksiyonlar bu shard'ın sunucusunda gerçekleştirilir. + +Bir işlevin sonucu genellikle hangi sunucuda gerçekleştirildiğine bağlı değildir. Ancak, bazen bu önemlidir. +Örneğin, sözlüklerle çalışan işlevler, üzerinde çalışmakta oldukları sunucuda bulunan sözlüğü kullanır. +Başka bir örnek ise `hostName` yapmak için üzerinde çalıştığı sunucunun adını döndüren işlev `GROUP BY` sunucular tarafından bir `SELECT` sorgu. + +Eğer sorguda bir işlevi istemcisi sunucu üzerinde yapılır, ama uzak sunucularda bunu gerçekleştirmek için ihtiyacınız varsa, bir saramaz mısın ‘any’ toplama işlevi veya bir anahtara ekleyin `GROUP BY`. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/) diff --git a/docs/tr/sql_reference/functions/introspection.md b/docs/tr/sql_reference/functions/introspection.md new file mode 100644 index 00000000000..fdc68fe76fb --- /dev/null +++ b/docs/tr/sql_reference/functions/introspection.md @@ -0,0 +1,310 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 65 +toc_title: "\u0130\xE7g\xF6zlem" +--- + +# İç Gözlem Fonksiyonları {#introspection-functions} + +İç gözlem için bu bölümde açıklanan işlevleri kullanabilirsiniz [ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format) ve [DWARF](https://en.wikipedia.org/wiki/DWARF) sorgu profilleme için. + +!!! warning "Uyarıcı" + Bu işlevler yavaştır ve güvenlik konuları getirebilir. + +İç gözlem fonksiyonlarının düzgün çalışması için: + +- Yüklemek `clickhouse-common-static-dbg` paket. + +- Ayarla... [allow\_introspection\_functions](../../operations/settings/settings.md#settings-allow_introspection_functions) ayar 1. + + For security reasons introspection functions are disabled by default. + +ClickHouse için profiler raporları kaydeder [trace\_log](../../operations/system_tables.md#system_tables-trace_log) sistem tablosu. Tablo ve profiler düzgün yapılandırıldığından emin olun. + +## addressToLine {#addresstoline} + +ClickHouse sunucu işleminin içindeki sanal bellek adresini dosya adına ve clickhouse kaynak kodundaki satır numarasına dönüştürür. + +Resmi ClickHouse paketleri kullanırsanız, yüklemeniz gerekir `clickhouse-common-static-dbg` paket. + +**Sözdizimi** + +``` sql +addressToLine(address_of_binary_instruction) +``` + +**Parametre** + +- `address_of_binary_instruction` ([Uİnt64](../../sql_reference/data_types/int_uint.md)) — Address of instruction in a running process. + +**Döndürülen değer** + +- Kaynak kodu dosya adı ve bu dosyadaki satır numarası iki nokta üst üste ile sınırlandırılmıştır. + + For example, `/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199`, where `199` is a line number. + +- Işlev hata ayıklama bilgilerini bulamadıysanız, bir ikili adı. + +- Adres geçerli değilse, boş dize. + +Tür: [Dize](../../sql_reference/data_types/string.md). + +**Örnek** + +İç gözlem işlevlerini etkinleştirme: + +``` sql +SET allow_introspection_functions=1 +``` + +İlk dizeyi seçme `trace_log` sistem tablosu: + +``` sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` + +``` text +Row 1: +────── +event_date: 2019-11-19 +event_time: 2019-11-19 18:57:23 +revision: 54429 +timer_type: Real +thread_number: 48 +query_id: 421b6855-1858-45a5-8f37-f383409d6d72 +trace: [140658411141617,94784174532828,94784076370703,94784076372094,94784076361020,94784175007680,140658411116251,140658403895439] +``` + +Bu `trace` alan, örnekleme anında yığın izini içerir. + +Tek bir adres için kaynak kodu dosya adını ve satır numarasını alma: + +``` sql +SELECT addressToLine(94784076370703) \G +``` + +``` text +Row 1: +────── +addressToLine(94784076370703): /build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199 +``` + +İşlevin tüm yığın izine uygulanması: + +``` sql +SELECT + arrayStringConcat(arrayMap(x -> addressToLine(x), trace), '\n') AS trace_source_code_lines +FROM system.trace_log +LIMIT 1 +\G +``` + +Bu [arrayMap](higher_order_functions.md#higher_order_functions-array-map) işlev, her bir elemanın işlenmesini sağlar `trace` ar arrayray by the `addressToLine` işlev. Gördüğünüz bu işlemin sonucu `trace_source_code_lines` çıktı sütunu. + +``` text +Row 1: +────── +trace_source_code_lines: /lib/x86_64-linux-gnu/libpthread-2.27.so +/usr/lib/debug/usr/bin/clickhouse +/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:199 +/build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.h:155 +/usr/include/c++/9/bits/atomic_base.h:551 +/usr/lib/debug/usr/bin/clickhouse +/lib/x86_64-linux-gnu/libpthread-2.27.so +/build/glibc-OTsEL5/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:97 +``` + +## addressToSymbol {#addresstosymbol} + +Clickhouse sunucu işlemi içindeki sanal bellek adresini ClickHouse nesne dosyalarından gelen simgeye dönüştürür. + +**Sözdizimi** + +``` sql +addressToSymbol(address_of_binary_instruction) +``` + +**Parametre** + +- `address_of_binary_instruction` ([Uİnt64](../../sql_reference/data_types/int_uint.md)) — Address of instruction in a running process. + +**Döndürülen değer** + +- ClickHouse nesne dosyalarından sembol. +- Adres geçerli değilse, boş dize. + +Tür: [Dize](../../sql_reference/data_types/string.md). + +**Örnek** + +İç gözlem işlevlerini etkinleştirme: + +``` sql +SET allow_introspection_functions=1 +``` + +İlk dizeyi seçme `trace_log` sistem tablosu: + +``` sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` + +``` text +Row 1: +────── +event_date: 2019-11-20 +event_time: 2019-11-20 16:57:59 +revision: 54429 +timer_type: Real +thread_number: 48 +query_id: 724028bf-f550-45aa-910d-2af6212b94ac +trace: [94138803686098,94138815010911,94138815096522,94138815101224,94138815102091,94138814222988,94138806823642,94138814457211,94138806823642,94138814457211,94138806823642,94138806795179,94138806796144,94138753770094,94138753771646,94138753760572,94138852407232,140399185266395,140399178045583] +``` + +Bu `trace` alan, örnekleme anında yığın izini içerir. + +Tek bir adres için sembol alma: + +``` sql +SELECT addressToSymbol(94138803686098) \G +``` + +``` text +Row 1: +────── +addressToSymbol(94138803686098): _ZNK2DB24IAggregateFunctionHelperINS_20AggregateFunctionSumImmNS_24AggregateFunctionSumDataImEEEEE19addBatchSinglePlaceEmPcPPKNS_7IColumnEPNS_5ArenaE +``` + +İşlevin tüm yığın izine uygulanması: + +``` sql +SELECT + arrayStringConcat(arrayMap(x -> addressToSymbol(x), trace), '\n') AS trace_symbols +FROM system.trace_log +LIMIT 1 +\G +``` + +Bu [arrayMap](higher_order_functions.md#higher_order_functions-array-map) işlev, her bir elemanın işlenmesini sağlar `trace` ar arrayray by the `addressToSymbols` işlev. Gördüğünüz bu işlemin sonucu `trace_symbols` çıktı sütunu. + +``` text +Row 1: +────── +trace_symbols: _ZNK2DB24IAggregateFunctionHelperINS_20AggregateFunctionSumImmNS_24AggregateFunctionSumDataImEEEEE19addBatchSinglePlaceEmPcPPKNS_7IColumnEPNS_5ArenaE +_ZNK2DB10Aggregator21executeWithoutKeyImplERPcmPNS0_28AggregateFunctionInstructionEPNS_5ArenaE +_ZN2DB10Aggregator14executeOnBlockESt6vectorIN3COWINS_7IColumnEE13immutable_ptrIS3_EESaIS6_EEmRNS_22AggregatedDataVariantsERS1_IPKS3_SaISC_EERS1_ISE_SaISE_EERb +_ZN2DB10Aggregator14executeOnBlockERKNS_5BlockERNS_22AggregatedDataVariantsERSt6vectorIPKNS_7IColumnESaIS9_EERS6_ISB_SaISB_EERb +_ZN2DB10Aggregator7executeERKSt10shared_ptrINS_17IBlockInputStreamEERNS_22AggregatedDataVariantsE +_ZN2DB27AggregatingBlockInputStream8readImplEv +_ZN2DB17IBlockInputStream4readEv +_ZN2DB26ExpressionBlockInputStream8readImplEv +_ZN2DB17IBlockInputStream4readEv +_ZN2DB26ExpressionBlockInputStream8readImplEv +_ZN2DB17IBlockInputStream4readEv +_ZN2DB28AsynchronousBlockInputStream9calculateEv +_ZNSt17_Function_handlerIFvvEZN2DB28AsynchronousBlockInputStream4nextEvEUlvE_E9_M_invokeERKSt9_Any_data +_ZN14ThreadPoolImplI20ThreadFromGlobalPoolE6workerESt14_List_iteratorIS0_E +_ZZN20ThreadFromGlobalPoolC4IZN14ThreadPoolImplIS_E12scheduleImplIvEET_St8functionIFvvEEiSt8optionalImEEUlvE1_JEEEOS4_DpOT0_ENKUlvE_clEv +_ZN14ThreadPoolImplISt6threadE6workerESt14_List_iteratorIS0_E +execute_native_thread_routine +start_thread +clone +``` + +## demangle {#demangle} + +Kullanarak alabileceğiniz bir sembolü dönüştürür [addressToSymbol](#addresstosymbol) C++ işlev adına işlev. + +**Sözdizimi** + +``` sql +demangle(symbol) +``` + +**Parametre** + +- `symbol` ([Dize](../../sql_reference/data_types/string.md)) — Symbol from an object file. + +**Döndürülen değer** + +- C++ işlevinin adı. +- Bir sembol geçerli değilse boş dize. + +Tür: [Dize](../../sql_reference/data_types/string.md). + +**Örnek** + +İç gözlem işlevlerini etkinleştirme: + +``` sql +SET allow_introspection_functions=1 +``` + +İlk dizeyi seçme `trace_log` sistem tablosu: + +``` sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` + +``` text +Row 1: +────── +event_date: 2019-11-20 +event_time: 2019-11-20 16:57:59 +revision: 54429 +timer_type: Real +thread_number: 48 +query_id: 724028bf-f550-45aa-910d-2af6212b94ac +trace: [94138803686098,94138815010911,94138815096522,94138815101224,94138815102091,94138814222988,94138806823642,94138814457211,94138806823642,94138814457211,94138806823642,94138806795179,94138806796144,94138753770094,94138753771646,94138753760572,94138852407232,140399185266395,140399178045583] +``` + +Bu `trace` alan, örnekleme anında yığın izini içerir. + +Tek bir adres için bir işlev adı alma: + +``` sql +SELECT demangle(addressToSymbol(94138803686098)) \G +``` + +``` text +Row 1: +────── +demangle(addressToSymbol(94138803686098)): DB::IAggregateFunctionHelper > >::addBatchSinglePlace(unsigned long, char*, DB::IColumn const**, DB::Arena*) const +``` + +İşlevin tüm yığın izine uygulanması: + +``` sql +SELECT + arrayStringConcat(arrayMap(x -> demangle(addressToSymbol(x)), trace), '\n') AS trace_functions +FROM system.trace_log +LIMIT 1 +\G +``` + +Bu [arrayMap](higher_order_functions.md#higher_order_functions-array-map) işlev, her bir elemanın işlenmesini sağlar `trace` ar arrayray by the `demangle` işlev. Gördüğünüz bu işlemin sonucu `trace_functions` çıktı sütunu. + +``` text +Row 1: +────── +trace_functions: DB::IAggregateFunctionHelper > >::addBatchSinglePlace(unsigned long, char*, DB::IColumn const**, DB::Arena*) const +DB::Aggregator::executeWithoutKeyImpl(char*&, unsigned long, DB::Aggregator::AggregateFunctionInstruction*, DB::Arena*) const +DB::Aggregator::executeOnBlock(std::vector::immutable_ptr, std::allocator::immutable_ptr > >, unsigned long, DB::AggregatedDataVariants&, std::vector >&, std::vector >, std::allocator > > >&, bool&) +DB::Aggregator::executeOnBlock(DB::Block const&, DB::AggregatedDataVariants&, std::vector >&, std::vector >, std::allocator > > >&, bool&) +DB::Aggregator::execute(std::shared_ptr const&, DB::AggregatedDataVariants&) +DB::AggregatingBlockInputStream::readImpl() +DB::IBlockInputStream::read() +DB::ExpressionBlockInputStream::readImpl() +DB::IBlockInputStream::read() +DB::ExpressionBlockInputStream::readImpl() +DB::IBlockInputStream::read() +DB::AsynchronousBlockInputStream::calculate() +std::_Function_handler::_M_invoke(std::_Any_data const&) +ThreadPoolImpl::worker(std::_List_iterator) +ThreadFromGlobalPool::ThreadFromGlobalPool::scheduleImpl(std::function, int, std::optional)::{lambda()#3}>(ThreadPoolImpl::scheduleImpl(std::function, int, std::optional)::{lambda()#3}&&)::{lambda()#1}::operator()() const +ThreadPoolImpl::worker(std::_List_iterator) +execute_native_thread_routine +start_thread +clone +``` diff --git a/docs/tr/sql_reference/functions/ip_address_functions.md b/docs/tr/sql_reference/functions/ip_address_functions.md new file mode 100644 index 00000000000..2caa7c49b2a --- /dev/null +++ b/docs/tr/sql_reference/functions/ip_address_functions.md @@ -0,0 +1,248 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 55 +toc_title: "IP adresleriyle \xE7al\u0131\u015Fma" +--- + +# IP adresleriyle çalışmak için işlevler {#functions-for-working-with-ip-addresses} + +## Ipv4numtostring (num) {#ipv4numtostringnum} + +Bir Uınt32 numarası alır. Big endian'da bir IPv4 adresi olarak yorumlar. Karşılık gelen IPv4 adresini a. B. C. d biçiminde içeren bir dize döndürür (ondalık formda nokta ile ayrılmış sayılar). + +## Ipv4stringtonum (s) {#ipv4stringtonums} + +IPv4NumToString ters işlevi. IPv4 adresi geçersiz bir biçime sahipse, 0 döndürür. + +## Ipv4numtostringclassc (num) {#ipv4numtostringclasscnum} + +Ipv4numtostring'e benzer, ancak son sekizli yerine xxx kullanıyor. + +Örnek: + +``` sql +SELECT + IPv4NumToStringClassC(ClientIP) AS k, + count() AS c +FROM test.hits +GROUP BY k +ORDER BY c DESC +LIMIT 10 +``` + +``` text +┌─k──────────────┬─────c─┐ +│ 83.149.9.xxx │ 26238 │ +│ 217.118.81.xxx │ 26074 │ +│ 213.87.129.xxx │ 25481 │ +│ 83.149.8.xxx │ 24984 │ +│ 217.118.83.xxx │ 22797 │ +│ 78.25.120.xxx │ 22354 │ +│ 213.87.131.xxx │ 21285 │ +│ 78.25.121.xxx │ 20887 │ +│ 188.162.65.xxx │ 19694 │ +│ 83.149.48.xxx │ 17406 │ +└────────────────┴───────┘ +``` + +Kullanıl sincedığından beri ‘xxx’ son derece sıradışı, bu gelecekte değiştirilebilir. Bu parçanın tam biçimine güvenmemenizi öneririz. + +### Ipv6numtostring (x) {#ipv6numtostringx} + +IPv6 adresini ikili biçimde içeren bir FixedString(16) değerini kabul eder. Bu adresi metin biçiminde içeren bir dize döndürür. +IPv6 eşlemeli IPv4 adresleri ::ffff:111.222.33.44 biçiminde çıktıdır. Örnekler: + +``` sql +SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'), 16)) AS addr +``` + +``` text +┌─addr─────────┐ +│ 2a02:6b8::11 │ +└──────────────┘ +``` + +``` sql +SELECT + IPv6NumToString(ClientIP6 AS k), + count() AS c +FROM hits_all +WHERE EventDate = today() AND substring(ClientIP6, 1, 12) != unhex('00000000000000000000FFFF') +GROUP BY k +ORDER BY c DESC +LIMIT 10 +``` + +``` text +┌─IPv6NumToString(ClientIP6)──────────────┬─────c─┐ +│ 2a02:2168:aaa:bbbb::2 │ 24695 │ +│ 2a02:2698:abcd:abcd:abcd:abcd:8888:5555 │ 22408 │ +│ 2a02:6b8:0:fff::ff │ 16389 │ +│ 2a01:4f8:111:6666::2 │ 16016 │ +│ 2a02:2168:888:222::1 │ 15896 │ +│ 2a01:7e00::ffff:ffff:ffff:222 │ 14774 │ +│ 2a02:8109:eee:ee:eeee:eeee:eeee:eeee │ 14443 │ +│ 2a02:810b:8888:888:8888:8888:8888:8888 │ 14345 │ +│ 2a02:6b8:0:444:4444:4444:4444:4444 │ 14279 │ +│ 2a01:7e00::ffff:ffff:ffff:ffff │ 13880 │ +└─────────────────────────────────────────┴───────┘ +``` + +``` sql +SELECT + IPv6NumToString(ClientIP6 AS k), + count() AS c +FROM hits_all +WHERE EventDate = today() +GROUP BY k +ORDER BY c DESC +LIMIT 10 +``` + +``` text +┌─IPv6NumToString(ClientIP6)─┬──────c─┐ +│ ::ffff:94.26.111.111 │ 747440 │ +│ ::ffff:37.143.222.4 │ 529483 │ +│ ::ffff:5.166.111.99 │ 317707 │ +│ ::ffff:46.38.11.77 │ 263086 │ +│ ::ffff:79.105.111.111 │ 186611 │ +│ ::ffff:93.92.111.88 │ 176773 │ +│ ::ffff:84.53.111.33 │ 158709 │ +│ ::ffff:217.118.11.22 │ 154004 │ +│ ::ffff:217.118.11.33 │ 148449 │ +│ ::ffff:217.118.11.44 │ 148243 │ +└────────────────────────────┴────────┘ +``` + +## Ipv6stringtonum (s) {#ipv6stringtonums} + +IPv6NumToString ters işlevi. IPv6 adresi geçersiz bir biçime sahipse, bir boş bayt dizesi döndürür. +HEX büyük veya küçük harf olabilir. + +## Ipv4toıpv6 (x) {#ipv4toipv6x} + +Alır bir `UInt32` numara. Bir IPv4 adresi olarak yorumlar [büyük endian](https://en.wikipedia.org/wiki/Endianness). Ret aur ANS a `FixedString(16)` IPv6 adresini ikili biçimde içeren değer. Örnekler: + +``` sql +SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr +``` + +``` text +┌─addr───────────────┐ +│ ::ffff:192.168.0.1 │ +└────────────────────┘ +``` + +## cutİPv6 (x, bytesToCutForİPv6, bytesToCutForİPv4) {#cutipv6x-bytestocutforipv6-bytestocutforipv4} + +IPv6 adresini ikili biçimde içeren bir FixedString(16) değerini kabul eder. Metin biçiminde kaldırılan belirtilen bayt sayısının adresini içeren bir dize döndürür. Mesela: + +``` sql +WITH + IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D') AS ipv6, + IPv4ToIPv6(IPv4StringToNum('192.168.0.1')) AS ipv4 +SELECT + cutIPv6(ipv6, 2, 0), + cutIPv6(ipv4, 0, 2) +``` + +``` text +┌─cutIPv6(ipv6, 2, 0)─────────────────┬─cutIPv6(ipv4, 0, 2)─┐ +│ 2001:db8:ac10:fe01:feed:babe:cafe:0 │ ::ffff:192.168.0.0 │ +└─────────────────────────────────────┴─────────────────────┘ +``` + +## Ipv4cidrtorange(ıpv4, cıdr), {#ipv4cidrtorangeipv4-cidr} + +İçeren bir IPv4 ve bir Uint8 değerini kabul eder [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). Alt ağın alt aralığını ve daha yüksek aralığını içeren iki IPv4 içeren bir tuple döndürür. + +``` sql +SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) +``` + +``` text +┌─IPv4CIDRToRange(toIPv4('192.168.5.2'), 16)─┐ +│ ('192.168.0.0','192.168.255.255') │ +└────────────────────────────────────────────┘ +``` + +## Ipv6cidrtorange(ıpv6, cıdr), {#ipv6cidrtorangeipv6-cidr} + +CIDR'Yİ içeren bir IPv6 ve bir Uİnt8 değerini kabul eder. Alt ağın alt aralığını ve daha yüksek aralığını içeren iki IPv6 içeren bir tuple döndürür. + +``` sql +SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32); +``` + +``` text +┌─IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32)─┐ +│ ('2001:db8::','2001:db8:ffff:ffff:ffff:ffff:ffff:ffff') │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +## toıpv4 (dize) {#toipv4string} + +İçin bir takma ad `IPv4StringToNum()` bu, IPv4 adresinin bir dize formunu alır ve değerini döndürür [Ipv44](../../sql_reference/data_types/domains/ipv4.md) tarafından döndürülen değere eşit ikili olan tür `IPv4StringToNum()`. + +``` sql +WITH + '171.225.130.45' as IPv4_string +SELECT + toTypeName(IPv4StringToNum(IPv4_string)), + toTypeName(toIPv4(IPv4_string)) +``` + +``` text +┌─toTypeName(IPv4StringToNum(IPv4_string))─┬─toTypeName(toIPv4(IPv4_string))─┐ +│ UInt32 │ IPv4 │ +└──────────────────────────────────────────┴─────────────────────────────────┘ +``` + +``` sql +WITH + '171.225.130.45' as IPv4_string +SELECT + hex(IPv4StringToNum(IPv4_string)), + hex(toIPv4(IPv4_string)) +``` + +``` text +┌─hex(IPv4StringToNum(IPv4_string))─┬─hex(toIPv4(IPv4_string))─┐ +│ ABE1822D │ ABE1822D │ +└───────────────────────────────────┴──────────────────────────┘ +``` + +## toıpv6 (dize) {#toipv6string} + +İçin bir takma ad `IPv6StringToNum()` bu, IPv6 adresinin bir dize formunu alır ve değerini döndürür [IPv6](../../sql_reference/data_types/domains/ipv6.md) tarafından döndürülen değere eşit ikili olan tür `IPv6StringToNum()`. + +``` sql +WITH + '2001:438:ffff::407d:1bc1' as IPv6_string +SELECT + toTypeName(IPv6StringToNum(IPv6_string)), + toTypeName(toIPv6(IPv6_string)) +``` + +``` text +┌─toTypeName(IPv6StringToNum(IPv6_string))─┬─toTypeName(toIPv6(IPv6_string))─┐ +│ FixedString(16) │ IPv6 │ +└──────────────────────────────────────────┴─────────────────────────────────┘ +``` + +``` sql +WITH + '2001:438:ffff::407d:1bc1' as IPv6_string +SELECT + hex(IPv6StringToNum(IPv6_string)), + hex(toIPv6(IPv6_string)) +``` + +``` text +┌─hex(IPv6StringToNum(IPv6_string))─┬─hex(toIPv6(IPv6_string))─────────┐ +│ 20010438FFFF000000000000407D1BC1 │ 20010438FFFF000000000000407D1BC1 │ +└───────────────────────────────────┴──────────────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/ip_address_functions/) diff --git a/docs/tr/sql_reference/functions/json_functions.md b/docs/tr/sql_reference/functions/json_functions.md new file mode 100644 index 00000000000..95aaa1768f2 --- /dev/null +++ b/docs/tr/sql_reference/functions/json_functions.md @@ -0,0 +1,231 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 56 +toc_title: "JSON ile \xE7al\u0131\u015Fmak." +--- + +# Json ile çalışmak için fonksiyonlar {#functions-for-working-with-json} + +Üye Olarak.Metrica, JSON kullanıcılar tarafından oturum parametreleri olarak iletilir. Bu JSON ile çalışmak için bazı özel fonksiyonlar var. (Çoğu durumda, JSONs ek olarak önceden işlenir ve elde edilen değerler işlenmiş biçimlerinde ayrı sütunlara konur .) Tüm bu işlevler, JSON'UN ne olabileceğine dair güçlü varsayımlara dayanır, ancak işi yapmak için mümkün olduğunca az şey yapmaya çalışırlar. + +Aşağıdaki varsayımlar yapılır: + +1. Alan adı (işlev bağımsız değişkeni) sabit olmalıdır. +2. Alan adı bir şekilde json'da kanonik olarak kodlanmıştır. Mesela: `visitParamHas('{"abc":"def"}', 'abc') = 1`, ama `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` +3. Alanlar, herhangi bir yuvalama düzeyinde, ayrım gözetmeksizin aranır. Birden çok eşleşen alan varsa, ilk olay kullanılır. +4. JSON, dize değişmezleri dışında boşluk karakterlerine sahip değildir. + +## visitParamHas (params, isim) {#visitparamhasparams-name} + +İle bir alan olup olmadığını denetler ‘name’ ad. + +## visitParamExtractUİnt (params, isim) {#visitparamextractuintparams-name} + +Uint64 adlı alanın değerinden ayrıştırır ‘name’. Bu bir dize alanı ise, dizenin başlangıcından itibaren bir sayıyı ayrıştırmaya çalışır. Alan yoksa veya varsa ancak bir sayı içermiyorsa, 0 döndürür. + +## visitParamExtractİnt (params, isim) {#visitparamextractintparams-name} + +Int64 için olduğu gibi. + +## visitParamExtractFloat (params, isim) {#visitparamextractfloatparams-name} + +Float64 için olduğu gibi. + +## visitParamExtractBool (params, isim) {#visitparamextractboolparams-name} + +True/false değerini ayrıştırır. Sonuç Uİnt8. + +## visitParamExtractRaw (params, isim) {#visitparamextractrawparams-name} + +Ayırıcılar da dahil olmak üzere bir alanın değerini döndürür. + +Örnekler: + +``` sql +visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"' +visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}' +``` + +## visitParamExtractString (params, isim) {#visitparamextractstringparams-name} + +Dizeyi çift tırnak içinde ayrıştırır. Değeri unescaped. Unescaping başarısız olursa, boş bir dize döndürür. + +Örnekler: + +``` sql +visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0' +visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺' +visitParamExtractString('{"abc":"\\u263"}', 'abc') = '' +visitParamExtractString('{"abc":"hello}', 'abc') = '' +``` + +Şu anda biçimdeki kod noktaları için destek yok `\uXXXX\uYYYY` bu temel çok dilli düzlemden değildir(UTF-8 yerine CESU-8'e dönüştürülürler). + +Aşağıdaki işlevler dayanmaktadır [simdjson](https://github.com/lemire/simdjson) daha karmaşık json ayrıştırma gereksinimleri için tasarlanmıştır. Yukarıda belirtilen varsayım 2 hala geçerlidir. + +## ısvalidjson(json) {#isvalidjsonjson} + +Dize geçirilen kontroller geçerli bir json'dur. + +Örnekler: + +``` sql +SELECT isValidJSON('{"a": "hello", "b": [-100, 200.0, 300]}') = 1 +SELECT isValidJSON('not a json') = 0 +``` + +## JSONHas(json\[, indices\_or\_keys\]…) {#jsonhasjson-indices-or-keys} + +Değer JSON belgesinde varsa, `1` iade edilecektir. + +Değer yoksa, `0` iade edilecektir. + +Örnekler: + +``` sql +SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = 1 +SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4) = 0 +``` + +`indices_or_keys` sıfır veya daha fazla argüman listesi her biri dize veya tamsayı olabilir. + +- String = nesne üyesine anahtarla erişin. +- Pozitif tamsayı = n-inci üyesine / anahtarına baştan erişin. +- Negatif tamsayı = sondan n-inci üye/anahtara erişin. + +Elemanın minimum Endeksi 1'dir. Böylece 0 öğesi mevcut değildir. + +Hem json dizilerine hem de JSON nesnelerine erişmek için tamsayılar kullanabilirsiniz. + +Bu yüzden, örneğin : + +``` sql +SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', 1) = 'a' +SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', 2) = 'b' +SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', -1) = 'b' +SELECT JSONExtractKey('{"a": "hello", "b": [-100, 200.0, 300]}', -2) = 'a' +SELECT JSONExtractString('{"a": "hello", "b": [-100, 200.0, 300]}', 1) = 'hello' +``` + +## JSONLength(json\[, indices\_or\_keys\]…) {#jsonlengthjson-indices-or-keys} + +Bir json dizisinin veya bir JSON nesnesinin uzunluğunu döndürür. + +Değer yoksa veya yanlış bir türe sahipse, `0` iade edilecektir. + +Örnekler: + +``` sql +SELECT JSONLength('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = 3 +SELECT JSONLength('{"a": "hello", "b": [-100, 200.0, 300]}') = 2 +``` + +## JSONType(json\[, indices\_or\_keys\]…) {#jsontypejson-indices-or-keys} + +Bir JSON değerinin türünü döndürür. + +Değer yoksa, `Null` iade edilecektir. + +Örnekler: + +``` sql +SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}') = 'Object' +SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}', 'a') = 'String' +SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = 'Array' +``` + +## JSONExtractUInt(json\[, indices\_or\_keys\]…) {#jsonextractuintjson-indices-or-keys} + +## JSONExtractInt(json\[, indices\_or\_keys\]…) {#jsonextractintjson-indices-or-keys} + +## JSONExtractFloat(json\[, indices\_or\_keys\]…) {#jsonextractfloatjson-indices-or-keys} + +## JSONExtractBool(json\[, indices\_or\_keys\]…) {#jsonextractbooljson-indices-or-keys} + +Bir JSON ayrıştırır ve bir değer ayıklayın. Bu işlevler benzer `visitParam` işlevler. + +Değer yoksa veya yanlış bir türe sahipse, `0` iade edilecektir. + +Örnekler: + +``` sql +SELECT JSONExtractInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 1) = -100 +SELECT JSONExtractFloat('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 2) = 200.0 +SELECT JSONExtractUInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', -1) = 300 +``` + +## JSONExtractString(json\[, indices\_or\_keys\]…) {#jsonextractstringjson-indices-or-keys} + +Bir json ayrıştırır ve bir dize ayıklayın. Bu işlev benzer `visitParamExtractString` işlevler. + +Değer yoksa veya yanlış bir tür varsa, boş bir dize döndürülür. + +Değeri unescaped. Unescaping başarısız olursa, boş bir dize döndürür. + +Örnekler: + +``` sql +SELECT JSONExtractString('{"a": "hello", "b": [-100, 200.0, 300]}', 'a') = 'hello' +SELECT JSONExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0' +SELECT JSONExtractString('{"abc":"\\u263a"}', 'abc') = '☺' +SELECT JSONExtractString('{"abc":"\\u263"}', 'abc') = '' +SELECT JSONExtractString('{"abc":"hello}', 'abc') = '' +``` + +## JSONExtract(json\[, indices\_or\_keys…\], return\_type) {#jsonextractjson-indices-or-keys-return-type} + +Bir Json ayrıştırır ve verilen ClickHouse veri türünün bir değerini çıkarır. + +Bu, önceki bir genellemedir `JSONExtract` işlevler. +Bu demektir +`JSONExtract(..., 'String')` tam olarak aynı döndürür `JSONExtractString()`, +`JSONExtract(..., 'Float64')` tam olarak aynı döndürür `JSONExtractFloat()`. + +Örnekler: + +``` sql +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(String, Array(Float64))') = ('hello',[-100,200,300]) +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(b Array(Float64), a String)') = ([-100,200,300],'hello') +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Nullable(Int8))') = [-100, NULL, NULL] +SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable(Int64)') = NULL +SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8') = 1 +SELECT JSONExtract('{"day": "Thursday"}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)') = 'Thursday' +SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)') = 'Friday' +``` + +## JSONExtractKeysAndValues(json\[, indices\_or\_keys…\], value\_type) {#jsonextractkeysandvaluesjson-indices-or-keys-value-type} + +Değerlerin verilen ClickHouse veri türünde olduğu bir json'dan anahtar değer çiftlerini ayrıştırın. + +Örnek: + +``` sql +SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)]; +``` + +## JSONExtractRaw(json\[, indices\_or\_keys\]…) {#jsonextractrawjson-indices-or-keys} + +Json'un bir bölümünü döndürür. + +Bölüm yoksa veya yanlış bir türe sahipse, boş bir dize döndürülür. + +Örnek: + +``` sql +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' +``` + +## JSONExtractArrayRaw(json\[, indices\_or\_keys\]…) {#jsonextractarrayrawjson-indices-or-keys} + +Her biri ayrıştırılmamış dize olarak temsil edilen json dizisinin öğeleriyle bir dizi döndürür. + +Bölüm yoksa veya dizi değilse, boş bir dizi döndürülür. + +Örnek: + +``` sql +SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']' +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/json_functions/) diff --git a/docs/tr/sql_reference/functions/logical_functions.md b/docs/tr/sql_reference/functions/logical_functions.md new file mode 100644 index 00000000000..543226319da --- /dev/null +++ b/docs/tr/sql_reference/functions/logical_functions.md @@ -0,0 +1,22 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 37 +toc_title: "Mant\u0131kl\u0131" +--- + +# Mantıksal fonksiyonlar {#logical-functions} + +Mantıksal işlevler herhangi bir sayısal türü kabul eder, ancak 0 veya 1'e eşit bir Uİnt8 numarası döndürür. + +Bir argüman olarak sıfır kabul edilir “false,” sıfır olmayan herhangi bir değer dikkate alınırken “true”. + +## ve, ve operatör {#and-and-operator} + +## or, or operat ,or {#or-or-operator} + +## değil, operatör değil {#not-not-operator} + +## xor {#xor} + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/logical_functions/) diff --git a/docs/tr/sql_reference/functions/machine_learning_functions.md b/docs/tr/sql_reference/functions/machine_learning_functions.md new file mode 100644 index 00000000000..a1d2857ed82 --- /dev/null +++ b/docs/tr/sql_reference/functions/machine_learning_functions.md @@ -0,0 +1,20 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 64 +toc_title: "Makine \xD6\u011Frenme Fonksiyonlar\u0131" +--- + +# Makine Öğrenme Fonksiyonları {#machine-learning-functions} + +## evalMLMethod (tahmin) {#machine_learning_methods-evalmlmethod} + +Tak fittedılmış regresyon model usinglerini kullanarak tahmin `evalMLMethod` işlev. Lin seeke bakınız `linearRegression`. + +### Stokastik Doğrusal Regresyon {#stochastic-linear-regression} + +Bu [stokastiklinearregression](../../sql_reference/aggregate_functions/reference.md#agg_functions-stochasticlinearregression) toplama fonksiyonu, doğrusal model ve MSE kayıp fonksiyonunu kullanarak stokastik Gradyan iniş yöntemini uygular. Kullanma `evalMLMethod` yeni veri üzerinde tahmin etmek için. + +### Stokastik Lojistik Regresyon {#stochastic-logistic-regression} + +Bu [stochasticLogisticRegression](../../sql_reference/aggregate_functions/reference.md#agg_functions-stochasticlogisticregression) toplama işlevi, ikili sınıflandırma problemi için stokastik Gradyan iniş yöntemini uygular. Kullanma `evalMLMethod` yeni veri üzerinde tahmin etmek için. diff --git a/docs/tr/sql_reference/functions/math_functions.md b/docs/tr/sql_reference/functions/math_functions.md new file mode 100644 index 00000000000..dee89681e73 --- /dev/null +++ b/docs/tr/sql_reference/functions/math_functions.md @@ -0,0 +1,116 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 44 +toc_title: Matematiksel +--- + +# Matematiksel fonksiyonlar {#mathematical-functions} + +Tüm işlevler bir Float64 numarası döndürür. Sonucun doğruluğu mümkün olan en yüksek hassasiyete yakındır, ancak sonuç, ilgili gerçek sayıya en yakın makine temsil edilebilir numarası ile çakışmayabilir. + +## e() {#e} + +E numarasına yakın bir Float64 numarası döndürür. + +## pi sayısı() {#pi} + +Returns a Float64 number that is close to the number π. + +## exp(x) {#expx} + +Sayısal bir bağımsız değişken kabul eder ve bir Float64 sayı argümanın üs yakın döndürür. + +## log (x), L (n(x) {#logx-lnx} + +Sayısal bir bağımsız değişken kabul eder ve bağımsız değişken doğal logaritma yakın bir Float64 sayı döndürür. + +## exp2 (x) {#exp2x} + +Sayısal bir bağımsız değişkeni kabul eder ve X gücüne 2'ye yakın bir Float64 numarası döndürür. + +## log2 (x) {#log2x} + +Sayısal bir bağımsız değişken kabul eder ve değişken ikili logaritma yakın bir Float64 sayı döndürür. + +## exp10 (x) {#exp10x} + +Sayısal bir bağımsız değişkeni kabul eder ve 10'a yakın Float64 numarasını x gücüne döndürür. + +## log10(x) {#log10x} + +Sayısal bir bağımsız değişken kabul eder ve bir float64 sayı bağımsız değişken ondalık logaritması yakın döndürür. + +## sqrt(x) {#sqrtx} + +Sayısal bir bağımsız değişken kabul eder ve bağımsız değişken kareköküne yakın bir Float64 numarası döndürür. + +## TCMB (x) {#cbrtx} + +Sayısal bir bağımsız değişkeni kabul eder ve bağımsız değişken kübik köküne yakın bir Float64 numarası döndürür. + +## erf (x) {#erfx} + +Eğer ‘x’ is non-negative, then erf(x / σ√2) standart sapma ile normal dağılıma sahip bir rasgele değişkenin olasılığı var mı ‘σ’ beklenen değerden daha fazla ayrılan değeri alır ‘x’. + +Örnek (üç sigma kuralı): + +``` sql +SELECT erf(3 / sqrt(2)) +``` + +``` text +┌─erf(divide(3, sqrt(2)))─┐ +│ 0.9973002039367398 │ +└─────────────────────────┘ +``` + +## erfc (x) {#erfcx} + +Sayısal bir bağımsız değişkeni kabul eder ve 1 - erf(x) yakın bir Float64 numarası döndürür, ancak büyük için hassasiyet kaybı olmadan ‘x’ değerler. + +## lgamma (x) {#lgammax} + +Gama fonksiyonunun logaritması. + +## tgamma (x) {#tgammax} + +Gama fonksiyonu. + +## günah(x) {#sinx} + +Sinüs. + +## C (os (x) {#cosx} + +Kosinüs. + +## tan (x) {#tanx} + +Teğet. + +## asin (x) {#asinx} + +Ark sinüsü. + +## acos (x) {#acosx} + +Ark kosinüsü. + +## atan (x) {#atanx} + +Ark teğet. + +## pow (x, y), güç (x, y)) {#powx-y-powerx-y} + +İki sayısal bağımsız değişken X ve y alır.X'e yakın bir Float64 numarasını y gücüne döndürür. + +## ıntexp2 {#intexp2} + +Sayısal bir bağımsız değişkeni kabul eder ve X'in gücüne 2'ye yakın bir uint64 numarası döndürür. + +## ıntexp10 {#intexp10} + +Sayısal bir bağımsız değişkeni kabul eder ve X gücüne 10'a yakın bir uint64 numarası döndürür. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/math_functions/) diff --git a/docs/tr/sql_reference/functions/other_functions.md b/docs/tr/sql_reference/functions/other_functions.md new file mode 100644 index 00000000000..052f289c64f --- /dev/null +++ b/docs/tr/sql_reference/functions/other_functions.md @@ -0,0 +1,1079 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 66 +toc_title: "Di\u011Fer" +--- + +# Diğer fonksiyonlar {#other-functions} + +## hostnamename() {#hostname} + +Bu işlevin gerçekleştirildiği ana bilgisayarın adını içeren bir dize döndürür. Dağıtılmış işlem için, bu işlev uzak bir sunucuda gerçekleştirilirse, uzak sunucu ana bilgisayarının adıdır. + +## FQDN {#fqdn} + +Tam etki alanı adını döndürür. + +**Sözdizimi** + +``` sql +fqdn(); +``` + +Bu işlev büyük / küçük harf duyarsızdır. + +**Döndürülen değer** + +- Tam etki alanı adı ile dize. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT FQDN(); +``` + +Sonuç: + +``` text +┌─FQDN()──────────────────────────┐ +│ clickhouse.ru-central1.internal │ +└─────────────────────────────────┘ +``` + +## basename {#basename} + +Son eğik çizgi veya ters eğik çizgiden sonra bir dizenin sondaki kısmını ayıklar. Bu işlev, genellikle bir yoldan dosya adını ayıklamak için kullanılır. + +``` sql +basename( expr ) +``` + +**Parametre** + +- `expr` — Expression resulting in a [Dize](../../sql_reference/data_types/string.md) type value. Tüm ters eğik çizgilerin ortaya çıkan değerden kaçması gerekir. + +**Döndürülen Değer** + +İçeren bir dize: + +- Son eğik çizgi veya ters eğik çizgiden sonra bir dizenin sondaki kısmı. + + If the input string contains a path ending with slash or backslash, for example, `/` or `c:\`, the function returns an empty string. + +- Eğik çizgi veya ters eğik çizgi yoksa orijinal dize. + +**Örnek** + +``` sql +SELECT 'some/long/path/to/file' AS a, basename(a) +``` + +``` text +┌─a──────────────────────┬─basename('some\\long\\path\\to\\file')─┐ +│ some\long\path\to\file │ file │ +└────────────────────────┴────────────────────────────────────────┘ +``` + +``` sql +SELECT 'some\\long\\path\\to\\file' AS a, basename(a) +``` + +``` text +┌─a──────────────────────┬─basename('some\\long\\path\\to\\file')─┐ +│ some\long\path\to\file │ file │ +└────────────────────────┴────────────────────────────────────────┘ +``` + +``` sql +SELECT 'some-file-name' AS a, basename(a) +``` + +``` text +┌─a──────────────┬─basename('some-file-name')─┐ +│ some-file-name │ some-file-name │ +└────────────────┴────────────────────────────┘ +``` + +## visibleWidth(x) {#visiblewidthx} + +Değerleri konsola metin biçiminde (sekmeyle ayrılmış) çıkarırken yaklaşık genişliği hesaplar. +Bu işlev, sistem tarafından güzel formatların uygulanması için kullanılır. + +`NULL` karşılık gelen bir dize olarak temsil edilir `NULL` içinde `Pretty` biçimliler. + +``` sql +SELECT visibleWidth(NULL) +``` + +``` text +┌─visibleWidth(NULL)─┐ +│ 4 │ +└────────────────────┘ +``` + +## toTypeName (x) {#totypenamex} + +Geçirilen bağımsız değişken türü adını içeren bir dize döndürür. + +Eğer `NULL` fonksiyona girdi olarak geçirilir, daha sonra `Nullable(Nothing)` bir iç karşılık gelen türü `NULL` Clickhouse'da temsil. + +## blockSize() {#function-blocksize} + +Bloğun boyutunu alır. +Clickhouse'da, sorgular her zaman bloklarda (sütun parçaları kümeleri) çalıştırılır. Bu işlev, aradığınız bloğun boyutunu almanızı sağlar. + +## materialize (x) {#materializex} + +Bir sabiti yalnızca bir değer içeren tam bir sütuna dönüştürür. +Clickhouse'da, tam sütunlar ve sabitler bellekte farklı şekilde temsil edilir. İşlevler, sabit argümanlar ve normal argümanlar için farklı şekilde çalışır (farklı kod yürütülür), ancak sonuç hemen hemen her zaman aynıdır. Bu işlev, bu davranış hata ayıklama içindir. + +## ignore(…) {#ignore} + +Dahil olmak üzere herhangi bir argümanı kabul eder `NULL`. Her zaman 0 döndürür. +Ancak, argüman hala değerlendirilir. Bu kriterler için kullanılabilir. + +## uyku (saniye) {#sleepseconds} + +Uykular ‘seconds’ her veri bloğunda saniye. Bir tamsayı veya kayan noktalı sayı belirtebilirsiniz. + +## sleepEachRow (saniye) {#sleepeachrowseconds} + +Uykular ‘seconds’ her satırda saniye. Bir tamsayı veya kayan noktalı sayı belirtebilirsiniz. + +## currentDatabase() {#currentdatabase} + +Geçerli veritabanının adını döndürür. +Bu işlevi, veritabanını belirtmeniz gereken bir tablo oluştur sorgusunda tablo altyapısı parametrelerinde kullanabilirsiniz. + +## currentUser() {#other-function-currentuser} + +Geçerli kullanıcının oturum açma döndürür. Kullanıcı girişi, bu başlatılan sorgu, durumda distibuted sorguda iade edilecektir. + +``` sql +SELECT currentUser(); +``` + +Takma ad: `user()`, `USER()`. + +**Döndürülen değerler** + +- Geçerli kullanıcının girişi. +- Disributed sorgu durumunda sorgu başlatılan kullanıcının giriş. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT currentUser(); +``` + +Sonuç: + +``` text +┌─currentUser()─┐ +│ default │ +└───────────────┘ +``` + +## isFinite (x) {#isfinitex} + +Float32 ve Float64 kabul eder ve bağımsız değişken sonsuz değilse ve bir NaN değilse, Uint8'i 1'e eşit olarak döndürür, aksi halde 0. + +## isİnfinite (x) {#isinfinitex} + +Float32 ve Float64 kabul eder ve bağımsız değişken sonsuz ise 1'e eşit Uİnt8 döndürür, aksi takdirde 0. Bir NaN için 0 döndürüldüğünü unutmayın. + +## ifNotFinite {#ifnotfinite} + +Kayan nokta değerinin sonlu olup olmadığını kontrol eder. + +**Sözdizimi** + + ifNotFinite(x,y) + +**Parametre** + +- `x` — Value to be checked for infinity. Type: [Yüzdürmek\*](../../sql_reference/data_types/float.md). +- `y` — Fallback value. Type: [Yüzdürmek\*](../../sql_reference/data_types/float.md). + +**Döndürülen değer** + +- `x` eğer `x` son isludur. +- `y` eğer `x` sonlu değildir. + +**Örnek** + +Sorgu: + + SELECT 1/0 as infimum, ifNotFinite(infimum,42) + +Sonuç: + + ┌─infimum─┬─ifNotFinite(divide(1, 0), 42)─┐ + │ inf │ 42 │ + └─────────┴───────────────────────────────┘ + +Kullanarak benzer sonuç alabilirsiniz [üçlü operatör](conditional_functions.md#ternary-operator): `isFinite(x) ? x : y`. + +## ısnan (x) {#isnanx} + +Float32 ve Float64 kabul eder ve bağımsız değişken bir NaN, aksi takdirde 0 ise 1'e eşit uint8 döndürür. + +## hasColumnİnTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) {#hascolumnintablehostname-username-password-database-table-column} + +Sabit dizeleri kabul eder: veritabanı adı, tablo adı ve sütun adı. Bir sütun varsa 1'e eşit bir uint8 sabit ifadesi döndürür, aksi halde 0. Hostname parametresi ayarlanmışsa, sınama uzak bir sunucuda çalışır. +Tablo yoksa, işlev bir özel durum atar. +İç içe veri yapısındaki öğeler için işlev, bir sütunun varlığını denetler. İç içe veri yapısının kendisi için işlev 0 döndürür. + +## bar {#function-bar} + +Unicode-art diyagramı oluşturmaya izin verir. + +`bar(x, min, max, width)` genişliği orantılı olan bir bant çizer `(x - min)` ve eşit `width` karakterler ne zaman `x = max`. + +Parametre: + +- `x` — Size to display. +- `min, max` — Integer constants. The value must fit in `Int64`. +- `width` — Constant, positive integer, can be fractional. + +Bant, bir sembolün sekizde birine doğrulukla çizilir. + +Örnek: + +``` sql +SELECT + toHour(EventTime) AS h, + count() AS c, + bar(c, 0, 600000, 20) AS bar +FROM test.hits +GROUP BY h +ORDER BY h ASC +``` + +``` text +┌──h─┬──────c─┬─bar────────────────┐ +│ 0 │ 292907 │ █████████▋ │ +│ 1 │ 180563 │ ██████ │ +│ 2 │ 114861 │ ███▋ │ +│ 3 │ 85069 │ ██▋ │ +│ 4 │ 68543 │ ██▎ │ +│ 5 │ 78116 │ ██▌ │ +│ 6 │ 113474 │ ███▋ │ +│ 7 │ 170678 │ █████▋ │ +│ 8 │ 278380 │ █████████▎ │ +│ 9 │ 391053 │ █████████████ │ +│ 10 │ 457681 │ ███████████████▎ │ +│ 11 │ 493667 │ ████████████████▍ │ +│ 12 │ 509641 │ ████████████████▊ │ +│ 13 │ 522947 │ █████████████████▍ │ +│ 14 │ 539954 │ █████████████████▊ │ +│ 15 │ 528460 │ █████████████████▌ │ +│ 16 │ 539201 │ █████████████████▊ │ +│ 17 │ 523539 │ █████████████████▍ │ +│ 18 │ 506467 │ ████████████████▊ │ +│ 19 │ 520915 │ █████████████████▎ │ +│ 20 │ 521665 │ █████████████████▍ │ +│ 21 │ 542078 │ ██████████████████ │ +│ 22 │ 493642 │ ████████████████▍ │ +│ 23 │ 400397 │ █████████████▎ │ +└────┴────────┴────────────────────┘ +``` + +## dönüştürmek {#transform} + +Bir değeri, bazı öğelerin açıkça tanımlanmış eşlemesine göre diğer öğelere dönüştürür. +Bu fonksiyonun iki varyasyonu vardır: + +### transform (x, array\_from, array\_to, varsayılan) {#transformx-array-from-array-to-default} + +`x` – What to transform. + +`array_from` – Constant array of values for converting. + +`array_to` – Constant array of values to convert the values in ‘from’ -e doğru. + +`default` – Which value to use if ‘x’ değer anylerden hiçbir equaline eşit değildir. ‘from’. + +`array_from` ve `array_to` – Arrays of the same size. + +Türler: + +`transform(T, Array(T), Array(U), U) -> U` + +`T` ve `U` sayısal, dize veya tarih veya DateTime türleri olabilir. +Aynı harfin belirtildiği (t veya U), sayısal türler için bunlar eşleşen türler değil, ortak bir türe sahip türler olabilir. +Örneğin, ilk bağımsız değişken Int64 türüne sahip olabilir, ikincisi ise Array(Uİnt16) türüne sahiptir. + +Eğer... ‘x’ değer, içindeki öğelerden birine eşittir. ‘array\_from’ array, varolan öğeyi döndürür (aynı numaralandırılır) ‘array\_to’ dizi. Aksi takdirde, döner ‘default’. İçinde birden fazla eşleşen öğe varsa ‘array\_from’, maçlardan birini döndürür. + +Örnek: + +``` sql +SELECT + transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other') AS title, + count() AS c +FROM test.hits +WHERE SearchEngineID != 0 +GROUP BY title +ORDER BY c DESC +``` + +``` text +┌─title─────┬──────c─┐ +│ Yandex │ 498635 │ +│ Google │ 229872 │ +│ Other │ 104472 │ +└───────────┴────────┘ +``` + +### transform (x, array\_from, array\_to) {#transformx-array-from-array-to} + +İlk vary thatasyon differsdan farklıdır. ‘default’ argüman atlandı. +Eğer... ‘x’ değer, içindeki öğelerden birine eşittir. ‘array\_from’ array, eşleşen öğeyi (aynı numaralandırılmış) döndürür ‘array\_to’ dizi. Aksi takdirde, döner ‘x’. + +Türler: + +`transform(T, Array(T), Array(T)) -> T` + +Örnek: + +``` sql +SELECT + transform(domain(Referer), ['yandex.ru', 'google.ru', 'vk.com'], ['www.yandex', 'example.com']) AS s, + count() AS c +FROM test.hits +GROUP BY domain(Referer) +ORDER BY count() DESC +LIMIT 10 +``` + +``` text +┌─s──────────────┬───────c─┐ +│ │ 2906259 │ +│ www.yandex │ 867767 │ +│ ███████.ru │ 313599 │ +│ mail.yandex.ru │ 107147 │ +│ ██████.ru │ 100355 │ +│ █████████.ru │ 65040 │ +│ news.yandex.ru │ 64515 │ +│ ██████.net │ 59141 │ +│ example.com │ 57316 │ +└────────────────┴─────────┘ +``` + +## formatReadableSize (x) {#formatreadablesizex} + +Boyutu (bayt sayısı) kabul eder. Bir sonek (KiB, MıB, vb.) ile yuvarlak bir boyut döndürür.) bir dize olarak. + +Örnek: + +``` sql +SELECT + arrayJoin([1, 1024, 1024*1024, 192851925]) AS filesize_bytes, + formatReadableSize(filesize_bytes) AS filesize +``` + +``` text +┌─filesize_bytes─┬─filesize───┐ +│ 1 │ 1.00 B │ +│ 1024 │ 1.00 KiB │ +│ 1048576 │ 1.00 MiB │ +│ 192851925 │ 183.92 MiB │ +└────────────────┴────────────┘ +``` + +## en az (a, b) {#leasta-b} + +A ve B'den en küçük değeri döndürür. + +## en büyük (a, b) {#greatesta-b} + +A ve B'nin en büyük değerini döndürür. + +## çalışma süresi() {#uptime} + +Sunucunun çalışma süresini saniyeler içinde döndürür. + +## sürüm() {#version} + +Sunucu sürümünü bir dize olarak döndürür. + +## saat dilimi() {#timezone} + +Sunucunun saat dilimini döndürür. + +## blockNumber {#blocknumber} + +Satırın bulunduğu veri bloğunun sıra numarasını döndürür. + +## rowNumberİnBlock {#function-rownumberinblock} + +Veri bloğundaki satırın sıra numarasını döndürür. Farklı veri blokları her zaman yeniden hesaplanır. + +## rownumberınallblocks() {#rownumberinallblocks} + +Veri bloğundaki satırın sıra numarasını döndürür. Bu işlev yalnızca etkilenen veri bloklarını dikkate alır. + +## komşuluk {#neighbor} + +Belirli bir sütunun geçerli satırından önce veya sonra gelen belirli bir ofsette bir satıra erişim sağlayan pencere işlevi. + +**Sözdizimi** + +``` sql +neighbor(column, offset[, default_value]) +``` + +İşlevin sonucu, etkilenen veri bloklarına ve bloktaki veri sırasına bağlıdır. +ORDER BY ile bir alt sorgu yaparsanız ve alt sorgunun dışından işlevi çağırırsanız, beklenen sonucu alabilirsiniz. + +**Parametre** + +- `column` — A column name or scalar expression. +- `offset` — The number of rows forwards or backwards from the current row of `column`. [Int64](../../sql_reference/data_types/int_uint.md). +- `default_value` — Optional. The value to be returned if offset goes beyond the scope of the block. Type of data blocks affected. + +**Döndürülen değerler** + +- İçin değer `column` içinde `offset` eğer geçerli satırdan uzaklık `offset` değer blok sınırları dışında değil. +- İçin varsayılan değer `column` eğer `offset` değer, blok sınırlarının dışındadır. Eğer `default_value` verilir, daha sonra kullanılacaktır. + +Tür: etkilenen veri bloklarının türü veya varsayılan değer türü. + +**Örnek** + +Sorgu: + +``` sql +SELECT number, neighbor(number, 2) FROM system.numbers LIMIT 10; +``` + +Sonuç: + +``` text +┌─number─┬─neighbor(number, 2)─┐ +│ 0 │ 2 │ +│ 1 │ 3 │ +│ 2 │ 4 │ +│ 3 │ 5 │ +│ 4 │ 6 │ +│ 5 │ 7 │ +│ 6 │ 8 │ +│ 7 │ 9 │ +│ 8 │ 0 │ +│ 9 │ 0 │ +└────────┴─────────────────────┘ +``` + +Sorgu: + +``` sql +SELECT number, neighbor(number, 2, 999) FROM system.numbers LIMIT 10; +``` + +Sonuç: + +``` text +┌─number─┬─neighbor(number, 2, 999)─┐ +│ 0 │ 2 │ +│ 1 │ 3 │ +│ 2 │ 4 │ +│ 3 │ 5 │ +│ 4 │ 6 │ +│ 5 │ 7 │ +│ 6 │ 8 │ +│ 7 │ 9 │ +│ 8 │ 999 │ +│ 9 │ 999 │ +└────────┴──────────────────────────┘ +``` + +Bu işlev, yıldan yıla metrik değeri hesaplamak için kullanılabilir: + +Sorgu: + +``` sql +WITH toDate('2018-01-01') AS start_date +SELECT + toStartOfMonth(start_date + (number * 32)) AS month, + toInt32(month) % 100 AS money, + neighbor(money, -12) AS prev_year, + round(prev_year / money, 2) AS year_over_year +FROM numbers(16) +``` + +Sonuç: + +``` text +┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ +│ 2018-01-01 │ 32 │ 0 │ 0 │ +│ 2018-02-01 │ 63 │ 0 │ 0 │ +│ 2018-03-01 │ 91 │ 0 │ 0 │ +│ 2018-04-01 │ 22 │ 0 │ 0 │ +│ 2018-05-01 │ 52 │ 0 │ 0 │ +│ 2018-06-01 │ 83 │ 0 │ 0 │ +│ 2018-07-01 │ 13 │ 0 │ 0 │ +│ 2018-08-01 │ 44 │ 0 │ 0 │ +│ 2018-09-01 │ 75 │ 0 │ 0 │ +│ 2018-10-01 │ 5 │ 0 │ 0 │ +│ 2018-11-01 │ 36 │ 0 │ 0 │ +│ 2018-12-01 │ 66 │ 0 │ 0 │ +│ 2019-01-01 │ 97 │ 32 │ 0.33 │ +│ 2019-02-01 │ 28 │ 63 │ 2.25 │ +│ 2019-03-01 │ 56 │ 91 │ 1.62 │ +│ 2019-04-01 │ 87 │ 22 │ 0.25 │ +└────────────┴───────┴───────────┴────────────────┘ +``` + +## runningDifference (x) {#other_functions-runningdifference} + +Calculates the difference between successive row values ​​in the data block. +İlk satır için 0 ve sonraki her satır için önceki satırdan farkı döndürür. + +İşlevin sonucu, etkilenen veri bloklarına ve bloktaki veri sırasına bağlıdır. +ORDER BY ile bir alt sorgu yaparsanız ve alt sorgunun dışından işlevi çağırırsanız, beklenen sonucu alabilirsiniz. + +Örnek: + +``` sql +SELECT + EventID, + EventTime, + runningDifference(EventTime) AS delta +FROM +( + SELECT + EventID, + EventTime + FROM events + WHERE EventDate = '2016-11-24' + ORDER BY EventTime ASC + LIMIT 5 +) +``` + +``` text +┌─EventID─┬───────────EventTime─┬─delta─┐ +│ 1106 │ 2016-11-24 00:00:04 │ 0 │ +│ 1107 │ 2016-11-24 00:00:05 │ 1 │ +│ 1108 │ 2016-11-24 00:00:05 │ 0 │ +│ 1109 │ 2016-11-24 00:00:09 │ 4 │ +│ 1110 │ 2016-11-24 00:00:10 │ 1 │ +└─────────┴─────────────────────┴───────┘ +``` + +Lütfen dikkat - blok boyutu sonucu etkiler. Her yeni blok ile, `runningDifference` durum sıfırlandı. + +``` sql +SELECT + number, + runningDifference(number + 1) AS diff +FROM numbers(100000) +WHERE diff != 1 +``` + +``` text +┌─number─┬─diff─┐ +│ 0 │ 0 │ +└────────┴──────┘ +┌─number─┬─diff─┐ +│ 65536 │ 0 │ +└────────┴──────┘ +``` + +``` sql +set max_block_size=100000 -- default value is 65536! + +SELECT + number, + runningDifference(number + 1) AS diff +FROM numbers(100000) +WHERE diff != 1 +``` + +``` text +┌─number─┬─diff─┐ +│ 0 │ 0 │ +└────────┴──────┘ +``` + +## runningDifferenceStartingWithFirstvalue {#runningdifferencestartingwithfirstvalue} + +İçin aynı [runningDifference](./other_functions.md#other_functions-runningdifference), fark ilk satırın değeridir, ilk satırın değerini döndürdü ve sonraki her satır önceki satırdan farkı döndürür. + +## MACNumToString (num) {#macnumtostringnum} + +Bir uınt64 numarasını kabul eder. Big endian'da bir MAC adresi olarak yorumlar. AA:BB:CC:DD:EE:FF biçiminde karşılık gelen MAC adresini içeren bir dize döndürür (onaltılık formda iki nokta üst üste ayrılmış sayılar). + +## MACStringToNum (s) {#macstringtonums} + +MACNumToString ters işlevi. MAC adresi geçersiz bir biçime sahipse, 0 döndürür. + +## MACStringToOUİ (s) {#macstringtoouis} + +AA:BB:CC:DD:EE:FF (onaltılık formda iki nokta üst üste ayrılmış sayılar) biçiminde bir MAC adresi kabul eder. İlk üç sekizli uint64 numarası olarak döndürür. MAC adresi geçersiz bir biçime sahipse, 0 döndürür. + +## getSizeOfEnumType {#getsizeofenumtype} + +Alan sayısını döndürür [Enum](../../sql_reference/data_types/enum.md). + +``` sql +getSizeOfEnumType(value) +``` + +**Parametre:** + +- `value` — Value of type `Enum`. + +**Döndürülen değerler** + +- İle alan sayısı `Enum` giriş değerleri. +- Tür değilse bir istisna atılır `Enum`. + +**Örnek** + +``` sql +SELECT getSizeOfEnumType( CAST('a' AS Enum8('a' = 1, 'b' = 2) ) ) AS x +``` + +``` text +┌─x─┐ +│ 2 │ +└───┘ +``` + +## blockSerializedSize {#blockserializedsize} + +Diskteki boyutu döndürür (sıkıştırmayı hesaba katmadan). + +``` sql +blockSerializedSize(value[, value[, ...]]) +``` + +**Parametre:** + +- `value` — Any value. + +**Döndürülen değerler** + +- (Sıkıştırma olmadan) değerler bloğu için diske yazılacak bayt sayısı. + +**Örnek** + +``` sql +SELECT blockSerializedSize(maxState(1)) as x +``` + +``` text +┌─x─┐ +│ 2 │ +└───┘ +``` + +## toColumnTypeName {#tocolumntypename} + +RAM'DEKİ sütunun veri türünü temsil eden sınıfın adını döndürür. + +``` sql +toColumnTypeName(value) +``` + +**Parametre:** + +- `value` — Any type of value. + +**Döndürülen değerler** + +- Temsil etmek için kullanılan sınıfın adını içeren bir dize `value` RAM veri türü. + +**Arasındaki fark örneği`toTypeName ' and ' toColumnTypeName`** + +``` sql +SELECT toTypeName(CAST('2018-01-01 01:02:03' AS DateTime)) +``` + +``` text +┌─toTypeName(CAST('2018-01-01 01:02:03', 'DateTime'))─┐ +│ DateTime │ +└─────────────────────────────────────────────────────┘ +``` + +``` sql +SELECT toColumnTypeName(CAST('2018-01-01 01:02:03' AS DateTime)) +``` + +``` text +┌─toColumnTypeName(CAST('2018-01-01 01:02:03', 'DateTime'))─┐ +│ Const(UInt32) │ +└───────────────────────────────────────────────────────────┘ +``` + +Örnek gösteriyor ki `DateTime` veri türü olarak bellekte saklanır `Const(UInt32)`. + +## dumpColumnStructure {#dumpcolumnstructure} + +Ram'deki veri yapılarının ayrıntılı bir açıklamasını verir + +``` sql +dumpColumnStructure(value) +``` + +**Parametre:** + +- `value` — Any type of value. + +**Döndürülen değerler** + +- Temsil etmek için kullanılan yapıyı açıklayan bir dize `value` RAM veri türü. + +**Örnek** + +``` sql +SELECT dumpColumnStructure(CAST('2018-01-01 01:02:03', 'DateTime')) +``` + +``` text +┌─dumpColumnStructure(CAST('2018-01-01 01:02:03', 'DateTime'))─┐ +│ DateTime, Const(size = 1, UInt32(size = 1)) │ +└──────────────────────────────────────────────────────────────┘ +``` + +## defaultValueOfArgumentType {#defaultvalueofargumenttype} + +Veri türü için varsayılan değeri verir. + +Kullanıcı tarafından ayarlanan özel sütunlar için varsayılan değerleri içermez. + +``` sql +defaultValueOfArgumentType(expression) +``` + +**Parametre:** + +- `expression` — Arbitrary type of value or an expression that results in a value of an arbitrary type. + +**Döndürülen değerler** + +- `0` sayılar için. +- Dizeler için boş dize. +- `ᴺᵁᴸᴸ` için [Nullable](../../sql_reference/data_types/nullable.md). + +**Örnek** + +``` sql +SELECT defaultValueOfArgumentType( CAST(1 AS Int8) ) +``` + +``` text +┌─defaultValueOfArgumentType(CAST(1, 'Int8'))─┐ +│ 0 │ +└─────────────────────────────────────────────┘ +``` + +``` sql +SELECT defaultValueOfArgumentType( CAST(1 AS Nullable(Int8) ) ) +``` + +``` text +┌─defaultValueOfArgumentType(CAST(1, 'Nullable(Int8)'))─┐ +│ ᴺᵁᴸᴸ │ +└───────────────────────────────────────────────────────┘ +``` + +## çoğaltmak {#other-functions-replicate} + +Tek bir değere sahip bir dizi oluşturur. + +İç uygulama için kullanılan [arrayJoin](array_join.md#functions_arrayjoin). + +``` sql +SELECT replicate(x, arr); +``` + +**Parametre:** + +- `arr` — Original array. ClickHouse creates a new array of the same length as the original and fills it with the value `x`. +- `x` — The value that the resulting array will be filled with. + +**Döndürülen değer** + +Değerle dolu bir dizi `x`. + +Tür: `Array`. + +**Örnek** + +Sorgu: + +``` sql +SELECT replicate(1, ['a', 'b', 'c']) +``` + +Sonuç: + +``` text +┌─replicate(1, ['a', 'b', 'c'])─┐ +│ [1,1,1] │ +└───────────────────────────────┘ +``` + +## filesystemAvailable {#filesystemavailable} + +Veritabanlarının dosyalarının bulunduğu dosya sisteminde kalan alan miktarını döndürür. Her zaman toplam boş alandan daha küçüktür ([filesystemFree](#filesystemfree)) çünkü OS için biraz alan ayrılmıştır. + +**Sözdizimi** + +``` sql +filesystemAvailable() +``` + +**Döndürülen değer** + +- Bayt olarak kullanılabilir kalan alan miktarı. + +Tür: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT formatReadableSize(filesystemAvailable()) AS "Available space", toTypeName(filesystemAvailable()) AS "Type"; +``` + +Sonuç: + +``` text +┌─Available space─┬─Type───┐ +│ 30.75 GiB │ UInt64 │ +└─────────────────┴────────┘ +``` + +## filesystemFree {#filesystemfree} + +Veritabanlarının dosyalarının bulunduğu dosya sistemindeki boş alanın toplam miktarını döndürür. Ayrıca bakınız `filesystemAvailable` + +**Sözdizimi** + +``` sql +filesystemFree() +``` + +**Döndürülen değer** + +- Bayt cinsinden boş alan miktarı. + +Tür: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT formatReadableSize(filesystemFree()) AS "Free space", toTypeName(filesystemFree()) AS "Type"; +``` + +Sonuç: + +``` text +┌─Free space─┬─Type───┐ +│ 32.39 GiB │ UInt64 │ +└────────────┴────────┘ +``` + +## filesystemCapacity {#filesystemcapacity} + +Dosya sisteminin kapasitesini bayt cinsinden döndürür. Değerlendirme için, [yol](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-path) veri dizinine yapılandırılmalıdır. + +**Sözdizimi** + +``` sql +filesystemCapacity() +``` + +**Döndürülen değer** + +- Dosya sisteminin bayt cinsinden kapasite bilgisi. + +Tür: [Uİnt64](../../sql_reference/data_types/int_uint.md). + +**Örnek** + +Sorgu: + +``` sql +SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesystemCapacity()) AS "Type" +``` + +Sonuç: + +``` text +┌─Capacity──┬─Type───┐ +│ 39.32 GiB │ UInt64 │ +└───────────┴────────┘ +``` + +## finalizeAggregation {#function-finalizeaggregation} + +Toplama işlevinin durumunu alır. Toplama sonucunu döndürür (kesinleşmiş durum). + +## runningAccumulate {#function-runningaccumulate} + +Toplama işlevinin durumlarını alır ve değerleri olan bir sütun döndürür, bu durumların bir dizi blok satırı için ilk satırdan geçerli satıra birikmesinin sonucudur. +Örneğin, toplama işlevinin durumunu alır (örnek runningAccumulate (uniqState (Userıd))) ve her blok satırı için, önceki tüm Satırların ve geçerli satırın durumlarının birleştirilmesinde toplama işlevinin sonucunu döndürür. +Bu nedenle, işlevin sonucu, verilerin bloklara bölünmesine ve blok içindeki verilerin sırasına bağlıdır. + +## joinGet {#joinget} + +İşlev, tablodan verileri bir tablodan aynı şekilde ayıklamanızı sağlar [sözlük](../../sql_reference/dictionaries/index.md). + +Veri alır [Katmak](../../engines/table_engines/special/join.md#creating-a-table) belirtilen birleştirme anahtarını kullanarak tablolar. + +Sadece ile oluşturulan tabloları destekler `ENGINE = Join(ANY, LEFT, )` deyim. + +**Sözdizimi** + +``` sql +joinGet(join_storage_table_name, `value_column`, join_keys) +``` + +**Parametre** + +- `join_storage_table_name` — an [tanıtıcı](../syntax.md#syntax-identifiers) aramanın nerede yapıldığını gösterir. Tanımlayıcı varsayılan veritabanında aranır (bkz. parametre `default_database` config dosyası). Varsayılan veritabanını geçersiz kılmak için `USE db_name` veya ayırıcı aracılığıyla veritabanını ve tabloyu belirtin `db_name.db_table` örnek bakın. +- `value_column` — name of the column of the table that contains required data. +- `join_keys` — list of keys. + +**Döndürülen değer** + +Anahtarların listesine karşılık gelen değerlerin listesini döndürür. + +Kaynak tabloda kesin yoksa o zaman `0` veya `null` esas alınarak iade edilecektir [join\_use\_nulls](../../operations/settings/settings.md#join_use_nulls) ayar. + +Hakkında daha fazla bilgi `join_use_nulls` içinde [Birleştirme işlemi](../../engines/table_engines/special/join.md). + +**Örnek** + +Giriş tablosu: + +``` sql +CREATE DATABASE db_test +CREATE TABLE db_test.id_val(`id` UInt32, `val` UInt32) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 1 +INSERT INTO db_test.id_val VALUES (1,11)(2,12)(4,13) +``` + +``` text +┌─id─┬─val─┐ +│ 4 │ 13 │ +│ 2 │ 12 │ +│ 1 │ 11 │ +└────┴─────┘ +``` + +Sorgu: + +``` sql +SELECT joinGet(db_test.id_val,'val',toUInt32(number)) from numbers(4) SETTINGS join_use_nulls = 1 +``` + +Sonuç: + +``` text +┌─joinGet(db_test.id_val, 'val', toUInt32(number))─┐ +│ 0 │ +│ 11 │ +│ 12 │ +│ 0 │ +└──────────────────────────────────────────────────┘ +``` + +## modelEvaluate(model\_name, …) {#function-modelevaluate} + +Dış modeli değerlendirin. +Bir model adı ve model bağımsız değişkenleri kabul eder. Float64 Döndürür. + +## throwİf(x \[, custom\_message\]) {#throwifx-custom-message} + +Argüman sıfır değilse bir istisna atın. +custom\_message-isteğe bağlı bir parametredir: sabit bir dize, bir hata mesajı sağlar + +``` sql +SELECT throwIf(number = 3, 'Too many') FROM numbers(10); +``` + +``` text +↙ Progress: 0.00 rows, 0.00 B (0.00 rows/s., 0.00 B/s.) Received exception from server (version 19.14.1): +Code: 395. DB::Exception: Received from localhost:9000. DB::Exception: Too many. +``` + +## kimlik {#identity} + +Bağımsız değişkeni olarak kullanılan aynı değeri döndürür. Hata ayıklama ve test için kullanılan, dizin kullanarak iptal ve tam bir tarama sorgu performansını almak için izin verir. Olası dizin kullanımı için sorgu analiz edildiğinde, analizör içeriye bakmaz `identity` işlevler. + +**Sözdizimi** + +``` sql +identity(x) +``` + +**Örnek** + +Sorgu: + +``` sql +SELECT identity(42) +``` + +Sonuç: + +``` text +┌─identity(42)─┐ +│ 42 │ +└──────────────┘ +``` + +## randomPrintableASCİİ {#randomascii} + +Rastgele bir dizi ile bir dize oluşturur [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) yazdırılabilir karakterler. + +**Sözdizimi** + +``` sql +randomPrintableASCII(length) +``` + +**Parametre** + +- `length` — Resulting string length. Positive integer. + + If you pass `length < 0`, behavior of the function is undefined. + +**Döndürülen değer** + +- Rastgele bir dizi dize [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) yazdırılabilir karakterler. + +Tür: [Dize](../../sql_reference/data_types/string.md) + +**Örnek** + +``` sql +SELECT number, randomPrintableASCII(30) as str, length(str) FROM system.numbers LIMIT 3 +``` + +``` text +┌─number─┬─str────────────────────────────┬─length(randomPrintableASCII(30))─┐ +│ 0 │ SuiCOSTvC0csfABSw=UcSzp2.`rv8x │ 30 │ +│ 1 │ 1Ag NlJ &RCN:*>HVPG;PE-nO"SUFD │ 30 │ +│ 2 │ /"+<"wUTh:=LjJ Vm!c&hI*m#XTfzz │ 30 │ +└────────┴────────────────────────────────┴──────────────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/other_functions/) diff --git a/docs/tr/sql_reference/functions/random_functions.md b/docs/tr/sql_reference/functions/random_functions.md new file mode 100644 index 00000000000..e57106426f4 --- /dev/null +++ b/docs/tr/sql_reference/functions/random_functions.md @@ -0,0 +1,30 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 51 +toc_title: "S\xF6zde Rasgele Say\u0131lar Olu\u015Fturma" +--- + +# Sözde rasgele sayılar üretmek için fonksiyonlar {#functions-for-generating-pseudo-random-numbers} + +Sözde rasgele sayıların kriptografik olmayan jeneratörleri kullanılır. + +Tüm işlevler sıfır bağımsız değişkeni veya bir bağımsız değişkeni kabul eder. +Bir argüman geçirilirse, herhangi bir tür olabilir ve değeri hiçbir şey için kullanılmaz. +Bu argümanın tek amacı, aynı işlevin iki farklı örneğinin farklı rasgele sayılarla farklı sütunlar döndürmesi için ortak alt ifade eliminasyonunu önlemektir. + +## Güney Afrika parası {#rand} + +Tüm uint32 tipi sayılar arasında eşit olarak dağıtılan bir sözde rasgele uint32 numarası döndürür. +Doğrusal bir uyumlu jeneratör kullanır. + +## rand64 {#rand64} + +Tüm uint64 tipi sayılar arasında eşit olarak dağıtılan sözde rasgele bir uint64 numarası döndürür. +Doğrusal bir uyumlu jeneratör kullanır. + +## randConstant {#randconstant} + +Bir sözde rastgele uint32 numarası döndürür, değer farklı bloklar için birdir. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/random_functions/) diff --git a/docs/tr/sql_reference/functions/rounding_functions.md b/docs/tr/sql_reference/functions/rounding_functions.md new file mode 100644 index 00000000000..6d47ae54dae --- /dev/null +++ b/docs/tr/sql_reference/functions/rounding_functions.md @@ -0,0 +1,190 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 45 +toc_title: "D\xF6nm\xFC\u015F" +--- + +# Yuvarlama fonksiyonları {#rounding-functions} + +## kat(x \[, N\]) {#floorx-n} + +Küçük veya eşit olan en büyük yuvarlak sayıyı döndürür `x`. Yuvarlak bir sayı, 1/10N'NİN katları veya 1 / 10N tam değilse, uygun veri türünün en yakın sayısıdır. +‘N’ bir tamsayı sabiti, isteğe bağlı parametredir. Varsayılan olarak sıfırdır, bu da bir tam sayıya yuvarlamak anlamına gelir. +‘N’ negatif olabilir. + +Örnekler: `floor(123.45, 1) = 123.4, floor(123.45, -1) = 120.` + +`x` herhangi bir sayısal türüdür. Sonuç aynı türden bir sayıdır. +Tamsayı argümanları için, bir negatif ile yuvarlamak mantıklıdır `N` değer (negatif olmayan için `N`, işlev hiçbir şey yapmaz). +Yuvarlama taşmasına neden olursa (örneğin, floor (-128, -1)), uygulamaya özgü bir sonuç döndürülür. + +## tavan(x \[, N\]), tavan (x \[, N\]) {#ceilx-n-ceilingx-n} + +Büyük veya eşit olan en küçük yuvarlak sayıyı döndürür `x`. Diğer her şekilde, aynı `floor` (yukarıda) işlevi. + +## trunc(x \[, N\]), truncate(x \[, N\]) {#truncx-n-truncatex-n} + +Mutlak değeri küçük veya eşit olan en büyük mutlak değere sahip yuvarlak sayıyı döndürür `x`‘s. In every other way, it is the same as the ’floor’ (yukarıda) işlevi. + +## Yuvarlak(x \[, N\]) {#rounding_functions-round} + +Belirtilen sayıda ondalık basamak için bir değer yuvarlar. + +İşlev, belirtilen siparişin en yakın numarasını döndürür. Verilen sayı çevreleyen sayılara eşit mesafeye sahip olduğunda, işlev, float sayı türleri için bankacının yuvarlamasını kullanır ve diğer sayı türleri için sıfırdan uzaklaşır. + +``` sql +round(expression [, decimal_places]) +``` + +**Parametre:** + +- `expression` — A number to be rounded. Can be any [ifade](../syntax.md#syntax-expressions) sayısal dönen [veri türü](../../sql_reference/data_types/index.md#data_types). +- `decimal-places` — An integer value. + - Eğer `decimal-places > 0` sonra işlev değeri ondalık noktanın sağına yuvarlar. + - Eğer `decimal-places < 0` ardından işlev değeri ondalık noktanın soluna yuvarlar. + - Eğer `decimal-places = 0` sonra işlev değeri tamsayı olarak yuvarlar. Bu durumda argüman ihmal edilebilir. + +**Döndürülen değer:** + +Giriş numarası ile aynı türden yuvarlatılmış sayı. + +### Örnekler {#examples} + +**Kullanım örneği** + +``` sql +SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3 +``` + +``` text +┌───x─┬─round(divide(number, 2))─┐ +│ 0 │ 0 │ +│ 0.5 │ 0 │ +│ 1 │ 1 │ +└─────┴──────────────────────────┘ +``` + +**Yuvarlama örnekleri** + +En yakın numaraya yuvarlama. + +``` text +round(3.2, 0) = 3 +round(4.1267, 2) = 4.13 +round(22,-1) = 20 +round(467,-2) = 500 +round(-467,-2) = -500 +``` + +Bankacı yuvarlanıyor. + +``` text +round(3.5) = 4 +round(4.5) = 4 +round(3.55, 1) = 3.6 +round(3.65, 1) = 3.6 +``` + +**Ayrıca Bakınız** + +- [roundBankers](#roundbankers) + +## roundBankers {#roundbankers} + +Bir sayıyı belirtilen ondalık konuma yuvarlar. + +- Yuvarlama sayısı iki sayı arasında yarıya ise, işlev banker yuvarlama kullanır. + + Banker's rounding is a method of rounding fractional numbers. When the rounding number is halfway between two numbers, it's rounded to the nearest even digit at the specified decimal position. For example: 3.5 rounds up to 4, 2.5 rounds down to 2. + + It's the default rounding method for floating point numbers defined in [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754#Roundings_to_nearest). The [round](#rounding_functions-round) function performs the same rounding for floating point numbers. The `roundBankers` function also rounds integers the same way, for example, `roundBankers(45, -1) = 40`. + +- Diğer durumlarda, işlev sayıları en yakın tam sayıya yuvarlar. + +Banker yuvarlama kullanarak, yuvarlama numaraları toplama veya bu sayıları çıkarma sonuçları üzerindeki etkisini azaltabilir. + +Örneğin, farklı yuvarlama ile 1.5, 2.5, 3.5, 4.5 sayılarını topla: + +- Yuvarlama yok: 1.5 + 2.5 + 3.5 + 4.5 = 12. +- Bankacı yuvarlama: 2 + 2 + 4 + 4 = 12. +- En yakın tam sayıya yuvarlama: 2 + 3 + 4 + 5 = 14. + +**Sözdizimi** + +``` sql +roundBankers(expression [, decimal_places]) +``` + +**Parametre** + +- `expression` — A number to be rounded. Can be any [ifade](../syntax.md#syntax-expressions) sayısal dönen [veri türü](../../sql_reference/data_types/index.md#data_types). +- `decimal-places` — Decimal places. An integer number. + - `decimal-places > 0` — The function rounds the number to the given position right of the decimal point. Example: `roundBankers(3.55, 1) = 3.6`. + - `decimal-places < 0` — The function rounds the number to the given position left of the decimal point. Example: `roundBankers(24.55, -1) = 20`. + - `decimal-places = 0` — The function rounds the number to an integer. In this case the argument can be omitted. Example: `roundBankers(2.5) = 2`. + +**Döndürülen değer** + +Banker yuvarlama yöntemi tarafından yuvarlanan bir değer. + +### Örnekler {#examples-1} + +**Kullanım örneği** + +Sorgu: + +``` sql + SELECT number / 2 AS x, roundBankers(x, 0) AS b fROM system.numbers limit 10 +``` + +Sonuç: + +``` text +┌───x─┬─b─┐ +│ 0 │ 0 │ +│ 0.5 │ 0 │ +│ 1 │ 1 │ +│ 1.5 │ 2 │ +│ 2 │ 2 │ +│ 2.5 │ 2 │ +│ 3 │ 3 │ +│ 3.5 │ 4 │ +│ 4 │ 4 │ +│ 4.5 │ 4 │ +└─────┴───┘ +``` + +**Bankacı yuvarlama örnekleri** + +``` text +roundBankers(0.4) = 0 +roundBankers(-3.5) = -4 +roundBankers(4.5) = 4 +roundBankers(3.55, 1) = 3.6 +roundBankers(3.65, 1) = 3.6 +roundBankers(10.35, 1) = 10.4 +roundBankers(10.755, 2) = 11,76 +``` + +**Ayrıca Bakınız** + +- [turlu](#rounding_functions-round) + +## roundToExp2 (num) {#roundtoexp2num} + +Bir sayı kabul eder. Sayı birden az ise, 0 döndürür. Aksi takdirde, sayıyı en yakın (negatif olmayan) iki dereceye yuvarlar. + +## roundDuration (num) {#rounddurationnum} + +Bir sayı kabul eder. Sayı birden az ise, 0 döndürür. Aksi takdirde, sayıyı kümeden sayılara yuvarlar: 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000. Bu fonksiyon (kayıt olmak için özeldir.Metrica ve oturum uzunluğu raporu uygulamak için kullanılır. + +## roundAge (num) {#roundagenum} + +Bir sayı kabul eder. Sayı 18'den küçükse, 0 döndürür. Aksi takdirde, sayıyı kümeden bir sayıya yuvarlar: 18, 25, 35, 45, 55. Bu fonksiyon (kayıt olmak için özeldir.Metrica ve kullanıcı yaş raporu uygulamak için kullanılır. + +## roundDown (num, arr) {#rounddownnum-arr} + +Bir sayıyı kabul eder ve belirtilen Dizideki bir öğeye yuvarlar. Değer en düşük sınırdan küçükse, en düşük sınır döndürülür. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/rounding_functions/) diff --git a/docs/tr/sql_reference/functions/splitting_merging_functions.md b/docs/tr/sql_reference/functions/splitting_merging_functions.md new file mode 100644 index 00000000000..81269606d98 --- /dev/null +++ b/docs/tr/sql_reference/functions/splitting_merging_functions.md @@ -0,0 +1,116 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 47 +toc_title: "Dizeleri ve dizileri b\xF6lme ve birle\u015Ftirme" +--- + +# Dizeleri ve dizileri bölme ve birleştirme işlevleri {#functions-for-splitting-and-merging-strings-and-arrays} + +## splitByChar (ayırıcı, s) {#splitbycharseparator-s} + +Bir dizeyi belirtilen bir karakterle ayrılmış alt dizelere böler. Sabit bir dize kullanır `separator` tam olarak bir karakterden oluşan. +Seçili alt dizelerin bir dizisini döndürür. Ayırıcı dizenin başında veya sonunda oluşursa veya ardışık birden çok ayırıcı varsa, boş alt dizeler seçilebilir. + +**Sözdizimi** + +``` sql +splitByChar(, ) +``` + +**Parametre** + +- `separator` — The separator which should contain exactly one character. [Dize](../../sql_reference/data_types/string.md). +- `s` — The string to split. [Dize](../../sql_reference/data_types/string.md). + +**Döndürülen değer (ler)** + +Seçili alt dizelerin bir dizisini döndürür. Boş alt dizeler şu durumlarda seçilebilir: + +- Dizenin başında veya sonunda bir ayırıcı oluşur; +- Birden fazla ardışık ayırıcı vardır; +- Orijinal dize `s` boş. + +Tür: [Dizi](../../sql_reference/data_types/array.md) -den [Dize](../../sql_reference/data_types/string.md). + +**Örnek** + +``` sql +SELECT splitByChar(',', '1,2,3,abcde') +``` + +``` text +┌─splitByChar(',', '1,2,3,abcde')─┐ +│ ['1','2','3','abcde'] │ +└─────────────────────────────────┘ +``` + +## splitByString (ayırıcı, s) {#splitbystringseparator-s} + +Bir dizeyi bir dizeyle ayrılmış alt dizelere böler. Sabit bir dize kullanır `separator` ayırıcı olarak birden fazla karakter. Eğer dize `separator` boş olduğunu, bu bölünmüş dize `s` tek karakter dizisine. + +**Sözdizimi** + +``` sql +splitByString(, ) +``` + +**Parametre** + +- `separator` — The separator. [Dize](../../sql_reference/data_types/string.md). +- `s` — The string to split. [Dize](../../sql_reference/data_types/string.md). + +**Döndürülen değer (ler)** + +Seçili alt dizelerin bir dizisini döndürür. Boş alt dizeler şu durumlarda seçilebilir: + +Tür: [Dizi](../../sql_reference/data_types/array.md) -den [Dize](../../sql_reference/data_types/string.md). + +- Boş olmayan bir ayırıcı dizenin başında veya sonunda oluşur; +- Birden fazla ardışık boş olmayan ayırıcı vardır; +- Orijinal dize `s` ayırıcı boş değilken boş. + +**Örnek** + +``` sql +SELECT splitByString(', ', '1, 2 3, 4,5, abcde') +``` + +``` text +┌─splitByString(', ', '1, 2 3, 4,5, abcde')─┐ +│ ['1','2 3','4,5','abcde'] │ +└───────────────────────────────────────────┘ +``` + +``` sql +SELECT splitByString('', 'abcde') +``` + +``` text +┌─splitByString('', 'abcde')─┐ +│ ['a','b','c','d','e'] │ +└────────────────────────────┘ +``` + +## arrayStringConcat(arr \[, ayırıcı\]) {#arraystringconcatarr-separator} + +Dizide listelenen dizeleri ayırıcı ile birleştirir.'separator' isteğe bağlı bir parametredir: varsayılan olarak boş bir dizeye ayarlanmış sabit bir dize. +Dizeyi döndürür. + +## alphaTokens (s) {#alphatokenss} + +A-z ve A-Z aralıklarından ardışık baytların alt dizelerini seçer. + +**Örnek** + +``` sql +SELECT alphaTokens('abca1abc') +``` + +``` text +┌─alphaTokens('abca1abc')─┐ +│ ['abca','abc'] │ +└─────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/splitting_merging_functions/) diff --git a/docs/tr/sql_reference/functions/string_functions.md b/docs/tr/sql_reference/functions/string_functions.md new file mode 100644 index 00000000000..012c8210537 --- /dev/null +++ b/docs/tr/sql_reference/functions/string_functions.md @@ -0,0 +1,489 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 40 +toc_title: "Dizeleri ile \xE7al\u0131\u015Fma" +--- + +# Dizelerle çalışmak için işlevler {#functions-for-working-with-strings} + +## boş {#empty} + +Boş bir dize için 1 veya boş olmayan bir dize için 0 döndürür. +Sonuç türü Uint8'dir. +Bir boşluk veya boş bayt olsa bile, en az bir bayt içeriyorsa, bir dize boş olarak kabul edilir. +İşlev ayrıca diziler için de çalışır. + +## notEmpty {#notempty} + +Boş bir dize için 0 veya boş olmayan bir dize için 1 döndürür. +Sonuç türü Uint8'dir. +İşlev ayrıca diziler için de çalışır. + +## uzunluk {#length} + +Bir dizenin uzunluğunu bayt cinsinden döndürür (karakterlerde değil, kod noktalarında değil). +Sonuç türü Uint64'tür. +İşlev ayrıca diziler için de çalışır. + +## lengthUTF8 {#lengthutf8} + +Dizenin UTF-8 kodlanmış metni oluşturan bir bayt kümesi içerdiğini varsayarak, Unicode kod noktalarında (karakterlerde değil) bir dizenin uzunluğunu döndürür. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). +Sonuç türü Uint64'tür. + +## char\_length, CHAR\_LENGTH {#char-length} + +Dizenin UTF-8 kodlanmış metni oluşturan bir bayt kümesi içerdiğini varsayarak, Unicode kod noktalarında (karakterlerde değil) bir dizenin uzunluğunu döndürür. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). +Sonuç türü Uint64'tür. + +## character\_length, CHARACTER\_LENGTH {#character-length} + +Dizenin UTF-8 kodlanmış metni oluşturan bir bayt kümesi içerdiğini varsayarak, Unicode kod noktalarında (karakterlerde değil) bir dizenin uzunluğunu döndürür. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). +Sonuç türü Uint64'tür. + +## alt, lcase {#lower} + +Bir dizedeki ASCII Latin sembollerini küçük harfe dönüştürür. + +## üst, ucase {#upper} + +Bir dizedeki ASCII Latin sembollerini büyük harfe dönüştürür. + +## lowerUTF8 {#lowerutf8} + +Dizenin UTF-8 kodlu bir metni oluşturan bir bayt kümesi içerdiğini varsayarak bir dizeyi küçük harfe dönüştürür. +Dili algılamaz. Yani Türkçe için sonuç tam olarak doğru olmayabilir. +UTF-8 bayt dizisinin uzunluğu bir kod noktasının büyük ve küçük harf için farklıysa, sonuç bu kod noktası için yanlış olabilir. +Dize, UTF-8 olmayan bir bayt kümesi içeriyorsa, davranış tanımsızdır. + +## upperUTF8 {#upperutf8} + +Dize, UTF-8 kodlanmış bir metni oluşturan bir bayt kümesi içerdiğini varsayarak bir dizeyi büyük harfe dönüştürür. +Dili algılamaz. Yani Türkçe için sonuç tam olarak doğru olmayabilir. +UTF-8 bayt dizisinin uzunluğu bir kod noktasının büyük ve küçük harf için farklıysa, sonuç bu kod noktası için yanlış olabilir. +Dize, UTF-8 olmayan bir bayt kümesi içeriyorsa, davranış tanımsızdır. + +## ısvalidutf8 {#isvalidutf8} + +Bayt kümesi geçerli UTF-8 kodlanmış, aksi takdirde 0 ise, 1 döndürür. + +## toValidUTF8 {#tovalidutf8} + +Geçersiz UTF-8 karakterlerini değiştirir `�` (U+FFFD) karakteri. Bir satırda çalışan tüm geçersiz karakterler bir yedek karaktere daraltılır. + +``` sql +toValidUTF8( input_string ) +``` + +Parametre: + +- input\_string — Any set of bytes represented as the [Dize](../../sql_reference/data_types/string.md) veri türü nesnesi. + +Döndürülen değer: geçerli UTF-8 dizesi. + +**Örnek** + +``` sql +SELECT toValidUTF8('\x61\xF0\x80\x80\x80b') +``` + +``` text +┌─toValidUTF8('a����b')─┐ +│ a�b │ +└───────────────────────┘ +``` + +## tekrarlama {#repeat} + +Bir dizeyi belirtilen kadar çok tekrarlar ve çoğaltılmış değerleri tek bir dize olarak birleştirir. + +**Sözdizimi** + +``` sql +repeat(s, n) +``` + +**Parametre** + +- `s` — The string to repeat. [Dize](../../sql_reference/data_types/string.md). +- `n` — The number of times to repeat the string. [Uİnt](../../sql_reference/data_types/int_uint.md). + +**Döndürülen değer** + +Dize içeren tek dize `s` tekrarlanan `n` kez. Eğer `n` \< 1, işlev boş dize döndürür. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT repeat('abc', 10) +``` + +Sonuç: + +``` text +┌─repeat('abc', 10)──────────────┐ +│ abcabcabcabcabcabcabcabcabcabc │ +└────────────────────────────────┘ +``` + +## tersi {#reverse} + +Dizeyi tersine çevirir (bayt dizisi olarak). + +## reverseUTF8 {#reverseutf8} + +Dizenin UTF-8 metnini temsil eden bir bayt kümesi içerdiğini varsayarak bir Unicode kod noktası dizisini tersine çevirir. Aksi takdirde, başka bir şey yapar(bir istisna atmaz). + +## format(pattern, s0, s1, …) {#format} + +Bağımsız değişkenlerde listelenen dize ile sabit desen biçimlendirme. `pattern` basitleştirilmiş bir Python biçimi desenidir. Biçim dizesi içerir “replacement fields” kıvırcık parantez ile çevrili `{}`. Parantez içinde bulunmayan herhangi bir şey, çıktıya değişmeden kopyalanan hazır metin olarak kabul edilir. Literal metne bir ayraç karakteri eklemeniz gerekiyorsa, iki katına çıkararak kaçabilir: `{{ '{{' }}` ve `{{ '}}' }}`. Alan adları sayılar (sıfırdan başlayarak) veya boş olabilir (daha sonra sonuç numaraları olarak kabul edilir). + +``` sql +SELECT format('{1} {0} {1}', 'World', 'Hello') +``` + +``` text +┌─format('{1} {0} {1}', 'World', 'Hello')─┐ +│ Hello World Hello │ +└─────────────────────────────────────────┘ +``` + +``` sql +SELECT format('{} {}', 'Hello', 'World') +``` + +``` text +┌─format('{} {}', 'Hello', 'World')─┐ +│ Hello World │ +└───────────────────────────────────┘ +``` + +## concat {#concat} + +Bağımsız değişkenlerde listelenen dizeleri ayırıcı olmadan birleştirir. + +**Sözdizimi** + +``` sql +concat(s1, s2, ...) +``` + +**Parametre** + +String veya FixedString türünün değerleri. + +**Döndürülen değerler** + +Bağımsız değişkenlerin birleştirilmesinden kaynaklanan dizeyi döndürür. + +Argüman değerlerinden herhangi biri ise `NULL`, `concat` dönüşler `NULL`. + +**Örnek** + +Sorgu: + +``` sql +SELECT concat('Hello, ', 'World!') +``` + +Sonuç: + +``` text +┌─concat('Hello, ', 'World!')─┐ +│ Hello, World! │ +└─────────────────────────────┘ +``` + +## concatassumeınjective {#concatassumeinjective} + +Aynı olarak [concat](#concat) emin olun bu ihtiyaç fark var `concat(s1, s2, ...) → sn` enjekte edilir, grup tarafından optimizasyonu için kullanılacaktır. + +İşlev adlı “injective” bağımsız değişkenlerin farklı değerleri için her zaman farklı sonuç döndürürse. Başka bir deyişle: farklı argümanlar asla aynı sonucu vermez. + +**Sözdizimi** + +``` sql +concatAssumeInjective(s1, s2, ...) +``` + +**Parametre** + +String veya FixedString türünün değerleri. + +**Döndürülen değerler** + +Bağımsız değişkenlerin birleştirilmesinden kaynaklanan dizeyi döndürür. + +Argüman değerlerinden herhangi biri ise `NULL`, `concatAssumeInjective` dönüşler `NULL`. + +**Örnek** + +Giriş tablosu: + +``` sql +CREATE TABLE key_val(`key1` String, `key2` String, `value` UInt32) ENGINE = TinyLog; +INSERT INTO key_val VALUES ('Hello, ','World',1), ('Hello, ','World',2), ('Hello, ','World!',3), ('Hello',', World!',2); +SELECT * from key_val; +``` + +``` text +┌─key1────┬─key2─────┬─value─┐ +│ Hello, │ World │ 1 │ +│ Hello, │ World │ 2 │ +│ Hello, │ World! │ 3 │ +│ Hello │ , World! │ 2 │ +└─────────┴──────────┴───────┘ +``` + +Sorgu: + +``` sql +SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY concatAssumeInjective(key1, key2) +``` + +Sonuç: + +``` text +┌─concat(key1, key2)─┬─sum(value)─┐ +│ Hello, World! │ 3 │ +│ Hello, World! │ 2 │ +│ Hello, World │ 3 │ +└────────────────────┴────────────┘ +``` + +## alt dize (s, ofset, uzunluk), orta (s, ofset, uzunluk), substr (s, ofset, uzunluk) {#substring} + +Bayttan başlayarak bir alt dize döndürür ‘offset’ ind thatex yani ‘length’ uzun bayt. Karakter indeksleme birinden başlar (standart SQL'DE olduğu gibi). Bu ‘offset’ ve ‘length’ bağımsız değişkenler sabit olmalıdır. + +## substringUTF8(s, ofset, uzunluk) {#substringutf8} + +Olarak aynı ‘substring’, ancak Unicode kod noktaları için. Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa, bir sonuç döndürür (bir istisna atmaz). + +## appendTrailingCharİfAbsent (s, c) {#appendtrailingcharifabsent} + +Eğer... ‘s’ dize boş değildir ve ‘c’ sonunda karakter, ekler ‘c’ sonuna kadar karakter. + +## convertCharset (s, from, to) {#convertcharset} + +Dize döndürür ‘s’ bu kodlamadan dönüştürüldü ‘from’ kod encodinglamaya ‘to’. + +## base64Encode (s) {#base64encode} + +Kodluyor ‘s’ Base64 içine dize + +## base64Decode (s) {#base64decode} + +Base64 kodlu dizeyi çözme ‘s’ orijinal dizeye. Başarısızlık durumunda bir istisna yükseltir. + +## tryBase64Decode (s) {#trybase64decode} + +Base64decode'a benzer, ancak hata durumunda boş bir dize döndürülür. + +## endsWith (s, sonek) {#endswith} + +Belirtilen sonek ile bitip bitmeyeceğini döndürür. Dize belirtilen sonek ile biterse 1 değerini döndürür, aksi takdirde 0 değerini döndürür. + +## startsWith (str, önek) {#startswith} + +Dize belirtilen önek ile başlayıp başlamadığını 1 döndürür, aksi halde 0 döndürür. + +``` sql +SELECT startsWith('Spider-Man', 'Spi'); +``` + +**Döndürülen değerler** + +- 1, dize belirtilen önek ile başlarsa. +- 0, dize belirtilen önek ile başlamazsa. + +**Örnek** + +Sorgu: + +``` sql +SELECT startsWith('Hello, world!', 'He'); +``` + +Sonuç: + +``` text +┌─startsWith('Hello, world!', 'He')─┐ +│ 1 │ +└───────────────────────────────────┘ +``` + +## kırpmak {#trim} + +Belirtilen tüm karakterleri bir dizenin başlangıcından veya sonundan kaldırır. +Varsayılan olarak, bir dizenin her iki ucundan ortak boşlukların (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. + +**Sözdizimi** + +``` sql +trim([[LEADING|TRAILING|BOTH] trim_character FROM] input_string) +``` + +**Parametre** + +- `trim_character` — specified characters for trim. [Dize](../../sql_reference/data_types/string.md). +- `input_string` — string for trim. [Dize](../../sql_reference/data_types/string.md). + +**Döndürülen değer** + +Önde gelen ve (veya) belirtilen karakterleri izleyen bir dize. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT trim(BOTH ' ()' FROM '( Hello, world! )') +``` + +Sonuç: + +``` text +┌─trim(BOTH ' ()' FROM '( Hello, world! )')─┐ +│ Hello, world! │ +└───────────────────────────────────────────────┘ +``` + +## trimLeft {#trimleft} + +Bir dizenin başlangıcından ortak boşluk (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. Diğer boşluk karakterlerini (sekme, boşluksuz boşluk, vb.) kaldırmaz.). + +**Sözdizimi** + +``` sql +trimLeft(input_string) +``` + +Takma ad: `ltrim(input_string)`. + +**Parametre** + +- `input_string` — string to trim. [Dize](../../sql_reference/data_types/string.md). + +**Döndürülen değer** + +Bir dize olmadan lider ortak whitespaces. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT trimLeft(' Hello, world! ') +``` + +Sonuç: + +``` text +┌─trimLeft(' Hello, world! ')─┐ +│ Hello, world! │ +└─────────────────────────────────────┘ +``` + +## trimRight {#trimright} + +Bir dizenin sonundan ortak boşluk (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. Diğer boşluk karakterlerini (sekme, boşluksuz boşluk, vb.) kaldırmaz.). + +**Sözdizimi** + +``` sql +trimRight(input_string) +``` + +Takma ad: `rtrim(input_string)`. + +**Parametre** + +- `input_string` — string to trim. [Dize](../../sql_reference/data_types/string.md). + +**Döndürülen değer** + +Ortak whitespaces firar olmadan bir dize. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT trimRight(' Hello, world! ') +``` + +Sonuç: + +``` text +┌─trimRight(' Hello, world! ')─┐ +│ Hello, world! │ +└──────────────────────────────────────┘ +``` + +## trimBoth {#trimboth} + +Bir dizenin her iki ucundan ortak boşluk (ASCII karakteri 32) tüm ardışık tekrarlarını kaldırır. Diğer boşluk karakterlerini (sekme, boşluksuz boşluk, vb.) kaldırmaz.). + +**Sözdizimi** + +``` sql +trimBoth(input_string) +``` + +Takma ad: `trim(input_string)`. + +**Parametre** + +- `input_string` — string to trim. [Dize](../../sql_reference/data_types/string.md). + +**Döndürülen değer** + +Bir dize olmadan lider ve sondaki ortak whitespaces. + +Tür: `String`. + +**Örnek** + +Sorgu: + +``` sql +SELECT trimBoth(' Hello, world! ') +``` + +Sonuç: + +``` text +┌─trimBoth(' Hello, world! ')─┐ +│ Hello, world! │ +└─────────────────────────────────────┘ +``` + +## CRC32 (s) {#crc32} + +CRC-32-IEEE 802.3 polinom ve başlangıç değerini kullanarak bir dizenin CRC32 sağlama toplamını döndürür `0xffffffff` (zlib uygulaması). + +Sonuç türü Uint32'dir. + +## Crc32ieee (s) {#crc32ieee} + +CRC-32-IEEE 802.3 polinomunu kullanarak bir dizenin CRC32 sağlama toplamını döndürür. + +Sonuç türü Uint32'dir. + +## CRC64 (s) {#crc64} + +CRC-64-ECMA polinomunu kullanarak bir dizenin CRC64 sağlama toplamını döndürür. + +Sonuç türü Uint64'tür. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) diff --git a/docs/tr/sql_reference/functions/string_replace_functions.md b/docs/tr/sql_reference/functions/string_replace_functions.md new file mode 100644 index 00000000000..6a6c0e56aec --- /dev/null +++ b/docs/tr/sql_reference/functions/string_replace_functions.md @@ -0,0 +1,94 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 42 +toc_title: "Dizelerde de\u011Fi\u015Ftirilmesi i\xE7in" +--- + +# Dizelerde arama ve değiştirme işlevleri {#functions-for-searching-and-replacing-in-strings} + +## replaceOne(Samanlık, desen, değiştirme) {#replaceonehaystack-pattern-replacement} + +Varsa, ilk oluş replacesumun yerini ‘pattern’ substring içinde ‘haystack’ ile... ‘replacement’ dize. +Ahiret, ‘pattern’ ve ‘replacement’ sabitleri olması gerekir. + +## replaceAll (Samanlık, desen, değiştirme), değiştirin (Samanlık, desen, değiştirme) {#replaceallhaystack-pattern-replacement-replacehaystack-pattern-replacement} + +Tüm oluşumları değiştirir ‘pattern’ substring içinde ‘haystack’ ile... ‘replacement’ dize. + +## replaceRegexpOne(Samanlık, desen, değiştirme) {#replaceregexponehaystack-pattern-replacement} + +Kullanarak değiştirme ‘pattern’ düzenli ifade. Re2 düzenli ifade. +Varsa, yalnızca ilk oluşumu değiştirir. +Bir desen olarak belirtilebilir ‘replacement’. Bu desen değiştirmeleri içerebilir `\0-\9`. +İkame `\0` tüm düzenli ifadeyi içerir. İkameler `\1-\9` alt desene karşılık gelir numbers.To use the `\` bir şablondaki karakter, kullanarak kaçış `\`. +Ayrıca, bir dize literalinin ekstra bir kaçış gerektirdiğini unutmayın. + +Örnek 1. Tarihi Amerikan format convertingına dönüştürme: + +``` sql +SELECT DISTINCT + EventDate, + replaceRegexpOne(toString(EventDate), '(\\d{4})-(\\d{2})-(\\d{2})', '\\2/\\3/\\1') AS res +FROM test.hits +LIMIT 7 +FORMAT TabSeparated +``` + +``` text +2014-03-17 03/17/2014 +2014-03-18 03/18/2014 +2014-03-19 03/19/2014 +2014-03-20 03/20/2014 +2014-03-21 03/21/2014 +2014-03-22 03/22/2014 +2014-03-23 03/23/2014 +``` + +Örnek 2. Bir dize on kez kopyalama: + +``` sql +SELECT replaceRegexpOne('Hello, World!', '.*', '\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0') AS res +``` + +``` text +┌─res────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World! │ +└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## replaceRegexpAll(Samanlık, desen, değiştirme) {#replaceregexpallhaystack-pattern-replacement} + +Bu aynı şeyi yapar, ancak tüm oluşumların yerini alır. Örnek: + +``` sql +SELECT replaceRegexpAll('Hello, World!', '.', '\\0\\0') AS res +``` + +``` text +┌─res────────────────────────┐ +│ HHeelllloo,, WWoorrlldd!! │ +└────────────────────────────┘ +``` + +Normal bir ifade boş bir alt dize üzerinde çalıştıysa, bir istisna olarak, değiştirme birden çok kez yapılmaz. +Örnek: + +``` sql +SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res +``` + +``` text +┌─res─────────────────┐ +│ here: Hello, World! │ +└─────────────────────┘ +``` + +## regexpQuoteMeta (s) {#regexpquotemetas} + +İşlev, dizedeki bazı önceden tanımlanmış karakterlerden önce bir ters eğik çizgi ekler. +Önceden tanımlanmış karakterler: ‘0’, ‘\\’, ‘\|’, ‘(’, ‘)’, ‘^’, ‘$’, ‘.’, ‘\[’, '\]', ‘?’, '\*‘,’+‘,’{‘,’:‘,’-'. +Bu uygulama biraz re2::RE2::QuoteMeta farklıdır. Sıfır bayttan 00 yerine \\0 olarak çıkar ve yalnızca gerekli karakterlerden kaçar. +Daha fazla bilgi için bağlantıya bakın: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473) + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/string_replace_functions/) diff --git a/docs/tr/sql_reference/functions/string_search_functions.md b/docs/tr/sql_reference/functions/string_search_functions.md new file mode 100644 index 00000000000..bfa3d8d0bd7 --- /dev/null +++ b/docs/tr/sql_reference/functions/string_search_functions.md @@ -0,0 +1,379 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 41 +toc_title: "Arama Dizeleri \u0130\xE7in" +--- + +# Dizeleri aramak için işlevler {#functions-for-searching-strings} + +Arama, tüm bu işlevlerde varsayılan olarak büyük / küçük harf duyarlıdır. Büyük / küçük harf duyarlı arama için ayrı Varyantlar vardır. + +## pozisyon (Samanlık, iğne), bulun (Samanlık, iğne) {#position} + +1'den başlayarak dizedeki bulunan alt dizenin konumunu (bayt cinsinden) döndürür. + +Dize, tek baytlık kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa ve bir karakter tek bir bayt kullanılarak temsil edilemezse, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki bayt kullanılarak temsil edilebilirse, iki bayt vb. kullanır. + +Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİnsensitive](#positioncaseinsensitive). + +**Sözdizimi** + +``` sql +position(haystack, needle) +``` + +Takma ad: `locate(haystack, needle)`. + +**Parametre** + +- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). +- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). + +**Döndürülen değerler** + +- Alt dize bulunursa, bayt cinsinden başlangıç pozisyonu (1'den sayma). +- 0, alt dize bulunamadı. + +Tür: `Integer`. + +**Örnekler** + +İfade “Hello, world!” tek baytla kodlanmış bir metni temsil eden bir bayt kümesi içerir. İşlev beklenen bazı sonuçları döndürür: + +Sorgu: + +``` sql +SELECT position('Hello, world!', '!') +``` + +Sonuç: + +``` text +┌─position('Hello, world!', '!')─┐ +│ 13 │ +└────────────────────────────────┘ +``` + +Rusça'daki aynı ifade, tek bir bayt kullanılarak temsil edilemeyen karakterler içerir. İşlev beklenmedik bir sonuç verir (kullanım [positionUTF8](#positionutf8) çok bayt kodlu metin için işlev): + +Sorgu: + +``` sql +SELECT position('Привет, мир!', '!') +``` + +Sonuç: + +``` text +┌─position('Привет, мир!', '!')─┐ +│ 21 │ +└───────────────────────────────┘ +``` + +## positionCaseİnsensitive {#positioncaseinsensitive} + +Olarak aynı [konum](#position) 1'den başlayarak dizedeki bulunan alt dizenin konumunu (bayt cinsinden) döndürür. Büyük / küçük harf duyarlı bir arama için işlevi kullanın. + +Dize, tek baytlık kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa ve bir karakter tek bir bayt kullanılarak temsil edilemezse, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki bayt kullanılarak temsil edilebilirse, iki bayt vb. kullanır. + +**Sözdizimi** + +``` sql +positionCaseInsensitive(haystack, needle) +``` + +**Parametre** + +- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). +- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). + +**Döndürülen değerler** + +- Alt dize bulunursa, bayt cinsinden başlangıç pozisyonu (1'den sayma). +- 0, alt dize bulunamadı. + +Tür: `Integer`. + +**Örnek** + +Sorgu: + +``` sql +SELECT positionCaseInsensitive('Hello, world!', 'hello') +``` + +Sonuç: + +``` text +┌─positionCaseInsensitive('Hello, world!', 'hello')─┐ +│ 1 │ +└───────────────────────────────────────────────────┘ +``` + +## positionUTF8 {#positionutf8} + +1'den başlayarak dizedeki bulunan alt dizenin konumunu (Unicode noktalarında) döndürür. + +Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki Unicode noktası kullanılarak temsil edilebilirse, iki vb. kullanır. + +Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİnsensitiveUTF8](#positioncaseinsensitiveutf8). + +**Sözdizimi** + +``` sql +positionUTF8(haystack, needle) +``` + +**Parametre** + +- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). +- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). + +**Döndürülen değerler** + +- Unicode noktalarında başlangıç pozisyonu (1'den sayma), eğer alt dize bulundu. +- 0, alt dize bulunamadı. + +Tür: `Integer`. + +**Örnekler** + +İfade “Hello, world!” rusça'da, tek noktalı kodlanmış bir metni temsil eden bir dizi Unicode noktası bulunur. İşlev beklenen bazı sonuçları döndürür: + +Sorgu: + +``` sql +SELECT positionUTF8('Привет, мир!', '!') +``` + +Sonuç: + +``` text +┌─positionUTF8('Привет, мир!', '!')─┐ +│ 12 │ +└───────────────────────────────────┘ +``` + +İfade “Salut, étudiante!” karakter nerede `é` bir nokta kullanılarak temsil edilebilir (`U+00E9`) veya iki puan (`U+0065U+0301`) fonksiyon bazı beklenmedik sonuç iade edilebilir: + +Mektup için sorgu `é` bir Unicode noktasını temsil eden `U+00E9`: + +``` sql +SELECT positionUTF8('Salut, étudiante!', '!') +``` + +Sonuç: + +``` text +┌─positionUTF8('Salut, étudiante!', '!')─┐ +│ 17 │ +└────────────────────────────────────────┘ +``` + +Mektup için sorgu `é`, iki Unicode noktası temsil edilen `U+0065U+0301`: + +``` sql +SELECT positionUTF8('Salut, étudiante!', '!') +``` + +Sonuç: + +``` text +┌─positionUTF8('Salut, étudiante!', '!')─┐ +│ 18 │ +└────────────────────────────────────────┘ +``` + +## positionCaseİnsensitiveUTF8 {#positioncaseinsensitiveutf8} + +Olarak aynı [positionUTF8](#positionutf8) ama büyük küçük harf duyarlı. 1'den başlayarak dizedeki bulunan alt dizenin konumunu (Unicode noktalarında) döndürür. + +Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği varsayımı altında çalışır. Bu varsayım karşılanmazsa, işlev bir istisna atmaz ve beklenmeyen bir sonuç döndürür. Karakter iki Unicode noktası kullanılarak temsil edilebilirse, iki vb. kullanır. + +**Sözdizimi** + +``` sql +positionCaseInsensitiveUTF8(haystack, needle) +``` + +**Parametre** + +- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). +- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). + +**Döndürülen değer** + +- Unicode noktalarında başlangıç pozisyonu (1'den sayma), eğer alt dize bulundu. +- 0, alt dize bulunamadı. + +Tür: `Integer`. + +**Örnek** + +Sorgu: + +``` sql +SELECT positionCaseInsensitiveUTF8('Привет, мир!', 'Мир') +``` + +Sonuç: + +``` text +┌─positionCaseInsensitiveUTF8('Привет, мир!', 'Мир')─┐ +│ 9 │ +└────────────────────────────────────────────────────┘ +``` + +## multiSearchAllPositions {#multisearchallpositions} + +Olarak aynı [konum](string_search_functions.md#position) ama döner `Array` dizede bulunan karşılık gelen alt dizelerin konumlarının (bayt cinsinden). Pozisyonlar 1'den başlayarak endekslenir. + +Arama, dize kodlaması ve harmanlama ile ilgili olmayan bayt dizileri üzerinde gerçekleştirilir. + +- Büyük / küçük harf duyarlı ASCII arama için işlevi kullanın `multiSearchAllPositionsCaseInsensitive`. +- UTF-8'de arama yapmak için işlevi kullanın [multiSearchAllPositionsUTF8](#multiSearchAllPositionsUTF8). +- Büyük / küçük harf duyarlı UTF-8 arama için multisearchallpositionscaseınsensitiveutf8 işlevini kullanın. + +**Sözdizimi** + +``` sql +multiSearchAllPositions(haystack, [needle1, needle2, ..., needlen]) +``` + +**Parametre** + +- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal). +- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal). + +**Döndürülen değerler** + +- Bayt cinsinden başlangıç pozisyonları dizisi (1'den sayma), karşılık gelen alt dize bulunursa ve 0 bulunmazsa. + +**Örnek** + +Sorgu: + +``` sql +SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']) +``` + +Sonuç: + +``` text +┌─multiSearchAllPositions('Hello, World!', ['hello', '!', 'world'])─┐ +│ [0,13,0] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## multiSearchAllPositionsUTF8 {#multiSearchAllPositionsUTF8} + +Görmek `multiSearchAllPositions`. + +## multiSearchFirstPosition (Samanlık, \[iğne1, iğne2, …, needleve\]) {#multisearchfirstposition} + +Olarak aynı `position` ancak dizenin en soldaki ofsetini döndürür `haystack` bu bazı iğnelerle eşleşti. + +Büyük/küçük harfe duyarsız arama veya / VE UTF-8 biçiminde kullanım işlevleri için `multiSearchFirstPositionCaseInsensitive, multiSearchFirstPositionUTF8, multiSearchFirstPositionCaseInsensitiveUTF8`. + +## multiSearchFirstİndex (Samanlık, \[iğne1, iğne2, …, needleve\]) {#multisearchfirstindexhaystack-needle1-needle2-needlen} + +Dizini döndürür `i` en soldaki bulunan iğnenin (1'den başlayarak)ben diz inede `haystack` ve 0 aksi takdirde. + +Büyük/küçük harfe duyarsız arama veya / VE UTF-8 biçiminde kullanım işlevleri için `multiSearchFirstIndexCaseInsensitive, multiSearchFirstIndexUTF8, multiSearchFirstIndexCaseInsensitiveUTF8`. + +## multiSearchAny (Samanlık, \[iğne1, iğne2, …, needleve\]) {#function-multisearchany} + +Döner 1, Eğer en az bir dize iğneben dize ile eşleşir `haystack` ve 0 aksi takdirde. + +Büyük/küçük harfe duyarsız arama veya / VE UTF-8 biçiminde kullanım işlevleri için `multiSearchAnyCaseInsensitive, multiSearchAnyUTF8, multiSearchAnyCaseInsensitiveUTF8`. + +!!! note "Not" + Tamamı `multiSearch*` fonksiyonlar iğne sayısı 2'den az olmalıdır8 uygulama şartname nedeniyle. + +## maç (Samanlık, desen) {#matchhaystack-pattern} + +Dize eşleşip eşleşmediğini denetler `pattern` düzenli ifade. Bir `re2` düzenli ifade. Bu [sözdizimi](https://github.com/google/re2/wiki/Syntax) of the `re2` düzenli ifadeler, Perl düzenli ifadelerin sözdiziminden daha sınırlıdır. + +Eşleşmezse 0 veya eşleşirse 1 değerini döndürür. + +Ters eğik çizgi sembolünün (`\`) normal ifadede kaçmak için kullanılır. Aynı sembol, dize değişmezlerinde kaçmak için kullanılır. Bu nedenle, normal bir ifadede sembolden kaçmak için, bir dize literalinde iki ters eğik çizgi (\\) yazmanız gerekir. + +Normal ifade, bir bayt kümesiymiş gibi dizeyle çalışır. Normal ifade boş bayt içeremez. +Bir dizedeki alt dizeleri aramak için desenler için, LİKE veya ‘position’, çok daha hızlı çalıştıkları için. + +## multiMatchAny (Samanlık, \[desen1, desen2, …, patternve\]) {#multimatchanyhaystack-pattern1-pattern2-patternn} + +Olarak aynı `match`, ancak normal ifadelerin hiçbiri eşleşmezse 0 ve desenlerden herhangi biri eşleşirse 1 değerini döndürür. Kullanır [hyperscan](https://github.com/intel/hyperscan) kitaplık. Bir dizede alt dizeleri aramak için desenler için, kullanmak daha iyidir `multiSearchAny` çok daha hızlı çalıştığı için. + +!!! note "Not" + Herhangi birinin uzunluğu `haystack` dize 2'den az olmalıdır32 bayt aksi takdirde özel durum atılır. Bu kısıtlama, hyperscan API nedeniyle gerçekleşir. + +## multimatchanyındex (haystack, \[desen1, desen2, …, patternve\]) {#multimatchanyindexhaystack-pattern1-pattern2-patternn} + +Olarak aynı `multiMatchAny`, ancak Samanlık eşleşen herhangi bir dizin döndürür. + +## multiMatchAllİndices (haystack, \[desen1, desen2, …, patternve\]) {#multimatchallindiceshaystack-pattern1-pattern2-patternn} + +Olarak aynı `multiMatchAny`, ancak herhangi bir sırada Samanlık eşleşen tüm indicies dizisini döndürür. + +## multiFuzzyMatchAny (Samanlık, mesafe, \[desen1, desen2, …, patternve\]) {#multifuzzymatchanyhaystack-distance-pattern1-pattern2-patternn} + +Olarak aynı `multiMatchAny`, ancak herhangi bir desen samanlıkta bir sabitle eşleşirse 1 döndürür [mesafeyi Düzenle](https://en.wikipedia.org/wiki/Edit_distance). Bu fonksiyon aynı zamanda deneysel bir moddadır ve son derece yavaş olabilir. Daha fazla bilgi için bkz. [hyperscan belgeleri](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching). + +## multifuzzymatchanyındex (Samanlık, mesafe, \[desen1, desen2, …, patternve\]) {#multifuzzymatchanyindexhaystack-distance-pattern1-pattern2-patternn} + +Olarak aynı `multiFuzzyMatchAny`, ancak sabit bir düzenleme mesafesi içinde Samanlık eşleşen herhangi bir dizin döndürür. + +## multiFuzzyMatchAllİndices (Samanlık, mesafe, \[desen1, desen2, …, patternve\]) {#multifuzzymatchallindiceshaystack-distance-pattern1-pattern2-patternn} + +Olarak aynı `multiFuzzyMatchAny`, ancak sabit bir düzenleme mesafesi içinde saman yığını ile eşleşen herhangi bir sırada tüm dizinlerin dizisini döndürür. + +!!! note "Not" + `multiFuzzyMatch*` işlevler UTF-8 normal ifadeleri desteklemez ve bu tür ifadeler hyperscan kısıtlaması nedeniyle bayt olarak kabul edilir. + +!!! note "Not" + Hyperscan kullanan tüm işlevleri kapatmak için, ayarı kullanın `SET allow_hyperscan = 0;`. + +## özü (Samanlık, desen) {#extracthaystack-pattern} + +Normal ifade kullanarak bir dize parçasını ayıklar. Eğer ‘haystack’ eşleşmiyor ‘pattern’ regex, boş bir dize döndürülür. Regex alt desenler içermiyorsa, tüm regex ile eşleşen parçayı alır. Aksi takdirde, ilk alt desenle eşleşen parçayı alır. + +## extractAll(Samanlık, desen) {#extractallhaystack-pattern} + +Normal bir ifade kullanarak bir dizenin tüm parçalarını ayıklar. Eğer ‘haystack’ eşleşmiyor ‘pattern’ regex, boş bir dize döndürülür. Regex için tüm eşleşmelerden oluşan bir dizi dizeyi döndürür. Genel olarak, davranış ile aynıdır ‘extract’ işlev (bir alt desen yoksa ilk alt deseni veya tüm ifadeyi alır). + +## gibi (Samanlık, desen), Samanlık gibi desen operatörü {#function-like} + +Bir dizenin basit bir normal ifadeyle eşleşip eşleşmediğini denetler. +Normal ifade metasymbols içerebilir `%` ve `_`. + +`%` herhangi bir bayt miktarını (sıfır karakter dahil) gösterir. + +`_` herhangi bir bayt gösterir. + +Ters eğik çizgi kullanın (`\`) metasimbollerden kaçmak için. Açıklamasında kaçan nota bakın ‘match’ işlev. + +Gibi düzenli ifadeler için `%needle%`, kod daha optimal ve hızlı olarak çalışır `position` işlev. +Diğer normal ifadeler için kod, ‘match’ işlev. + +## notLike (Samanlık, desen), Samanlık desen operatörü gibi değil {#function-notlike} + +Aynı şey ‘like’ ama negatif. + +## ngramDistance(Samanlık, iğne) {#ngramdistancehaystack-needle} + +Arasındaki 4 gram distancelık mesaf theeyi hesaplar `haystack` ve `needle`: counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns float number from 0 to 1 – the closer to zero, the more strings are similar to each other. If the constant `needle` veya `haystack` 32kb'den fazla, bir istisna atar. Sabit olmayan bazı `haystack` veya `needle` dizeler 32kb'den daha fazladır, mesafe her zaman birdir. + +Büyük/küçük harf duyarsız arama veya / VE UTF-8 formatında kullanım işlevleri için `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8`. + +## ngramsearch(Samanlık, iğne) {#ngramsearchhaystack-needle} + +Aynı olarak `ngramDistance` ama arasındaki simetrik olmayan farkı hesaplar `needle` ve `haystack` – the number of n-grams from needle minus the common number of n-grams normalized by the number of `needle` n-büyükanne. Daha yakın, daha `needle` is in the `haystack`. Bulanık dize arama için yararlı olabilir. + +Büyük/küçük harf duyarsız arama veya / VE UTF-8 formatında kullanım işlevleri için `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8`. + +!!! note "Not" + For UTF-8 case we use 3-gram distance. All these are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) diff --git a/docs/tr/sql_reference/functions/type_conversion_functions.md b/docs/tr/sql_reference/functions/type_conversion_functions.md new file mode 100644 index 00000000000..643ddf9dbc0 --- /dev/null +++ b/docs/tr/sql_reference/functions/type_conversion_functions.md @@ -0,0 +1,534 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 38 +toc_title: "Tip D\xF6n\xFC\u015Ft\xFCrme" +--- + +# Tip Dönüştürme Fonksiyonları {#type-conversion-functions} + +## Sayısal dönüşümlerin ortak sorunları {#numeric-conversion-issues} + +Bir değeri birinden başka bir veri türüne dönüştürdüğünüzde, ortak durumda, veri kaybına neden olabilecek güvenli olmayan bir işlem olduğunu unutmamalısınız. Değeri daha büyük bir veri türünden daha küçük bir veri türüne sığdırmaya çalışırsanız veya değerleri farklı veri türleri arasında dönüştürürseniz, veri kaybı oluşabilir. + +ClickHouse vardır [C++ programları ile aynı davranış](https://en.cppreference.com/w/cpp/language/implicit_conversion). + +## toİnt(8/16/32/64) {#toint8163264} + +Bir giriş değeri dönüştürür [Tamsayı](../../sql_reference/data_types/int_uint.md) veri türü. Bu işlev ailesi şunları içerir: + +- `toInt8(expr)` — Results in the `Int8` veri türü. +- `toInt16(expr)` — Results in the `Int16` veri türü. +- `toInt32(expr)` — Results in the `Int32` veri türü. +- `toInt64(expr)` — Results in the `Int64` veri türü. + +**Parametre** + +- `expr` — [İfade](../syntax.md#syntax-expressions) bir sayının ondalık gösterimiyle bir sayı veya dize döndürülmesi. Sayıların ikili, sekizli ve onaltılık gösterimleri desteklenmez. Önde gelen sıfırlar soyulur. + +**Döndürülen değer** + +Tamsayı değeri `Int8`, `Int16`, `Int32`, veya `Int64` veri türü. + +Fonksiyonlar kullanımı [sıfıra doğru yuvarlama](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), yani sayıların kesirli rakamlarını keserler. + +Fonksiyon behaviorların davranışı [N andan ve In andf](../../sql_reference/data_types/float.md#data_type-float-nan-inf) argümanlar tanımsızdır. Hakkında hatırla [sayısal convertions sorunları](#numeric-conversion-issues), fonksiyonları kullanırken. + +**Örnek** + +``` sql +SELECT toInt64(nan), toInt32(32), toInt16('16'), toInt8(8.8) +``` + +``` text +┌─────────toInt64(nan)─┬─toInt32(32)─┬─toInt16('16')─┬─toInt8(8.8)─┐ +│ -9223372036854775808 │ 32 │ 16 │ 8 │ +└──────────────────────┴─────────────┴───────────────┴─────────────┘ +``` + +## toİnt (8/16/32/64)OrZero {#toint8163264orzero} + +String türünde bir argüman alır ve İnt içine ayrıştırmaya çalışır(8 \| 16 \| 32 \| 64). Başarısız olursa, 0 döndürür. + +**Örnek** + +``` sql +select toInt64OrZero('123123'), toInt8OrZero('123qwe123') +``` + +``` text +┌─toInt64OrZero('123123')─┬─toInt8OrZero('123qwe123')─┐ +│ 123123 │ 0 │ +└─────────────────────────┴───────────────────────────┘ +``` + +## toİnt(8/16/32/64) OrNull {#toint8163264ornull} + +String türünde bir argüman alır ve İnt içine ayrıştırmaya çalışır(8 \| 16 \| 32 \| 64). Başarısız olursa, NULL döndürür. + +**Örnek** + +``` sql +select toInt64OrNull('123123'), toInt8OrNull('123qwe123') +``` + +``` text +┌─toInt64OrNull('123123')─┬─toInt8OrNull('123qwe123')─┐ +│ 123123 │ ᴺᵁᴸᴸ │ +└─────────────────────────┴───────────────────────────┘ +``` + +## toUİnt(8/16/32/64) {#touint8163264} + +Bir giriş değeri dönüştürür [Uİnt](../../sql_reference/data_types/int_uint.md) veri türü. Bu işlev ailesi şunları içerir: + +- `toUInt8(expr)` — Results in the `UInt8` veri türü. +- `toUInt16(expr)` — Results in the `UInt16` veri türü. +- `toUInt32(expr)` — Results in the `UInt32` veri türü. +- `toUInt64(expr)` — Results in the `UInt64` veri türü. + +**Parametre** + +- `expr` — [İfade](../syntax.md#syntax-expressions) bir sayının ondalık gösterimiyle bir sayı veya dize döndürülmesi. Sayıların ikili, sekizli ve onaltılık gösterimleri desteklenmez. Önde gelen sıfırlar soyulur. + +**Döndürülen değer** + +Tamsayı değeri `UInt8`, `UInt16`, `UInt32`, veya `UInt64` veri türü. + +Fonksiyonlar kullanımı [sıfıra doğru yuvarlama](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), yani sayıların kesirli rakamlarını keserler. + +Olumsuz agruments için işlevlerin davranışı ve [N andan ve In andf](../../sql_reference/data_types/float.md#data_type-float-nan-inf) argümanlar tanımsızdır. Örneğin, negatif bir sayı ile bir dize geçirirseniz `'-32'`, ClickHouse bir özel durum yükseltir. Hakkında hatırla [sayısal convertions sorunları](#numeric-conversion-issues), fonksiyonları kullanırken. + +**Örnek** + +``` sql +SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8) +``` + +``` text +┌───────toUInt64(nan)─┬─toUInt32(-32)─┬─toUInt16('16')─┬─toUInt8(8.8)─┐ +│ 9223372036854775808 │ 4294967264 │ 16 │ 8 │ +└─────────────────────┴───────────────┴────────────────┴──────────────┘ +``` + +## toUİnt (8/16/32/64)OrZero {#touint8163264orzero} + +## toUİnt(8/16/32/64) OrNull {#touint8163264ornull} + +## toFloat(32/64) {#tofloat3264} + +## toFloat (32/64)OrZero {#tofloat3264orzero} + +## toFloat(32/64) OrNull {#tofloat3264ornull} + +## toDate {#todate} + +## toDateOrZero {#todateorzero} + +## toDateOrNull {#todateornull} + +## toDateTime {#todatetime} + +## toDateTimeOrZero {#todatetimeorzero} + +## toDateTimeOrNull {#todatetimeornull} + +## toDecimal(32/64/128) {#todecimal3264128} + +Dönüşüyo `value` to the [Ondalık](../../sql_reference/data_types/decimal.md) hassas veri türü `S`. Bu `value` bir sayı veya bir dize olabilir. Bu `S` (scale) parametresi ondalık basamak sayısını belirtir. + +- `toDecimal32(value, S)` +- `toDecimal64(value, S)` +- `toDecimal128(value, S)` + +## toDecimal(32/64/128) OrNull {#todecimal3264128ornull} + +Bir giriş dizesini bir [Nullable (Ondalık (P, S))](../../sql_reference/data_types/decimal.md) veri türü değeri. Bu işlev ailesi şunları içerir: + +- `toDecimal32OrNull(expr, S)` — Results in `Nullable(Decimal32(S))` veri türü. +- `toDecimal64OrNull(expr, S)` — Results in `Nullable(Decimal64(S))` veri türü. +- `toDecimal128OrNull(expr, S)` — Results in `Nullable(Decimal128(S))` veri türü. + +Bu işlevler yerine kullanılmalıdır `toDecimal*()` fonksiyonlar, eğer bir almak için tercih `NULL` bir giriş değeri ayrıştırma hatası durumunda bir özel durum yerine değer. + +**Parametre** + +- `expr` — [İfade](../syntax.md#syntax-expressions) bir değeri döndürür [Dize](../../sql_reference/data_types/string.md) veri türü. ClickHouse ondalık sayının metinsel temsilini bekler. Mesela, `'1.111'`. +- `S` — Scale, the number of decimal places in the resulting value. + +**Döndürülen değer** + +İçinde bir değer `Nullable(Decimal(P,S))` veri türü. Değeri içerir: + +- İle sayı `S` ondalık basamaklar, ClickHouse giriş dizesi bir sayı olarak yorumlar. +- `NULL`, ClickHouse giriş dizesini bir sayı olarak yorumlayamazsa veya giriş numarası birden fazla içeriyorsa `S` ondalık basamaklar. + +**Örnekler** + +``` sql +SELECT toDecimal32OrNull(toString(-1.111), 5) AS val, toTypeName(val) +``` + +``` text +┌──────val─┬─toTypeName(toDecimal32OrNull(toString(-1.111), 5))─┐ +│ -1.11100 │ Nullable(Decimal(9, 5)) │ +└──────────┴────────────────────────────────────────────────────┘ +``` + +``` sql +SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val) +``` + +``` text +┌──val─┬─toTypeName(toDecimal32OrNull(toString(-1.111), 2))─┐ +│ ᴺᵁᴸᴸ │ Nullable(Decimal(9, 2)) │ +└──────┴────────────────────────────────────────────────────┘ +``` + +## toDecimal (32/64/128)OrZero {#todecimal3264128orzero} + +Bir giriş değeri dönüştürür [Ondalık(P, S)](../../sql_reference/data_types/decimal.md) veri türü. Bu işlev ailesi şunları içerir: + +- `toDecimal32OrZero( expr, S)` — Results in `Decimal32(S)` veri türü. +- `toDecimal64OrZero( expr, S)` — Results in `Decimal64(S)` veri türü. +- `toDecimal128OrZero( expr, S)` — Results in `Decimal128(S)` veri türü. + +Bu işlevler yerine kullanılmalıdır `toDecimal*()` fonksiyonlar, eğer bir almak için tercih `0` bir giriş değeri ayrıştırma hatası durumunda bir özel durum yerine değer. + +**Parametre** + +- `expr` — [İfade](../syntax.md#syntax-expressions) bir değeri döndürür [Dize](../../sql_reference/data_types/string.md) veri türü. ClickHouse ondalık sayının metinsel temsilini bekler. Mesela, `'1.111'`. +- `S` — Scale, the number of decimal places in the resulting value. + +**Döndürülen değer** + +İçinde bir değer `Nullable(Decimal(P,S))` veri türü. Değeri içerir: + +- İle sayı `S` ondalık basamaklar, ClickHouse giriş dizesi bir sayı olarak yorumlar. +- 0 ile `S` ondalık basamaklar, ClickHouse giriş dizesini bir sayı olarak yorumlayamazsa veya giriş numarası birden fazla içeriyorsa `S` ondalık basamaklar. + +**Örnek** + +``` sql +SELECT toDecimal32OrZero(toString(-1.111), 5) AS val, toTypeName(val) +``` + +``` text +┌──────val─┬─toTypeName(toDecimal32OrZero(toString(-1.111), 5))─┐ +│ -1.11100 │ Decimal(9, 5) │ +└──────────┴────────────────────────────────────────────────────┘ +``` + +``` sql +SELECT toDecimal32OrZero(toString(-1.111), 2) AS val, toTypeName(val) +``` + +``` text +┌──val─┬─toTypeName(toDecimal32OrZero(toString(-1.111), 2))─┐ +│ 0.00 │ Decimal(9, 2) │ +└──────┴────────────────────────────────────────────────────┘ +``` + +## toString {#tostring} + +Sayılar, dizeler (ancak sabit olmayan dizeler), tarihler ve tarihlerle saatler arasında dönüştürme işlevleri. +Tüm bu işlevler bir argümanı kabul eder. + +Bir dizeye veya dizeye dönüştürürken, değer, sekmeyle aynı kuralları kullanarak biçimlendirilir veya ayrıştırılır. ayrı biçim (ve hemen hemen tüm diğer metin biçimleri). Dize ayrıştırılamazsa, bir istisna atılır ve istek iptal edilir. + +Tarihleri sayılara dönüştürürken veya tam tersi, Tarih Unix döneminin başlangıcından bu yana geçen gün sayısına karşılık gelir. +Tarihleri zamanlarla sayılara dönüştürürken veya tam tersi olduğunda, zaman ile tarih, Unix döneminin başlangıcından bu yana geçen saniye sayısına karşılık gelir. + +ToDate / toDateTime işlevleri için tarih ve saatli tarih biçimleri aşağıdaki gibi tanımlanır: + +``` text +YYYY-MM-DD +YYYY-MM-DD hh:mm:ss +``` + +Özel durum olarak, uınt32, Int32, Uınt64 veya Int64 sayısal türlerinden bugüne dönüştürme ve sayı 65536'dan büyük veya eşitse, sayı Unıx zaman damgası (ve gün sayısı olarak değil) olarak yorumlanır ve tarihe yuvarlanır. Bu, yaygın yazı oluşumu için destek sağlar ‘toDate(unix\_timestamp)’, aksi takdirde bir hata olur ve daha hantal yazmayı gerektirir ‘toDate(toDateTime(unix\_timestamp))’. + +Bir tarih ve tarih ile saat arasında dönüştürme doğal bir şekilde gerçekleştirilir: boş bir zaman ekleyerek veya saati bırakarak. + +Sayısal türler arasındaki dönüştürme, C++ ' daki farklı sayısal türler arasındaki atamalarla aynı kuralları kullanır. + +Ayrıca, Tostring işlevi DateTime bağımsız değişkeni, saat dilimi adını içeren ikinci bir dize bağımsız değişkeni alabilir. Örnek: `Asia/Yekaterinburg` Bu durumda, saat belirtilen saat dilimine göre biçimlendirilir. + +``` sql +SELECT + now() AS now_local, + toString(now(), 'Asia/Yekaterinburg') AS now_yekat +``` + +``` text +┌───────────now_local─┬─now_yekat───────────┐ +│ 2016-06-15 00:11:21 │ 2016-06-15 02:11:21 │ +└─────────────────────┴─────────────────────┘ +``` + +Ayrıca bakınız `toUnixTimestamp` işlev. + +## toFixedString(s, N) {#tofixedstrings-n} + +Bir dize türü bağımsız değişkeni dönüştürür bir FixedString(N) türü (sabit uzunlukta bir dize N). N sabit olmalıdır. +Dize n'den daha az bayt varsa, sağa boş bayt ile geçirilir. Dize n'den daha fazla bayt varsa, bir özel durum atılır. + +## tostringcuttozero (s) {#tostringcuttozeros} + +Bir dize veya fixedstring bağımsız değişkeni kabul eder. Bulunan ilk sıfır baytta kesilmiş içeriği olan dizeyi döndürür. + +Örnek: + +``` sql +SELECT toFixedString('foo', 8) AS s, toStringCutToZero(s) AS s_cut +``` + +``` text +┌─s─────────────┬─s_cut─┐ +│ foo\0\0\0\0\0 │ foo │ +└───────────────┴───────┘ +``` + +``` sql +SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut +``` + +``` text +┌─s──────────┬─s_cut─┐ +│ foo\0bar\0 │ foo │ +└────────────┴───────┘ +``` + +## reinterpretAsUİnt(8/16/32/64) {#reinterpretasuint8163264} + +## reinterpretAsİnt(8/16/32/64) {#reinterpretasint8163264} + +## reinterpretAsFloat (32/64) {#reinterpretasfloat3264} + +## reinterpretAsDate {#reinterpretasdate} + +## reinterpretAsDateTime {#reinterpretasdatetime} + +Bu işlevler bir dizeyi kabul eder ve dizenin başına yerleştirilen baytları ana bilgisayar düzeninde (little endian) bir sayı olarak yorumlar. Dize yeterince uzun değilse, işlevler dize gerekli sayıda boş baytla doldurulmuş gibi çalışır. Dize gerekenden daha uzunsa, ek bayt yoksayılır. Bir tarih, Unix döneminin başlangıcından bu yana geçen gün sayısı olarak yorumlanır ve zamana sahip bir tarih, Unix döneminin başlangıcından bu yana geçen saniye sayısı olarak yorumlanır. + +## reinterpretAsString {#type_conversion_functions-reinterpretAsString} + +Bu işlev, bir sayı veya tarih veya tarih saat ile kabul eder ve ana bilgisayar düzeninde (little endian) karşılık gelen değeri temsil eden bayt içeren bir dize döndürür. Boş bayt sondan bırakılır. Örneğin, 255 uint32 türü değeri bir bayt uzunluğunda bir dizedir. + +## reinterpretAsFixedString {#reinterpretasfixedstring} + +Bu işlev, bir sayı veya tarih veya tarih saat ile kabul eder ve karşılık gelen değeri ana bilgisayar sırasına (little endian) temsil eden bayt içeren bir FixedString döndürür. Boş bayt sondan bırakılır. Örneğin, 255 uint32 türü değeri bir bayt uzunluğunda bir FixedString. + +## CAS (t(x, t) {#type_conversion_function-cast} + +Dönüşüyo ‘x’ to the ‘t’ veri türü. Sözdizimi CAST (x AS t) da desteklenmektedir. + +Örnek: + +``` sql +SELECT + '2016-06-15 23:00:00' AS timestamp, + CAST(timestamp AS DateTime) AS datetime, + CAST(timestamp AS Date) AS date, + CAST(timestamp, 'String') AS string, + CAST(timestamp, 'FixedString(22)') AS fixed_string +``` + +``` text +┌─timestamp───────────┬────────────datetime─┬───────date─┬─string──────────────┬─fixed_string──────────────┐ +│ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00\0\0\0 │ +└─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘ +``` + +Fixedstring(N) ' ye dönüştürme yalnızca String veya FixedString(N) türünde argümanlar için çalışır. + +Type con conversionvers conversionion to [Nullable](../../sql_reference/data_types/nullable.md) ve geri desteklenmektedir. Örnek: + +``` sql +SELECT toTypeName(x) FROM t_null +``` + +``` text +┌─toTypeName(x)─┐ +│ Int8 │ +│ Int8 │ +└───────────────┘ +``` + +``` sql +SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null +``` + +``` text +┌─toTypeName(CAST(x, 'Nullable(UInt16)'))─┐ +│ Nullable(UInt16) │ +│ Nullable(UInt16) │ +└─────────────────────────────────────────┘ +``` + +## toİnterval(yıl\|Çeyrek\|Ay\|hafta\|Gün\|Saat\|Dakika / Saniye) {#function-tointerval} + +Bir sayı türü argümanını bir [Aralıklı](../../sql_reference/data_types/special_data_types/interval.md) veri türü. + +**Sözdizimi** + +``` sql +toIntervalSecond(number) +toIntervalMinute(number) +toIntervalHour(number) +toIntervalDay(number) +toIntervalWeek(number) +toIntervalMonth(number) +toIntervalQuarter(number) +toIntervalYear(number) +``` + +**Parametre** + +- `number` — Duration of interval. Positive integer number. + +**Döndürülen değerler** + +- Değeri `Interval` veri türü. + +**Örnek** + +``` sql +WITH + toDate('2019-01-01') AS date, + INTERVAL 1 WEEK AS interval_week, + toIntervalWeek(1) AS interval_to_week +SELECT + date + interval_week, + date + interval_to_week +``` + +``` text +┌─plus(date, interval_week)─┬─plus(date, interval_to_week)─┐ +│ 2019-01-08 │ 2019-01-08 │ +└───────────────────────────┴──────────────────────────────┘ +``` + +## parseDateTimeBestEffort {#parsedatetimebesteffort} + +Bir tarih ve saati dönüştürür [Dize](../../sql_reference/data_types/string.md) temsil etmek [DateTime](../../sql_reference/data_types/datetime.md#data_type-datetime) veri türü. + +İşlev ayrıştırır [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), [RFC 1123-5.2.14 RFC-822 Tarih ve Saat özellikleri](https://tools.ietf.org/html/rfc1123#page-55), ClickHouse ve diğer bazı tarih ve saat biçimleri. + +**Sözdizimi** + +``` sql +parseDateTimeBestEffort(time_string [, time_zone]); +``` + +**Parametre** + +- `time_string` — String containing a date and time to convert. [Dize](../../sql_reference/data_types/string.md). +- `time_zone` — Time zone. The function parses `time_string` saat dilimine göre. [Dize](../../sql_reference/data_types/string.md). + +**Desteklenen standart dışı formatlar** + +- 9 içeren bir dize..10 haneli [unix zaman damgası](https://en.wikipedia.org/wiki/Unix_time). +- Tarih ve saat bileşeni olan bir dize: `YYYYMMDDhhmmss`, `DD/MM/YYYY hh:mm:ss`, `DD-MM-YY hh:mm`, `YYYY-MM-DD hh:mm:ss` vb. +- Bir tarih, ancak hiçbir zaman bileşeni ile bir dize: `YYYY`, `YYYYMM`, `YYYY*MM`, `DD/MM/YYYY`, `DD-MM-YY` vb. +- Bir gün ve Saat ile bir dize: `DD`, `DD hh`, `DD hh:mm`. Bu durumda `YYYY-MM` olarak ikame edilir `2000-01`. +- Tarih ve Saat Saat Dilimi uzaklık bilgileri ile birlikte içeren bir dize: `YYYY-MM-DD hh:mm:ss ±h:mm` vb. Mesela, `2020-12-12 17:36:00 -5:00`. + +Ayırıcılı tüm formatlar için işlev, tam adlarıyla veya bir ay adının ilk üç harfiyle ifade edilen ay adlarını ayrıştırır. Örnekler: `24/DEC/18`, `24-Dec-18`, `01-September-2018`. + +**Döndürülen değer** + +- `time_string` dönüştürül thedü `DateTime` veri türü. + +**Örnekler** + +Sorgu: + +``` sql +SELECT parseDateTimeBestEffort('12/12/2020 12:12:57') +AS parseDateTimeBestEffort; +``` + +Sonuç: + +``` text +┌─parseDateTimeBestEffort─┐ +│ 2020-12-12 12:12:57 │ +└─────────────────────────┘ +``` + +Sorgu: + +``` sql +SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Europe/Moscow') +AS parseDateTimeBestEffort +``` + +Sonuç: + +``` text +┌─parseDateTimeBestEffort─┐ +│ 2018-08-18 10:22:16 │ +└─────────────────────────┘ +``` + +Sorgu: + +``` sql +SELECT parseDateTimeBestEffort('1284101485') +AS parseDateTimeBestEffort +``` + +Sonuç: + +``` text +┌─parseDateTimeBestEffort─┐ +│ 2015-07-07 12:04:41 │ +└─────────────────────────┘ +``` + +Sorgu: + +``` sql +SELECT parseDateTimeBestEffort('2018-12-12 10:12:12') +AS parseDateTimeBestEffort +``` + +Sonuç: + +``` text +┌─parseDateTimeBestEffort─┐ +│ 2018-12-12 10:12:12 │ +└─────────────────────────┘ +``` + +Sorgu: + +``` sql +SELECT parseDateTimeBestEffort('10 20:19') +``` + +Sonuç: + +``` text +┌─parseDateTimeBestEffort('10 20:19')─┐ +│ 2000-01-10 20:19:00 │ +└─────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- \[ISO 8601 announcement by @xkcd\](https://xkcd.com/1179/) +- [RFC 1123](https://tools.ietf.org/html/rfc1123) +- [toDate](#todate) +- [toDateTime](#todatetime) + +## parseDateTimeBestEffortOrNull {#parsedatetimebesteffortornull} + +İçin aynı [parseDateTimeBestEffort](#parsedatetimebesteffort) işlenemeyen bir tarih biçimiyle karşılaştığında null döndürmesi dışında. + +## parseDateTimeBestEffortOrZero {#parsedatetimebesteffortorzero} + +İçin aynı [parseDateTimeBestEffort](#parsedatetimebesteffort) bunun dışında, işlenemeyen bir tarih biçimiyle karşılaştığında sıfır tarih veya sıfır tarih saati döndürür. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) diff --git a/docs/tr/sql_reference/functions/url_functions.md b/docs/tr/sql_reference/functions/url_functions.md new file mode 100644 index 00000000000..47f127f1667 --- /dev/null +++ b/docs/tr/sql_reference/functions/url_functions.md @@ -0,0 +1,209 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 54 +toc_title: "URL'ler ile \xE7al\u0131\u015Fma" +--- + +# URL'ler ile çalışmak için işlevler {#functions-for-working-with-urls} + +Tüm bu işlevler RFC'Yİ takip etmez. Geliştirilmiş performans için maksimum derecede basitleştirilmişlerdir. + +## Bir URL'nin bölümlerini Ayıklayan işlevler {#functions-that-extract-parts-of-a-url} + +İlgili bölüm bir URL'de yoksa, boş bir dize döndürülür. + +### protokol {#protocol} + +Protokolü bir URL'den ayıklar. + +Examples of typical returned values: http, https, ftp, mailto, tel, magnet… + +### etki {#domain} + +Ana bilgisayar adını bir URL'den ayıklar. + +``` sql +domain(url) +``` + +**Parametre** + +- `url` — URL. Type: [Dize](../../sql_reference/data_types/string.md). + +URL, bir şema ile veya şema olmadan belirtilebilir. Örnekler: + +``` text +svn+ssh://some.svn-hosting.com:80/repo/trunk +some.svn-hosting.com:80/repo/trunk +https://yandex.com/time/ +``` + +Bu örnekler için, `domain` işlev aşağıdaki sonuçları döndürür: + +``` text +some.svn-hosting.com +some.svn-hosting.com +yandex.com +``` + +**Döndürülen değerler** + +- Adı ana. ClickHouse giriş dizesini bir URL olarak ayrıştırırsa. +- Boş dize. ClickHouse giriş dizesini bir URL olarak ayrıştıramazsa. + +Tür: `String`. + +**Örnek** + +``` sql +SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') +``` + +``` text +┌─domain('svn+ssh://some.svn-hosting.com:80/repo/trunk')─┐ +│ some.svn-hosting.com │ +└────────────────────────────────────────────────────────┘ +``` + +### domainWithoutWWW {#domainwithoutwww} + +Etki alanını döndürür ve birden fazla kaldırır ‘www.’ başlangıcına, eğer var dan. + +### topLevelDomain {#topleveldomain} + +Üst düzey etki alanını bir URL'den ayıklar. + +``` sql +topLevelDomain(url) +``` + +**Parametre** + +- `url` — URL. Type: [Dize](../../sql_reference/data_types/string.md). + +URL, bir şema ile veya şema olmadan belirtilebilir. Örnekler: + +``` text +svn+ssh://some.svn-hosting.com:80/repo/trunk +some.svn-hosting.com:80/repo/trunk +https://yandex.com/time/ +``` + +**Döndürülen değerler** + +- Etki alanı adı. ClickHouse giriş dizesini bir URL olarak ayrıştırırsa. +- Boş dize. ClickHouse giriş dizesini bir URL olarak ayrıştıramazsa. + +Tür: `String`. + +**Örnek** + +``` sql +SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') +``` + +``` text +┌─topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk')─┐ +│ com │ +└────────────────────────────────────────────────────────────────────┘ +``` + +### firstSignificantSubdomain {#firstsignificantsubdomain} + +Ret theur thens the “first significant subdomain”. Bu, Yandex'e özgü standart olmayan bir kavramdır.Metrica. İlk önemli alt etki alanı ise ikinci düzey bir etki alanıdır ‘com’, ‘net’, ‘org’, veya ‘co’. Aksi takdirde, üçüncü düzey bir alandır. Mesela, `firstSignificantSubdomain (‘https://news.yandex.ru/’) = ‘yandex’, firstSignificantSubdomain (‘https://news.yandex.com.tr/’) = ‘yandex’`. Listesi “insignificant” ikinci düzey etki alanları ve diğer uygulama ayrıntıları gelecekte değişebilir. + +### cutToFirstSignificantSubdomain {#cuttofirstsignificantsubdomain} + +En üst düzey alt etki alanlarını içeren etki alanının bir bölümünü döndürür. “first significant subdomain” (yukarıdaki açıklamaya bakınız). + +Mesela, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. + +### yol {#path} + +Yolu döndürür. Örnek: `/top/news.html` Yol sorgu dizesini içermez. + +### pathFull {#pathfull} + +Yukarıdaki ile aynı, ancak sorgu dizesi ve parça dahil. Örnek: / top / haberler.html?Sayfa = 2 \# yorumlar + +### queryString {#querystring} + +Sorgu dizesini döndürür. Örnek: Sayfa = 1 & lr = 213. sorgu dizesi, ilk soru işaretinin yanı sıra \# ve \# sonrası her şeyi içermez. + +### parça {#fragment} + +Parça tanımlayıcısını döndürür. fragment ilk karma sembolü içermez. + +### queryStringAndFragment {#querystringandfragment} + +Sorgu dizesini ve parça tanımlayıcısını döndürür. Örnek: Sayfa = 1\#29390. + +### extractURLParameter (URL, isim) {#extracturlparameterurl-name} + +Değerini döndürür ‘name’ varsa, URL'DEKİ parametre. Aksi takdirde, boş bir dize. Bu ada sahip birçok parametre varsa, ilk oluşumu döndürür. Bu işlev, parametre adının URL'de geçirilen bağımsız değişkenle aynı şekilde kodlandığı varsayımı altında çalışır. + +### extractURLParameters (URL) {#extracturlparametersurl} + +Bir dizi döndürür name = URL parametrelerine karşılık gelen değer dizeleri. Değerler hiçbir şekilde deşifre edilmez. + +### extractURLParameterNames(URL) {#extracturlparameternamesurl} + +URL parametrelerinin adlarına karşılık gelen bir dizi ad dizesi döndürür. Değerler hiçbir şekilde deşifre edilmez. + +### URLHierarchy(URL) {#urlhierarchyurl} + +Sonunda/,? simgeleriyle kesilen URL'yi içeren bir dizi döndürür yol ve sorgu dizesinde. Ardışık ayırıcı karakterler bir olarak sayılır. Kesim, tüm ardışık ayırıcı karakterlerden sonra pozisyonda yapılır. + +### URLPathHierarchy(URL) {#urlpathhierarchyurl} + +Yukarıdaki ile aynı, ancak sonuçta protokol ve ana bilgisayar olmadan. / Eleman (kök) dahil değildir. Örnek: işlev, yandex'te URL'yi ağaç raporları uygulamak için kullanılır. Ölçü. + +``` text +URLPathHierarchy('https://example.com/browse/CONV-6788') = +[ + '/browse/', + '/browse/CONV-6788' +] +``` + +### decodeURLComponent (URL) {#decodeurlcomponenturl} + +Çözülmüş URL'yi döndürür. +Örnek: + +``` sql +SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS DecodedURL; +``` + +``` text +┌─DecodedURL─────────────────────────────┐ +│ http://127.0.0.1:8123/?query=SELECT 1; │ +└────────────────────────────────────────┘ +``` + +## URL'nin bir bölümünü kaldıran işlevler {#functions-that-remove-part-of-a-url} + +URL'de benzer bir şey yoksa, URL değişmeden kalır. + +### cutWWW {#cutwww} + +Birden fazla kaldırır ‘www.’ varsa, URL'nin etki alanının başından itibaren. + +### cutQueryString {#cutquerystring} + +Sorgu dizesini kaldırır. Soru işareti de kaldırılır. + +### cutFragment {#cutfragment} + +Parça tanımlayıcısını kaldırır. Sayı işareti de kaldırılır. + +### cutQueryStringAndFragment {#cutquerystringandfragment} + +Sorgu dizesini ve parça tanımlayıcısını kaldırır. Soru işareti ve sayı işareti de kaldırılır. + +### cutURLParameter (URL, isim) {#cuturlparameterurl-name} + +Kaldırır ‘name’ Varsa URL parametresi. Bu işlev, parametre adının URL'de geçirilen bağımsız değişkenle aynı şekilde kodlandığı varsayımı altında çalışır. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/url_functions/) diff --git a/docs/tr/sql_reference/functions/uuid_functions.md b/docs/tr/sql_reference/functions/uuid_functions.md new file mode 100644 index 00000000000..93969dd4640 --- /dev/null +++ b/docs/tr/sql_reference/functions/uuid_functions.md @@ -0,0 +1,122 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 53 +toc_title: "UUID ile \xE7al\u0131\u015Fma" +--- + +# UUID ile çalışmak için fonksiyonlar {#functions-for-working-with-uuid} + +UUID ile çalışmak için işlevler aşağıda listelenmiştir. + +## generateuuıdv4 {#uuid-function-generate} + +Üretir [UUID](../../sql_reference/data_types/uuid.md) -den [sürüm 4](https://tools.ietf.org/html/rfc4122#section-4.4). + +``` sql +generateUUIDv4() +``` + +**Döndürülen değer** + +UUID türü değeri. + +**Kullanım örneği** + +Bu örnek, UUID türü sütunuyla bir tablo oluşturma ve tabloya bir değer ekleme gösterir. + +``` sql +CREATE TABLE t_uuid (x UUID) ENGINE=TinyLog + +INSERT INTO t_uuid SELECT generateUUIDv4() + +SELECT * FROM t_uuid +``` + +``` text +┌────────────────────────────────────x─┐ +│ f4bf890f-f9dc-4332-ad5c-0c18e73f28e9 │ +└──────────────────────────────────────┘ +``` + +## toUUİD (x) {#touuid-x} + +Dize türü değerini UUID türüne dönüştürür. + +``` sql +toUUID(String) +``` + +**Döndürülen değer** + +UUID türü değeri. + +**Kullanım örneği** + +``` sql +SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') AS uuid +``` + +``` text +┌─────────────────────────────────uuid─┐ +│ 61f0c404-5cb3-11e7-907b-a6006ad3dba0 │ +└──────────────────────────────────────┘ +``` + +## UUİDStringToNum {#uuidstringtonum} + +Biçiminde 36 karakter içeren bir dize kabul eder `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` ve bir bayt kümesi olarak döndürür [FixedString (16)](../../sql_reference/data_types/fixedstring.md). + +``` sql +UUIDStringToNum(String) +``` + +**Döndürülen değer** + +FixedString (16) + +**Kullanım örnekleri** + +``` sql +SELECT + '612f3c40-5d3b-217e-707b-6a546a3d7b29' AS uuid, + UUIDStringToNum(uuid) AS bytes +``` + +``` text +┌─uuid─────────────────────────────────┬─bytes────────────┐ +│ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ a/<@];!~p{jTj={) │ +└──────────────────────────────────────┴──────────────────┘ +``` + +## UUİDNumToString {#uuidnumtostring} + +Kabul eder bir [FixedString (16)](../../sql_reference/data_types/fixedstring.md) değer ve metin biçiminde 36 karakter içeren bir dize döndürür. + +``` sql +UUIDNumToString(FixedString(16)) +``` + +**Döndürülen değer** + +Dize. + +**Kullanım örneği** + +``` sql +SELECT + 'a/<@];!~p{jTj={)' AS bytes, + UUIDNumToString(toFixedString(bytes, 16)) AS uuid +``` + +``` text +┌─bytes────────────┬─uuid─────────────────────────────────┐ +│ a/<@];!~p{jTj={) │ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ +└──────────────────┴──────────────────────────────────────┘ +``` + +## Ayrıca bakınız {#see-also} + +- [dictGetUUİD](ext_dict_functions.md#ext_dict_functions-other) + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/uuid_function/) diff --git a/docs/tr/sql_reference/functions/ym_dict_functions.md b/docs/tr/sql_reference/functions/ym_dict_functions.md new file mode 100644 index 00000000000..46384107029 --- /dev/null +++ b/docs/tr/sql_reference/functions/ym_dict_functions.md @@ -0,0 +1,155 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 59 +toc_title: "Yandex ile \xE7al\u0131\u015Fmak.Metrica S\xF6zl\xFCkleri" +--- + +# Yandex ile çalışmak için fonksiyonlar.Metrica sözlükleri {#functions-for-working-with-yandex-metrica-dictionaries} + +Aşağıdaki işlevlerin çalışması için, sunucu yapılandırmasının tüm Yandex'i almak için yolları ve adresleri belirtmesi gerekir.Metrica sözlükler. Sözlükler, bu işlevlerden herhangi birinin ilk çağrısında yüklenir. Başvuru listeleri yüklenemiyorsa, bir özel durum atılır. + +Başvuru listeleri oluşturma hakkında daha fazla bilgi için bölüme bakın “Dictionaries”. + +## Çoklu geobazlar {#multiple-geobases} + +ClickHouse, belirli bölgelerin hangi ülkelere ait olduğu konusunda çeşitli perspektifleri desteklemek için aynı anda birden fazla alternatif jeobaz (bölgesel hiyerarşiler) ile çalışmayı destekler. + +Bu ‘clickhouse-server’ config, dosyayı bölgesel hiyerarşi ile belirtir::`/opt/geo/regions_hierarchy.txt` + +Bu dosyanın yanı sıra, yakındaki \_ sembolüne ve isme eklenen herhangi bir sonek (dosya uzantısından önce) olan dosyaları da arar. +Örneğin, dosyayı da bulacaktır `/opt/geo/regions_hierarchy_ua.txt` varsa. + +`ua` sözlük anahtarı denir. Soneksiz bir sözlük için anahtar boş bir dizedir. + +Tüm sözlükler çalışma zamanında yeniden yüklenir (buıltın\_dıctıonarıes\_reload\_ınterval yapılandırma parametresinde tanımlandığı gibi belirli sayıda saniyede bir kez veya varsayılan olarak saatte bir kez). Ancak, sunucu başladığında kullanılabilir sözlüklerin listesi bir kez tanımlanır. + +All functions for working with regions have an optional argument at the end – the dictionary key. It is referred to as the geobase. +Örnek: + +``` sql +regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txt +regionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txt +regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt +``` + +### regionToCity (id \[, geobase\]) {#regiontocityid-geobase} + +Accepts a UInt32 number – the region ID from the Yandex geobase. If this region is a city or part of a city, it returns the region ID for the appropriate city. Otherwise, returns 0. + +### regionToArea (id \[, geobase\]) {#regiontoareaid-geobase} + +Bir bölgeyi bir alana dönüştürür (geobase içinde 5 yazın). Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. + +``` sql +SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua')) +FROM system.numbers +LIMIT 15 +``` + +``` text +┌─regionToName(regionToArea(toUInt32(number), \'ua\'))─┐ +│ │ +│ Moscow and Moscow region │ +│ St. Petersburg and Leningrad region │ +│ Belgorod region │ +│ Ivanovsk region │ +│ Kaluga region │ +│ Kostroma region │ +│ Kursk region │ +│ Lipetsk region │ +│ Orlov region │ +│ Ryazan region │ +│ Smolensk region │ +│ Tambov region │ +│ Tver region │ +│ Tula region │ +└──────────────────────────────────────────────────────┘ +``` + +### regionToDistrict (id \[, geobase\]) {#regiontodistrictid-geobase} + +Bir bölgeyi federal bir bölgeye dönüştürür (geobase içinde tip 4). Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. + +``` sql +SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua')) +FROM system.numbers +LIMIT 15 +``` + +``` text +┌─regionToName(regionToDistrict(toUInt32(number), \'ua\'))─┐ +│ │ +│ Central federal district │ +│ Northwest federal district │ +│ South federal district │ +│ North Caucases federal district │ +│ Privolga federal district │ +│ Ural federal district │ +│ Siberian federal district │ +│ Far East federal district │ +│ Scotland │ +│ Faroe Islands │ +│ Flemish region │ +│ Brussels capital region │ +│ Wallonia │ +│ Federation of Bosnia and Herzegovina │ +└──────────────────────────────────────────────────────────┘ +``` + +### regionToCountry (ıd \[, geobase\]) {#regiontocountryid-geobase} + +Bir bölgeyi bir ülkeye dönüştürür. Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. +Örnek: `regionToCountry(toUInt32(213)) = 225` Moskova'yı (213) Rusya'ya (225) dönüştürür. + +### regionToContinent (id \[, geobase\]) {#regiontocontinentid-geobase} + +Bir bölgeyi bir kıtaya dönüştürür. Diğer her şekilde, bu işlev aynıdır ‘regionToCity’. +Örnek: `regionToContinent(toUInt32(213)) = 10001` Moskova'yı (213) Avrasya'ya (10001) dönüştürür. + +### regionToTopContinent (\#regiontotopcontinent) {#regiontotopcontinent-regiontotopcontinent} + +Bölgenin hiyerarşisinde en yüksek kıtayı bulur. + +**Sözdizimi** + +``` sql +regionToTopContinent(id[, geobase]); +``` + +**Parametre** + +- `id` — Region ID from the Yandex geobase. [Uİnt32](../../sql_reference/data_types/int_uint.md). +- `geobase` — Dictionary key. See [Çoklu Geobazlar](#multiple-geobases). [Dize](../../sql_reference/data_types/string.md). İsteğe bağlı. + +**Döndürülen değer** + +- Üst düzey kıtanın tanımlayıcısı (bölgeler hiyerarşisine tırmandığınızda ikincisi). +- 0, yoksa. + +Tür: `UInt32`. + +### regionToPopulation (id \[, geobase\]) {#regiontopopulationid-geobase} + +Bir bölge için nüfusu alır. +Nüfus geobase ile dosyalarda kaydedilebilir. Bölümüne bakınız “External dictionaries”. +Bölge için nüfus kaydedilmezse, 0 döndürür. +Yandex geobase'de, nüfus alt bölgeler için kaydedilebilir, ancak üst bölgeler için kaydedilemez. + +### regionİn (lhs, rhs \[, geobase\]) {#regioninlhs-rhs-geobase} + +Olup olmadığını denetler bir ‘lhs’ bölge bir ‘rhs’ bölge. Aitse 1'e eşit bir Uİnt8 numarası veya ait değilse 0 döndürür. +The relationship is reflexive – any region also belongs to itself. + +### regionHierarchy (id \[, geobase\]) {#regionhierarchyid-geobase} + +Accepts a UInt32 number – the region ID from the Yandex geobase. Returns an array of region IDs consisting of the passed region and all parents along the chain. +Örnek: `regionHierarchy(toUInt32(213)) = [213,1,3,225,10001,10000]`. + +### regionToName (id \[, lang\]) {#regiontonameid-lang} + +Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID doesn't exist, an empty string is returned. + +`ua` ve `uk` hem Ukrayna demek. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/functions/ym_dict_functions/) diff --git a/docs/tr/sql_reference/index.md b/docs/tr/sql_reference/index.md new file mode 100644 index 00000000000..25b886b9c9d --- /dev/null +++ b/docs/tr/sql_reference/index.md @@ -0,0 +1,18 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "SQL ba\u015Fvurusu" +toc_hidden: true +toc_priority: 28 +toc_title: "gizlenmi\u015F" +--- + +# SQL başvurusu {#sql-reference} + +- [SELECT](statements/select.md) +- [INSERT INTO](statements/insert_into.md) +- [CREATE](statements/create.md) +- [ALTER](statements/alter.md#query_language_queries_alter) +- [Diğer sorgu türleri](statements/misc.md) + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/) diff --git a/docs/tr/sql_reference/operators.md b/docs/tr/sql_reference/operators.md new file mode 100644 index 00000000000..63100e9e9ea --- /dev/null +++ b/docs/tr/sql_reference/operators.md @@ -0,0 +1,277 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 37 +toc_title: "Operat\xF6rler" +--- + +# Operatörler {#operators} + +ClickHouse onların öncelik, öncelik ve ilişkilendirme göre sorgu ayrıştırma aşamasında karşılık gelen işlevlere işleçleri dönüştürür. + +## Erişim Operatörleri {#access-operators} + +`a[N]` – Access to an element of an array. The `arrayElement(a, N)` işlev. + +`a.N` – Access to a tuple element. The `tupleElement(a, N)` işlev. + +## Sayısal Olumsuzlama Operatörü {#numeric-negation-operator} + +`-a` – The `negate (a)` işlev. + +## Çarpma ve bölme operatörleri {#multiplication-and-division-operators} + +`a * b` – The `multiply (a, b)` işlev. + +`a / b` – The `divide(a, b)` işlev. + +`a % b` – The `modulo(a, b)` işlev. + +## Toplama ve çıkarma operatörleri {#addition-and-subtraction-operators} + +`a + b` – The `plus(a, b)` işlev. + +`a - b` – The `minus(a, b)` işlev. + +## Karşılaştırma Operatörleri {#comparison-operators} + +`a = b` – The `equals(a, b)` işlev. + +`a == b` – The `equals(a, b)` işlev. + +`a != b` – The `notEquals(a, b)` işlev. + +`a <> b` – The `notEquals(a, b)` işlev. + +`a <= b` – The `lessOrEquals(a, b)` işlev. + +`a >= b` – The `greaterOrEquals(a, b)` işlev. + +`a < b` – The `less(a, b)` işlev. + +`a > b` – The `greater(a, b)` işlev. + +`a LIKE s` – The `like(a, b)` işlev. + +`a NOT LIKE s` – The `notLike(a, b)` işlev. + +`a BETWEEN b AND c` – The same as `a >= b AND a <= c`. + +`a NOT BETWEEN b AND c` – The same as `a < b OR a > c`. + +## Veri kümeleriyle çalışmak için operatörler {#operators-for-working-with-data-sets} + +*Görmek [Operatör İNLERDE](statements/select.md#select-in-operators).* + +`a IN ...` – The `in(a, b)` işlev. + +`a NOT IN ...` – The `notIn(a, b)` işlev. + +`a GLOBAL IN ...` – The `globalIn(a, b)` işlev. + +`a GLOBAL NOT IN ...` – The `globalNotIn(a, b)` işlev. + +## Tarih ve Saatlerle çalışmak için operatörler {#operators-datetime} + +### EXTRACT {#operator-extract} + +``` sql +EXTRACT(part FROM date); +``` + +Belirli bir tarihten parçaları ayıklayın. Örneğin, belirli bir tarihten bir ay veya bir zamandan bir saniye alabilirsiniz. + +Bu `part` parametre almak için tarihin hangi bölümünü belirtir. Aşağıdaki değerler kullanılabilir: + +- `DAY` — The day of the month. Possible values: 1–31. +- `MONTH` — The number of a month. Possible values: 1–12. +- `YEAR` — The year. +- `SECOND` — The second. Possible values: 0–59. +- `MINUTE` — The minute. Possible values: 0–59. +- `HOUR` — The hour. Possible values: 0–23. + +Bu `part` parametre büyük / küçük harf duyarsızdır. + +Bu `date` parametre, işlenecek tarihi veya saati belirtir. Ya [Tarihli](../sql_reference/data_types/date.md) veya [DateTime](../sql_reference/data_types/datetime.md) türü desteklenir. + +Örnekler: + +``` sql +SELECT EXTRACT(DAY FROM toDate('2017-06-15')); +SELECT EXTRACT(MONTH FROM toDate('2017-06-15')); +SELECT EXTRACT(YEAR FROM toDate('2017-06-15')); +``` + +Aşağıdaki örnekte bir tablo oluşturuyoruz ve içine bir değer ekliyoruz `DateTime` tür. + +``` sql +CREATE TABLE test.Orders +( + OrderId UInt64, + OrderName String, + OrderDate DateTime +) +ENGINE = Log; +``` + +``` sql +INSERT INTO test.Orders VALUES (1, 'Jarlsberg Cheese', toDateTime('2008-10-11 13:23:44')); +``` + +``` sql +SELECT + toYear(OrderDate) AS OrderYear, + toMonth(OrderDate) AS OrderMonth, + toDayOfMonth(OrderDate) AS OrderDay, + toHour(OrderDate) AS OrderHour, + toMinute(OrderDate) AS OrderMinute, + toSecond(OrderDate) AS OrderSecond +FROM test.Orders; +``` + +``` text +┌─OrderYear─┬─OrderMonth─┬─OrderDay─┬─OrderHour─┬─OrderMinute─┬─OrderSecond─┐ +│ 2008 │ 10 │ 11 │ 13 │ 23 │ 44 │ +└───────────┴────────────┴──────────┴───────────┴─────────────┴─────────────┘ +``` + +Daha fazla örnek görebilirsiniz [testler](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00619_extract.sql). + +### INTERVAL {#operator-interval} + +Oluşturur bir [Aralıklı](../sql_reference/data_types/special_data_types/interval.md)- aritmetik işlemlerde kullanılması gereken tip değeri [Tarihli](../sql_reference/data_types/date.md) ve [DateTime](../sql_reference/data_types/datetime.md)- tip değerleri. + +Aralık türleri: +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `QUARTER` +- `YEAR` + +!!! warning "Uyarıcı" + Farklı tiplere sahip aralıklar birleştirilemez. Gibi ifadeler kullanamazsınız `INTERVAL 4 DAY 1 HOUR`. Aralıkların, örneğin aralığın en küçük birimine eşit veya daha küçük olan birimlerdeki aralıkları belirtin, `INTERVAL 25 HOUR`. Aşağıdaki örnekte olduğu gibi ardışık işlemleri kullanabilirsiniz. + +Örnek: + +``` sql +SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR +``` + +``` text +┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ +│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +``` + +**Ayrıca Bakınız** + +- [Aralıklı](../sql_reference/data_types/special_data_types/interval.md) veri türü +- [toİnterval](../sql_reference/functions/type_conversion_functions.md#function-tointerval) tip dönüştürme işlevleri + +## Mantıksal Olumsuzlama Operatörü {#logical-negation-operator} + +`NOT a` – The `not(a)` işlev. + +## Mantıksal ve operatör {#logical-and-operator} + +`a AND b` – The`and(a, b)` işlev. + +## Mantıksal veya operatör {#logical-or-operator} + +`a OR b` – The `or(a, b)` işlev. + +## Koşullu Operatör {#conditional-operator} + +`a ? b : c` – The `if(a, b, c)` işlev. + +Not: + +Koşullu işleç B ve c değerlerini hesaplar, ardından a koşulunun karşılanıp karşılanmadığını kontrol eder ve ardından karşılık gelen değeri döndürür. Eğer `b` veya `C` is an [arrayJoin()](../sql_reference/functions/array_join.md#functions_arrayjoin) işlev, her satır ne olursa olsun çoğaltılır “a” koşul. + +## Koşullu İfade {#operator_case} + +``` sql +CASE [x] + WHEN a THEN b + [WHEN ... THEN ...] + [ELSE c] +END +``` + +Eğer `x` belirtilen sonra `transform(x, [a, ...], [b, ...], c)` function is used. Otherwise – `multiIf(a, b, ..., c)`. + +Eğer herhangi bir `ELSE c` ifadedeki yan tümce, varsayılan değer `NULL`. + +Bu `transform` fonksiyonu ile çalışmıyor `NULL`. + +## Birleştirme Operatörü {#concatenation-operator} + +`s1 || s2` – The `concat(s1, s2) function.` + +## Lambda Oluşturma Operatörü {#lambda-creation-operator} + +`x -> expr` – The `lambda(x, expr) function.` + +Parantez oldukları için aşağıdaki operatörler bir önceliğe sahip değildir: + +## Dizi Oluşturma Operatörü {#array-creation-operator} + +`[x1, ...]` – The `array(x1, ...) function.` + +## Tuple Oluşturma Operatörü {#tuple-creation-operator} + +`(x1, x2, ...)` – The `tuple(x2, x2, ...) function.` + +## İlişkisellik {#associativity} + +Tüm ikili operatörler ilişkisellikten ayrıldı. Mesela, `1 + 2 + 3` dönüştür toülür `plus(plus(1, 2), 3)`. +Bazen bu beklediğiniz gibi çalışmaz. Mesela, `SELECT 4 > 2 > 3` 0 ile sonuç willlanır. + +Verimlilik için, `and` ve `or` işlevler herhangi bir sayıda bağımsız değişkeni kabul eder. İlgili zincirler `AND` ve `OR` operatörler bu işlevlerin tek bir çağrısına dönüştürülür. + +## İçin kontrol `NULL` {#checking-for-null} + +ClickHouse destekler `IS NULL` ve `IS NOT NULL` operatörler. + +### IS NULL {#operator-is-null} + +- İçin [Nullable](../sql_reference/data_types/nullable.md) türü değerleri `IS NULL` operatör döner: + - `1` değeri ise `NULL`. + - `0` başka. +- Diğer değerler için, `IS NULL` operatör her zaman döner `0`. + + + +``` sql +SELECT x+100 FROM t_null WHERE y IS NULL +``` + +``` text +┌─plus(x, 100)─┐ +│ 101 │ +└──────────────┘ +``` + +### IS NOT NULL {#is-not-null} + +- İçin [Nullable](../sql_reference/data_types/nullable.md) türü değerleri `IS NOT NULL` operatör döner: + - `0` değeri ise `NULL`. + - `1` başka. +- Diğer değerler için, `IS NOT NULL` operatör her zaman döner `1`. + + + +``` sql +SELECT * FROM t_null WHERE y IS NOT NULL +``` + +``` text +┌─x─┬─y─┐ +│ 2 │ 3 │ +└───┴───┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/operators/) diff --git a/docs/tr/sql_reference/statements/alter.md b/docs/tr/sql_reference/statements/alter.md new file mode 100644 index 00000000000..b61a3784af9 --- /dev/null +++ b/docs/tr/sql_reference/statements/alter.md @@ -0,0 +1,504 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 36 +toc_title: ALTER +--- + +## ALTER {#query_language_queries_alter} + +Bu `ALTER` sorgu yalnızca için desteklenir `*MergeTree` tablo gibi `Merge`ve`Distributed`. Sorgunun çeşitli varyasyonları vardır. + +### Sütun Manipülasyonları {#column-manipulations} + +Tablo yapısını değiştirme. + +``` sql +ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ... +``` + +Sorguda, bir veya daha fazla virgülle ayrılmış eylemlerin bir listesini belirtin. +Her eylem bir sütun üzerinde bir işlemdir. + +Aşağıdaki eylemler desteklenir: + +- [ADD COLUMN](#alter_add-column) — Adds a new column to the table. +- [DROP COLUMN](#alter_drop-column) — Deletes the column. +- [CLEAR COLUMN](#alter_clear-column) — Resets column values. +- [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. +- [MODIFY COLUMN](#alter_modify-column) — Changes column's type, default expression and TTL. + +Bu eylemler aşağıda ayrıntılı olarak açıklanmıştır. + +#### ADD COLUMN {#alter_add-column} + +``` sql +ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] +``` + +Belirtilen tabloya yeni bir sütun ekler `name`, `type`, [`codec`](create.md#codecs) ve `default_expr` (bkz [Varsayılan ifadeler](create.md#create-default-values)). + +Eğer... `IF NOT EXISTS` yan tümcesi dahil, sütun zaten varsa sorgu bir hata döndürmez. Belirtir specifyseniz `AFTER name_after` (başka bir sütunun adı), sütun tablo sütunları listesinde belirtilen sonra eklenir. Aksi takdirde, sütun tablonun sonuna eklenir. Bir tablonun başına bir sütun eklemek için bir yol olduğunu unutmayın. Bir eylem zinciri için, `name_after` önceki eylemlerden birine eklenen bir sütunun adı olabilir. + +Bir sütun eklemek, verilerle herhangi bir işlem yapmadan tablo yapısını değiştirir. Sonra veriler diskte görünmüyor `ALTER`. Tablodan okurken bir sütun için veri eksikse, varsayılan değerlerle doldurulur (varsa, varsayılan ifadeyi gerçekleştirerek veya sıfır veya boş dizeler kullanarak). Sütun, veri parçalarını birleştirdikten sonra diskte görünür (bkz. [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)). + +Bu yaklaşım bize tamamlamak için izin verir `ALTER` eski verilerin hacmini arttırmadan anında sorgulayın. + +Örnek: + +``` sql +ALTER TABLE visits ADD COLUMN browser String AFTER user_id +``` + +#### DROP COLUMN {#alter_drop-column} + +``` sql +DROP COLUMN [IF EXISTS] name +``` + +Sütun adı ile siler `name`. Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. + +Dosya sisteminden veri siler. Bu, tüm dosyaları sildiğinden, sorgu neredeyse anında tamamlanır. + +Örnek: + +``` sql +ALTER TABLE visits DROP COLUMN browser +``` + +#### CLEAR COLUMN {#alter_clear-column} + +``` sql +CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name +``` + +Belirtilen bölüm için bir sütundaki tüm verileri sıfırlar. Bölümdeki bölüm adını ayarlama hakkında daha fazla bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). + +Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. + +Örnek: + +``` sql +ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple() +``` + +#### COMMENT COLUMN {#alter_comment-column} + +``` sql +COMMENT COLUMN [IF EXISTS] name 'comment' +``` + +Sütuna bir yorum ekler. Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. + +Her sütunun bir yorumu olabilir. Sütun için bir yorum zaten varsa, yeni bir yorum önceki yorumun üzerine yazar. + +Yorumlar saklanır `comment_expression` tarafından döndürülen sütun [DESCRIBE TABLE](misc.md#misc-describe-table) sorgu. + +Örnek: + +``` sql +ALTER TABLE visits COMMENT COLUMN browser 'The table shows the browser used for accessing the site.' +``` + +#### MODIFY COLUMN {#alter_modify-column} + +``` sql +MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] +``` + +Bu sorgu değişiklikleri `name` sütun özellikleri: + +- Tür + +- Varsayılan ifade + +- TTL + + For examples of columns TTL modifying, see [Column TTL](../engines/table_engines/mergetree_family/mergetree.md#mergetree-column-ttl). + +Eğer... `IF EXISTS` yan tümcesi belirtilir, sütun yoksa sorgu bir hata döndürmez. + +Türü değiştirirken, değerler sanki [toType](../../sql_reference/functions/type_conversion_functions.md) fonksiyonlar onlara uygulandı. Yalnızca varsayılan ifade değiştirilirse, sorgu karmaşık bir şey yapmaz ve neredeyse anında tamamlanır. + +Örnek: + +``` sql +ALTER TABLE visits MODIFY COLUMN browser Array(String) +``` + +Changing the column type is the only complex action – it changes the contents of files with data. For large tables, this may take a long time. + +Birkaç işlem aşaması vardır: + +- Geçici (yeni) dosyaları değiştirilmiş verilerle hazırlama. +- Eski dosyaları yeniden adlandırma. +- Geçici (yeni) dosyaları eski adlara yeniden adlandırma. +- Eski dosyaları silme. + +Sadece ilk aşama zaman alır. Bu aşamada bir hata varsa, veriler değişmez. +Ardışık aşamalardan biri sırasında bir hata varsa, veriler el ile geri yüklenebilir. Eski dosyalar dosya sisteminden silindi, ancak yeni dosyaların verileri diske yazılmadı ve kaybolduysa istisnadır. + +Bu `ALTER` sütunları değiştirmek için sorgu çoğaltılır. Talimatlar ZooKeeper kaydedilir, daha sonra her kopya bunları uygular. Tüm `ALTER` sorgular aynı sırada çalıştırılır. Sorgu, diğer yinelemeler üzerinde tamamlanması uygun eylemleri bekler. Ancak, yinelenen bir tablodaki sütunları değiştirmek için bir sorgu kesilebilir ve tüm eylemler zaman uyumsuz olarak gerçekleştirilir. + +#### Sorgu sınırlamalarını değiştir {#alter-query-limitations} + +Bu `ALTER` sorgu oluşturmak ve iç içe veri yapıları, ancak tüm iç içe veri yapıları ayrı öğeleri (sütunlar) silmenizi sağlar. İç içe geçmiş bir veri yapısı eklemek için, aşağıdaki gibi bir ada sahip sütunlar ekleyebilirsiniz `name.nested_name` ve türü `Array(T)`. İç içe geçmiş bir veri yapısı, noktadan önce aynı öneki olan bir ada sahip birden çok dizi sütununa eşdeğerdir. + +Birincil anahtardaki veya örnekleme anahtarındaki sütunları silmek için destek yoktur. `ENGINE` ifade). Birincil anahtarda bulunan sütunların türünü değiştirmek, yalnızca bu değişiklik verilerin değiştirilmesine neden olmazsa mümkündür (örneğin, bir numaraya değer eklemenize veya bir türden değiştirmenize izin verilir `DateTime` -e doğru `UInt32`). + +Eğer... `ALTER` sorgu, ihtiyacınız olan tablo değişikliklerini yapmak için yeterli değildir, yeni bir tablo oluşturabilir, verileri kullanarak kopyalayabilirsiniz. [INSERT SELECT](insert_into.md#insert_query_insert-select) sorgu, daha sonra tabloları kullanarak geçiş [RENAME](misc.md#misc_operations-rename) sorgu ve eski tabloyu silin. Kullanabilirsiniz [clickhouse-fotokopi makinesi](../../operations/utilities/clickhouse-copier.md) bir alternatif olarak `INSERT SELECT` sorgu. + +Bu `ALTER` sorgu tüm okur ve tablo için yazar engeller. Başka bir deyişle, Eğer uzun `SELECT` zamanda çalışıyor `ALTER` sorgu `ALTER` sorgu tamamlanmasını bekleyecektir. Aynı zamanda, aynı tablodaki tüm yeni sorgular bu sırada bekleyecektir `ALTER` çalışıyor. + +Verileri kendileri saklamayan tablolar için (örneğin `Merge` ve `Distributed`), `ALTER` sadece tablo yapısını değiştirir ve alt tabloların yapısını değiştirmez. Örneğin, ALTER for a çalıştırırken `Distributed` tablo, ayrıca çalıştırmak gerekir `ALTER` tüm uzak sunuculardaki tablolar için. + +### Anahtar İfadelerle Manipülasyonlar {#manipulations-with-key-expressions} + +Aşağıdaki komut desteklenir: + +``` sql +MODIFY ORDER BY new_expression +``` + +Sadece tablolar için çalışır [`MergeTree`](../../engines/table_engines/mergetree_family/mergetree.md) aile (dahil +[çoğaltıyordu](../../engines/table_engines/mergetree_family/replication.md) Tablolar). Komutu değiştirir +[sıralama anahtarı](../../engines/table_engines/mergetree_family/mergetree.md) tablonun +-e doğru `new_expression` (bir ifade veya ifadelerin bir tuple). Birincil anahtar aynı kalır. + +Komut, yalnızca meta verileri değiştirdiği bir anlamda hafiftir. Veri parçası özelliği tutmak için +satırlar sıralama anahtarı ifadesi tarafından sıralanır varolan sütunları içeren ifadeler ekleyemezsiniz +sıralama anahtarına (yalnızca sütun tarafından eklenen `ADD COLUMN` aynı komut `ALTER` sorgu). + +### Veri Atlama Endeksleri İle Manipülasyonlar {#manipulations-with-data-skipping-indices} + +Sadece tablolar için çalışır [`*MergeTree`](../../engines/table_engines/mergetree_family/mergetree.md) aile (dahil +[çoğaltıyordu](../../engines/table_engines/mergetree_family/replication.md) Tablolar). Aşağıdaki işlemler +mevcuttur: + +- `ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value AFTER name [AFTER name2]` - Tablolar meta dizin açıklama ekler. + +- `ALTER TABLE [db].name DROP INDEX name` - Tablolar meta dizin açıklama kaldırır ve diskten dizin dosyalarını siler. + +Bu komutlar, yalnızca meta verileri değiştirdikleri veya dosyaları kaldırdıkları bir anlamda hafiftir. +Ayrıca, çoğaltılırlar (ZooKeeper aracılığıyla indeks meta verilerini senkronize etme). + +### Kısıtlamalar İle Manipülasyonlar {#manipulations-with-constraints} + +Daha fazla görmek [kısıtlamalar](create.md#constraints) + +Kısıtlamalar eklenebilir veya aşağıdaki sözdizimi kullanılarak silinebilir: + +``` sql +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Sorgular eklemek veya hemen işlenir, böylece tablodan kısıtlamaları hakkında meta verileri kaldırın. + +Kısıtlama kontrolü *idam edilm willeyecek* eklen .mişse mevcut ver .ilerde + +Çoğaltılmış tablolardaki tüm değişiklikler Zookeeper'a yayınlanır, bu nedenle diğer kopyalara uygulanır. + +### Bölümler ve parçalar ile manipülasyonlar {#alter_manipulations-with-partitions} + +Aşağıdaki işlemler ile [bölümler](../../engines/table_engines/mergetree_family/custom_partitioning_key.md) mevcuttur: + +- [DETACH PARTITION](#alter_detach-partition) – Moves a partition to the `detached` dizin ve unutun. +- [DROP PARTITION](#alter_drop-partition) – Deletes a partition. +- [ATTACH PART\|PARTITION](#alter_attach-partition) – Adds a part or partition from the `detached` tabloya dizin. +- [ATTACH PARTITION FROM](#alter_attach-partition-from) – Copies the data partition from one table to another and adds. +- [REPLACE PARTITION](#alter_replace-partition) - Veri bölümünü bir tablodan diğerine kopyalar ve değiştirir. +- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) (\#alter\_move\_to\_table-partition) - veri bölümünü bir tablodan diğerine taşıyın. +- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) - Bir bölümdeki belirtilen sütunun değerini sıfırlar. +- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) - Bir bölümde belirtilen ikincil dizini sıfırlar. +- [FREEZE PARTITION](#alter_freeze-partition) – Creates a backup of a partition. +- [FETCH PARTITION](#alter_fetch-partition) – Downloads a partition from another server. +- [MOVE PARTITION\|PART](#alter_move-partition) – Move partition/data part to another disk or volume. + + + +#### Bölüm ayırmak {\#alter\_detach-bölüm} {#detach-partition-alter-detach-partition} + +``` sql +ALTER TABLE table_name DETACH PARTITION partition_expr +``` + +Belirtilen bölüm için tüm verileri `detached` dizin. Sunucu, yok gibi ayrılmış veri Bölümü hakkında unutur. Sunucu, bu verileri siz yapana kadar bilmeyecektir. [ATTACH](#alter_attach-partition) sorgu. + +Örnek: + +``` sql +ALTER TABLE visits DETACH PARTITION 201901 +``` + +Bir bölümdeki bölüm ifadesini ayarlama hakkında bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). + +Sorgu yürütüldükten sonra, veri ile istediğiniz her şeyi yapabilirsiniz `detached` directory — delete it from the file system, or just leave it. + +This query is replicated – it moves the data to the `detached` tüm kopyalarda dizin. Bu sorguyu yalnızca bir lider yinelemesinde yürütebileceğinizi unutmayın. Bir kopya bir lider olup olmadığını öğrenmek için `SELECT` sorgu için [sistem.yinelemeler](../../operations/system_tables.md#system_tables-replicas) Tablo. Alternatif olarak, bir yapmak daha kolaydır `DETACH` tüm yinelemelerde sorgu - tüm yinelemeler, lider yinelemesi dışında bir özel durum oluşturur. + +#### DROP PARTITION {#alter_drop-partition} + +``` sql +ALTER TABLE table_name DROP PARTITION partition_expr +``` + +Belirtilen bölümü tablodan siler. Bu sorgu bölümü etkin olarak etiketler ve verileri tamamen yaklaşık 10 dakika içinde siler. + +Bir bölümdeki bölüm ifadesini ayarlama hakkında bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). + +The query is replicated – it deletes data on all replicas. + +#### DROP DETACHED PARTITION\|PART {#alter_drop-detached} + +``` sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Belirtilen bölümü veya belirtilen bölümün tüm bölümlerini kaldırır `detached`. +Bir bölümdeki bölüm ifadesini ayarlama hakkında daha fazla bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). + +#### ATTACH PARTITION\|PART {#alter_attach-partition} + +``` sql +ALTER TABLE table_name ATTACH PARTITION|PART partition_expr +``` + +Tablodan veri ekler `detached` dizin. Tüm bir bölüm veya ayrı bir bölüm için veri eklemek mümkündür. Örnekler: + +``` sql +ALTER TABLE visits ATTACH PARTITION 201901; +ALTER TABLE visits ATTACH PART 201901_2_2_0; +``` + +Bir bölümdeki bölüm ifadesini ayarlama hakkında daha fazla bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). + +Bu sorgu çoğaltılır. Çoğaltma başlatıcısı, veri olup olmadığını denetler. `detached` dizin. Veri varsa, sorgu bütünlüğünü denetler. Her şey doğruysa, sorgu verileri tabloya ekler. Diğer tüm yinelemeler, çoğaltma başlatıcısından verileri karşıdan yükleyin. + +Böylece veri koyabilirsiniz `detached` bir kopya üzerinde dizin ve `ALTER ... ATTACH` tüm yinelemelerde tabloya eklemek için sorgu. + +#### ATTACH PARTITION FROM {#alter_attach-partition-from} + +``` sql +ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 +``` + +Bu sorgu, veri bölümünü `table1` -e doğru `table2` exsisting için veri ekler `table2`. Verilerin silinmeyeceğini unutmayın `table1`. + +Sorgunun başarıyla çalışması için aşağıdaki koşulların karşılanması gerekir: + +- Her iki tablo da aynı yapıya sahip olmalıdır. +- Her iki tablo da aynı bölüm anahtarına sahip olmalıdır. + +#### REPLACE PARTITION {#alter_replace-partition} + +``` sql +ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 +``` + +Bu sorgu, veri bölümünü `table1` -e doğru `table2` ve mevcut bölümün yerini alır `table2`. Verilerin silinmeyeceğini unutmayın `table1`. + +Sorgunun başarıyla çalışması için aşağıdaki koşulların karşılanması gerekir: + +- Her iki tablo da aynı yapıya sahip olmalıdır. +- Her iki tablo da aynı bölüm anahtarına sahip olmalıdır. + +#### MOVE PARTITION TO TABLE {#alter_move_to_table-partition} + +``` sql +ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest +``` + +Bu sorgu, veri bölümünü `table_source` -e doğru `table_dest` verileri silme ile `table_source`. + +Sorgunun başarıyla çalışması için aşağıdaki koşulların karşılanması gerekir: + +- Her iki tablo da aynı yapıya sahip olmalıdır. +- Her iki tablo da aynı bölüm anahtarına sahip olmalıdır. +- Her iki tablo da aynı motor ailesi olmalıdır. (çoğaltılmış veya çoğaltılmamış) +- Her iki tablo da aynı depolama ilkesine sahip olmalıdır. + +#### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} + +``` sql +ALTER TABLE table_name CLEAR COLUMN column_name IN PARTITION partition_expr +``` + +Bir bölümdeki belirtilen sütundaki tüm değerleri sıfırlar. Eğer... `DEFAULT` bir tablo oluştururken yan tümcesi belirlendi, bu sorgu sütun değerini belirtilen varsayılan değere ayarlar. + +Örnek: + +``` sql +ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902 +``` + +#### FREEZE PARTITION {#alter_freeze-partition} + +``` sql +ALTER TABLE table_name FREEZE [PARTITION partition_expr] +``` + +Bu sorgu, belirtilen bir bölümün yerel yedeğini oluşturur. Eğer... `PARTITION` yan tümcesi atlandı, sorgu aynı anda tüm bölümlerin yedeğini oluşturur. + +!!! note "Not" + Tüm yedekleme işlemi sunucuyu durdurmadan gerçekleştirilir. + +Eski tarz tablolar için bölüm adının önekini belirtebileceğinizi unutmayın (örneğin, ‘2019’)- daha sonra sorgu tüm ilgili bölümler için yedek oluşturur. Bir bölümdeki bölüm ifadesini ayarlama hakkında bilgi edinin [Bölüm ifadesi nasıl belirlenir](#alter-how-to-specify-part-expr). + +Yürütme sırasında, bir veri anlık görüntüsü için sorgu, bir tablo verilerine sabit bağlantılar oluşturur. Hardlinks dizine yerleştirilir `/var/lib/clickhouse/shadow/N/...`, nere: + +- `/var/lib/clickhouse/` yapılandırmada belirtilen çalışma ClickHouse dizinidir. +- `N` yedeklemenin artımlı sayısıdır. + +!!! note "Not" + Kullanıyorsanız [bir tablodaki veri depolama için disk kümesi](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes), bu `shadow/N` dizin tarafından eşleşen veri parçalarını depolamak, her diskte görünür `PARTITION` ifade. + +Dizinlerin aynı yapısı, içinde olduğu gibi yedek içinde oluşturulur `/var/lib/clickhouse/`. Sorgu gerçekleştirir ‘chmod’ tüm dosyalar için, onlara yazmayı yasaklamak. + +Yedeklemeyi oluşturduktan sonra, verileri `/var/lib/clickhouse/shadow/` uzak sunucuya ve sonra yerel sunucudan silin. Not `ALTER t FREEZE PARTITION` sorgu çoğaltılmaz. Yalnızca yerel sunucuda yerel bir yedekleme oluşturur. + +Sorgu neredeyse anında yedekleme oluşturur (ancak önce geçerli sorguları ilgili tabloya çalışmayı bitirmek için bekler). + +`ALTER TABLE t FREEZE PARTITION` tablo meta verilerini değil, yalnızca verileri kopyalar. Tablo meta verilerinin yedeğini almak için dosyayı kopyalayın `/var/lib/clickhouse/metadata/database/table.sql` + +Bir yedekten veri geri yüklemek için aşağıdakileri yapın: + +1. Yoksa tablo oluşturun. Sorguyu görüntülemek için kullanın .sql dosyası (değiştir `ATTACH` içinde ile `CREATE`). +2. Veri kopyalama `data/database/table/` yedekleme içindeki dizin `/var/lib/clickhouse/data/database/table/detached/` dizin. +3. Koşmak `ALTER TABLE t ATTACH PARTITION` verileri bir tabloya eklemek için sorgular. + +Yedeklemeden geri yükleme, sunucuyu durdurmayı gerektirmez. + +Yedekleme ve geri yükleme verileri hakkında daha fazla bilgi için bkz: [Veri Yedekleme](../../operations/backup.md) bölme. + +#### CLEAR INDEX IN PARTITION {#alter_clear-index-partition} + +``` sql +ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr +``` + +Sorgu benzer çalışır `CLEAR COLUMN`, ancak bir sütun verileri yerine bir dizini sıfırlar. + +#### FETCH PARTITION {#alter_fetch-partition} + +``` sql +ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper' +``` + +Başka bir sunucudan bir bölüm indirir. Bu sorgu yalnızca çoğaltılmış tablolar için çalışır. + +Sorgu aşağıdakileri yapar: + +1. Bölümü belirtilen parçadan indirir. İçinde ‘path-in-zookeeper’ zookeeper içinde shard için bir yol belirtmeniz gerekir. +2. Sonra sorgu indirilen verileri `detached` directory of the `table_name` Tablo. Kullan... [ATTACH PARTITION\|PART](#alter_attach-partition) tabloya veri eklemek için sorgu. + +Mesela: + +``` sql +ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits'; +ALTER TABLE users ATTACH PARTITION 201902; +``` + +Not thate that: + +- Bu `ALTER ... FETCH PARTITION` sorgu çoğaltılmaz. Bu bölüm için yerleştirir `detached` yalnızca yerel sunucuda dizin. +- Bu `ALTER TABLE ... ATTACH` sorgu çoğaltılır. Verileri tüm yinelemelere ekler. Veriler, kopyalardan birine eklenir. `detached` dizin ve diğerlerine - komşu kopyalardan. + +İndirmeden önce, sistem bölümün olup olmadığını ve tablo yapısının eşleşip eşleşmediğini kontrol eder. En uygun yineleme, sağlıklı yinelemeler otomatik olarak seçilir. + +Sorgu çağrılsa da `ALTER TABLE`, tablo yapısını değiştirmez ve tabloda bulunan verileri hemen değiştirmez. + +#### MOVE PARTITION\|PART {#alter_move-partition} + +Bölümleri veya veri parçalarını başka bir birime veya diske taşır. `MergeTree`- motor masaları. Görmek [Veri depolama için birden fazla blok cihazı kullanma](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-multiple-volumes). + +``` sql +ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name' +``` + +Bu `ALTER TABLE t MOVE` sorgu: + +- Çoğaltılamaz, çünkü farklı çoğaltmalar farklı depolama ilkelerine sahip olabilir. +- Belirtilen disk veya birim yapılandırılmamışsa bir hata döndürür. Depolama ilkesinde belirtilen veri taşıma koşulları uygulanamazsa, sorgu da bir hata döndürür. +- Durumda bir hata döndürebilir, taşınacak veriler zaten bir arka plan işlemi tarafından taşındığında, eşzamanlı `ALTER TABLE t MOVE` sorgu veya arka plan veri birleştirme sonucu. Bir kullanıcı bu durumda herhangi bir ek eylem gerçekleştirmemelidir. + +Örnek: + +``` sql +ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow' +ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd' +``` + +#### Bölüm İfadesi Nasıl Ayarlanır {#alter-how-to-specify-part-expr} + +Bölüm ifadesini şu şekilde belirtebilirsiniz `ALTER ... PARTITION` farklı şekillerde sorgular: + +- Bu gibi bir değer `partition` sütun `system.parts` Tablo. Mesela, `ALTER TABLE visits DETACH PARTITION 201901`. +- Tablo sütunundan ifade olarak. Sabitler ve sabit ifadeler desteklenir. Mesela, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. +- Bölüm kimliğini kullanma. Partition ID, dosya sistemindeki ve Zookeeper'daki bölümlerin adları olarak kullanılan bölümün (mümkünse insan tarafından okunabilir) bir dize tanımlayıcısıdır. Bölüm kimliği belirtilmelidir `PARTITION ID` fık .ra, tek tırnak içinde. Mesela, `ALTER TABLE visits DETACH PARTITION ID '201901'`. +- İn the [ALTER ATTACH PART](#alter_attach-partition) ve [DROP DETACHED PART](#alter_drop-detached) sorgu, bir parçanın adını belirtmek için, bir değer ile dize literal kullanın `name` sütun [sistem.detached\_parts](../../operations/system_tables.md#system_tables-detached_parts) Tablo. Mesela, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. + +Bölüm belirtilirken tırnak kullanımı bölüm ifadesi türüne bağlıdır. Örneğin, için `String` yazın, adını tırnak içinde belirtmeniz gerekir (`'`). İçin `Date` ve `Int*` türleri hiçbir tırnak gereklidir. + +Eski stil tablolar için, bölümü bir sayı olarak belirtebilirsiniz `201901` veya bir dize `'201901'`. Yeni stil tabloları için sözdizimi türleri ile daha sıkı (değerleri giriş biçimi için ayrıştırıcı benzer). + +Yukarıdaki tüm kurallar için de geçerlidir [OPTIMIZE](misc.md#misc_operations-optimize) sorgu. Bölümlenmemiş bir tabloyu en iyi duruma getirirken tek bölümü belirtmeniz gerekiyorsa, ifadeyi ayarlayın `PARTITION tuple()`. Mesela: + +``` sql +OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; +``` + +Örnekleri `ALTER ... PARTITION` sorgular testlerde gösterilmiştir [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) ve [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). + +### Tablo TTL ile manipülasyonlar {#manipulations-with-table-ttl} + +Değiştirebilirsiniz [tablo TTL](../../engines/table_engines/mergetree_family/mergetree.md#mergetree-table-ttl) aşağıdaki formun bir isteği ile: + +``` sql +ALTER TABLE table-name MODIFY TTL ttl-expression +``` + +### Alter sorgularının eşzamanlılığı {#synchronicity-of-alter-queries} + +Replicatable olmayan tablolar için, tüm `ALTER` sorgular eşzamanlı olarak gerçekleştirilir. Replicatable tablolar için, sorgu yalnızca uygun eylemler için yönergeler ekler `ZooKeeper` ve eylemlerin kendileri mümkün olan en kısa sürede gerçekleştirilir. Ancak, sorgu tüm yinelemeler üzerinde tamamlanması için bu eylemleri bekleyebilir. + +İçin `ALTER ... ATTACH|DETACH|DROP` sorgular, kullanabilirsiniz `replication_alter_partitions_sync` bekleyen kurmak için ayarlama. +Olası değerler: `0` – do not wait; `1` – only wait for own execution (default); `2` – wait for all. + +### Mutasyonlar {#alter-mutations} + +Mutasyonlar, bir tablodaki satırların değiştirilmesine veya silinmesine izin veren bir alter query varyantıdır. Standart aksine `UPDATE` ve `DELETE` nokta veri değişikliklerine yönelik sorgular, mutasyonlar, bir tablodaki çok sayıda satırı değiştiren ağır işlemler için tasarlanmıştır. İçin desteklenen `MergeTree` çoğaltma desteği olan motorlar da dahil olmak üzere tablo motorları ailesi. + +Varolan tablolar olduğu gibi mutasyonlar için hazırdır(dönüştürme gerekmez), ancak ilk mutasyon bir tabloya uygulandıktan sonra Meta Veri formatı önceki sunucu sürümleriyle uyumsuz hale gelir ve önceki bir sürüme geri dönmek imkansız hale gelir. + +Şu anda mevcut komutlar: + +``` sql +ALTER TABLE [db.]table DELETE WHERE filter_expr +``` + +Bu `filter_expr` tip olmalıdır `UInt8`. Sorgu, bu ifadenin sıfır olmayan bir değer aldığı tablodaki satırları siler. + +``` sql +ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr +``` + +Bu `filter_expr` tip olmalıdır `UInt8`. Bu sorgu, belirtilen sütunların değerlerini, satırlardaki karşılık gelen ifadelerin değerlerine güncelleştirir. `filter_expr` sıfır olmayan bir değer alır. Değerleri kullanarak sütun türüne döküm `CAST` operatör. Birincil veya bölüm anahtarının hesaplanmasında kullanılan sütunları güncelleştirme desteklenmiyor. + +``` sql +ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name +``` + +Sorgu ikincil dizini yeniden oluşturur `name` bölümünde `partition_name`. + +Bir sorgu virgülle ayrılmış birkaç komut içerebilir. + +\* MergeTree tabloları mutasyonları için tüm veri parçalarını yeniden yazarak yürütün. Atomiklik yoktur-parçalar, hazır oldukları anda mutasyona uğramış parçalar için ikame edilir ve bir `SELECT` bir mutasyon sırasında yürütülmeye başlayan sorgu, henüz mutasyona uğramamış olan parçalardan gelen verilerle birlikte mutasyona uğramış olan parçalardan gelen verileri görecektir. + +Mutasyonlar tamamen yaratılış sırasına göre sıralanır ve her bir parçaya bu sırayla uygulanır. Mutasyonlar da kısmen ekler ile sıralanır-mutasyon gönderilmeden önce tabloya eklenen veriler mutasyona uğrayacak ve bundan sonra eklenen veriler mutasyona uğramayacaktır. Mutasyonların ekleri hiçbir şekilde engellemediğini unutmayın. + +Mutasyon girişi eklendikten hemen sonra bir mutasyon sorgusu döner(çoğaltılmış tablolar Zookeeper'a, çoğaltılmamış tablolar için-dosya sistemine). Mutasyonun kendisi sistem profili ayarlarını kullanarak eşzamansız olarak yürütür. Mutasyonların ilerlemesini izlemek için kullanabilirsiniz [`system.mutations`](../../operations/system_tables.md#system_tables-mutations) Tablo. Başarıyla gönderilen BIR mutasyon, ClickHouse sunucuları yeniden başlatılmış olsa bile yürütmeye devam edecektir. Gönderildikten sonra mutasyonu geri almanın bir yolu yoktur, ancak mutasyon herhangi bir nedenle sıkışmışsa, [`KILL MUTATION`](misc.md#kill-mutation) sorgu. + +Bitmiş mutasyonlar için girişler hemen silinmez (korunmuş girişlerin sayısı, `finished_mutations_to_keep` depolama motoru parametresi). Eski mutasyon girişleri silinir. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/alter/) diff --git a/docs/tr/sql_reference/statements/create.md b/docs/tr/sql_reference/statements/create.md new file mode 100644 index 00000000000..5479034ad98 --- /dev/null +++ b/docs/tr/sql_reference/statements/create.md @@ -0,0 +1,305 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 35 +toc_title: CREATE +--- + +# Sorgu oluştur {#create-queries} + +## CREATE DATABASE {#query-language-create-database} + +Veritabanı oluşturur. + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(...)] +``` + +### Yanlar {#clauses} + +- `IF NOT EXISTS` + Eğer... `db_name` veritabanı zaten var, daha sonra ClickHouse yeni bir veritabanı oluşturmuyor ve: + + - If yan tümcesi belirtilmişse bir istisna atmaz. + - Bir istisna atar if yan tümcesi belirtilmemiş. + +- `ON CLUSTER` + ClickHouse oluşturur `db_name` belirtilen bir kümenin tüm sunucularında veritabanı. + +- `ENGINE` + + - [MySQL](../../engines/database_engines/mysql.md) + Uzak MySQL sunucusundan veri almanızı sağlar. + Varsayılan olarak, ClickHouse kendi kullanır [Veritabanı Altyapısı](../../engines/database_engines/index.md). + +## CREATE TABLE {#create-table-query} + +Bu `CREATE TABLE` sorgu çeşitli formlara sahip olabilir. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2], + ... +) ENGINE = engine +``` + +Adlı bir tablo oluşturur ‘name’ in the ‘db’ veritabanı veya geçerli veritabanı ise ‘db’ küme değil, parantez içinde belirtilen yapı ve ‘engine’ motor. +Tablonun yapısı sütun açıklamalarının bir listesidir. Dizinler altyapısı tarafından destekleniyorsa, tablo altyapısı için parametreler olarak gösterilir. + +Bir sütun açıklaması `name type` en basit durumda. Örnek: `RegionID UInt32`. +İfadeler varsayılan değerler için de tanımlanabilir (aşağıya bakın). + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine] +``` + +Başka bir tablo ile aynı yapıya sahip bir tablo oluşturur. Tablo için farklı bir motor belirtebilirsiniz. Motor belirtilmemişse, aynı motor için olduğu gibi kullanılacaktır `db2.name2` Tablo. + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name AS table_function() +``` + +Yapısı ve veri tarafından döndürülen bir tablo oluşturur. [tablo fonksiyonu](../table_functions/index.md). + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... +``` + +Sonucu gibi bir yapıya sahip bir tablo oluşturur `SELECT` Sorgu, ile ‘engine’ motor ve SELECT verilerle doldurur. + +Her durumda, eğer `IF NOT EXISTS` tablo zaten varsa, sorgu bir hata döndürmez. Bu durumda, sorgu hiçbir şey yapmaz. + +Sonra başka maddeler olabilir `ENGINE` sorguda yan tümcesi. Açıklamalarda tabloların nasıl oluşturulacağına ilişkin ayrıntılı belgelere bakın [masa motorları](../../engines/table_engines/index.md#table_engines). + +### Varsayılan Değerler {#create-default-values} + +Sütun açıklaması, aşağıdaki yollardan biriyle varsayılan değer için bir ifade belirtebilir:`DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`. +Örnek: `URLDomain String DEFAULT domain(URL)`. + +Varsayılan değer için bir ifade tanımlanmamışsa, varsayılan değerler sayılar için sıfırlar, dizeler için boş dizeler, diziler için boş diziler ve `0000-00-00` tarihler için veya `0000-00-00 00:00:00` zamanla tarihler için. Boş alanlar desteklenmez. + +Varsayılan ifade tanımlanmışsa, sütun türü isteğe bağlıdır. Açıkça tanımlanmış bir tür yoksa, varsayılan ifade türü kullanılır. Örnek: `EventDate DEFAULT toDate(EventTime)` – the ‘Date’ türü için kullanılacak ‘EventDate’ sütun. + +Veri türü ve varsayılan ifade açıkça tanımlanırsa, bu ifade type casting işlevleri kullanılarak belirtilen türe aktarılır. Örnek: `Hits UInt32 DEFAULT 0` aynı şeyi ifade eder `Hits UInt32 DEFAULT toUInt32(0)`. + +Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions don't contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. + +`DEFAULT expr` + +Normal varsayılan değer. INSERT sorgusu karşılık gelen sütunu belirtmezse, ilgili ifadeyi hesaplayarak doldurulur. + +`MATERIALIZED expr` + +Somut ifade. Böyle bir sütun INSERT için belirtilemez, çünkü her zaman hesaplanır. +Sütun listesi olmayan bir ekleme için bu sütunlar dikkate alınmaz. +Buna ek olarak, bir SELECT sorgusunda Yıldız İşareti kullanıldığında bu sütun değiştirilmez. Bu, dökümü kullanarak elde edilen değişmezi korumaktır `SELECT *` sütun listesini belirtmeden INSERT kullanarak tabloya geri eklenebilir. + +`ALIAS expr` + +Eşanlamlı sözcük. Böyle bir sütun tabloda hiç depolanmaz. +Değerleri bir tabloya eklenemez ve bir SELECT sorgusunda Yıldız İşareti kullanılırken değiştirilmez. +Sorgu ayrıştırma sırasında diğer ad genişletilirse, seçimlerde kullanılabilir. + +Yeni sütunlar eklemek için ALTER sorgusunu kullanırken, bu sütunlar için eski veriler yazılmaz. Bunun yerine, yeni sütunlar için değerleri olmayan eski verileri okurken, ifadeler varsayılan olarak anında hesaplanır. Ancak, ifadeleri çalıştırmak sorguda belirtilmeyen farklı sütunlar gerektiriyorsa, bu sütunlar ayrıca okunur, ancak yalnızca buna ihtiyaç duyan veri blokları için okunur. + +Bir tabloya yeni bir sütun eklerseniz, ancak daha sonra varsayılan ifadesini değiştirirseniz, eski veriler için kullanılan değerler değişir (değerlerin diskte depolanmadığı veriler için). Arka plan birleştirmeleri çalıştırırken, birleştirme parçalarından birinde eksik olan sütunların verileri birleştirilmiş parçaya yazıldığını unutmayın. + +İç içe geçmiş veri yapılarındaki öğeler için varsayılan değerleri ayarlamak mümkün değildir. + +### Kısıtlamalar {#constraints} + +Sütun açıklamaları kısıtlamaları ile birlikte tanımlanabilir: + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` herhangi bir Boole ifadesi ile olabilir. Tablo için kısıtlamalar tanımlanırsa, her biri her satır için kontrol edilir `INSERT` query. If any constraint is not satisfied — server will raise an exception with constraint name and checking expression. + +Büyük miktarda kısıtlama eklemek, büyük `INSERT` sorgular. + +### TTL ifadesi {#ttl-expression} + +Değerler için depolama süresini tanımlar. Sadece MergeTree-family tabloları için belirtilebilir. Ayrıntılı açıklama için, bkz. [Sütunlar ve tablolar için TTL](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-ttl). + +### Sütun Sıkıştırma Kodekleri {#codecs} + +Varsayılan olarak, ClickHouse `lz4` sıkıştırma yöntemi. İçin `MergeTree`- motor ailesi varsayılan sıkıştırma yöntemini değiştirebilirsiniz [sıkıştırma](../../operations/server_configuration_parameters/settings.md#server-settings-compression) bir sunucu yapılandırması bölümü. Her bir sütun için sıkıştırma yöntemini de tanımlayabilirsiniz. `CREATE TABLE` sorgu. + +``` sql +CREATE TABLE codec_example +( + dt Date CODEC(ZSTD), + ts DateTime CODEC(LZ4HC), + float_value Float32 CODEC(NONE), + double_value Float64 CODEC(LZ4HC(9)) + value Float32 CODEC(Delta, ZSTD) +) +ENGINE = +... +``` + +Bir codec bileşeni belirtilmişse, varsayılan codec bileşeni geçerli değildir. Kodekler bir boru hattında birleştirilebilir, örneğin, `CODEC(Delta, ZSTD)`. Projeniz için en iyi codec kombinasyonunu seçmek için, Altınlıkta açıklanana benzer kriterler geçirin [ClickHouse verimliliğini artırmak için yeni Kodlamalar](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) makale. + +!!! warning "Uyarıcı" + ClickHouse veritabanı dosyalarını harici yardımcı programlarla açamazsınız `lz4`. Bunun yerine, özel kullanın [clickhouse-kompresör](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) program. + +Sıkıştırma Aşağıdaki tablo motorları için desteklenir: + +- [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) aile. Sütun sıkıştırma kodeklerini destekler ve varsayılan sıkıştırma yöntemini seçerek [sıkıştırma](../../operations/server_configuration_parameters/settings.md#server-settings-compression) ayarlar. +- [Günlük](../../engines/table_engines/log_family/log_family.md) aile. Kullanır `lz4` sıkıştırma yöntemi varsayılan olarak ve sütun sıkıştırma codec destekler. +- [Koymak](../../engines/table_engines/special/set.md). Yalnızca varsayılan sıkıştırmayı destekledi. +- [Katmak](../../engines/table_engines/special/join.md). Yalnızca varsayılan sıkıştırmayı destekledi. + +ClickHouse ortak amaçlı codec ve özel codec destekler. + +#### Özel Kodekler {#create-query-specialized-codecs} + +Bu kodekler, verilerin belirli özelliklerini kullanarak sıkıştırmayı daha etkili hale getirmek için tasarlanmıştır. Bu kodeklerden bazıları verileri kendileri sıkıştırmaz. Bunun yerine, verileri ortak bir amaç için hazırlarlar codec, bu hazırlık olmadan daha iyi sıkıştırır. + +Özel kodekler: + +- `Delta(delta_bytes)` — Compression approach in which raw values are replaced by the difference of two neighboring values, except for the first value that stays unchanged. Up to `delta_bytes` delta değerlerini saklamak için kullanılır, böylece `delta_bytes` ham değerlerin maksimum boyutudur. Olası `delta_bytes` değerler: 1, 2, 4, 8. İçin varsayılan değer `delta_bytes` oluyor `sizeof(type)` 1, 2, 4 veya 8'e eşitse. Diğer tüm durumlarda, 1. +- `DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: Hızlı, Ölçeklenebilir, Bellek İçi Zaman Serisi Veritabanı](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: Hızlı, Ölçeklenebilir, Bellek İçi Zaman Serisi Veritabanı](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` ve `DateTime`). Algoritmasının her adımında, codec 64 değerden oluşan bir blok alır, 64x64 bit matrisine koyar, aktarır, kullanılmayan değer bitlerini kırpar ve gerisini bir dizi olarak döndürür. Kullanılmayan bitler, sıkıştırmanın kullanıldığı tüm veri bölümündeki maksimum ve minimum değerler arasında farklılık göstermeyen bitlerdir. + +`DoubleDelta` ve `Gorilla` kodekler, Gorilla TSDB'DE sıkıştırma algoritmasının bileşenleri olarak kullanılır. Gorilla yaklaşımı, zaman damgaları ile yavaş yavaş değişen değerler dizisi olduğunda senaryolarda etkilidir. Zaman damgaları tarafından etkili bir şekilde sıkıştırılır `DoubleDelta` codec ve değerler etkin bir şekilde sıkıştırılır `Gorilla` codec. Örneğin, etkili bir şekilde saklanan bir tablo elde etmek için, aşağıdaki yapılandırmada oluşturabilirsiniz: + +``` sql +CREATE TABLE codec_example +( + timestamp DateTime CODEC(DoubleDelta), + slow_values Float32 CODEC(Gorilla) +) +ENGINE = MergeTree() +``` + +#### Ortak Amaç {#create-query-common-purpose-codecs} + +Cod codecsec codecs'ler: + +- `NONE` — No compression. +- `LZ4` — Lossless [veri sıkıştırma algoritması](https://github.com/lz4/lz4) varsayılan olarak kullanılır. Lz4 hızlı sıkıştırma uygular. +- `LZ4HC[(level)]` — LZ4 HC (high compression) algorithm with configurable level. Default level: 9. Setting `level <= 0` varsayılan düzeyi uygular. Olası seviyeleri: \[1, 12\]. Önerilen seviye aralığı: \[4, 9\]. +- `ZSTD[(level)]` — [Zstd sıkıştırma algoritması](https://en.wikipedia.org/wiki/Zstandard) yapılandırılabilir ile `level`. Olası seviyeler: \[1, 22\]. Varsayılan değer: 1. + +Yüksek Sıkıştırma seviyeleri asimetrik senaryolar için kullanışlıdır, örneğin bir kez sıkıştırın, tekrar tekrar sıkıştırın. Daha yüksek seviyeler daha iyi sıkıştırma ve daha yüksek CPU kullanımı anlamına gelir. + +## Geçici Tablolar {#temporary-tables} + +ClickHouse aşağıdaki özelliklere sahip geçici tabloları destekler: + +- Bağlantı kaybolursa da dahil olmak üzere oturum sona erdiğinde geçici tablolar kaybolur. +- Geçici bir tablo yalnızca bellek altyapısını kullanır. +- DB geçici bir tablo için belirtilemez. Veritabanları dışında oluşturulur. +- Tüm küme sunucularında dağıtılmış DDL sorgusu ile geçici bir tablo oluşturmak imkansız (kullanarak `ON CLUSTER`): bu tablo yalnızca geçerli oturumda bulunur. +- Geçici bir tablo başka bir ile aynı ada sahip ve bir sorgu DB belirtmeden tablo adını belirtir, geçici tablo kullanılır. +- Dağıtılmış sorgu işleme için bir sorguda kullanılan geçici tablolar uzak sunuculara geçirilir. + +Geçici bir tablo oluşturmak için aşağıdaki sözdizimini kullanın: + +``` sql +CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) +``` + +Çoğu durumda, geçici tablolar el ile oluşturulmaz, ancak bir sorgu için veya dağıtılmış için dış verileri kullanırken `(GLOBAL) IN`. Daha fazla bilgi için uygun bölümlere bakın + +İle tabloları kullanmak mümkündür [Motor = bellek](../../engines/table_engines/special/memory.md) geçici tablolar yerine. + +## Dağıtılmış DDL sorguları (küme yan tümcesinde) {#distributed-ddl-queries-on-cluster-clause} + +Bu `CREATE`, `DROP`, `ALTER`, ve `RENAME` sorgular, bir kümede dağıtılmış yürütmeyi destekler. +Örneğin, aşağıdaki sorgu oluşturur `all_hits` `Distributed` her ana bilgisayarda tablo `cluster`: + +``` sql +CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE = Distributed(cluster, default, hits) +``` + +Bu sorguları doğru bir şekilde çalıştırmak için, her ana bilgisayarın aynı küme tanımına sahip olması gerekir (senkronizasyon yapılandırmalarını basitleştirmek için zookeeper'dan değiştirmeleri kullanabilirsiniz). Ayrıca ZooKeeper sunucularına bağlanmaları gerekir. +Bazı ana bilgisayarlar şu anda mevcut olmasa bile, sorgunun yerel sürümü sonunda kümedeki her ana bilgisayarda uygulanır. Tek bir ana makine içinde sorguları yürütme sırası garanti edilir. + +## CREATE VIEW {#create-view} + +``` sql +CREATE [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]table_name [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... +``` + +Bir görünüm oluşturur. İki tür görüş vardır: normal ve SOMUTLAŞTIRILMIŞ. + +Normal görünümler herhangi bir veri depolamaz, ancak başka bir tablodan bir okuma gerçekleştirir. Başka bir deyişle, normal bir görünüm kaydedilmiş bir sorgudan başka bir şey değildir. Bir görünümden okurken, bu kaydedilmiş sorgu FROM yan tümcesinde bir alt sorgu olarak kullanılır. + +Örnek olarak, bir görünüm oluşturduğunuzu varsayalım: + +``` sql +CREATE VIEW view AS SELECT ... +``` + +ve bir sorgu yazdı: + +``` sql +SELECT a, b, c FROM view +``` + +Bu sorgu, alt sorguyu kullanmaya tam olarak eşdeğerdir: + +``` sql +SELECT a, b, c FROM (SELECT ...) +``` + +Materialized görünümler, ilgili SELECT sorgusu tarafından dönüştürülmüş verileri depolar. + +Olmadan hayata bir görünüm oluştururken `TO [db].[table]`, you must specify ENGINE – the table engine for storing data. + +İle somutlaştırılmış bir görünüm oluştururken `TO [db].[table]`, kullanma mustmalısınız `POPULATE`. + +Materialized görünüm aşağıdaki gibi düzenlenmiştir: SELECT belirtilen tabloya veri eklerken, eklenen verilerin bir kısmı bu SELECT sorgusu tarafından dönüştürülür ve sonuç görünümde eklenir. + +Doldur belirtirseniz, varolan tablo verilerini oluştururken görünümde, sanki bir `CREATE TABLE ... AS SELECT ...` . Aksi takdirde, sorgu yalnızca görünümü oluşturduktan sonra tabloya eklenen verileri içerir. Görünüm oluşturma sırasında tabloya eklenen veriler EKLENMEYECEĞİNDEN, doldur kullanmanızı önermiyoruz. + +A `SELECT` sorgu içerebilir `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` ayarlanır, veri ekleme sırasında toplanır, ancak yalnızca tek bir eklenen veri paketi içinde toplanır. Veriler daha fazla toplanmayacaktır. Özel durum, bağımsız olarak veri toplama, gibi gerçekleştiren bir motor kullanırken olur `SummingMergeTree`. + +Yürütme `ALTER` somut görünümlerle ilgili sorgular tam olarak geliştirilmemiştir, bu nedenle rahatsız edici olabilirler. Eğer hayata görünüm inşaat kullanıyorsa `TO [db.]name` yapabilirsiniz `DETACH` the view, run `ALTER` hedef tablo için ve sonra `ATTACH` daha önce müstakil (`DETACH`) görünüm. + +Görünümler normal tablolarla aynı görünür. Örneğin, bunlar sonucu listelenir `SHOW TABLES` sorgu. + +Görünümleri silmek için ayrı bir sorgu yok. Bir görünümü silmek için şunları kullanın `DROP TABLE`. + +## CREATE DICTIONARY {#create-dictionary-query} + +``` sql +CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] +( + key1 type1 [DEFAULT|EXPRESSION expr1] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], + key2 type2 [DEFAULT|EXPRESSION expr2] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], + attr1 type2 [DEFAULT|EXPRESSION expr3], + attr2 type2 [DEFAULT|EXPRESSION expr4] +) +PRIMARY KEY key1, key2 +SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) +LAYOUT(LAYOUT_NAME([param_name param_value])) +LIFETIME([MIN val1] MAX val2) +``` + +Oluşturuyor [dış sözlük](../../sql_reference/dictionaries/external_dictionaries/external_dicts.md) verilen ile [yapılı](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md), [kaynaklı](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md), [düzen](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md) ve [ömür](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md). + +Dış sözlük yapısı özniteliklerden oluşur. Sözlük öznitelikleri tablo sütunlarına benzer şekilde belirtilir. Tek gerekli öznitelik özelliği türüdür, diğer tüm özelliklerin varsayılan değerleri olabilir. + +Sözlüğe bağlı olarak [düzen](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md) bir veya daha fazla öznitelik sözlük anahtarları olarak belirtilebilir. + +Daha fazla bilgi için, bkz. [Dış Söz Dictionarieslükler](../../sql_reference/dictionaries/external_dictionaries/external_dicts.md) bölme. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/create/) diff --git a/docs/tr/sql_reference/statements/index.md b/docs/tr/sql_reference/statements/index.md new file mode 100644 index 00000000000..0298948b26e --- /dev/null +++ b/docs/tr/sql_reference/statements/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Deyimler +toc_priority: 31 +--- + + diff --git a/docs/tr/sql_reference/statements/insert_into.md b/docs/tr/sql_reference/statements/insert_into.md new file mode 100644 index 00000000000..1aafdc368e9 --- /dev/null +++ b/docs/tr/sql_reference/statements/insert_into.md @@ -0,0 +1,80 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 34 +toc_title: INSERT INTO +--- + +## INSERT {#insert} + +Veri ekleme. + +Temel sorgu biçimi: + +``` sql +INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... +``` + +Sorgu eklemek için sütunların bir listesini belirtebilirsiniz `[(c1, c2, c3)]`. Bu durumda, sütunların geri kalanı ile doldurulur: + +- Hesaplanan değerler `DEFAULT` tablo tanımında belirtilen ifadeler. +- Sıfırlar ve boş dizeler, eğer `DEFAULT` ifadeler tanımlanmamıştır. + +Eğer [strict\_ınsert\_defaults = 1](../../operations/settings/settings.md), sahip olmayan sütunlar `DEFAULT` tanımlanan sorguda listelenmelidir. + +Veri herhangi bir İNSERT geçirilebilir [biçimli](../../interfaces/formats.md#formats) ClickHouse tarafından desteklenmektedir. Biçim sorguda açıkça belirtilmelidir: + +``` sql +INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set +``` + +For example, the following query format is identical to the basic version of INSERT … VALUES: + +``` sql +INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ... +``` + +ClickHouse, veriden önce tüm boşlukları ve bir satır beslemesini (varsa) kaldırır. Bir sorgu oluştururken, sorgu işleçlerinden sonra verileri yeni bir satıra koymanızı öneririz (veriler boşluklarla başlarsa bu önemlidir). + +Örnek: + +``` sql +INSERT INTO t FORMAT TabSeparated +11 Hello, world! +22 Qwerty +``` + +Komut satırı istemcisini veya HTTP arabirimini kullanarak verileri sorgudan ayrı olarak ekleyebilirsiniz. Daha fazla bilgi için bölüme bakın “[Arabirimler](../../interfaces/index.md#interfaces)”. + +### Kısıtlamalar {#constraints} + +Tablo varsa [kısıtlamalar](create.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. + +### Sonuçları Ekleme `SELECT` {#insert_query_insert-select} + +``` sql +INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... +``` + +Sütunlar, SELECT yan tümcesindeki konumlarına göre eşleştirilir. Ancak, SELECT ifadesi ve INSERT için tablo adları farklı olabilir. Gerekirse, tip döküm yapılır. + +Değerler dışındaki veri biçimlerinin hiçbiri, aşağıdaki gibi ifadelere değerler ayarlamasına izin vermez `now()`, `1 + 2` ve bu yüzden. Değerler biçimi, ifadelerin sınırlı kullanımına izin verir, ancak bu önerilmez, çünkü bu durumda verimsiz kod yürütme için kullanılır. + +Veri bölümlerini değiştirmek için diğer sorgular desteklenmiyor: `UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`. +Ancak, eski verileri kullanarak silebilirsiniz `ALTER TABLE ... DROP PARTITION`. + +`FORMAT` yan tümcesi sorgu sonunda belirtilmelidir eğer `SELECT` yan tümcesi tablo işlevi içerir [girdi()](../table_functions/input.md). + +### Performans Konuları {#performance-considerations} + +`INSERT` giriş verilerini birincil anahtarla sıralar ve bunları bir bölüm anahtarı ile bölümlere ayırır. Bir kerede birkaç bölüme veri eklerseniz, bu veri tabanının performansını önemli ölçüde azaltabilir. `INSERT` sorgu. Bunu önlemek için: + +- Bir seferde 100.000 satır gibi oldukça büyük gruplar halinde veri ekleyin. +- Clickhouse'a yüklemeden önce verileri bir bölüm anahtarıyla gruplandırın. + +Eğer performans azalmaz: + +- Veri gerçek zamanlı olarak eklenir. +- Genellikle zamana göre sıralanır veri yükleyin. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/insert_into/) diff --git a/docs/tr/sql_reference/statements/misc.md b/docs/tr/sql_reference/statements/misc.md new file mode 100644 index 00000000000..689fc8cd77d --- /dev/null +++ b/docs/tr/sql_reference/statements/misc.md @@ -0,0 +1,252 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 39 +toc_title: "Di\u011Fer" +--- + +# Çeşitli Sorgular {#miscellaneous-queries} + +## ATTACH {#attach} + +Bu sorgu tam olarak aynıdır `CREATE`, ama + +- Kelime yerine `CREATE` kelime kullanır `ATTACH`. +- Sorgu diskte veri oluşturmaz, ancak verilerin zaten uygun yerlerde olduğunu ve yalnızca tablo hakkında bilgi sunucuya eklediğini varsayar. + Bir ekleme sorgusu çalıştırdıktan sonra, sunucu tablonun varlığı hakkında bilgi sahibi olacaktır. + +Tablo daha önce ayrılmış olsaydı (`DETACH`), yapısının bilindiği anlamına gelir, yapıyı tanımlamadan steno kullanabilirsiniz. + +``` sql +ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] +``` + +Bu sorgu, sunucu başlatılırken kullanılır. Sunucu, tablo meta verilerini dosyalar olarak depolar `ATTACH` başlangıçta çalıştığı sorgular (sunucuda açıkça oluşturulan sistem tabloları hariç). + +## CHECK TABLE {#check-table} + +Tablodaki verilerin bozuk olup olmadığını denetler. + +``` sql +CHECK TABLE [db.]name +``` + +Bu `CHECK TABLE` sorgu, gerçek dosya boyutlarını sunucuda depolanan beklenen değerlerle karşılaştırır. Dosya boyutları depolanan değerlerle eşleşmiyorsa, verilerin bozuk olduğu anlamına gelir. Bu, örneğin, sorgu yürütme sırasında bir sistem çökmesine neden olabilir. + +Sorgu yanıtı içerir `result` tek satırlı sütun. Satır bir değere sahiptir +[Boeanoleanean](../../sql_reference/data_types/boolean.md) tür: + +- 0-tablodaki veriler bozuk. +- 1 - veri bütünlüğünü korur. + +Bu `CHECK TABLE` sorgu Aşağıdaki tablo motorlarını destekler: + +- [Günlük](../../engines/table_engines/log_family/log.md) +- [TinyLog](../../engines/table_engines/log_family/tinylog.md) +- [StripeLog](../../engines/table_engines/log_family/stripelog.md) +- [MergeTree ailesi](../../engines/table_engines/mergetree_family/mergetree.md) + +Başka bir tablo motorları ile tablolar üzerinde gerçekleştirilen bir özel duruma neden olur. + +Motor fromlardan `*Log` aile başarısızlık otomatik veri kurtarma sağlamaz. Kullan... `CHECK TABLE` veri kaybını zamanında izlemek için sorgu. + +İçin `MergeTree` aile motorları, `CHECK TABLE` sorgu, yerel sunucudaki bir tablonun her bir veri bölümü için bir kontrol durumunu gösterir. + +**Veri bozuksa** + +Tablo bozuksa, bozuk olmayan verileri başka bir tabloya kopyalayabilirsiniz. Bunu yapmak için : + +1. Bozuk tablo ile aynı yapıya sahip yeni bir tablo oluşturun. Bunu yapmak için sorguyu yürütün `CREATE TABLE AS `. +2. Ayarla... [max\_threads](../../operations/settings/settings.md#settings-max_threads) bir sonraki sorguyu tek bir iş parçacığında işlemek için 1 değeri. Bunu yapmak için sorguyu çalıştırın `SET max_threads = 1`. +3. Sorgu yürütme `INSERT INTO SELECT * FROM `. Bu istek bozuk olmayan verileri bozuk tablodan başka bir tabloya kopyalar. Yalnızca bozuk parçadan önceki veriler kopyalanır. +4. Yeniden Başlat `clickhouse-client` sıfırlamak için `max_threads` değer. + +## DESCRIBE TABLE {#misc-describe-table} + +``` sql +DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] +``` + +Aşağıdaki döndürür `String` sütun tipi: + +- `name` — Column name. +- `type`— Column type. +- `default_type` — Clause that is used in [varsayılan ifade](create.md#create-default-values) (`DEFAULT`, `MATERIALIZED` veya `ALIAS`). Varsayılan ifade belirtilmemişse, sütun boş bir dize içerir. +- `default_expression` — Value specified in the `DEFAULT` yan. +- `comment_expression` — Comment text. + +İç içe veri yapıları çıktı “expanded” biçimli. Her sütun ayrı ayrı gösterilir, bir noktadan sonra adı ile. + +## DETACH {#detach} + +Hakkında bilgi siler ‘name’ sunucudan tablo. Sunucu, tablonun varlığını bilmeyi durdurur. + +``` sql +DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +``` + +Bu tablonun veri veya meta verileri silmez. Bir sonraki sunucu lansmanında, sunucu meta verileri okuyacak ve tablo hakkında tekrar bilgi edinecektir. +Benzer şekilde, bir “detached” tablo kullanılarak yeniden eklenebilir `ATTACH` sorgu (bunlar için depolanan meta verilere sahip olmayan sistem tabloları hariç). + +Hiç yok... `DETACH DATABASE` sorgu. + +## DROP {#drop} + +Bu sorgu iki türü vardır: `DROP DATABASE` ve `DROP TABLE`. + +``` sql +DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] +``` + +İçindeki tüm tabloları siler ‘db’ veritabanı, daha sonra siler ‘db’ veritabanı kendisi. +Eğer `IF EXISTS` belirtilen, veritabanı yoksa bir hata döndürmez. + +``` sql +DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +``` + +Tabloyu siler. +Eğer `IF EXISTS` belirtilmişse, tablo yoksa veya veritabanı yoksa bir hata döndürmez. + + DROP DICTIONARY [IF EXISTS] [db.]name + +Sözlük Delets. +Eğer `IF EXISTS` belirtilmişse, tablo yoksa veya veritabanı yoksa bir hata döndürmez. + +## EXISTS {#exists} + +``` sql +EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] +``` + +Bir tek döndürür `UInt8`- tek değeri içeren sütun yazın `0` tablo veya veritabanı yoksa veya `1` tablo belirtilen veritabanında varsa. + +## KILL QUERY {#kill-query} + +``` sql +KILL QUERY [ON CLUSTER cluster] + WHERE + [SYNC|ASYNC|TEST] + [FORMAT format] +``` + +Şu anda çalışan sorguları zorla sonlandırmaya çalışır. +Sonlandırılacak sorgular sistemden seçilir.tanımlanan kriterleri kullanarak işlemler tablosu `WHERE` fıkra ofsı `KILL` sorgu. + +Örnekler: + +``` sql +-- Forcibly terminates all queries with the specified query_id: +KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' + +-- Synchronously terminates all queries run by 'username': +KILL QUERY WHERE user='username' SYNC +``` + +Salt okunur kullanıcılar yalnızca kendi sorgularını durdurabilir. + +Varsayılan olarak, sorguların zaman uyumsuz sürümü kullanılır (`ASYNC`), sorguların durduğuna dair onay beklemez. + +Senkron versiyonu (`SYNC`) tüm sorguların durmasını bekler ve durduğunda her işlem hakkında bilgi görüntüler. +Yanıt içerir `kill_status` aşağıdaki değerleri alabilen sütun: + +1. ‘finished’ – The query was terminated successfully. +2. ‘waiting’ – Waiting for the query to end after sending it a signal to terminate. +3. The other values ​​explain why the query can't be stopped. + +Bir test sorgusu (`TEST`) yalnızca kullanıcının haklarını denetler ve durdurulacak sorguların bir listesini görüntüler. + +## KILL MUTATION {#kill-mutation} + +``` sql +KILL MUTATION [ON CLUSTER cluster] + WHERE + [TEST] + [FORMAT format] +``` + +İptal etmek ve kaldırmak için çalışır [mutasyonlar](alter.md#alter-mutations) şu anda yürütülüyor. İptal etmek için mutationsasyonlar seçilir [`system.mutations`](../../operations/system_tables.md#system_tables-mutations) tablo tarafından belirtilen filtreyi kullanarak `WHERE` fıkra ofsı `KILL` sorgu. + +Bir test sorgusu (`TEST`) yalnızca kullanıcının haklarını denetler ve durdurulacak sorguların bir listesini görüntüler. + +Örnekler: + +``` sql +-- Cancel and remove all mutations of the single table: +KILL MUTATION WHERE database = 'default' AND table = 'table' + +-- Cancel the specific mutation: +KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = 'mutation_3.txt' +``` + +The query is useful when a mutation is stuck and cannot finish (e.g. if some function in the mutation query throws an exception when applied to the data contained in the table). + +Mutasyon tarafından yapılan değişiklikler geri alınmaz. + +## OPTIMIZE {#misc_operations-optimize} + +``` sql +OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] +``` + +Bu sorgu, bir tablo altyapısı ile tablolar için veri parçaları planlanmamış birleştirme başlatmaya çalışır. [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) aile. + +Bu `OPTMIZE` sorgu için de desteklenmektedir [MaterializedView](../../engines/table_engines/special/materializedview.md) ve... [Arabellek](../../engines/table_engines/special/buffer.md) motorlar. Diğer tablo motorları desteklenmiyor. + +Ne zaman `OPTIMIZE` ile kullanılır [ReplicatedMergeTree](../../engines/table_engines/mergetree_family/replication.md) Tablo motorları ailesi, ClickHouse birleştirme için bir görev oluşturur ve tüm düğümlerde yürütülmeyi bekler (eğer `replication_alter_partitions_sync` ayar etkinse) ' dir. + +- Eğer `OPTIMIZE` herhangi bir nedenle bir birleştirme gerçekleştirmez, müşteriye bildirmez. Bildirimleri etkinleştirmek için [optimize\_throw\_if\_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) ayar. +- Belirtir aseniz bir `PARTITION`, sadece belirtilen bölüm optimize edilmiştir. [Bölüm ifadesi nasıl ayarlanır](alter.md#alter-how-to-specify-part-expr). +- Belirtir specifyseniz `FINAL`, optimizasyon, tüm veriler zaten bir parçada olsa bile gerçekleştirilir. +- Belirtir specifyseniz `DEDUPLICATE`, sonra tamamen aynı satırlar tekilleştirilecektir (tüm sütunlar karşılaştırılır), sadece MergeTree motoru için anlamlıdır. + +!!! warning "Uyarıcı" + `OPTIMIZE` Düzelt canemiyorum “Too many parts” hatasız. + +## RENAME {#misc_operations-rename} + +Bir veya daha fazla tabloyu yeniden adlandırır. + +``` sql +RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... [ON CLUSTER cluster] +``` + +Tüm tablolar genel kilitleme altında yeniden adlandırılır. Tabloları yeniden adlandırma hafif bir işlemdir. İÇİN'DEN sonra başka bir veritabanı belirttiyseniz, tablo bu veritabanına taşınacaktır. Ancak, veritabanlarına sahip dizinlerin aynı dosya sisteminde bulunması gerekir (aksi takdirde bir hata döndürülür). + +## SET {#query-set} + +``` sql +SET param = value +``` + +Atıyor `value` to the `param` [ayar](../../operations/settings/index.md) geçerli oturum için. Değiştiremezsiniz [sunucu ayarları](../../operations/server_configuration_parameters/index.md) bu şekilde. + +Belirtilen ayarlar profilindeki tüm değerleri tek bir sorguda da ayarlayabilirsiniz. + +``` sql +SET profile = 'profile-name-from-the-settings-file' +``` + +Daha fazla bilgi için, bkz. [Ayarlar](../../operations/settings/settings.md). + +## TRUNCATE {#truncate} + +``` sql +TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +``` + +Bir tablodaki tüm verileri kaldırır. Fık thera ne zaman `IF EXISTS` tablo yoksa, sorgu bir hata döndürür. + +Bu `TRUNCATE` sorgu için desteklenmiyor [Görünüm](../../engines/table_engines/special/view.md), [Dosya](../../engines/table_engines/special/file.md), [URL](../../engines/table_engines/special/url.md) ve [Boş](../../engines/table_engines/special/null.md) masa motorları. + +## USE {#use} + +``` sql +USE db +``` + +Oturum için geçerli veritabanını ayarlamanızı sağlar. +Geçerli veritabanı, veritabanı sorguda tablo adından önce bir nokta ile açıkça tanımlanmamışsa, tabloları aramak için kullanılır. +Bir oturum kavramı olmadığından, bu sorgu HTTP protokolünü kullanırken yapılamaz. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/misc/) diff --git a/docs/tr/sql_reference/statements/select.md b/docs/tr/sql_reference/statements/select.md new file mode 100644 index 00000000000..287a8029ee9 --- /dev/null +++ b/docs/tr/sql_reference/statements/select.md @@ -0,0 +1,610 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 33 +toc_title: SELECT +--- + +# Select Queries sözdizimi {#select-queries-syntax} + +`SELECT` veri alma gerçekleştirir. + +``` sql +[WITH expr_list|(subquery)] +SELECT [DISTINCT] expr_list +[FROM [db.]table | (subquery) | table_function] [FINAL] +[SAMPLE sample_coeff] +[ARRAY JOIN ...] +[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list +[PREWHERE expr] +[WHERE expr] +[GROUP BY expr_list] [WITH TOTALS] +[HAVING expr] +[ORDER BY expr_list] +[LIMIT [offset_value, ]n BY columns] +[LIMIT [n, ]m] +[UNION ALL ...] +[INTO OUTFILE filename] +[FORMAT format] +``` + +Tüm yan tümceleri isteğe bağlıdır, hemen sonra ifadelerin gerekli listesi hariç seçin. +Aşağıdaki yan tümceleri sorgu yürütme konveyör hemen hemen aynı sırada açıklanmıştır. + +Sorgu atlarsa `DISTINCT`, `GROUP BY` ve `ORDER BY` CLA andus Andes and the `IN` ve `JOIN` alt sorgular, sorgu o (1) RAM miktarını kullanarak tamamen akış işlenecektir. +Aksi takdirde, uygun kısıtlamalar belirtilmezse, sorgu çok fazla RAM tüketebilir: `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. Daha fazla bilgi için bölüme bakın “Settings”. Harici sıralama (geçici tabloları bir diske kaydetme) ve harici toplama kullanmak mümkündür. `The system does not have "merge join"`. + +### Fık WİTHRA ile {#with-clause} + +Bu bölüm, ortak tablo ifadeleri için destek sağlar ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), bazı sınırlamalar ile: +1. Özyinelemeli sorgular desteklenmiyor +2. Alt sorgu bölüm ile birlikte kullanıldığında, sonuç tam olarak bir satır ile skaler olmalıdır +3. İfadenin sonuçları alt sorgularda kullanılamaz +WITH yan tümcesi ifadeleri sonuçları SELECT yan tümcesi içinde kullanılabilir. + +Örnek 1: Sabit ifadeyi aşağıdaki gibi kullanma “variable” + +``` sql +WITH '2019-08-01 15:23:00' as ts_upper_bound +SELECT * +FROM hits +WHERE + EventDate = toDate(ts_upper_bound) AND + EventTime <= ts_upper_bound +``` + +Örnek 2: SELECT yan tümcesi sütun listesinden toplam(bayt) ifade sonucunu çıkarma + +``` sql +WITH sum(bytes) as s +SELECT + formatReadableSize(s), + table +FROM system.parts +GROUP BY table +ORDER BY s +``` + +Örnek 3: skaler alt sorgu sonuçlarını kullanma + +``` sql +/* this example would return TOP 10 of most huge tables */ +WITH + ( + SELECT sum(bytes) + FROM system.parts + WHERE active + ) AS total_disk_usage +SELECT + (sum(bytes) / total_disk_usage) * 100 AS table_disk_usage, + table +FROM system.parts +GROUP BY table +ORDER BY table_disk_usage DESC +LIMIT 10 +``` + +Örnek 4: alt sorguda ifadeyi yeniden kullanma +Alt sorgularda ifade kullanımı için geçerli sınırlama için bir geçici çözüm olarak çoğaltabilirsiniz. + +``` sql +WITH ['hello'] AS hello +SELECT + hello, + * +FROM +( + WITH ['hello'] AS hello + SELECT hello +) +``` + +``` text +┌─hello─────┬─hello─────┐ +│ ['hello'] │ ['hello'] │ +└───────────┴───────────┘ +``` + +### Fık FROMRAS FROMINDAN {#select-from} + +FROM yan tümcesi atlanırsa, veriler `system.one` Tablo. +Bu `system.one` tablo tam olarak bir satır içerir (bu tablo diğer Dbms'lerde bulunan çift tablo ile aynı amacı yerine getirir). + +Bu `FROM` yan tümcesi veri okumak için kaynak belirtir: + +- Tablo +- Alt sorgu +- [Tablo fonksiyonu](../table_functions/index.md) + +`ARRAY JOIN` ve düzenli `JOIN` ayrıca dahil edilebilir (aşağıya bakınız). + +Bunun yerine bir tablo, `SELECT` alt sorgu parantez içinde belirtilebilir. +Standart SQL aksine, bir eşanlamlı bir alt sorgudan sonra belirtilmesi gerekmez. + +Bir sorguyu yürütmek için, sorguda listelenen tüm sütunlar uygun tablodan ayıklanır. Dış sorgu için gerekli olmayan tüm sütunlar alt sorgulardan atılır. +Bir sorgu herhangi bir sütun listelemezse (örneğin, `SELECT count() FROM t`), satır sayısını hesaplamak için yine de tablodan bir sütun çıkarılır (en küçük olanı tercih edilir). + +#### Son değiştirici {#select-from-final} + +Tablolardan veri seçerken uygulanabilir [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)- motor ailesi dışında `GraphiteMergeTree`. Ne zaman `FINAL` belirtilen, ClickHouse sonucu döndürmeden önce verileri tam olarak birleştirir ve böylece verilen tablo altyapısı için birleştirmeler sırasında gerçekleşen tüm veri dönüşümlerini gerçekleştirir. + +Ayrıca için desteklenen: +- [Çoğaltıyordu](../../engines/table_engines/mergetree_family/replication.md) sürümleri `MergeTree` motorlar. +- [Görünüm](../../engines/table_engines/special/view.md), [Arabellek](../../engines/table_engines/special/buffer.md), [Dağılı](../../engines/table_engines/special/distributed.md), ve [MaterializedView](../../engines/table_engines/special/materializedview.md) üzerinden oluşturul ,maları koşuluyla diğer motorlar üzerinde çalışan motorlar `MergeTree`- motor masaları. + +Kullanan sorgular `FINAL` olmayan benzer sorgular kadar hızlı Yürüt ,ülür, çünkü: + +- Sorgu tek bir iş parçacığında yürütülür ve veri sorgu yürütme sırasında birleştirilir. +- İle sorgular `FINAL` sorguda belirtilen sütunlara ek olarak birincil anahtar sütunlarını okuyun. + +Çoğu durumda, kullanmaktan kaçının `FINAL`. + +### Örnek Madde {#select-sample-clause} + +Bu `SAMPLE` yan tümcesi yaklaşık sorgu işleme için izin verir. + +Veri örneklemesi etkinleştirildiğinde, sorgu tüm veriler üzerinde değil, yalnızca belirli bir veri kesirinde (örnek) gerçekleştirilir. Örneğin, tüm ziyaretler için istatistikleri hesaplamanız gerekiyorsa, sorguyu tüm ziyaretlerin 1/10 kesirinde yürütmek ve ardından sonucu 10 ile çarpmak yeterlidir. + +Yaklaşık sorgu işleme aşağıdaki durumlarda yararlı olabilir: + +- Sıkı zamanlama gereksinimleriniz olduğunda (\<100ms gibi), ancak bunları karşılamak için ek donanım kaynaklarının maliyetini haklı çıkaramazsınız. +- Ham verileriniz doğru olmadığında, yaklaşım kaliteyi belirgin şekilde düşürmez. +- İş gereksinimleri yaklaşık sonuçları hedef alır (maliyet etkinliği için veya kesin sonuçları premium kullanıcılara pazarlamak için). + +!!! note "Not" + Örneklemeyi yalnızca aşağıdaki tablolarla kullanabilirsiniz: [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md) tablo oluşturma sırasında örnekleme ifadesi belirtilmişse (bkz [MergeTree motoru](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table)). + +Veri örneklemesinin özellikleri aşağıda listelenmiştir: + +- Veri örneklemesi deterministik bir mekanizmadır. Aynı sonucu `SELECT .. SAMPLE` sorgu her zaman aynıdır. +- Örnekleme, farklı tablolar için sürekli olarak çalışır. Tek bir örnekleme anahtarına sahip tablolar için, aynı katsayıya sahip bir örnek her zaman olası verilerin aynı alt kümesini seçer. Örneğin, kullanıcı kimlikleri örneği, farklı tablolardan olası tüm kullanıcı kimliklerinin aynı alt kümesine sahip satırları alır. Bu, örneği alt sorgularda kullanabileceğiniz anlamına gelir. [IN](#select-in-operators) yan. Ayrıca, kullanarak örnekleri katılabilir [JOIN](#select-join) yan. +- Örnekleme, bir diskten daha az veri okumayı sağlar. Örnekleme anahtarını doğru belirtmeniz gerektiğini unutmayın. Daha fazla bilgi için, bkz. [MergeTree tablosu oluşturma](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table). + +İçin `SAMPLE` yan tümcesi aşağıdaki sözdizimi desteklenir: + +| SAMPLE Clause Syntax | Açıklama | +|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SAMPLE k` | Burada `k` 0'dan 1'e kadar olan sayıdır.
Sorgu üzerinde yürütülür `k` verilerin kesir. Mesela, `SAMPLE 0.1` sorguyu verilerin %10'unda çalıştırır. [Daha fazla bilgi edinin](#select-sample-k) | +| `SAMPLE n` | Burada `n` yeterince büyük bir tamsayıdır.
Sorgu en az bir örnek üzerinde yürütülür `n` satırlar (ancak bundan önemli ölçüde daha fazla değil). Mesela, `SAMPLE 10000000` sorguyu en az 10.000.000 satır çalıştırır. [Daha fazla bilgi edinin](#select-sample-n) | +| `SAMPLE k OFFSET m` | Burada `k` ve `m` 0'dan 1'e kadar olan sayılardır.
Sorgu bir örnek üzerinde yürütülür `k` verilerin kesir. Örnek için kullanılan veriler, `m` bölme. [Daha fazla bilgi edinin](#select-sample-offset) | + +#### SAMPLE K {#select-sample-k} + +Burada `k` 0'dan 1'e kadar olan sayıdır (hem kesirli hem de ondalık gösterimler desteklenir). Mesela, `SAMPLE 1/2` veya `SAMPLE 0.5`. + +İn a `SAMPLE k` fık ,ra, örnek alınır `k` verilerin kesir. Örnek aşağıda gösterilmiştir: + +``` sql +SELECT + Title, + count() * 10 AS PageViews +FROM hits_distributed +SAMPLE 0.1 +WHERE + CounterID = 34 +GROUP BY Title +ORDER BY PageViews DESC LIMIT 1000 +``` + +Bu örnekte, sorgu 0.1 (%10) veri örneği üzerinde yürütülür. Toplam fonksiyonların değerleri otomatik olarak düzeltilmez, bu nedenle yaklaşık bir sonuç elde etmek için, değer `count()` elle 10 ile çarpılır. + +#### SAMPLE N {#select-sample-n} + +Burada `n` yeterince büyük bir tamsayıdır. Mesela, `SAMPLE 10000000`. + +Bu durumda, sorgu en az bir örnek üzerinde yürütülür `n` satırlar (ancak bundan önemli ölçüde daha fazla değil). Mesela, `SAMPLE 10000000` sorguyu en az 10.000.000 satır çalıştırır. + +Veri okuma için minimum birim bir granül olduğundan (boyutu `index_granularity` ayar), granülün boyutundan çok daha büyük bir örnek ayarlamak mantıklıdır. + +Kullanırken `SAMPLE n` yan tümce, verilerin hangi göreli yüzde işlendiğini bilmiyorsunuz. Yani toplam fonksiyonların çarpılması gereken katsayıyı bilmiyorsunuz. Kullan... `_sample_factor` sanal sütun yaklaşık sonucu almak için. + +Bu `_sample_factor` sütun dinamik olarak hesaplanan göreli katsayıları içerir. Bu sütun otomatik olarak oluşturulduğunda [oluşturmak](../../engines/table_engines/mergetree_family/mergetree.md#table_engine-mergetree-creating-a-table) belirtilen örnekleme anahtarına sahip bir tablo. Kullanım örnekleri `_sample_factor` sütun aşağıda gösterilmiştir. + +Masayı düşünelim `visits`, site ziyaretleri ile ilgili istatistikleri içerir. İlk örnek, sayfa görünümlerinin sayısını nasıl hesaplayacağınızı gösterir: + +``` sql +SELECT sum(PageViews * _sample_factor) +FROM visits +SAMPLE 10000000 +``` + +Bir sonraki örnek, toplam ziyaret sayısını nasıl hesaplayacağınızı gösterir: + +``` sql +SELECT sum(_sample_factor) +FROM visits +SAMPLE 10000000 +``` + +Aşağıdaki örnek, ortalama oturum süresinin nasıl hesaplanacağını göstermektedir. Ortalama değerleri hesaplamak için göreli katsayıyı kullanmanız gerekmediğini unutmayın. + +``` sql +SELECT avg(Duration) +FROM visits +SAMPLE 10000000 +``` + +#### SAMPLE K OFFSET M {#select-sample-offset} + +Burada `k` ve `m` 0'dan 1'e kadar olan sayılardır. Örnekler aşağıda gösterilmiştir. + +**Örnek 1** + +``` sql +SAMPLE 1/10 +``` + +Bu örnekte, örnek tüm verilerin 1/10'udur: + +`[++------------]` + +**Örnek 2** + +``` sql +SAMPLE 1/10 OFFSET 1/2 +``` + +Burada, verilerin ikinci yarısından %10'luk bir örnek alınır. + +`[------++------]` + +### Dizi Jo JOİNİN yan tüm Clausecesi {#select-array-join-clause} + +Yürüt allowsmeyi sağlar `JOIN` bir dizi veya iç içe veri yapısı ile. Niyet benzer [arrayJoin](../../sql_reference/functions/array_join.md#functions_arrayjoin) işlev, ancak işlevselliği daha geniştir. + +``` sql +SELECT +FROM +[LEFT] ARRAY JOIN +[WHERE|PREWHERE ] +... +``` + +Yalnızca bir tek belirtebilirsiniz `ARRAY JOIN` bir sorguda yan tümcesi. + +Sorgu yürütme sırası çalışırken en iyi duruma getirilmiştir `ARRAY JOIN`. Rağmen `ARRAY JOIN` her zaman önce belirtilmelidir `WHERE/PREWHERE` fık ,ra, daha önce de yapılabilir `WHERE/PREWHERE` (sonuç bu maddede gerekliyse) veya tamamladıktan sonra (hesaplamaların hacmini azaltmak için). İşlem sırası sorgu iyileştiricisi tarafından denetlenir. + +Desteklenen türleri `ARRAY JOIN` aşağıda listelenmiştir: + +- `ARRAY JOIN` - Bu durumda, boş diziler sonucu dahil değildir `JOIN`. +- `LEFT ARRAY JOIN` Bunun sonucu `JOIN` boş dizilere sahip satırlar içerir. Boş bir dizinin değeri, dizi öğesi türü için varsayılan değere ayarlanır (genellikle 0, boş dize veya NULL). + +Aşağıdaki örnekler kullanımını göstermektedir `ARRAY JOIN` ve `LEFT ARRAY JOIN` yanlar. Bir tablo oluşturalım [Dizi](../../sql_reference/data_types/array.md) sütun yazın ve içine değerler ekleyin: + +``` sql +CREATE TABLE arrays_test +( + s String, + arr Array(UInt8) +) ENGINE = Memory; + +INSERT INTO arrays_test +VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []); +``` + +``` text +┌─s───────────┬─arr─────┐ +│ Hello │ [1,2] │ +│ World │ [3,4,5] │ +│ Goodbye │ [] │ +└─────────────┴─────────┘ +``` + +Aşağıdaki örnek kullanır `ARRAY JOIN` yan: + +``` sql +SELECT s, arr +FROM arrays_test +ARRAY JOIN arr; +``` + +``` text +┌─s─────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +└───────┴─────┘ +``` + +Sonraki örnek kullanımlar `LEFT ARRAY JOIN` yan: + +``` sql +SELECT s, arr +FROM arrays_test +LEFT ARRAY JOIN arr; +``` + +``` text +┌─s───────────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +│ Goodbye │ 0 │ +└─────────────┴─────┘ +``` + +#### Takma Ad Kullanma {#using-aliases} + +Bir dizi için bir diğer ad belirtilebilir `ARRAY JOIN` yan. Bu durumda, bir dizi öğesine bu diğer adla erişilebilir, ancak dizinin kendisine özgün adla erişilir. Örnek: + +``` sql +SELECT s, arr, a +FROM arrays_test +ARRAY JOIN arr AS a; +``` + +``` text +┌─s─────┬─arr─────┬─a─┐ +│ Hello │ [1,2] │ 1 │ +│ Hello │ [1,2] │ 2 │ +│ World │ [3,4,5] │ 3 │ +│ World │ [3,4,5] │ 4 │ +│ World │ [3,4,5] │ 5 │ +└───────┴─────────┴───┘ +``` + +Takma adlar kullanarak şunları yapabilirsiniz `ARRAY JOIN` harici bir dizi ile. Mesela: + +``` sql +SELECT s, arr_external +FROM arrays_test +ARRAY JOIN [1, 2, 3] AS arr_external; +``` + +``` text +┌─s───────────┬─arr_external─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ Hello │ 3 │ +│ World │ 1 │ +│ World │ 2 │ +│ World │ 3 │ +│ Goodbye │ 1 │ +│ Goodbye │ 2 │ +│ Goodbye │ 3 │ +└─────────────┴──────────────┘ +``` + +Birden çok diziler virgülle ayrılmış olabilir `ARRAY JOIN` yan. Bu durumda, `JOIN` onlarla aynı anda gerçekleştirilir (doğrudan toplam, kartezyen ürün değil). Tüm dizilerin aynı boyuta sahip olması gerektiğini unutmayın. Örnek: + +``` sql +SELECT s, arr, a, num, mapped +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ +│ Hello │ [1,2] │ 1 │ 1 │ 2 │ +│ Hello │ [1,2] │ 2 │ 2 │ 3 │ +│ World │ [3,4,5] │ 3 │ 1 │ 4 │ +│ World │ [3,4,5] │ 4 │ 2 │ 5 │ +│ World │ [3,4,5] │ 5 │ 3 │ 6 │ +└───────┴─────────┴───┴─────┴────────┘ +``` + +Aşağıdaki örnek kullanır [arrayEnumerate](../../sql_reference/functions/array_functions.md#array_functions-arrayenumerate) işlev: + +``` sql +SELECT s, arr, a, num, arrayEnumerate(arr) +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; +``` + +``` text +┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ +│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ +│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ +│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ +│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ +│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ +└───────┴─────────┴───┴─────┴─────────────────────┘ +``` + +#### İç içe veri yapısı ile dizi birleştirme {#array-join-with-nested-data-structure} + +`ARRAY`Jo "in " ile de çalışır [iç içe veri yapıları](../../sql_reference/data_types/nested_data_structures/nested.md). Örnek: + +``` sql +CREATE TABLE nested_test +( + s String, + nest Nested( + x UInt8, + y UInt32) +) ENGINE = Memory; + +INSERT INTO nested_test +VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []); +``` + +``` text +┌─s───────┬─nest.x──┬─nest.y─────┐ +│ Hello │ [1,2] │ [10,20] │ +│ World │ [3,4,5] │ [30,40,50] │ +│ Goodbye │ [] │ [] │ +└─────────┴─────────┴────────────┘ +``` + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +İç içe geçmiş veri yapılarının adlarını belirtirken `ARRAY JOIN` anlam aynıdır `ARRAY JOIN` içerdiği tüm dizi öğeleri ile. Örnekler aşağıda listelenmiştir: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`, `nest.y`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ +``` + +Bu varyasyon da mantıklı: + +``` sql +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`; +``` + +``` text +┌─s─────┬─nest.x─┬─nest.y─────┐ +│ Hello │ 1 │ [10,20] │ +│ Hello │ 2 │ [10,20] │ +│ World │ 3 │ [30,40,50] │ +│ World │ 4 │ [30,40,50] │ +│ World │ 5 │ [30,40,50] │ +└───────┴────────┴────────────┘ +``` + +Bir diğer ad, iç içe geçmiş bir veri yapısı için kullanılabilir. `JOIN` sonuç veya kaynak dizi. Örnek: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest AS n; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ +└───────┴─────┴─────┴─────────┴────────────┘ +``` + +Kullanma örneği [arrayEnumerate](../../sql_reference/functions/array_functions.md#array_functions-arrayenumerate) işlev: + +``` sql +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num +FROM nested_test +ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num; +``` + +``` text +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ +└───────┴─────┴─────┴─────────┴────────────┴─────┘ +``` + +### Jo {#select-join} + +Verileri normal olarak birleştirir [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) anlama. + +!!! info "Not" + İlgili [ARRAY JOIN](#select-array-join-clause). + +``` sql +SELECT +FROM +[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN +(ON )|(USING ) ... +``` + +Tablo adları yerine belirtilebilir `` ve ``. Bu eşdeğerdir `SELECT * FROM table` alt sorgu, tablonun sahip olduğu özel bir durum dışında [Katmak](../../engines/table_engines/special/join.md) engine – an array prepared for joining. + +#### Desteklenen Türleri `JOIN` {#select-join-types} + +- `INNER JOIN` (veya `JOIN`) +- `LEFT JOIN` (veya `LEFT OUTER JOIN`) +- `RIGHT JOIN` (veya `RIGHT OUTER JOIN`) +- `FULL JOIN` (veya `FULL OUTER JOIN`) +- `CROSS JOIN` (veya `,` ) + +Standarda bakın [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)) açıklama. + +#### Çoklu birleştirme {#multiple-join} + +Sorguları gerçekleştiren ClickHouse, çoklu tablo birleşimlerini iki tablo birleşimlerinin sırasına yeniden yazar. Örneğin, JOIN ClickHouse için dört tablo varsa birinci ve ikinci katılır, ardından üçüncü tablo ile sonuç katılır ve son adımda dördüncü bir katılır. + +Bir sorgu içeriyorsa `WHERE` yan tümcesi, ClickHouse Ara birleştirme aracılığıyla bu yan tümcesi filtreleri pushdown çalışır. Filtreyi her Ara birleşime uygulayamazsa, tüm birleşimler tamamlandıktan sonra clickhouse filtreleri uygular. + +Biz tavsiye `JOIN ON` veya `JOIN USING` sorguları oluşturmak için sözdizimi. Mesela: + +``` sql +SELECT * FROM t1 JOIN t2 ON t1.a = t2.a JOIN t3 ON t1.a = t3.a +``` + +Virgülle ayrılmış tablo listelerini kullanabilirsiniz. `FROM` yan. Mesela: + +``` sql +SELECT * FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a +``` + +Bu sözdizimleri karıştırmayın. + +ClickHouse virgülle sözdizimini doğrudan desteklemez, bu yüzden bunları kullanmanızı önermiyoruz. Algoritma, sorguyu şu şekilde yeniden yazmaya çalışır: `CROSS JOIN` ve `INNER JOIN` yan tümceleri ve sonra sorgu işleme devam eder. Sorguyu yeniden yazarken, ClickHouse performansı ve bellek tüketimini en iyi duruma getirmeye çalışır. Varsayılan olarak, ClickHouse virgülleri bir `INNER JOIN` CLA anduse and conver andts `INNER JOIN` -e doğru `CROSS JOIN` algoritma bunu garanti edemez zaman `INNER JOIN` gerekli verileri döndürür. + +#### Katılık {#select-join-strictness} + +- `ALL` — If the right table has several matching rows, ClickHouse creates a [Kartezyen ürün](https://en.wikipedia.org/wiki/Cartesian_product) eşleşen satırlardan. Bu standart `JOIN` SQL davranış. +- `ANY` — If the right table has several matching rows, only the first one found is joined. If the right table has only one matching row, the results of queries with `ANY` ve `ALL` anahtar kelimeler aynıdır. +- `ASOF` — For joining sequences with a non-exact match. `ASOF JOIN` kullanım aşağıda açıklanmıştır. + +**ASOF JOIN kullanımı** + +`ASOF JOIN` tam olarak eşleşmeyen kayıtlara katılmanız gerektiğinde kullanışlıdır. + +İçin tablolar `ASOF JOIN` sıralı bir sıra sütunu olmalıdır. Bu sütun bir tabloda tek başına olamaz ve veri türlerinden biri olmalıdır: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date`, ve `DateTime`. + +Sözdizimi `ASOF JOIN ... ON`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF LEFT JOIN table_2 +ON equi_cond AND closest_match_cond +``` + +Herhangi bir sayıda eşitlik koşulunu ve tam olarak en yakın eşleşme koşulunu kullanabilirsiniz. Mesela, `SELECT count() FROM table_1 ASOF LEFT JOIN table_2 ON table_1.a == table_2.b AND table_2.t <= table_1.t`. + +En yakın maç için desteklenen koşullar: `>`, `>=`, `<`, `<=`. + +Sözdizimi `ASOF JOIN ... USING`: + +``` sql +SELECT expressions_list +FROM table_1 +ASOF JOIN table_2 +USING (equi_column1, ... equi_columnN, asof_column) +``` + +`ASOF JOIN` kullanma `equi_columnX` eşit onliğe katılma ve `asof_column` ile en yakın maça katılmak için `table_1.asof_column >= table_2.asof_column` koşul. Bu `asof_column` sütun her zaman sonuncusu `USING` yan. + +Örneğin, aşağıdaki tabloları göz önünde bulundurun: + +\`\`\` Metin +table\_1 table\_2 + +olay / ev\_time / user\_id olay / ev\_time / user\_id diff --git a/docs/tr/sql_reference/statements/show.md b/docs/tr/sql_reference/statements/show.md new file mode 100644 index 00000000000..155b28886d9 --- /dev/null +++ b/docs/tr/sql_reference/statements/show.md @@ -0,0 +1,105 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 38 +toc_title: SHOW +--- + +# Sorguları göster {#show-queries} + +## SHOW CREATE TABLE {#show-create-table} + +``` sql +SHOW CREATE [TEMPORARY] [TABLE|DICTIONARY] [db.]table [INTO OUTFILE filename] [FORMAT format] +``` + +Bir tek döndürür `String`-tür ‘statement’ column, which contains a single value – the `CREATE` belirtilen nesneyi oluşturmak için kullanılan sorgu. + +## SHOW DATABASES {#show-databases} + +``` sql +SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] +``` + +Tüm veritabanlarının bir listesini yazdırır. +Bu sorgu ile aynıdır `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`. + +## SHOW PROCESSLIST {#show-processlist} + +``` sql +SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] +``` + +İçeriği verir [sistem.işleyişler](../../operations/system_tables.md#system_tables-processes) şu anda işlenmekte olan sorguların bir listesini içeren tablo, hariç `SHOW PROCESSLIST` sorgular. + +Bu `SELECT * FROM system.processes` sorgu, geçerli tüm sorgular hakkında veri döndürür. + +İpucu (konsolda Yürüt): + +``` bash +$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" +``` + +## SHOW TABLES {#show-tables} + +Tablo listesini görüntüler. + +``` sql +SHOW [TEMPORARY] TABLES [{FROM | IN} ] [LIKE '' | WHERE expr] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +Eğer... `FROM` yan tümcesi belirtilmemiş, sorgu geçerli veritabanından tabloların listesini döndürür. + +Aynı sonuçları elde edebilirsiniz `SHOW TABLES` aşağıdaki şekilde sorgu: + +``` sql +SELECT name FROM system.tables WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +**Örnek** + +Aşağıdaki sorgu, tablo listesinden ilk iki satırı seçer. `system` adları içeren veritabanı `co`. + +``` sql +SHOW TABLES FROM system LIKE '%co%' LIMIT 2 +``` + +``` text +┌─name───────────────────────────┐ +│ aggregate_function_combinators │ +│ collations │ +└────────────────────────────────┘ +``` + +## SHOW DICTIONARIES {#show-dictionaries} + +Bir listesini görüntüler [dış söz dictionarieslükler](../../sql_reference/dictionaries/external_dictionaries/external_dicts.md). + +``` sql +SHOW DICTIONARIES [FROM ] [LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +Eğer... `FROM` yan tümcesi belirtilmemiş, sorgu geçerli veritabanından sözlükler listesini döndürür. + +Aynı sonuçları elde edebilirsiniz `SHOW DICTIONARIES` aşağıdaki şekilde sorgu: + +``` sql +SELECT name FROM system.dictionaries WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +**Örnek** + +Aşağıdaki sorgu, tablo listesinden ilk iki satırı seçer. `system` adları içeren veritabanı `reg`. + +``` sql +SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 +``` + +``` text +┌─name─────────┐ +│ regions │ +│ region_names │ +└──────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/show/) diff --git a/docs/tr/sql_reference/statements/system.md b/docs/tr/sql_reference/statements/system.md new file mode 100644 index 00000000000..761f6e77737 --- /dev/null +++ b/docs/tr/sql_reference/statements/system.md @@ -0,0 +1,113 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 37 +toc_title: SYSTEM +--- + +# Sistem sorguları {#query-language-system} + +- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) +- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) +- [DROP DNS CACHE](#query_language-system-drop-dns-cache) +- [DROP MARK CACHE](#query_language-system-drop-mark-cache) +- [FLUSH LOGS](#query_language-system-flush_logs) +- [RELOAD CONFIG](#query_language-system-reload-config) +- [SHUTDOWN](#query_language-system-shutdown) +- [KILL](#query_language-system-kill) +- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) +- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) +- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) +- [STOP MERGES](#query_language-system-stop-merges) +- [START MERGES](#query_language-system-start-merges) + +## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} + +Daha önce başarıyla yüklenen tüm sözlükleri yeniden yükler. +Varsayılan olarak, sözlükler tembel yüklenir (bkz [dictionaries\_lazy\_load](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load)), bu nedenle başlangıçta otomatik olarak yüklenmek yerine, dictGet işlevi aracılığıyla ilk erişimde başlatılır veya ENGİNE = Dictionary ile tablolardan seçim yapılır. Bu `SYSTEM RELOAD DICTIONARIES` sorgu bu sözlükleri yeniden yükler (yüklü). +Her zaman döner `Ok.` sözlük güncellemesinin sonucu ne olursa olsun. + +## Sözlük Dictionary\_name yeniden yükle {#query_language-system-reload-dictionary} + +Tamamen bir sözlük reloads `dictionary_name`, sözlük durumuna bakılmaksızın (LOADED / NOT\_LOADED / FAİLED). +Her zaman döner `Ok.` ne olursa olsun sözlük güncelleme sonucu. +Sözlüğün durumu sorgulanarak kontrol edilebilir `system.dictionaries` Tablo. + +``` sql +SELECT name, status FROM system.dictionaries; +``` + +## DROP DNS CACHE {#query_language-system-drop-dns-cache} + +Clickhouse'un iç DNS önbelleğini sıfırlar. Bazen (eski ClickHouse sürümleri için) altyapıyı değiştirirken (başka bir ClickHouse sunucusunun IP adresini veya sözlükler tarafından kullanılan sunucuyu değiştirirken) bu komutu kullanmak gerekir. + +Daha uygun (otomatik) önbellek yönetimi için bkz: disable\_internal\_dns\_cache, dns\_cache\_update\_period parametreleri. + +## DROP MARK CACHE {#query_language-system-drop-mark-cache} + +İşaret önbelleğini sıfırlar. ClickHouse ve performans testlerinin geliştirilmesinde kullanılır. + +## FLUSH LOGS {#query_language-system-flush_logs} + +Flushes buffers of log messages to system tables (e.g. system.query\_log). Allows you to not wait 7.5 seconds when debugging. + +## RELOAD CONFIG {#query_language-system-reload-config} + +ClickHouse yapılandırmasını yeniden yükler. Yapılandırma ZooKeeeper saklandığında kullanılır. + +## SHUTDOWN {#query_language-system-shutdown} + +Normalde Clickhouse'u kapatır (gibi `service clickhouse-server stop` / `kill {$pid_clickhouse-server}`) + +## KILL {#query_language-system-kill} + +ClickHouse işlemini iptal eder (gibi `kill -9 {$ pid_clickhouse-server}`) + +## Dağıtılmış Tabloları Yönetme {#query-language-system-distributed} + +ClickHouse yönetebilir [dağılı](../../engines/table_engines/special/distributed.md) Tablolar. Bir kullanıcı bu tablolara veri eklediğinde, ClickHouse önce küme düğümlerine gönderilmesi gereken verilerin bir sırası oluşturur, sonra zaman uyumsuz olarak gönderir. İle kuyruk işleme yönetebilirsiniz [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [FLUSH DISTRIBUTED](#query_language-system-flush-distributed), ve [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) sorgular. Ayrıca, dağıtılmış verileri eşzamanlı olarak `insert_distributed_sync` ayar. + +### STOP DISTRIBUTED SENDS {#query_language-system-stop-distributed-sends} + +Dağıtılmış tablolara veri eklerken arka plan veri dağıtımını devre dışı bırakır. + +``` sql +SYSTEM STOP DISTRIBUTED SENDS [db.] +``` + +### FLUSH DISTRIBUTED {#query_language-system-flush-distributed} + +Küme düğümlerine eşzamanlı olarak veri göndermek için Clickhouse'u zorlar. Herhangi bir düğüm kullanılamıyorsa, ClickHouse bir özel durum atar ve sorgu yürütülmesini durdurur. Tüm düğümler tekrar çevrimiçi olduğunda gerçekleşecek olan başarılı olana kadar sorguyu yeniden deneyebilirsiniz. + +``` sql +SYSTEM FLUSH DISTRIBUTED [db.] +``` + +### START DISTRIBUTED SENDS {#query_language-system-start-distributed-sends} + +Dağıtılmış tablolara veri eklerken arka plan veri dağıtımını etkinleştirir. + +``` sql +SYSTEM START DISTRIBUTED SENDS [db.] +``` + +### STOP MERGES {#query_language-system-stop-merges} + +MergeTree ailesindeki tablolar için arka plan birleşmelerini durdurma imkanı sağlar: + +``` sql +SYSTEM STOP MERGES [[db.]merge_tree_family_table_name] +``` + +!!! note "Not" + `DETACH / ATTACH` tablo, daha önce tüm MergeTree tabloları için birleştirmeler durdurulduğunda bile tablo için arka plan birleştirmelerini başlatır. + +### START MERGES {#query_language-system-start-merges} + +MergeTree ailesindeki tablolar için arka plan birleştirmelerini başlatma imkanı sağlar: + +``` sql +SYSTEM START MERGES [[db.]merge_tree_family_table_name] +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/system/) diff --git a/docs/tr/sql_reference/syntax.md b/docs/tr/sql_reference/syntax.md new file mode 100644 index 00000000000..a7d725fe696 --- /dev/null +++ b/docs/tr/sql_reference/syntax.md @@ -0,0 +1,187 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 31 +toc_title: "S\xF6zdizimi" +--- + +# Sözdizimi {#syntax} + +Sistemde iki tür ayrıştırıcı vardır: tam SQL ayrıştırıcısı (özyinelemeli bir iniş ayrıştırıcısı) ve veri biçimi ayrıştırıcısı (hızlı akış ayrıştırıcısı). +Dışında her durumda `INSERT` sorgu, sadece tam SQL ayrıştırıcı kullanılır. +Bu `INSERT` sorgu her iki ayrıştırıcıyı da kullanır: + +``` sql +INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') +``` + +Bu `INSERT INTO t VALUES` parça tam ayrıştırıcı tarafından ayrıştırılır ve veriler `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` hızlı akış ayrıştırıcısı tarafından ayrıştırılır. Ayrıca kullanarak veriler için tam ayrıştırıcı açabilirsiniz [ınput\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) ayar. Ne zaman `input_format_values_interpret_expressions = 1`, ClickHouse önce hızlı akış ayrıştırıcısı ile değerleri ayrıştırmaya çalışır. Başarısız olursa, ClickHouse veriler için tam ayrıştırıcıyı kullanmaya çalışır ve bir SQL gibi davranır [ifade](#syntax-expressions). + +Veri herhangi bir biçime sahip olabilir. Bir sorgu alındığında, sunucu daha fazla hesaplar [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) istek bayt RAM (varsayılan olarak, 1 MB) ve geri kalanı akış ayrıştırılır. +Bu büyük sorunları önlemek için izin verir `INSERT` sorgular. + +Kullanırken `Values` biçim içinde bir `INSERT` sorgu, verilerin bir ifadedeki ifadelerle aynı şekilde ayrıştırıldığı görünebilir `SELECT` sorgu, ancak bu doğru değil. Bu `Values` biçim çok daha sınırlıdır. + +Bu makalenin geri kalanı tam çözümleyici kapsar. Biçim ayrıştırıcıları hakkında daha fazla bilgi için bkz: [Biçimliler](../interfaces/formats.md) bölme. + +## Alanlar {#spaces} + +Sözdizimsel yapılar arasında (bir sorgunun başlangıcı ve sonu dahil) herhangi bir sayıda boşluk simgesi olabilir. Boşluk sembolleri boşluk, sekme, satır beslemesi, CR ve form beslemesini içerir. + +## Yorumlar {#comments} + +ClickHouse, SQL stili ve C stili yorumlarını destekler. +SQL tarzı yorumlar ile başlar `--` ve hattın sonuna kadar devam, bir boşluk sonra `--` atlanmış olabilir. +C-style dan `/*` -e doğru `*/`ve çok satırlı olabilir, boşluklar da gerekli değildir. + +## Kelimeler {#syntax-keywords} + +Anahtar kelimeler karşılık geldiğinde büyük / küçük harf duyarsızdır: + +- SQL standardı. Mesela, `SELECT`, `select` ve `SeLeCt` hepsi geçerlidir. +- Bazı popüler DBMS'DE (MySQL veya Postgres) uygulama. Mesela, `DateTime` ile aynıdır `datetime`. + +Veri türü adı büyük / küçük harf duyarlı olup olmadığını denetlenebilir `system.data_type_families` Tablo. + +Standart SQL'İN aksine, diğer tüm anahtar kelimeler (işlev adları dahil) şunlardır **büyük küçük harf duyarlı**. + +Anahtar kelimeler ayrılmış değildir; sadece karşılık gelen bağlamda bu şekilde ele alınır. Kullanıyorsanız [tanıtıcılar](#syntax-identifiers) anahtar kelimelerle aynı ada sahip olarak, bunları çift tırnak veya backticks içine alın. Örneğin, sorgu `SELECT "FROM" FROM table_name` tablo geçerli ise `table_name` adı ile sütun vardır `"FROM"`. + +## Tanıtıcılar {#syntax-identifiers} + +Tanımlay areıcılar: + +- Küme, veritabanı, tablo, bölüm ve sütun adları. +- İşlevler. +- Veri türleri. +- [İfade takma adları](#syntax-expression_aliases). + +Tanımlayıcılar alıntılanabilir veya alıntılanamaz. İkincisi tercih edilir. + +Alıntılanmamış tanımlayıcılar regex ile eşleşmelidir `^[a-zA-Z_][0-9a-zA-Z_]*$` ve eşit olamaz [kelimeler](#syntax-keywords). Örnekler: `x, _1, X_y__Z123_.` + +Tanımlayıcıları anahtar kelimelerle aynı şekilde kullanmak istiyorsanız veya tanımlayıcılarda başka semboller kullanmak istiyorsanız, örneğin çift tırnak işaretleri veya backticks kullanarak alıntı yapın, `"id"`, `` `id` ``. + +## Harfler {#literals} + +Sayısal, dize, bileşik ve `NULL` harfler. + +### Sayısal {#numeric} + +Sayısal literal ayrıştırılmaya çalışılıyor: + +- İlk olarak, 64-bit imzalı bir sayı olarak, [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) işlev. +- Başarısız olursa, 64-bit imzasız bir sayı olarak, [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) işlev. +- Başarısız olursa, kayan noktalı sayı olarak [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) işlev. +- Aksi takdirde, bir hata döndürür. + +Hazır bilgi değeri, değerin sığdığı en küçük türe sahiptir. +Örneğin, 1 olarak ayrıştırılır `UInt8`, ancak 256 olarak ayrıştırılır `UInt16`. Daha fazla bilgi için, bkz. [Veri türleri](../sql_reference/data_types/index.md). + +Örnekler: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. + +### Dize {#syntax-string-literal} + +Tek tırnak yalnızca dize değişmezleri desteklenir. Kapalı karakterler ters eğik çizgi kaçabilir. Aşağıdaki kaçış dizileri karşılık gelen özel bir değere sahiptir: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. Diğer tüm durumlarda, çıkış dizileri biçiminde `\c`, nere `c` herhangi bir karakter, dönüştürülür `c`. Bu dizileri kullanabileceğiniz anlamına gelir `\'`ve`\\`. Değeri olacak [Dize](../sql_reference/data_types/string.md) tür. + +Dize değişmezlerinde, en azından kaçmanız gerekir `'` ve `\`. Tek tırnak tek Alıntı ile kaçabilir, değişmez `'It\'s'` ve `'It''s'` eşittir. + +### Bileşik {#compound} + +Diziler köşeli parantez ile inşa edilmiştir `[1, 2, 3]`. Nuples yuvarlak parantez ile inşa edilmiştir `(1, 'Hello, world!', 2)`. +Teknik olarak bunlar değişmezler değil, sırasıyla dizi oluşturma işleci ve tuple oluşturma işleci ile ifadeler. +Bir dizi en az bir öğeden oluşmalı ve bir tuple en az iki öğeye sahip olmalıdır. +İçinde tuples göründüğünde ayrı bir durum var `IN` CLA ause of a `SELECT` sorgu. Sorgu sonuçları tuples içerebilir, ancak tuples bir veritabanına kaydedilemez (tablolar hariç [Bellek](../engines/table_engines/special/memory.md) motor). + +### NULL {#null-literal} + +Değerin eksik olduğunu gösterir. + +Saklamak için `NULL` bir tablo alanında, bu olmalıdır [Nullable](../sql_reference/data_types/nullable.md) tür. + +Veri formatına bağlı olarak (giriş veya çıkış), `NULL` farklı bir temsili olabilir. Daha fazla bilgi için belgelere bakın [veri formatları](../interfaces/formats.md#formats). + +İşleme için birçok nüans var `NULL`. Örneğin, bir karşılaştırma işleminin argümanlarından en az biri ise `NULL`, bu işlemin sonucu da `NULL`. Aynı şey çarpma, toplama ve diğer işlemler için de geçerlidir. Daha fazla bilgi için her işlem için belgeleri okuyun. + +Sorgularda, kontrol edebilirsiniz `NULL` kullanarak [IS NULL](operators.md#operator-is-null) ve [IS NOT NULL](operators.md) operatörler ve ilgili fonksiyonlar `isNull` ve `isNotNull`. + +## İşlevler {#functions} + +İşlev çağrıları, yuvarlak parantez içinde bir argüman listesi (muhtemelen boş) olan bir tanımlayıcı gibi yazılır. Standart SQL'İN aksine, boş bir argüman listesi için bile parantezler gereklidir. Örnek: `now()`. +Düzenli ve agrega işlevleri vardır (bkz. “Aggregate functions”). Bazı toplama işlevleri parantez içinde iki bağımsız değişken listesi içerebilir. Örnek: `quantile (0.9) (x)`. Bu toplama fonksiyonları denir “parametric” fonksiyonlar ve ilk listedeki argümanlar çağrılır “parameters”. Parametresiz toplama işlevlerinin sözdizimi, normal işlevlerle aynıdır. + +## Operatörler {#operators} + +Operatörler, sorgu ayrıştırma sırasında önceliklerini ve ilişkilendirmelerini dikkate alarak karşılık gelen işlevlerine dönüştürülür. +Örneğin, ifade `1 + 2 * 3 + 4` dönüştür toülür `plus(plus(1, multiply(2, 3)), 4)`. + +## Veri türleri ve veritabanı tablosu motorları {#data_types-and-database-table-engines} + +Veri türleri ve tablo motorları `CREATE` sorgu tanımlayıcıları veya işlevleri aynı şekilde yazılır. Başka bir deyişle, parantez içinde bir argüman listesi içerebilir veya içermeyebilir. Daha fazla bilgi için bölümlere bakın “Data types,” “Table engines,” ve “CREATE”. + +## İfade Takma Adları {#syntax-expression_aliases} + +Diğer ad, sorgudaki ifade için kullanıcı tanımlı bir addır. + +``` sql +expr AS alias +``` + +- `AS` — The keyword for defining aliases. You can define the alias for a table name or a column name in a `SELECT` kullanmadan fık thera `AS` kelime. + + For example, `SELECT table_name_alias.column_name FROM table_name table_name_alias`. + + In the [CAST](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) function, the `AS` keyword has another meaning. See the description of the function. + +- `expr` — Any expression supported by ClickHouse. + + For example, `SELECT column_name * 2 AS double FROM some_table`. + +- `alias` — Name for `expr`. Takma adlar ile uyumlu olmalıdır [tanıtıcılar](#syntax-identifiers) sözdizimi. + + For example, `SELECT "table t".column_name FROM table_name AS "table t"`. + +### Kullanımı ile ilgili notlar {#notes-on-usage} + +Diğer adlar bir sorgu veya alt sorgu için geneldir ve herhangi bir ifade için sorgunun herhangi bir bölümünde bir diğer ad tanımlayabilirsiniz. Mesela, `SELECT (1 AS n) + 2, n`. + +Diğer adlar alt sorgularda ve alt sorgular arasında görünmez. Örneğin, sorgu yürütülürken `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ClickHouse istisna oluşturur `Unknown identifier: num`. + +Sonuç sütunları için bir diğer ad tanımlanmışsa `SELECT` bir alt sorgunun yan tümcesi, bu sütunlar dış sorguda görülebilir. Mesela, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. + +Sütun veya tablo adlarıyla aynı olan diğer adlara dikkat edin. Aşağıdaki örneği ele alalım: + +``` sql +CREATE TABLE t +( + a Int, + b Int +) +ENGINE = TinyLog() +``` + +``` sql +SELECT + argMax(a, b), + sum(b) AS b +FROM t +``` + +``` text +Received exception from server (version 18.14.17): +Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: Aggregate function sum(b) is found inside another aggregate function in query. +``` + +Bu örnekte, tablo ilan ettik `t` sütun ile `b`. Ardından, veri seçerken, `sum(b) AS b` takma ad. Takma adlar küresel olduğundan, ClickHouse literal yerine `b` ifad theesinde `argMax(a, b)` ifad theesiyle `sum(b)`. Bu ikame istisnaya neden oldu. + +## Yıldız işareti {#asterisk} + +İn a `SELECT` sorgu, bir yıldız ifadesinin yerini alabilir. Daha fazla bilgi için bölüme bakın “SELECT”. + +## İfadeler {#syntax-expressions} + +Bir ifade, bir işlev, tanımlayıcı, değişmez, bir operatörün uygulaması, parantez içindeki ifade, alt sorgu veya yıldız işaretidir. Ayrıca bir takma ad içerebilir. +İfadelerin listesi, virgülle ayrılmış bir veya daha fazla ifadedir. +Fonksiyonlar ve operatörler, sırayla, argüman olarak ifadelere sahip olabilirler. + +[Orijinal makale](https://clickhouse.tech/docs/en/sql_reference/syntax/) diff --git a/docs/tr/sql_reference/table_functions/file.md b/docs/tr/sql_reference/table_functions/file.md new file mode 100644 index 00000000000..67d67ccd7cd --- /dev/null +++ b/docs/tr/sql_reference/table_functions/file.md @@ -0,0 +1,121 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 37 +toc_title: Dosya +--- + +# Dosya {#file} + +Bir dosyadan bir tablo oluşturur. Bu tablo işlevi benzer [url](url.md) ve [hdf'ler](hdfs.md) biri. + +``` sql +file(path, format, structure) +``` + +**Giriş parametreleri** + +- `path` — The relative path to the file from [user\_files\_path](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-user_files_path). Readonly modunda glob'ları takip eden dosya desteğine giden yol: `*`, `?`, `{abc,def}` ve `{N..M}` nerede `N`, `M` — numbers, \``'abc', 'def'` — strings. +- `format` — The [biçimli](../../interfaces/formats.md#formats) dosya. +- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. + +**Döndürülen değer** + +Belirtilen dosyada veri okumak veya yazmak için belirtilen yapıya sahip bir tablo. + +**Örnek** + +Ayar `user_files_path` ve dosyanın içeriği `test.csv`: + +``` bash +$ grep user_files_path /etc/clickhouse-server/config.xml + /var/lib/clickhouse/user_files/ + +$ cat /var/lib/clickhouse/user_files/test.csv + 1,2,3 + 3,2,1 + 78,43,45 +``` + +Tablo fromdan`test.csv` ve ondan ilk iki satır seçimi: + +``` sql +SELECT * +FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') +LIMIT 2 +``` + +``` text +┌─column1─┬─column2─┬─column3─┐ +│ 1 │ 2 │ 3 │ +│ 3 │ 2 │ 1 │ +└─────────┴─────────┴─────────┘ +``` + +``` sql +-- getting the first 10 lines of a table that contains 3 columns of UInt32 type from a CSV file +SELECT * FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') LIMIT 10 +``` + +**Yolda Globs** + +Birden çok yol bileşenleri globs olabilir. İşlenmek için dosya var olmalı ve tüm yol deseniyle eşleşmelidir (sadece sonek veya önek değil). + +- `*` — Substitutes any number of any characters except `/` boş dize dahil. +- `?` — Substitutes any single character. +- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. +- `{N..M}` — Substitutes any number in range from N to M including both borders. + +İle yapılar `{}` benzer olan [uzaktan masa fonksiyonu](../../sql_reference/table_functions/remote.md)). + +**Örnek** + +1. Aşağıdaki göreli yollara sahip birkaç dosyamız olduğunu varsayalım: + +- ‘some\_dir/some\_file\_1’ +- ‘some\_dir/some\_file\_2’ +- ‘some\_dir/some\_file\_3’ +- ‘another\_dir/some\_file\_1’ +- ‘another\_dir/some\_file\_2’ +- ‘another\_dir/some\_file\_3’ + +1. Bu dosyalardaki satır miktarını sorgula: + + + +``` sql +SELECT count(*) +FROM file('{some,another}_dir/some_file_{1..3}', 'TSV', 'name String, value UInt32') +``` + +1. Bu iki dizinin tüm dosyalarındaki satır miktarını sorgula: + + + +``` sql +SELECT count(*) +FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32') +``` + +!!! warning "Uyarıcı" + Dosya listenizde önde gelen sıfırlar içeren sayı aralıkları varsa, her basamak için parantez içeren yapıyı ayrı ayrı kullanın veya kullanın `?`. + +**Örnek** + +Adlı dosy thealardan verileri sorgu thelamak `file000`, `file001`, … , `file999`: + +``` sql +SELECT count(*) +FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') +``` + +## Sanal Sütunlar {#virtual-columns} + +- `_path` — Path to the file. +- `_file` — Name of the file. + +**Ayrıca Bakınız** + +- [Sanal sütunlar](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/file/) diff --git a/docs/tr/sql_reference/table_functions/generate.md b/docs/tr/sql_reference/table_functions/generate.md new file mode 100644 index 00000000000..f9fc1fc9b21 --- /dev/null +++ b/docs/tr/sql_reference/table_functions/generate.md @@ -0,0 +1,45 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 47 +toc_title: generateRandom +--- + +# generateRandom {#generaterandom} + +Verilen şema ile rastgele veri üretir. +Test tablolarını verilerle doldurmaya izin verir. +Dışında tabloda saklanabilir tüm veri türlerini destekler `LowCardinality` ve `AggregateFunction`. + +``` sql +generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]]); +``` + +**Parametre** + +- `name` — Name of corresponding column. +- `TypeName` — Type of corresponding column. +- `limit` — Number of rows to generate. +- `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`. +- `max_string_length` — Maximum string length for all generated strings. Defaults to `10`. +- `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated. + +**Döndürülen Değer** + +Istenen şema ile bir tablo nesnesi. + +## Kullanım Örneği {#usage-example} + +``` sql +SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2); +``` + +``` text +┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ +│ [77] │ -124167.6723 │ ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') │ +│ [32,110] │ -141397.7312 │ ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') │ +│ [68] │ -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │ +└──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/generate/) diff --git a/docs/tr/sql_reference/table_functions/hdfs.md b/docs/tr/sql_reference/table_functions/hdfs.md new file mode 100644 index 00000000000..e15d721135a --- /dev/null +++ b/docs/tr/sql_reference/table_functions/hdfs.md @@ -0,0 +1,104 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 45 +toc_title: hdf'ler +--- + +# hdf'ler {#hdfs} + +Hdfs'deki dosyalardan bir tablo oluşturur. Bu tablo işlevi benzer [url](url.md) ve [Dosya](file.md) biri. + +``` sql +hdfs(URI, format, structure) +``` + +**Giriş parametreleri** + +- `URI` — The relative URI to the file in HDFS. Path to file support following globs in readonly mode: `*`, `?`, `{abc,def}` ve `{N..M}` nerede `N`, `M` — numbers, \``'abc', 'def'` — strings. +- `format` — The [biçimli](../../interfaces/formats.md#formats) dosya. +- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. + +**Döndürülen değer** + +Belirtilen dosyada veri okumak veya yazmak için belirtilen yapıya sahip bir tablo. + +**Örnek** + +Tablo fromdan `hdfs://hdfs1:9000/test` ve ondan ilk iki satır seçimi: + +``` sql +SELECT * +FROM hdfs('hdfs://hdfs1:9000/test', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') +LIMIT 2 +``` + +``` text +┌─column1─┬─column2─┬─column3─┐ +│ 1 │ 2 │ 3 │ +│ 3 │ 2 │ 1 │ +└─────────┴─────────┴─────────┘ +``` + +**Yolda Globs** + +Birden çok yol bileşenleri globs olabilir. İşlenmek için dosya var olmalı ve tüm yol deseniyle eşleşmelidir (sadece sonek veya önek değil). + +- `*` — Substitutes any number of any characters except `/` boş dize dahil. +- `?` — Substitutes any single character. +- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. +- `{N..M}` — Substitutes any number in range from N to M including both borders. + +İle yapılar `{}` benzer olan [uzaktan masa fonksiyonu](../../sql_reference/table_functions/remote.md)). + +**Örnek** + +1. HDFS'DE aşağıdaki Urı'lere sahip birkaç dosyamız olduğunu varsayalım: + +- ‘hdfs://hdfs1:9000/some\_dir/some\_file\_1’ +- ‘hdfs://hdfs1:9000/some\_dir/some\_file\_2’ +- ‘hdfs://hdfs1:9000/some\_dir/some\_file\_3’ +- ‘hdfs://hdfs1:9000/another\_dir/some\_file\_1’ +- ‘hdfs://hdfs1:9000/another\_dir/some\_file\_2’ +- ‘hdfs://hdfs1:9000/another\_dir/some\_file\_3’ + +1. Bu dosyalardaki satır miktarını sorgula: + + + +``` sql +SELECT count(*) +FROM hdfs('hdfs://hdfs1:9000/{some,another}_dir/some_file_{1..3}', 'TSV', 'name String, value UInt32') +``` + +1. Bu iki dizinin tüm dosyalarındaki satır miktarını sorgula: + + + +``` sql +SELECT count(*) +FROM hdfs('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32') +``` + +!!! warning "Uyarıcı" + Dosya listenizde önde gelen sıfırlar içeren sayı aralıkları varsa, her basamak için parantez içeren yapıyı ayrı ayrı kullanın veya kullanın `?`. + +**Örnek** + +Adlı dosy thealardan verileri sorgu thelamak `file000`, `file001`, … , `file999`: + +``` sql +SELECT count(*) +FROM hdfs('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') +``` + +## Sanal Sütunlar {#virtual-columns} + +- `_path` — Path to the file. +- `_file` — Name of the file. + +**Ayrıca Bakınız** + +- [Sanal sütunlar](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/hdfs/) diff --git a/docs/tr/sql_reference/table_functions/index.md b/docs/tr/sql_reference/table_functions/index.md new file mode 100644 index 00000000000..3108903713f --- /dev/null +++ b/docs/tr/sql_reference/table_functions/index.md @@ -0,0 +1,38 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: "Tablo Fonksiyonlar\u0131" +toc_priority: 34 +toc_title: "Giri\u015F" +--- + +# Tablo Fonksiyonları {#table-functions} + +Tablo işlevleri tabloları oluşturmak için yöntemlerdir. + +Tablo işlevlerini kullanabilirsiniz: + +- [FROM](../statements/select.md#select-from) fıkra ofsı `SELECT` sorgu. + + The method for creating a temporary table that is available only in the current query. The table is deleted when the query finishes. + +- [Tablo oluştur \](../statements/create.md#create-table-query) sorgu. + + It's one of the methods of creating a table. + +!!! warning "Uyarıcı" + Eğer tablo işlevlerini kullanamazsınız [allow\_ddl](../../operations/settings/permissions_for_queries.md#settings_allow_ddl) ayarı devre dışı. + +| İşlev | Açıklama | +|--------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| [Dosya](file.md) | Oluşturur bir [Dosya](../../engines/table_engines/special/file.md)- motor masası. | +| [birleştirmek](merge.md) | Oluşturur bir [Birleştirmek](../../engines/table_engines/special/merge.md)- motor masası. | +| [şiir](numbers.md) | Tamsayı sayılarla dolu tek bir sütun içeren bir tablo oluşturur. | +| [uzak](remote.md) | Oluşturmadan uzak sunuculara erişmenizi sağlar. [Dağılı](../../engines/table_engines/special/distributed.md)- motor masası. | +| [url](url.md) | Oluşturur bir [Url](../../engines/table_engines/special/url.md)- motor masası. | +| [mysql](mysql.md) | Oluşturur bir [MySQL](../../engines/table_engines/integrations/mysql.md)- motor masası. | +| [jdbc](jdbc.md) | Oluşturur bir [JDBC](../../engines/table_engines/integrations/jdbc.md)- motor masası. | +| [odbc](odbc.md) | Oluşturur bir [ODBC](../../engines/table_engines/integrations/odbc.md)- motor masası. | +| [hdf'ler](hdfs.md) | Oluşturur bir [HDFS](../../engines/table_engines/integrations/hdfs.md)- motor masası. | + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/) diff --git a/docs/tr/sql_reference/table_functions/input.md b/docs/tr/sql_reference/table_functions/input.md new file mode 100644 index 00000000000..5639e05eb81 --- /dev/null +++ b/docs/tr/sql_reference/table_functions/input.md @@ -0,0 +1,47 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 46 +toc_title: girdi +--- + +# girdi {#input} + +`input(structure)` - etkin bir şekilde dönüştürmek ve veri eklemek sağlar tablo fonksiyonu gönderilen +başka bir yapıya sahip tabloya verilen yapıya sahip sunucu. + +`structure` - aşağıdaki formatta sunucuya gönderilen verilerin yapısı `'column1_name column1_type, column2_name column2_type, ...'`. +Mesela, `'id UInt32, name String'`. + +Bu işlev yalnızca kullanılabilir `INSERT SELECT` sorgu ve sadece bir kez ama aksi takdirde sıradan tablo işlevi gibi davranır +(örneğin, alt sorguda vb.kullanılabilir.). + +Veri sıradan gibi herhangi bir şekilde gönderilebilir `INSERT` sorgu ve herhangi bir kullanılabilir geçti [biçimli](../../interfaces/formats.md#formats) +bu sorgu sonunda belirtilmelidir (sıradan aksine `INSERT SELECT`). + +Bu işlevin ana özelliği, sunucu istemciden veri aldığında aynı anda onu dönüştürmesidir +ifadeler listesine göre `SELECT` yan tümcesi ve hedef tabloya ekler. Geçici tablo +aktarılan tüm veriler ile oluşturulmaz. + +**Örnekler** + +- L letet the `test` tablo aşağıdaki yapıya sahiptir `(a String, b String)` + ve veri `data.csv` farklı bir yapıya sahiptir `(col1 String, col2 Date, col3 Int32)`. Insert sorgusu + bu verileri `data.csv` içine `test` eşzamanlı dönüşüm ile tablo şöyle görünüyor: + + + +``` bash +$ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT lower(col1), col3 * col3 FROM input('col1 String, col2 Date, col3 Int32') FORMAT CSV"; +``` + +- Eğer `data.csv` aynı yapının verilerini içerir `test_structure` tablo olarak `test` sonra bu iki sorgu eşittir: + + + +``` bash +$ cat data.csv | clickhouse-client --query="INSERT INTO test FORMAT CSV" +$ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT * FROM input('test_structure') FORMAT CSV" +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/input/) diff --git a/docs/tr/sql_reference/table_functions/jdbc.md b/docs/tr/sql_reference/table_functions/jdbc.md new file mode 100644 index 00000000000..451fdefc013 --- /dev/null +++ b/docs/tr/sql_reference/table_functions/jdbc.md @@ -0,0 +1,29 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 43 +toc_title: jdbc +--- + +# jdbc {#table-function-jdbc} + +`jdbc(jdbc_connection_uri, schema, table)` - JDBC sürücüsü ile bağlı döner tablo. + +Bu tablo işlevi ayrı gerektirir `clickhouse-jdbc-bridge` program çalıştırılacak. +Bu (sorgulanan uzak tablonun DDL dayalı) null türleri destekler. + +**Örnekler** + +``` sql +SELECT * FROM jdbc('jdbc:mysql://localhost:3306/?user=root&password=root', 'schema', 'table') +``` + +``` sql +SELECT * FROM jdbc('mysql://localhost:3306/?user=root&password=root', 'schema', 'table') +``` + +``` sql +SELECT * FROM jdbc('datasource://mysql-local', 'schema', 'table') +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) diff --git a/docs/tr/sql_reference/table_functions/merge.md b/docs/tr/sql_reference/table_functions/merge.md new file mode 100644 index 00000000000..67e1355383c --- /dev/null +++ b/docs/tr/sql_reference/table_functions/merge.md @@ -0,0 +1,14 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 38 +toc_title: "birle\u015Ftirmek" +--- + +# birleştirmek {#merge} + +`merge(db_name, 'tables_regexp')` – Creates a temporary Merge table. For more information, see the section “Table engines, Merge”. + +Tablo yapısı, normal ifadeyle eşleşen ilk tablodan alınır. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/merge/) diff --git a/docs/tr/sql_reference/table_functions/mysql.md b/docs/tr/sql_reference/table_functions/mysql.md new file mode 100644 index 00000000000..aee7311dc56 --- /dev/null +++ b/docs/tr/sql_reference/table_functions/mysql.md @@ -0,0 +1,86 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 42 +toc_title: mysql +--- + +# mysql {#mysql} + +Veriyor `SELECT` uzak bir MySQL sunucusunda depolanan veriler üzerinde gerçekleştirilecek sorgular. + +``` sql +mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); +``` + +**Parametre** + +- `host:port` — MySQL server address. + +- `database` — Remote database name. + +- `table` — Remote table name. + +- `user` — MySQL user. + +- `password` — User password. + +- `replace_query` — Flag that converts `INSERT INTO` için sorgular `REPLACE INTO`. Eğer `replace_query=1`, sorgu değiştirilir. + +- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` eklenen ifade `INSERT` sorgu. + + Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause. + + To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception. + +Basit `WHERE` gibi maddeler `=, !=, >, >=, <, <=` şu anda MySQL sunucusunda yürütülür. + +Geri kalan şartlar ve `LIMIT` örnekleme kısıtlaması, yalnızca MySQL sorgusu bittikten sonra Clickhouse'da yürütülür. + +**Döndürülen Değer** + +Orijinal MySQL tablosu ile aynı sütunlara sahip bir tablo nesnesi. + +## Kullanım Örneği {#usage-example} + +MySQL tablo: + +``` text +mysql> CREATE TABLE `test`.`test` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `int_nullable` INT NULL DEFAULT NULL, + -> `float` FLOAT NOT NULL, + -> `float_nullable` FLOAT NULL DEFAULT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into test (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from test; ++------+----------+-----+----------+ +| int_id | int_nullable | float | float_nullable | ++------+----------+-----+----------+ +| 1 | NULL | 2 | NULL | ++------+----------+-----+----------+ +1 row in set (0,00 sec) +``` + +Clickhouse'dan veri seçme: + +``` sql +SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123') +``` + +``` text +┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ 2 │ ᴺᵁᴸᴸ │ +└────────┴──────────────┴───────┴────────────────┘ +``` + +## Ayrıca Bakınız {#see-also} + +- [Bu ‘MySQL’ masa motoru](../../engines/table_engines/integrations/mysql.md) +- [Harici sözlük kaynağı olarak MySQL kullanma](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql) + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/mysql/) diff --git a/docs/tr/sql_reference/table_functions/numbers.md b/docs/tr/sql_reference/table_functions/numbers.md new file mode 100644 index 00000000000..45ca2f5f45d --- /dev/null +++ b/docs/tr/sql_reference/table_functions/numbers.md @@ -0,0 +1,30 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 39 +toc_title: "\u015Fiir" +--- + +# şiir {#numbers} + +`numbers(N)` – Returns a table with the single ‘number’ 0'dan n-1'e kadar tamsayılar içeren sütun (Uİnt64). +`numbers(N, M)` - Tek bir tablo döndürür ‘number’ n'den (n + M - 1) tamsayıları içeren sütun (Uİnt64). + +Benzer `system.numbers` tablo, ardışık değerleri test etmek ve üretmek için kullanılabilir, `numbers(N, M)` daha verimli `system.numbers`. + +Aşağıdaki sorgular eşdeğerdir: + +``` sql +SELECT * FROM numbers(10); +SELECT * FROM numbers(0, 10); +SELECT * FROM system.numbers LIMIT 10; +``` + +Örnekler: + +``` sql +-- Generate a sequence of dates from 2010-01-01 to 2010-12-31 +select toDate('2010-01-01') + number as d FROM numbers(365); +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/numbers/) diff --git a/docs/tr/sql_reference/table_functions/odbc.md b/docs/tr/sql_reference/table_functions/odbc.md new file mode 100644 index 00000000000..d250ce21311 --- /dev/null +++ b/docs/tr/sql_reference/table_functions/odbc.md @@ -0,0 +1,108 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 44 +toc_title: odbc +--- + +# odbc {#table-functions-odbc} + +Üzerinden bağlanan tabloyu döndürür [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). + +``` sql +odbc(connection_settings, external_database, external_table) +``` + +Parametre: + +- `connection_settings` — Name of the section with connection settings in the `odbc.ini` Dosya. +- `external_database` — Name of a database in an external DBMS. +- `external_table` — Name of a table in the `external_database`. + +ODBC bağlantılarını güvenli bir şekilde uygulamak için ClickHouse ayrı bir program kullanır `clickhouse-odbc-bridge`. ODBC sürücüsü doğrudan yüklenmişse `clickhouse-server`, sürücü sorunları ClickHouse sunucu çökmesine neden olabilir. ClickHouse otomatik olarak başlar `clickhouse-odbc-bridge` gerekli olduğunda. ODBC Köprüsü programı aynı paketten yüklenir `clickhouse-server`. + +Alanları ile `NULL` dış tablodaki değerler, temel veri türü için varsayılan değerlere dönüştürülür. Örneğin, uzak bir MySQL tablo alanı `INT NULL` yazın 0'a dönüştürülür (ClickHouse için varsayılan değer `Int32` veri türü). + +## Kullanım örneği {#usage-example} + +**ODBC üzerinden yerel MySQL kurulumundan veri alma** + +Bu örnek Ubuntu Linux 18.04 ve MySQL server 5.7 için kontrol edilir. + +UnixODBC ve MySQL Connector yüklü olduğundan emin olun. + +Varsayılan olarak (paketlerden yüklüyse), ClickHouse kullanıcı olarak başlar `clickhouse`. Bu nedenle, bu kullanıcıyı MySQL sunucusunda oluşturmanız ve yapılandırmanız gerekir. + +``` bash +$ sudo mysql +``` + +``` sql +mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse'; +mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION; +``` + +Sonra bağlantıyı yapılandırın `/etc/odbc.ini`. + +``` bash +$ cat /etc/odbc.ini +[mysqlconn] +DRIVER = /usr/local/lib/libmyodbc5w.so +SERVER = 127.0.0.1 +PORT = 3306 +DATABASE = test +USERNAME = clickhouse +PASSWORD = clickhouse +``` + +Kullanarak bağlantıyı kontrol edebilirsiniz `isql` unixodbc yüklemesinden yardımcı program. + +``` bash +$ isql -v mysqlconn ++-------------------------+ +| Connected! | +| | +... +``` + +MySQL tablo: + +``` text +mysql> CREATE TABLE `test`.`test` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `int_nullable` INT NULL DEFAULT NULL, + -> `float` FLOAT NOT NULL, + -> `float_nullable` FLOAT NULL DEFAULT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into test (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from test; ++------+----------+-----+----------+ +| int_id | int_nullable | float | float_nullable | ++------+----------+-----+----------+ +| 1 | NULL | 2 | NULL | ++------+----------+-----+----------+ +1 row in set (0,00 sec) +``` + +Clickhouse'daki MySQL tablosundan veri alma: + +``` sql +SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test') +``` + +``` text +┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐ +│ 1 │ 0 │ 2 │ 0 │ +└────────┴──────────────┴───────┴────────────────┘ +``` + +## Ayrıca Bakınız {#see-also} + +- [ODBC harici sözlükler](../../sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc) +- [ODBC tablo motoru](../../engines/table_engines/integrations/odbc.md). + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) diff --git a/docs/tr/sql_reference/table_functions/remote.md b/docs/tr/sql_reference/table_functions/remote.md new file mode 100644 index 00000000000..58c4154643c --- /dev/null +++ b/docs/tr/sql_reference/table_functions/remote.md @@ -0,0 +1,83 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 40 +toc_title: uzak +--- + +# uzaktan, remoteSecure {#remote-remotesecure} + +Oluşturmadan uzak sunuculara erişmenizi sağlar. `Distributed` Tablo. + +İmzalar: + +``` sql +remote('addresses_expr', db, table[, 'user'[, 'password']]) +remote('addresses_expr', db.table[, 'user'[, 'password']]) +``` + +`addresses_expr` – An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port` ya da sadece `host`. Ana bilgisayar sunucu adı veya IPv4 veya IPv6 adresi olarak belirtilebilir. Köşeli parantez içinde bir IPv6 adresi belirtilir. Bağlantı noktası, uzak sunucudaki TCP bağlantı noktasıdır. Bağlantı noktası atlanırsa, kullanır `tcp_port` sunucunun yapılandırma dosyasından (varsayılan olarak, 9000). + +!!! important "Önemli" + Bir IPv6 adresi için bağlantı noktası gereklidir. + +Örnekler: + +``` text +example01-01-1 +example01-01-1:9000 +localhost +127.0.0.1 +[::]:9000 +[2a02:6b8:0:1111::11]:9000 +``` + +Birden çok Adres virgülle ayrılmış olabilir. Bu durumda, ClickHouse dağıtılmış işleme kullanır, bu nedenle sorguyu belirtilen tüm adreslere gönderir (farklı verilerle kırıklar gibi). + +Örnek: + +``` text +example01-01-1,example01-02-1 +``` + +İfadenin bir kısmı kıvırcık parantez içinde belirtilebilir. Önceki örnek aşağıdaki gibi yazılabilir: + +``` text +example01-0{1,2}-1 +``` + +Kıvırcık parantez iki nokta (negatif olmayan tamsayılar) ile ayrılmış bir sayı aralığı içerebilir. Bu durumda, Aralık, shard adresleri üreten bir değer kümesine genişletilir. İlk sayı sıfır ile başlarsa, değerler aynı sıfır hizalamasıyla oluşturulur. Önceki örnek aşağıdaki gibi yazılabilir: + +``` text +example01-{01..02}-1 +``` + +Birden fazla kıvırcık parantez çiftiniz varsa, ilgili kümelerin doğrudan ürününü oluşturur. + +Adresler ve kıvırcık parantez içindeki adreslerin parçaları boru sembolü (\|) ile ayrılabilir. Bu durumda, karşılık gelen Adres kümeleri yinelemeler olarak yorumlanır ve sorgu ilk sağlıklı yinelemeye gönderilir. Ancak, yinelemeler şu anda ayarlanmış sırayla yinelenir [dengeleme](../../operations/settings/settings.md) ayar. + +Örnek: + +``` text +example01-{01..02}-{1|2} +``` + +Bu örnek, her birinin iki kopyası olan iki parçayı belirtir. + +Oluşturulan Adres sayısı bir sabit tarafından sınırlıdır. Şu anda bu 1000 Adres. + +Kullanarak `remote` tablo işlevi, bir `Distributed` tablo, çünkü bu durumda, her istek için sunucu bağlantısı yeniden kurulur. Buna ek olarak, ana bilgisayar adları ayarlanmışsa, adlar giderilir ve çeşitli yinelemelerle çalışırken hatalar sayılmaz. Çok sayıda sorgu işlerken, her zaman `Distributed` masa vaktinden önce, ve kullanmayın `remote` tablo işlevi. + +Bu `remote` tablo işlevi aşağıdaki durumlarda yararlı olabilir: + +- Veri karşılaştırma, hata ayıklama ve sınama için belirli bir sunucuya erişme. +- Araştırma amaçlı çeşitli ClickHouse kümeleri arasındaki sorgular. +- El ile yapılan seyrek dağıtılmış istekler. +- Sunucu kümesinin her seferinde yeniden tanımlandığı dağıtılmış istekler. + +Kullanıcı belirtilmemişse, `default` kullanılır. +Parola belirtilmezse, boş bir parola kullanılır. + +`remoteSecure` - aynı `remote` but with secured connection. Default port — [tcp\_port\_secure](../../operations/server_configuration_parameters/settings.md#server_configuration_parameters-tcp_port_secure) yapılandırma veya 9440'ten. + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/remote/) diff --git a/docs/tr/sql_reference/table_functions/url.md b/docs/tr/sql_reference/table_functions/url.md new file mode 100644 index 00000000000..bb841fd344c --- /dev/null +++ b/docs/tr/sql_reference/table_functions/url.md @@ -0,0 +1,26 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 41 +toc_title: url +--- + +# url {#url} + +`url(URL, format, structure)` - oluşturulan bir tablo döndürür `URL` verilen ile +`format` ve `structure`. + +Kabul edebilen URL - HTTP veya HTTPS sunucu adresi `GET` ve / veya `POST` istemler. + +biçimli - [biçimli](../../interfaces/formats.md#formats) verilerin. + +yapı-tablo yapısı `'UserID UInt64, Name String'` biçimli. Sütun adlarını ve türlerini belirler. + +**Örnek** + +``` sql +-- getting the first 3 lines of a table that contains columns of String and UInt32 type from HTTP-server which answers in CSV format. +SELECT * FROM url('http://127.0.0.1:12345/', CSV, 'column1 String, column2 UInt32') LIMIT 3 +``` + +[Orijinal makale](https://clickhouse.tech/docs/en/query_language/table_functions/url/) diff --git a/docs/tr/whats_new/changelog/2017.md b/docs/tr/whats_new/changelog/2017.md new file mode 100644 index 00000000000..2f31be7f6de --- /dev/null +++ b/docs/tr/whats_new/changelog/2017.md @@ -0,0 +1,268 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 79 +toc_title: '2017' +--- + +### ClickHouse sürüm 1.1.54327, 2017-12-21 {#clickhouse-release-1-1-54327-2017-12-21} + +Bu sürüm önceki sürüm 1.1.54318 için hata düzeltmeleri içerir: + +- Veri kaybına yol açabilir çoğaltma Olası yarış koşulu ile Sabit hata. Bu sorun sürümleri 1.1.54310 ve 1.1.54318 etkiler. Bu sürümlerden birini çoğaltılmış tablolarla kullanırsanız, güncelleştirme önerilir. Bu sorun, aşağıdaki gibi uyarı iletilerindeki günlüklerde gösterilir `Part ... from own log doesn't exist.` Bu iletileri günlüklerde görmeseniz bile sorun geçerlidir. + +### ClickHouse sürümü 1.1.54318, 2017-11-30 {#clickhouse-release-1-1-54318-2017-11-30} + +Bu sürüm önceki sürüm 1.1.54310 için hata düzeltmeleri içerir: + +- SummingMergeTree motorunda birleştirmeler sırasında hatalı satır silme işlemleri düzeltildi +- Unreplicated MergeTree motorlarında bir bellek sızıntısı düzeltildi +- MergeTree motorlarında sık sık eklerle sabit performans düşüşü +- Çoğaltma kuyruğunun çalışmayı durdurmasına neden olan bir sorun giderildi +- Sunucu günlüklerinin sabit rotasyonu ve arşivlenmesi + +### ClickHouse sürümü 1.1.54310, 2017-11-01 {#clickhouse-release-1-1-54310-2017-11-01} + +#### Yenilik: {#new-features} + +- Tablo motorları MergeTree ailesi için özel bölümleme anahtarı. +- [Kafka](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) masa motoru. +- Yükleme için destek eklendi [CatBoost](https://catboost.yandex/) modelleri ve ClickHouse saklanan verilere uygulayarak. +- UTC olmayan tamsayı uzaklıklar ile saat dilimleri için destek eklendi. +- Zaman aralıklarıyla aritmetik işlemler için destek eklendi. +- Tarih ve DateTime türleri için değer aralığı 2105 yılına genişletilir. +- Add theed the `CREATE MATERIALIZED VIEW x TO y` sorgu (materyalleştirilmiş bir görünümün verilerini depolamak için varolan bir tabloyu belirtir). +- Add theed the `ATTACH TABLE` argüman olmadan sorgu. +- Bir SummingMergeTree tablosunda-Map biten adları ile iç içe sütunlar için işleme mantığı için ayıklandı sumMap toplama işlevi. Şimdi bu tür sütunları açıkça belirtebilirsiniz. +- IP trie sözlüğünün maksimum boyutu 128M girişlerine yükseltilir. +- GetSizeOfEnumType işlevi eklendi. +- SumWithOverflow toplama işlevi eklendi. +- Cap'n Proto giriş biçimi için destek eklendi. +- Artık zstd algoritmasını kullanırken sıkıştırma seviyesini özelleştirebilirsiniz. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes} + +- Bellek dışında bir motor ile geçici tablolar oluşturulmasına izin verilmez. +- View veya MaterializedView altyapısı ile tabloların açık oluşturulmasına izin verilmez. +- Tablo oluşturma sırasında yeni bir onay örnekleme anahtar ifadesinin birincil anahtara dahil edildiğini doğrular. + +#### Hata düzeltmeleri: {#bug-fixes} + +- Sabit hangups zaman eşzamanlı bir dağıtılmış tabloya ekleme. +- Sabit atomik olmayan ekleme ve çoğaltılmış tablolarda parçaların çıkarılması. +- Materialized görünümüne eklenen veriler gereksiz tekilleştirme tabi değildir. +- Yerel yinelemenin geciktiği ve uzak yinelemelerin kullanılamadığı dağıtılmış bir tabloya bir sorgu yürütmek artık bir hataya neden olmaz. +- Kullanıcıların erişim izinlerine ihtiyacı yoktur `default` veritabanı artık geçici tablolar oluşturmak için. +- Bağımsız değişkenler olmadan dizi türünü belirtirken çökmesini düzeltildi. +- Sunucu günlüklerini içeren disk birimi dolu olduğunda sabit hangups. +- Unix döneminin ilk haftası için toRelativeWeekNum işlevinde bir taşma düzeltildi. + +#### İyileştirmeler oluşturun: {#build-improvements} + +- Birkaç üçüncü taraf Kütüphanesi (özellikle Poco) güncellendi ve git alt modüllerine dönüştürüldü. + +### ClickHouse sürümü 1.1.54304, 2017-10-19 {#clickhouse-release-1-1-54304-2017-10-19} + +#### Yenilik: {#new-features-1} + +- Yerel protokolde TLS desteği (etkinleştirmek, ayarlamak için `tcp_ssl_port` içinde `config.xml` ). + +#### Hata düzeltmeleri: {#bug-fixes-1} + +- `ALTER` çoğaltılmış tablolar için şimdi en kısa sürede çalışmaya başlamak çalışır. +- Ayarı ile veri okurken çökmesini sabit `preferred_block_size_bytes=0.` +- Sabit çöker `clickhouse-client` bas whenarken `Page Down` +- Bazı karmaşık sorguların doğru yorumlanması `GLOBAL IN` ve `UNION ALL` +- `FREEZE PARTITION` her zaman atomik olarak çalışır. +- Boş posta istekleri şimdi 411 koduyla bir yanıt döndürür. +- Gibi ifadeler için sabit yorumlama hataları `CAST(1 AS Nullable(UInt8)).` +- Okurken bir hata düzeltildi `Array(Nullable(String))` Col fromum fromns from `MergeTree` Tablolar. +- Gibi sorguları ayrıştırırken sabit çökmesini `SELECT dummy AS dummy, dummy AS b` +- Kullanıcılar geçersiz ile doğru güncellenir `users.xml` +- Yürütülebilir bir sözlük sıfır olmayan bir yanıt kodu döndürdüğünde doğru işleme. + +### ClickHouse yayın 1.1.54292, 2017-09-20 {#clickhouse-release-1-1-54292-2017-09-20} + +#### Yenilik: {#new-features-2} + +- Add theed the `pointInPolygon` koordinat düzleminde koordinatlarla çalışmak için işlev. +- Add theed the `sumMap` benzer dizilerin toplamını hesaplamak için toplama işlevi `SummingMergeTree`. +- Add theed the `trunc` işlev. Yuvarlama fonksiyonlarının geliştirilmiş performansı (`round`, `floor`, `ceil`, `roundToExp2`) ve nasıl çalıştıklarının mantığını düzeltti. Mantığını değiştirdi `roundToExp2` kesirler ve negatif sayılar için işlev. +- ClickHouse yürütülebilir dosyası artık libc sürümüne daha az bağımlıdır. Aynı ClickHouse yürütülebilir dosya Linux sistemleri çok çeşitli çalıştırabilirsiniz. Derlenmiş sorguları kullanırken hala bir bağımlılık var (ayar ile `compile = 1` , varsayılan olarak kullanılmaz). +- Sorguların dinamik derlenmesi için gereken süreyi azalttı. + +#### Hata düzeltmeleri: {#bug-fixes-2} + +- Bazen üretilen bir hata düzeltildi `part ... intersects previous part` mesajlar ve kopya tutarlılığı zayıfladı. +- ZooKeeper kapatma sırasında kullanılamıyorsa sunucunun kilitlenmesine neden olan bir hata düzeltildi. +- Kopyaları geri yüklerken aşırı günlüğü kaldırıldı. +- Birlik tüm uygulamasında bir hata düzeltildi. +- Bir bloktaki ilk sütun dizi türüne sahipse, concat işlevinde bir hata düzeltildi. +- İlerleme şimdi sistemde doğru görüntülenir.tablo birleştirir. + +### ClickHouse yayın 1.1.54289, 2017-09-13 {#clickhouse-release-1-1-54289-2017-09-13} + +#### Yenilik: {#new-features-3} + +- `SYSTEM` sunucu yönetimi için sorgular: `SYSTEM RELOAD DICTIONARY`, `SYSTEM RELOAD DICTIONARIES`, `SYSTEM DROP DNS CACHE`, `SYSTEM SHUTDOWN`, `SYSTEM KILL`. +- Dizilerle çalışmak için işlevler eklendi: `concat`, `arraySlice`, `arrayPushBack`, `arrayPushFront`, `arrayPopBack`, `arrayPopFront`. +- Katma `root` ve `identity` ZooKeeper yapılandırması için parametreler. Bu, aynı ZooKeeper kümesinde tek tek kullanıcıları izole etmenizi sağlar. +- Toplam fonksiyonları eklendi `groupBitAnd`, `groupBitOr`, ve `groupBitXor` (uyumluluk için, isimler altında da mevcuttur `BIT_AND`, `BIT_OR`, ve `BIT_XOR`). +- Dış sözlükler dosya sisteminde bir soket belirterek Mysql'den yüklenebilir. +- Harici sözlükler SSL üzerinden MySQL yüklenebilir (`ssl_cert`, `ssl_key`, `ssl_ca` parametre). +- Add theed the `max_network_bandwidth_for_user` kullanıcı başına sorgular için genel bant genişliği kullanımını kısıtlamak için ayarlama. +- İçin destek `DROP TABLE` geçici tablolar için. +- Okuma desteği `DateTime` Unix zaman damgası biçimindeki değerler `CSV` ve `JSONEachRow` biçimliler. +- Dağıtılmış sorgularda gecikmeli yinelemeler artık varsayılan olarak dışlanır (varsayılan eşik 5 dakikadır). +- FIFO kilitleme sırasında kullanılır ALTER: bir ALTER sorgusu sürekli çalışan sorgular için süresiz olarak engellenmez. +- Seçeneği ayarlamak için `umask` yapılandırma dosyasında. +- İle sorgular için geliştirilmiş performans `DISTINCT` . + +#### Hata düzeltmeleri: {#bug-fixes-3} + +- ZooKeeper eski düğümleri silme işlemi geliştirildi. Daha önce, eski düğümler bazen çok sık ekler varsa silinmedi, bu da sunucunun diğer şeylerin yanı sıra kapanması için yavaş olmasına neden oldu. +- ZooKeeper bağlantı için ana seçerken sabit randomizasyon. +- Çoğaltma localhost ise, dağıtılmış sorgularda gecikmiş kopyaların dışlanması düzeltildi. +- Bir veri parçası bir hata düzeltildi `ReplicatedMergeTree` tablo çalıştırdıktan sonra kırık olabilir `ALTER MODIFY` bir element üzerinde `Nested` yapılı. +- SELECT sorgularına neden olabilecek bir hata düzeltildi “hang”. +- Dağıtılmış DDL sorguları için iyileştirmeler. +- Sorgu düzeltildi `CREATE TABLE ... AS `. +- Çık themaz theı Çöz thedü `ALTER ... CLEAR COLUMN IN PARTITION` sorgu için `Buffer` Tablolar. +- İçin geçersiz varsayılan değer düzeltildi `Enum` kullanırken s (minimum yerine 0) `JSONEachRow` ve `TSKV` biçimliler. +- Bir sözlük ile bir sözlük kullanırken zombi süreçlerinin görünümünü çözdü `executable` kaynaklı. +- Kafa sorgusu için sabit segfault. + +#### ClickHouse geliştirmek ve birleştirmek için geliştirilmiş iş akışı: {#improved-workflow-for-developing-and-assembling-clickhouse} + +- Kullanabilirsiniz `pbuilder` ClickHouse inşa etmek. +- Kullanabilirsiniz `libc++` yerine `libstdc++` Linux üzerine inşa edilmiştir. +- Statik kod analiz araçlarını kullanma talimatları eklendi: `Coverage`, `clang-tidy`, `cppcheck`. + +#### Yükseltme yaparken lütfen unutmayın: {#please-note-when-upgrading} + +- MergeTree ayarı için artık daha yüksek bir varsayılan değer var `max_bytes_to_merge_at_max_space_in_pool` (bayt olarak birleştirilecek veri parçalarının maksimum toplam boyutu): 100 Gib'den 150 Gib'e yükseldi. Bu, disk Alt Sisteminde artan bir yüke neden olabilecek sunucu yükseltmesinden sonra çalışan büyük birleştirmelere neden olabilir. Sunucuda kullanılabilir boş alan, çalışan birleştirmelerin toplam miktarının iki katından azsa, bu, diğer tüm birleştirmelerin, küçük veri parçalarının birleştirmeleri de dahil olmak üzere çalışmayı durdurmasına neden olur. Sonuç olarak, INSERT sorguları iletiyle başarısız olur “Merges are processing significantly slower than inserts.” Kullan... `SELECT * FROM system.merges` durumu izlemek için sorgu. Ayrıca kontrol edebilirsiniz `DiskSpaceReservedForMerge` metr theik `system.metrics` tablo veya Grafit. Bunu düzeltmek için hiçbir şey yapmanıza gerek yoktur, çünkü büyük birleşimler bittikten sonra sorun kendiliğinden çözülür. Bu kabul edilemez bulursanız, önceki değeri geri yükleyebilirsiniz `max_bytes_to_merge_at_max_space_in_pool` ayar. Bunu yapmak için, gidin config bölümünde.xml, set ``` ``107374182400 ``` ve sunucuyu yeniden başlatın. + +### ClickHouse sürümü 1.1.54284, 2017-08-29 {#clickhouse-release-1-1-54284-2017-08-29} + +- Bu, önceki 1.1.54282 sürümü için bir hata düzeltme sürümüdür. Bu ZooKeeper parçalar dizininde sızıntıları giderir. + +### ClickHouse yayın 1.1.54282, 2017-08-23 {#clickhouse-release-1-1-54282-2017-08-23} + +Bu sürüm önceki sürüm 1.1.54276 için hata düzeltmeleri içerir: + +- Sabit `DB::Exception: Assertion violation: !_path.empty()` dağıtılmış bir tabloya eklerken. +- Giriş verileri'; ' ile başlarsa RowBinary biçiminde eklerken ayrıştırma düzeltildi. +- Errors during runtime compilation of certain aggregate functions (e.g. `groupArray()`). + +### ClickHouse Yayın 1.1.54276, 2017-08-16 {#clickhouse-release-1-1-54276-2017-08-16} + +#### Yenilik: {#new-features-4} + +- Bir seçme sorgusu için bölüm ile isteğe bağlı eklendi. Örnek sorgu: `WITH 1+1 AS a SELECT a, a*a` +- INSERT, dağıtılmış bir tabloda eşzamanlı olarak gerçekleştirilebilir: Tamam, yalnızca tüm veriler tüm parçalara kaydedildikten sonra döndürülür. Bu ayar tarafından etkinleştirilir ınsert\_distributed\_sync = 1. +- 16 baytlık tanımlayıcılarla çalışmak için UUID veri türü eklendi. +- Tablo ile uyumluluk için CHAR, FLOAT ve diğer türlerin takma adları eklendi. +- Sayılara zaman dönüştürmek için fonksiyonları toYYYYMM, toYYYYMMDD ve toYYYYMMDDhhmmss eklendi. +- Kümelenmiş DDL sorguları için sunucuları tanımlamak için IP adreslerini (ana bilgisayar adı ile birlikte) kullanabilirsiniz. +- İşlevde sabit olmayan argümanlar ve negatif uzaklıklar için destek eklendi `substring(str, pos, len).` +- İçin max\_size parametresi eklendi `groupArray(max_size)(column)` toplama işlevi ve performansını optimize etti. + +#### Ana değişiklikler: {#main-changes} + +- Güvenlik iyileştirmeleri: tüm sunucu dosyaları 0640 izinleriyle oluşturulur (üzerinden değiştirilebilir config parametresi). +- Geçersiz sözdizimi ile sorgular için geliştirilmiş hata mesajları. +- MergeTree verilerinin büyük bölümlerini birleştirirken bellek tüketimini önemli ölçüde azalttı ve performansı artırdı. +- ReplacingMergeTree motoru için veri birleştirmelerinin performansını önemli ölçüde artırdı. +- Birden çok kaynak ekler birleştirerek dağıtılmış bir tablodan zaman uyumsuz ekler için geliştirilmiş performans. Bu işlevi etkinleştirmek için distributed\_directory\_monitor\_batch\_ınserts = 1 ayarını kullanın. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-1} + +- Toplam durumların ikili biçimini değiştirdi `groupArray(array_column)` diziler için fonksiyonlar. + +#### Değişikliklerin tam listesi: {#complete-list-of-changes} + +- Add theed the `output_format_json_quote_denormals` json formatında nan ve ınf değerleri çıktısı sağlayan ayar. +- Dağıtılmış bir tablodan okurken optimize edilmiş akış tahsisi. +- Değer değişmezse ayarlar salt okunur modda yapılandırılabilir. +- Preferred\_block\_size\_bytes ayarında belirtilen blok boyutu üzerindeki kısıtlamaları karşılamak için mergetree motorunun tamsayı olmayan granüllerini alma yeteneği eklendi. Amaç, RAM tüketimini azaltmak ve büyük sütunlu tablolardan gelen sorguları işlerken önbellek konumunu arttırmaktır. +- Gibi ifadeler içeren dizinlerin verimli kullanımı `toStartOfHour(x)` gibi koşullar için `toStartOfHour(x) op сonstexpr.` +- MergeTree motorları için yeni ayarlar eklendi (yapılandırmada merge\_tree bölümü.xml): + - replicated\_deduplication\_window\_seconds yinelenen tablolar ekler tekilleştirme için izin verilen saniye sayısını ayarlar. + - cleanup\_delay\_period, eski verileri kaldırmak için temizleme işleminin ne sıklıkta başlatılacağını ayarlar. + - replicated\_can\_become\_leader, bir kopyanın lider olmasını (ve birleştirme atamasını) engelleyebilir. +- Hızlandırılmış Temizleme ZooKeeper eski verileri kaldırmak için. +- Kümelenmiş DDL sorguları için birden fazla iyileştirme ve düzeltme. Özellikle ilgi çekici olan yeni ayardır distributed\_ddl\_task\_timeout, kümedeki sunuculardan bir yanıt beklemek için zamanı sınırlar. Tüm ana bilgisayarlarda bir ddl isteği gerçekleştirilmediyse, bir yanıt bir zaman aşımı hatası içerir ve bir zaman uyumsuz modunda bir istek yürütülür. +- Sunucu günlüklerinde yığın izlerinin geliştirilmiş gösterimi. +- Add theed the “none” sıkıştırma yöntemi için değer. +- Yapılandırmada birden çok dictionaries\_config bölümünü kullanabilirsiniz.xml. +- Bu dosya sisteminde bir soket üzerinden MySQL bağlanmak mümkündür. +- Sistem.parçalar tablosu, bayt cinsinden işaretlerin boyutu hakkında bilgi içeren yeni bir sütuna sahiptir. + +#### Hata düzeltmeleri: {#bug-fixes-4} + +- Bir birleştirme tablosu kullanarak dağıtılmış tablolar artık bir koşulla bir SELECT sorgusu için doğru şekilde çalışır. `_table` alan. +- Veri parçalarını kontrol ederken ReplicatedMergeTree nadir bir yarış durumu düzeltildi. +- Sabit Olası donma “leader election” bir sunucu başlatırken. +- Veri kaynağının yerel bir kopyasını kullanırken max\_replica\_delay\_for\_distributed\_queries ayarı göz ardı edildi. Bu sorun giderildi. +- Sabit yanlış davranış `ALTER TABLE CLEAR COLUMN IN PARTITION` varolan olmayan bir sütunu temizlemeye çalışırken. +- Boş diziler veya dizeleri kullanırken multiİf işlevinde bir istisna düzeltildi. +- Sabit aşırı bellek ayırmaları yerel biçimi serisini kaldırırken. +- Trie sözlüklerin sabit yanlış otomatik güncelleme. +- Örnek kullanırken bir birleştirme tablosundan bir GROUP BY yan tümcesi ile sorguları çalıştırırken bir özel durum düzeltildi. +- Distributed\_aggregation\_memory\_efficient = 1 kullanırken grup çökmesi düzeltildi. +- Şimdi veritabanını belirtebilirsiniz.ın ve JOİN sağ tarafında tablo. +- Paralel toplama için çok fazla iplik kullanıldı. Bu sorun giderildi. +- Sabit nasıl “if” işlev FixedString argümanları ile çalışır. +- 0 ağırlığında kırıkları için dağıtılmış bir tablodan yanlış çalıştı seçin. Bu sorun giderildi. +- Çalışma `CREATE VIEW IF EXISTS no longer causes crashes.` +- Sabit yanlış davranış input\_format\_skip\_unknown\_fields=1 ayarlanır ve negatif sayılar vardır. +- Sabit bir sonsuz döngü içinde `dictGetHierarchy()` sözlükte bazı geçersiz veriler varsa işlev. +- Sabit `Syntax error: unexpected (...)` bir In veya JOIN yan tümcesi ve birleştirme tablolarındaki alt sorgularla dağıtılmış sorguları çalıştırırken hatalar. +- Sözlük tablolarından bir seçme sorgusunun yanlış yorumlanması düzeltildi. +- Sabit “Cannot mremap” 2 milyardan fazla öğe içeren ın ve JOIN yan tümcelerinde diziler kullanılırken hata oluştu. +- Kaynak olarak MySQL ile sözlükler için yük devretme düzeltildi. + +#### ClickHouse geliştirmek ve birleştirmek için geliştirilmiş iş akışı: {#improved-workflow-for-developing-and-assembling-clickhouse-1} + +- Yapılar Arcadia'da monte edilebilir. +- Clickhouse'u derlemek için gcc 7'yi kullanabilirsiniz. +- Ccache+distcc kullanarak paralel yapılar artık daha hızlı. + +### ClickHouse yayın 1.1.54245, 2017-07-04 {#clickhouse-release-1-1-54245-2017-07-04} + +#### Yenilik: {#new-features-5} + +- Dağıtılmış DDL (örneğin, `CREATE TABLE ON CLUSTER`) +- Çoğaltılan sorgu `ALTER TABLE CLEAR COLUMN IN PARTITION.` +- Sözlük tabloları için motor (bir tablo şeklinde sözlük verilerine erişim). +- Sözlük veritabanı motoru (bu tür veritabanı otomatik olarak bağlı tüm dış sözlükler için sözlük tabloları vardır). +- Kaynağa bir istek göndererek sözlükteki güncellemeleri kontrol edebilirsiniz. +- Nitelikli sütun adları +- Çift tırnak işaretleri kullanarak tanımlayıcıları alıntı. +- HTTP arabirimindeki oturumlar. +- Çoğaltılmış bir tablo için en iyi duruma getirme sorgusu yalnızca lider üzerinde çalışabilir. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-2} + +- Kaldırıldı set GLOBAL. + +#### Küçük değişiklikler: {#minor-changes} + +- Şimdi bir uyarı tetiklendikten sonra günlük tam yığın izleme yazdırır. +- Başlangıçta hasarlı / ekstra veri parçalarının sayısının doğrulanmasını rahatlattı(çok fazla yanlış pozitif vardı). + +#### Hata düzeltmeleri: {#bug-fixes-5} + +- Kötü bir bağlantı düzeltildi “sticking” dağıtılmış bir tabloya eklerken. +- GLOBAL IN şimdi dağıtılmış bir tabloya bakan bir birleştirme tablosundan bir sorgu için çalışır. +- Bir Google Compute Engine sanal makinesinde yanlış sayıda çekirdek tespit edildi. Bu sorun giderildi. +- Önbelleğe alınmış harici sözlüklerin yürütülebilir bir kaynağının nasıl çalıştığındaki değişiklikler. +- Null karakter içeren dizelerin karşılaştırması düzeltildi. +- Float32 birincil anahtar alanlarının sabitler ile karşılaştırılması düzeltildi. +- Önceden, bir alanın boyutunun yanlış bir tahmini, aşırı büyük tahsisatlara neden olabilir. +- Alter kullanarak bir tabloya eklenen null bir sütun sorgularken bir kilitlenme düzeltildi. +- Satır sayısı sınırdan az ise, null bir sütuna göre sıralama yaparken bir kilitlenme düzeltildi. +- Yalnızca sabit değerlerden oluşan alt sorgu ile bir sipariş düzeltildi. +- Daha önce, çoğaltılmış bir tablo başarısız bir bırakma tablosundan sonra geçersiz durumda kalabilir. +- Boş sonuçları olan skaler alt sorgular için takma adlar artık kaybolmaz. +- Şimdi derleme kullanılan bir sorgu .so dosyası zarar görürse bir hata ile başarısız değil. diff --git a/docs/tr/whats_new/changelog/2018.md b/docs/tr/whats_new/changelog/2018.md new file mode 100644 index 00000000000..884c1725bbd --- /dev/null +++ b/docs/tr/whats_new/changelog/2018.md @@ -0,0 +1,1063 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 78 +toc_title: '2018' +--- + +## ClickHouse sürümü 18.16 {#clickhouse-release-18-16} + +### ClickHouse sürümü 18.16.1, 2018-12-21 {#clickhouse-release-18-16-1-2018-12-21} + +#### Hata düzeltmeleri: {#bug-fixes} + +- ODBC kaynağı ile sözlükleri güncelleme ile ilgili sorunlara yol açan bir hata düzeltildi. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) +- Toplama işlevlerinin JIT derlemesi artık LowCardinality sütunlarıyla çalışır. [\#3838](https://github.com/ClickHouse/ClickHouse/issues/3838) + +#### Geliştirmeler: {#improvements} + +- Add theed the `low_cardinality_allow_in_native_format` ayar (varsayılan: etkin). Devre dışı bırakıldığında, LOWCARDİNALİTY sütunları, SELECT sorguları için sıradan sütunlara dönüştürülür ve INSERT sorguları için sıradan sütunlar beklenir. [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) + +#### İyileştirmeler oluşturun: {#build-improvements} + +- MacOS ve ARM üzerine inşa için düzeltmeler. + +### ClickHouse yayın 18.16.0, 2018-12-14 {#clickhouse-release-18-16-0-2018-12-14} + +#### Yenilik: {#new-features} + +- `DEFAULT` ifadeler, yarı yapılandırılmış giriş biçimlerinde veri yüklenirken eksik alanlar için değerlendirilir (`JSONEachRow`, `TSKV`). Özelliği ile etkin `insert_sample_with_metadata` ayar. [\#3555](https://github.com/ClickHouse/ClickHouse/pull/3555) +- Bu `ALTER TABLE` sorgu şimdi var `MODIFY ORDER BY` bir tablo sütunu eklerken veya kaldırırken sıralama anahtarını değiştirme eylemi. Bu tablolar için yararlıdır `MergeTree` bu sıralama anahtarına dayalı olarak birleştirilirken ek görevler gerçekleştiren aile `SummingMergeTree`, `AggregatingMergeTree` ve bu yüzden. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) [\#3755](https://github.com/ClickHouse/ClickHouse/pull/3755) +- Tablolar için `MergeTree` aile, şimdi farklı bir sıralama anahtarı belirtebilirsiniz (`ORDER BY` ve dizin (`PRIMARY KEY`). Sıralama anahtarı dizin daha uzun olabilir. [\#3581](https://github.com/ClickHouse/ClickHouse/pull/3581) +- Add theed the `hdfs` tablo fonksiyonu ve `HDFS` hdfs'ye veri içe ve dışa aktarmak için tablo motoru. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/3617) +- Base64 ile çalışmak için fonksiyonlar eklendi: `base64Encode`, `base64Decode`, `tryBase64Decode`. [Alexander Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3350) +- Şimdi hassasiyetini yapılandırmak için bir parametre kullanabilirsiniz `uniqCombined` toplama işlevi (HyperLogLog hücrelerinin sayısını seçin). [\#3406](https://github.com/ClickHouse/ClickHouse/pull/3406) +- Add theed the `system.contributors` ClickHouse taahhüt yapılan herkesin adlarını içeren tablo. [\#3452](https://github.com/ClickHouse/ClickHouse/pull/3452) +- Bölüm için ihmal yeteneği eklendi `ALTER TABLE ... FREEZE` tüm bölümleri bir kerede yedeklemek için sorgu. [\#3514](https://github.com/ClickHouse/ClickHouse/pull/3514) +- Katma `dictGet` ve `dictGetOrDefault` dönüş değeri türünü belirtmeyi gerektirmeyen işlevler. Tür sözlük açıklamasından otomatik olarak belirlenir. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3564) +- Şimdi tablo açıklamasında bir sütun için yorum belirtmek ve kullanarak değiştirebilirsiniz `ALTER`. [\#3377](https://github.com/ClickHouse/ClickHouse/pull/3377) +- Okuma için desteklenir `Join` basit tuşlarla tabloları yazın. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3728) +- Şimdi seçenekleri belirtebilirsiniz `join_use_nulls`, `max_rows_in_join`, `max_bytes_in_join`, ve `join_overflow_mode` oluştururken bir `Join` tablo yazın. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3728) +- Add theed the `joinGet` kullan allowsmanıza olanak sağlayan bir işlev `Join` bir sözlük gibi tablo yazın. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3728) +- Add theed the `partition_key`, `sorting_key`, `primary_key`, ve `sampling_key` Col theum thens to the `system.tables` tablo tuşları hakkında bilgi vermek için tablo. [\#3609](https://github.com/ClickHouse/ClickHouse/pull/3609) +- Add theed the `is_in_partition_key`, `is_in_sorting_key`, `is_in_primary_key`, ve `is_in_sampling_key` Col theum thens to the `system.columns` Tablo. [\#3609](https://github.com/ClickHouse/ClickHouse/pull/3609) +- Add theed the `min_time` ve `max_time` Col theum thens to the `system.parts` Tablo. Bölümleme anahtarı aşağıdakilerden oluşan bir ifade olduğunda bu sütunlar doldurulur `DateTime` sütun. [Emmanuel Donin de Rosière](https://github.com/ClickHouse/ClickHouse/pull/3800) + +#### Hata düzeltmeleri: {#bug-fixes-1} + +- Düzeltmeler ve performans iyileştirmeleri için `LowCardinality` veri türü. `GROUP BY` kullanım `LowCardinality(Nullable(...))`. Değerlerini almak `extremes`. Yüksek mertebeden fonksiyonların işlenmesi. `LEFT ARRAY JOIN`. Dağılı `GROUP BY`. Dönen işlevler `Array`. Yürütme `ORDER BY`. Yazma `Distributed` tablolar (nicelulu). Geriye dönük uyumluluk için `INSERT` uygulayan eski istemcilerden gelen sorgular `Native` protokol. İçin destek `LowCardinality` için `JOIN`. Tek bir akışta çalışırken geliştirilmiş performans. [\#3823](https://github.com/ClickHouse/ClickHouse/pull/3823) [\#3803](https://github.com/ClickHouse/ClickHouse/pull/3803) [\#3799](https://github.com/ClickHouse/ClickHouse/pull/3799) [\#3769](https://github.com/ClickHouse/ClickHouse/pull/3769) [\#3744](https://github.com/ClickHouse/ClickHouse/pull/3744) [\#3681](https://github.com/ClickHouse/ClickHouse/pull/3681) [\#3651](https://github.com/ClickHouse/ClickHouse/pull/3651) [\#3649](https://github.com/ClickHouse/ClickHouse/pull/3649) [\#3641](https://github.com/ClickHouse/ClickHouse/pull/3641) [\#3632](https://github.com/ClickHouse/ClickHouse/pull/3632) [\#3568](https://github.com/ClickHouse/ClickHouse/pull/3568) [\#3523](https://github.com/ClickHouse/ClickHouse/pull/3523) [\#3518](https://github.com/ClickHouse/ClickHouse/pull/3518) +- Sabit nasıl `select_sequential_consistency` seçenek çalışır. Daha önce, bu ayar etkinleştirildiğinde, bazen yeni bir bölüme yazmaya başladıktan sonra tamamlanmamış bir sonuç döndürüldü. [\#2863](https://github.com/ClickHouse/ClickHouse/pull/2863) +- DDL yürütülürken veritabanları doğru belirtilir `ON CLUSTER` sorgular ve `ALTER UPDATE/DELETE`. [\#3772](https://github.com/ClickHouse/ClickHouse/pull/3772) [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) +- Veritabanları doğru bir görünüm içinde alt sorgular için belirtilir. [\#3521](https://github.com/ClickHouse/ClickHouse/pull/3521) +- Bir hata düzeltildi `PREWHERE` ile `FINAL` için `VersionedCollapsingMergeTree`. [7167bfd7](https://github.com/ClickHouse/ClickHouse/commit/7167bfd7b365538f7a91c4307ad77e552ab4e8c1) +- Şimdi kullanabilirsiniz `KILL QUERY` henüz başlatılmamış olan sorguları iptal etmek için, tablonun kilitlenmesini bekliyorlar. [\#3517](https://github.com/ClickHouse/ClickHouse/pull/3517) +- Saatlerin gece yarısı geri hareket ettirilmesi durumunda düzeltilmiş tarih ve saat hesaplamaları (bu İran'da olur ve Moskova'da 1981'den 1983'e kadar olur). Önceden, bu, gerekenden bir gün önce sıfırlanma süresine yol açtı ve ayrıca tarih ve saatin metin biçiminde yanlış biçimlendirilmesine neden oldu. [\#3819](https://github.com/ClickHouse/ClickHouse/pull/3819) +- Bazı durumlarda sabit hatalar `VIEW` ve veritabanını atlayan alt sorgular. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3521) +- Aynı anda bir yarıştan okurken bir yarış durumu düzeltildi `MATERIALIZED VIEW` ve Silme bir `MATERIALIZED VIEW` nedeniyle değil kilitleme iç `MATERIALIZED VIEW`. [\#3404](https://github.com/ClickHouse/ClickHouse/pull/3404) [\#3694](https://github.com/ClickHouse/ClickHouse/pull/3694) +- Hata düzeltildi `Lock handler cannot be nullptr.` [\#3689](https://github.com/ClickHouse/ClickHouse/pull/3689) +- Sabit sorgu işleme zaman `compile_expressions` seçenek etkindir (varsayılan olarak etkindir). Gibi Nondeterministic sabit ifadeler `now` işlev artık açılmıyor. [\#3457](https://github.com/ClickHouse/ClickHouse/pull/3457) +- Sabit olmayan bir ölçek argümanı belirtilirken bir kilitlenme düzeltildi `toDecimal32/64/128` işlevler. +- Bir dizi eklemeye çalışırken bir hata düzeltildi `NULL` element inler `Values` bir sütuna Biçimlendir türü `Array` olarak `Nullable` (eğer `input_format_values_interpret_expressions` = 1). [\#3487](https://github.com/ClickHouse/ClickHouse/pull/3487) [\#3503](https://github.com/ClickHouse/ClickHouse/pull/3503) +- Sabit sürekli hata günlüğü `DDLWorker` ZooKeeper mevcut değilse. [8f50c620](https://github.com/ClickHouse/ClickHouse/commit/8f50c620334988b28018213ec0092fe6423847e2) +- Sabit dönüş türü için `quantile*` gelen fonksiyonlar `Date` ve `DateTime` argüman türleri. [\#3580](https://github.com/ClickHouse/ClickHouse/pull/3580) +- Sabit `WITH` Ifade olmadan basit bir takma ad belirtirse yan tümcesi. [\#3570](https://github.com/ClickHouse/ClickHouse/pull/3570) +- Adlandırılmış alt sorgular ve nitelikli sütun adları ile sorguların sabit işleme `enable_optimize_predicate_expression` etkindir. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3588) +- Hata düzeltildi `Attempt to attach to nullptr thread group` maddi görüşlerle çalışırken. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3623) +- Belirli yanlış argümanları iletirken bir kilitlenme düzeltildi `arrayReverse` işlev. [73e3a7b6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) +- Tampon taş theması Düzelt theildi `extractURLParameter` işlev. Geliştirilmiş performans. Sıfır bayt içeren dizelerin doğru işlenmesi eklendi. [141e9799](https://github.com/ClickHouse/ClickHouse/commit/141e9799e49201d84ea8e951d1bed4fb6d3dacb5) +- Sabit tampon taşması `lowerUTF8` ve `upperUTF8` işlevler. Üzerinde bu işlevleri yürütmek için yeteneği kaldırıldı `FixedString` bağımsız değişkenleri yazın. [\#3662](https://github.com/ClickHouse/ClickHouse/pull/3662) +- Silerken nadir bir yarış durumu düzeltildi `MergeTree` Tablolar. [\#3680](https://github.com/ClickHouse/ClickHouse/pull/3680) +- Okuma sırasında bir yarış durumu düzeltildi `Buffer` tablolar ve aynı anda performans `ALTER` veya `DROP` hedef masalarda. [\#3719](https://github.com/ClickHouse/ClickHouse/pull/3719) +- Sabit bir segfault eğer `max_temporary_non_const_columns` sınır aşıldı. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) + +#### Geliştirmeler: {#improvements-1} + +- Sunucu için işlenmiş yapılandırma dosyalarını yazmaz `/etc/clickhouse-server/` dizin. Bunun yerine, onları kaydeder `preprocessed_configs` içindeki dizin `path`. Bu demektir `/etc/clickhouse-server/` dizin için yazma erişimi yok `clickhouse` güvenliği artıran kullanıcı. [\#2443](https://github.com/ClickHouse/ClickHouse/pull/2443) +- Bu `min_merge_bytes_to_use_direct_io` seçenek varsayılan olarak 10 GiB olarak ayarlanır. Mergetree ailesinden tabloların büyük bölümlerini oluşturan bir birleştirme gerçekleştirilir `O_DIRECT` aşırı sayfa önbellek tahliyesini önleyen mod. [\#3504](https://github.com/ClickHouse/ClickHouse/pull/3504) +- Hızlandırılmış sunucu, çok sayıda tablo olduğunda başlar. [\#3398](https://github.com/ClickHouse/ClickHouse/pull/3398) +- Bir bağlantı havuzu ve HTTP eklendi `Keep-Alive` yinelemeler arasındaki bağlantılar için. [\#3594](https://github.com/ClickHouse/ClickHouse/pull/3594) +- Sorgu sözdizimi geçersiz ise, `400 Bad Request` kod iade edilir `HTTP` arabirim (500 daha önce iade edildi). [31bc680a](https://github.com/ClickHouse/ClickHouse/commit/31bc680ac5f4bb1d0360a8ba4696fa84bb47d6ab) +- Bu `join_default_strictness` option is set to `ALL` uyumluluk için varsayılan olarak. [120e2cbe](https://github.com/ClickHouse/ClickHouse/commit/120e2cbe2ff4fbad626c28042d9b28781c805afe) +- Kaldırılan günlüğü `stderr` şuradan `re2` geçersiz veya karmaşık düzenli ifadeler için kütüphane. [\#3723](https://github.com/ClickHouse/ClickHouse/pull/3723) +- İçin eklendi `Kafka` tablo altyapısı: kafka'dan okumaya başlamadan önce abonelikleri denetler; tablo için kafka\_max\_block\_size ayarı. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3396) +- Bu `cityHash64`, `farmHash64`, `metroHash64`, `sipHash64`, `halfMD5`, `murmurHash2_32`, `murmurHash2_64`, `murmurHash3_32`, ve `murmurHash3_64` işlevler artık herhangi bir sayıda bağımsız değişken ve bağımsız değişkenler için tuples şeklinde çalışır. [\#3451](https://github.com/ClickHouse/ClickHouse/pull/3451) [\#3519](https://github.com/ClickHouse/ClickHouse/pull/3519) +- Bu `arrayReverse` işlev artık herhangi bir dizi türü ile çalışır. [73e3a7b6](https://github.com/ClickHouse/ClickHouse/commit/73e3a7b662161d6005e7727d8a711b930386b871) +- İsteğe bağlı bir parametre eklendi: yuva boyutu için `timeSlots` işlev. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3724) +- İçin `FULL` ve `RIGHT JOIN`, bu `max_block_size` ayar, sağ tablodan birleştirilmemiş veri akışı için kullanılır. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3699) +- Add theed the `--secure` komut satırı parametresi `clickhouse-benchmark` ve `clickhouse-performance-test` TLS'Yİ etkinleştirmek için. [\#3688](https://github.com/ClickHouse/ClickHouse/pull/3688) [\#3690](https://github.com/ClickHouse/ClickHouse/pull/3690) +- Tür dönüştürme zaman yapısı bir `Buffer` tür tablo hedef tablonun yapısıyla eşleşmiyor. [Vitaly Baranov](https://github.com/ClickHouse/ClickHouse/pull/3603) +- Add theed the `tcp_keep_alive_timeout` belirtilen zaman aralığı için hareketsizlikten sonra canlı tutma paketlerini etkinleştirme seçeneği. [\#3441](https://github.com/ClickHouse/ClickHouse/pull/3441) +- Bölüm anahtarı için değerlerin gereksiz alıntı kaldırıldı `system.parts` tablo tek bir sütundan oluşuyorsa. [\#3652](https://github.com/ClickHouse/ClickHouse/pull/3652) +- Modulo fonksiyonu için çalışır `Date` ve `DateTime` veri türleri. [\#3385](https://github.com/ClickHouse/ClickHouse/pull/3385) +- İçin eş anlamlı eklendi `POWER`, `LN`, `LCASE`, `UCASE`, `REPLACE`, `LOCATE`, `SUBSTR`, ve `MID` işlevler. [\#3774](https://github.com/ClickHouse/ClickHouse/pull/3774) [\#3763](https://github.com/ClickHouse/ClickHouse/pull/3763) Bazı işlev adları, SQL standardı ile uyumluluk için büyük / küçük harf duyarsızdır. Sözdizimsel şeker eklendi `SUBSTRING(expr FROM start FOR length)` SQL ile uyumluluk için. [\#3804](https://github.com/ClickHouse/ClickHouse/pull/3804) +- Yeteneği eklendi `mlock` karşılık gelen bellek sayfaları `clickhouse-server` bellek yetersiz zorla önlemek için yürütülebilir kod. Bu özellik varsayılan olarak devre dışıdır. [\#3553](https://github.com/ClickHouse/ClickHouse/pull/3553) +- Okuma sırasında geliştirilmiş performans `O_DIRECT` (ile `min_bytes_to_use_direct_io` seçeneği etkin). [\#3405](https://github.com/ClickHouse/ClickHouse/pull/3405) +- Geliştirilmiş performans `dictGet...OrDefault` sabit anahtar bağımsız değişkeni ve sabit olmayan bir varsayılan bağımsız değişken için işlev. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3563) +- Bu `firstSignificantSubdomain` işlev artık etki alanlarını işler `gov`, `mil`, ve `edu`. [Igor Hatarist](https://github.com/ClickHouse/ClickHouse/pull/3601) Geliştirilmiş performans. [\#3628](https://github.com/ClickHouse/ClickHouse/pull/3628) +- Başlangıç için özel ortam değişkenlerini belirleme yeteneği `clickhouse-server` kullanarak `SYS-V init.d` tanım bylayarak script `CLICKHOUSE_PROGRAM_ENV` içinde `/etc/default/clickhouse`. + [Pavlo Bashynskyi](https://github.com/ClickHouse/ClickHouse/pull/3612) +- Clickhouse-server init komut dosyası için doğru dönüş kodu. [\#3516](https://github.com/ClickHouse/ClickHouse/pull/3516) +- Bu `system.metrics` tablo şimdi var `VersionInteger` metr andik ve `system.build_options` eklenen satır var `VERSION_INTEGER`, ClickHouse sürümünün sayısal formunu içeren, örneğin `18016000`. [\#3644](https://github.com/ClickHouse/ClickHouse/pull/3644) +- Karşılaştırma yeteneği kaldırıldı `Date` gibi olası hataları önlemek için bir sayı ile yazın `date = 2018-12-17`, tarih etrafında tırnak yanlışlıkla ihmal nerede. [\#3687](https://github.com/ClickHouse/ClickHouse/pull/3687) +- Durumsal işlevlerin davranışı gibi düzeltildi `rowNumberInAllBlocks`. Daha önce sorgu analizi sırasında başlatma nedeniyle bir sayı daha büyük olan bir sonuç çıkardılar. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3729) +- Eğer... `force_restore_data` dosya silinemez, bir hata mesajı görüntülenir. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3794) + +#### İyileştirmeler oluşturun: {#build-improvements-1} + +- Güncelleme `jemalloc` olası bir bellek sızıntısını gideren kitaplık. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3557) +- İle profil profilingleme `jemalloc` hata ayıklama yapıları için varsayılan olarak etkindir. [2cc82f5c](https://github.com/ClickHouse/ClickHouse/commit/2cc82f5cbe266421cd4c1165286c2c47e5ffcb15) +- Yalnızca entegrasyon testlerini çalıştırma yeteneği eklendi `Docker` sistemde yüklü. [\#3650](https://github.com/ClickHouse/ClickHouse/pull/3650) +- Select sorgularında fuzz ifade testi eklendi. [\#3442](https://github.com/ClickHouse/ClickHouse/pull/3442) +- Daha fazla yarış koşullarını tespit etmek için paralel ve rastgele sırayla fonksiyonel testler gerçekleştiren taahhütler için bir stres testi eklendi. [\#3438](https://github.com/ClickHouse/ClickHouse/pull/3438) +- Bir Docker görüntüsünde clickhouse-server başlatmak için yöntem geliştirildi. [Elghazal Ahmed](https://github.com/ClickHouse/ClickHouse/pull/3663) +- Bir Docker görüntüsü için, dosyaları kullanarak veritabanlarını başlatmak için destek eklendi `/docker-entrypoint-initdb.d` dizin. [Konstantin Lebedev](https://github.com/ClickHouse/ClickHouse/pull/3695) +- Arm üzerine inşa için düzeltmeler. [\#3709](https://github.com/ClickHouse/ClickHouse/pull/3709) + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes} + +- Karşılaştırma yeteneği kaldırıldı `Date` bir sayı ile yazın. Yerine `toDate('2018-12-18') = 17883`, açık tür dönüştürme kullanmanız gerekir `= toDate(17883)` [\#3687](https://github.com/ClickHouse/ClickHouse/pull/3687) + +## ClickHouse sürümü 18.14 {#clickhouse-release-18-14} + +### ClickHouse yayın 18.14.19, 2018-12-19 {#clickhouse-release-18-14-19-2018-12-19} + +#### Hata düzeltmeleri: {#bug-fixes-2} + +- ODBC kaynağı ile sözlükleri güncelleme ile ilgili sorunlara yol açan bir hata düzeltildi. [\#3825](https://github.com/ClickHouse/ClickHouse/issues/3825), [\#3829](https://github.com/ClickHouse/ClickHouse/issues/3829) +- DDL yürütülürken veritabanları doğru belirtilir `ON CLUSTER` sorgular. [\#3460](https://github.com/ClickHouse/ClickHouse/pull/3460) +- Sabit bir segfault eğer `max_temporary_non_const_columns` sınır aşıldı. [\#3788](https://github.com/ClickHouse/ClickHouse/pull/3788) + +#### İyileştirmeler oluşturun: {#build-improvements-2} + +- Arm üzerine inşa için düzeltmeler. + +### ClickHouse yayın 18.14.18, 2018-12-04 {#clickhouse-release-18-14-18-2018-12-04} + +#### Hata düzeltmeleri: {#bug-fixes-3} + +- Sabit hata `dictGet...` tip sözlükler için işlev `range`, argümanlardan biri sabit ve diğeri değilse. [\#3751](https://github.com/ClickHouse/ClickHouse/pull/3751) +- Mesajlara neden olan Sabit hata `netlink: '...': attribute type 1 has an invalid length` Linux çekirdeği günlüğüne yazdırılmak üzere, bu sadece Linux çekirdeğinin yeterince taze sürümlerinde gerçekleşiyordu. [\#3749](https://github.com/ClickHouse/ClickHouse/pull/3749) +- Fonksiyon sabit segfault `empty` argüman için `FixedString` tür. [Daniel, Dao Quang Minh.](https://github.com/ClickHouse/ClickHouse/pull/3703) +- Büyük bir değer kullanırken sabit aşırı bellek ayırma `max_query_size` ayar (bir bellek yığını `max_query_size` bayt bir kerede önceden tahsis edildi). [\#3720](https://github.com/ClickHouse/ClickHouse/pull/3720) + +#### Yapı değişiklikleri: {#build-changes} + +- OS paketlerinden sürüm 7 LLVM/Clang kütüphaneleri ile sabit yapı (bu kütüphaneler çalışma zamanı sorgu derleme için kullanılır). [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) + +### ClickHouse yayın 18.14.17, 2018-11-30 {#clickhouse-release-18-14-17-2018-11-30} + +#### Hata düzeltmeleri: {#bug-fixes-4} + +- ODBC köprü işlemi ana sunucu işlemi ile sonlandırmak değil sabit durumlar. [\#3642](https://github.com/ClickHouse/ClickHouse/pull/3642) +- Sabit senkron ekleme içine `Distributed` uzak tablonun sütun listesinden farklı bir sütun listesi içeren tablo. [\#3673](https://github.com/ClickHouse/ClickHouse/pull/3673) +- Bir MergeTree tablosunu düşürürken bir kazaya yol açabilecek nadir bir yarış durumu düzeltildi. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) +- Sorgu iş parçacığı oluşturma ile başarısız olduğunda bir sorgu kilitlenme düzeltildi `Resource temporarily unavailable` hatasız. [\#3643](https://github.com/ClickHouse/ClickHouse/pull/3643) +- Sabit ayrıştırma `ENGINE` fık thera ne zaman `CREATE AS table` sözdizimi kullanıldı ve `ENGINE` fık thera daha önce belirt theilmişti `AS table` (hata, belirtilen motoru yok saymakla sonuçlandı). [\#3692](https://github.com/ClickHouse/ClickHouse/pull/3692) + +### ClickHouse yayın 18.14.15, 2018-11-21 {#clickhouse-release-18-14-15-2018-11-21} + +#### Hata düzeltmeleri: {#bug-fixes-5} + +- Bellek yığınının boyutu, türün sütununu seri hale getirirken fazla tahmin edildi `Array(String)` bu yol açar “Memory limit exceeded” hatasızlar. Sorun 18.12.13 sürümünde ortaya çıktı. [\#3589](https://github.com/ClickHouse/ClickHouse/issues/3589) + +### ClickHouse yayın 18.14.14, 2018-11-20 {#clickhouse-release-18-14-14-2018-11-20} + +#### Hata düzeltmeleri: {#bug-fixes-6} + +- Sabit `ON CLUSTER` küme güvenli (bayrak) olarak yapılandırıldığında sorgular ``). [\#3599](https://github.com/ClickHouse/ClickHouse/pull/3599) + +#### Yapı değişiklikleri: {#build-changes-1} + +- Sabit sorunlar (sistemden llvm-7, macos) [\#3582](https://github.com/ClickHouse/ClickHouse/pull/3582) + +### ClickHouse yayın 18.14.13, 2018-11-08 {#clickhouse-release-18-14-13-2018-11-08} + +#### Hata düzeltmeleri: {#bug-fixes-7} + +- Sabit `Block structure mismatch in MergingSorted stream` hatasız. [\#3162](https://github.com/ClickHouse/ClickHouse/issues/3162) +- Sabit `ON CLUSTER` küme yapılandırmasında güvenli bağlantıların açık olması durumunda sorgular ( `` bayrak). [\#3465](https://github.com/ClickHouse/ClickHouse/pull/3465) +- Kullanılan sorgularda bir hata düzeltildi `SAMPLE`, `PREWHERE` ve alias sütunları. [\#3543](https://github.com/ClickHouse/ClickHouse/pull/3543) +- Sabit bir nadir `unknown compression method` hata ne zaman `min_bytes_to_use_direct_io` ayar etkinleştirildi. [3544](https://github.com/ClickHouse/ClickHouse/pull/3544) + +#### Performans iyileştirmeleri: {#performance-improvements} + +- Sorguların sabit performans gerilemesi `GROUP BY` AMD EPYC işlemciler üzerinde yürütülürken uint16 veya tarih türü sütunların. [Igor Lapko](https://github.com/ClickHouse/ClickHouse/pull/3512) +- Uzun dizeleri işleyen sorguların sabit performans gerilemesi. [\#3530](https://github.com/ClickHouse/ClickHouse/pull/3530) + +#### İyileştirmeler oluşturun: {#build-improvements-3} + +- Arcadia yapı basitleştirilmesi için iyileştirmeler. [\#3475](https://github.com/ClickHouse/ClickHouse/pull/3475), [\#3535](https://github.com/ClickHouse/ClickHouse/pull/3535) + +### ClickHouse yayın 18.14.12, 2018-11-02 {#clickhouse-release-18-14-12-2018-11-02} + +#### Hata düzeltmeleri: {#bug-fixes-8} + +- İki isimsiz alt sorgu katılmadan bir kilitlenme düzeltildi. [\#3505](https://github.com/ClickHouse/ClickHouse/pull/3505) +- Hatalı sorgular oluşturma sabit (boş `WHERE` yan tümcesi) dış veritabanlarını sorgularken. [hotid](https://github.com/ClickHouse/ClickHouse/pull/3477) +- ODBC sözlüklerinde yanlış bir zaman aşımı değeri kullanılarak düzeltildi. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3511) + +### ClickHouse yayın 18.14.11, 2018-10-29 {#clickhouse-release-18-14-11-2018-10-29} + +#### Hata düzeltmeleri: {#bug-fixes-9} + +- Hata düzeltildi `Block structure mismatch in UNION stream: different number of columns` sınır sorgularında. [\#2156](https://github.com/ClickHouse/ClickHouse/issues/2156) +- İç içe geçmiş yapılar içinde diziler içeren tablolarda veri birleştirirken sabit hatalar. [\#3397](https://github.com/ClickHouse/ClickHouse/pull/3397) +- Sabit yanlış sorgu sonuçları eğer `merge_tree_uniform_read_distribution` ayar devre dışı (varsayılan: etkin). [\#3429](https://github.com/ClickHouse/ClickHouse/pull/3429) +- Yerel formatta dağıtılmış bir tabloya ekler üzerinde bir hata düzeltildi. [\#3411](https://github.com/ClickHouse/ClickHouse/issues/3411) + +### ClickHouse yayın 18.14.10, 2018-10-23 {#clickhouse-release-18-14-10-2018-10-23} + +- Bu `compile_expressions` ayar (ifadelerin JIT derlemesi) varsayılan olarak devre dışıdır. [\#3410](https://github.com/ClickHouse/ClickHouse/pull/3410) +- Bu `enable_optimize_predicate_expression` ayar varsayılan olarak devre dışıdır. + +### ClickHouse yayın 18.14.9, 2018-10-16 {#clickhouse-release-18-14-9-2018-10-16} + +#### Yenilik: {#new-features-1} + +- Bu `WITH CUBE` değiştirici için `GROUP BY` (alternatif sözdizimi `GROUP BY CUBE(...)` ayrıca kullanılabilir) vardır. [\#3172](https://github.com/ClickHouse/ClickHouse/pull/3172) +- Add theed the `formatDateTime` işlev. [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/2770) +- Add theed the `JDBC` masa motoru ve `jdbc` tablo işlevi (clickhouse-jdbc-bridge yüklenmesini gerektirir). [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) +- ISO hafta numarası ile çalışmak için fonksiyonlar eklendi: `toISOWeek`, `toISOYear`, `toStartOfISOYear`, ve `toDayOfYear`. [\#3146](https://github.com/ClickHouse/ClickHouse/pull/3146) +- Şimdi kullanabilirsiniz `Nullable` Col forum forns for `MySQL` ve `ODBC` Tablolar. [\#3362](https://github.com/ClickHouse/ClickHouse/pull/3362) +- İç içe geçmiş veri yapıları, iç içe geçmiş nesneler olarak okunabilir `JSONEachRow` biçimli. Add theed the `input_format_import_nested_json` ayar. [Veloman Yunkan](https://github.com/ClickHouse/ClickHouse/pull/3144) +- Paralel işleme birçok kişi için kullanılabilir `MATERIALIZED VIEW`s veri eklerken. Görmek `parallel_view_processing` ayar. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3208) +- Add theed the `SYSTEM FLUSH LOGS` sorgu (sistem tablolarına zorla günlük basması gibi `query_log`) [\#3321](https://github.com/ClickHouse/ClickHouse/pull/3321) +- Şimdi önceden tanımlanmış kullanabilirsiniz `database` ve `table` bildirirken makrolar `Replicated` Tablolar. [\#3251](https://github.com/ClickHouse/ClickHouse/pull/3251) +- Okuma yeteneği eklendi `Decimal` mühendislik notasyonunda değerleri yazın (on yetkilerini gösterir). [\#3153](https://github.com/ClickHouse/ClickHouse/pull/3153) + +#### Deneysel özellikler: {#experimental-features} + +- GROUP BY CLA clauseuse for `LowCardinality data types.` [\#3138](https://github.com/ClickHouse/ClickHouse/pull/3138) +- İçin ifadelerin optimize hesaplama `LowCardinality data types.` [\#3200](https://github.com/ClickHouse/ClickHouse/pull/3200) + +#### Geliştirmeler: {#improvements-2} + +- İle sorgular için önemli ölçüde azaltılmış bellek tüketimi `ORDER BY` ve `LIMIT`. Görmek `max_bytes_before_remerge_sort` ayar. [\#3205](https://github.com/ClickHouse/ClickHouse/pull/3205) +- Yokluğ theunda `JOIN` (`LEFT`, `INNER`, …), `INNER JOIN` kabul edilir. [\#3147](https://github.com/ClickHouse/ClickHouse/pull/3147) +- Nitelikli yıldız işaretleri ile sorgularda düzgün çalışır `JOIN`. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3202) +- Bu `ODBC` table engine doğru bir uzak veritabanı SQL lehçesinde tanımlayıcıları alıntı yöntemi seçer. [Alexandr Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/3210) +- Bu `compile_expressions` ayar (ifadelerin JIT derlemesi) varsayılan olarak etkindir. +- Varsa eşzamanlı damla veritabanı/tablo için sabit davranış ve varsa veritabanı/tablo oluşturun. Daha önce, bir `CREATE DATABASE ... IF NOT EXISTS` sorgu hata iletisi döndürebilir “File … already exists” ve `CREATE TABLE ... IF NOT EXISTS` ve `DROP TABLE IF EXISTS` sorgular dönebilir `Table ... is creating or attaching right now`. [\#3101](https://github.com/ClickHouse/ClickHouse/pull/3101) +- MySQL veya ODBC tablolarından sorgularken sabit bir sağ yarıya sahip ifadelerde ve ifadelerde uzak sunucuya geçirilir. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) +- MySQL ve ODBC tablolarından sorgularken bir WHERE yan tümcesinde sabit ifadelerle karşılaştırmalar uzak sunucuya geçirilir. Önceden, sadece sabitler ile karşılaştırmalar geçirildi. [\#3182](https://github.com/ClickHouse/ClickHouse/pull/3182) +- Terminalde satır genişliğinin doğru hesaplanması `Pretty` hiyeroglifli dizeler de dahil olmak üzere formatlar. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/3257). +- `ON CLUSTER` için belirt forilebilir `ALTER UPDATE` sorgular. +- Verileri okumak için geliştirilmiş performans `JSONEachRow` biçimli. [\#3332](https://github.com/ClickHouse/ClickHouse/pull/3332) +- İçin eş anlamlı eklendi `LENGTH` ve `CHARACTER_LENGTH` uyumluluk için fonksiyonlar. Bu `CONCAT` işlev artık büyük / küçük harfe duyarlı değildir. [\#3306](https://github.com/ClickHouse/ClickHouse/pull/3306) +- Add theed the `TIMESTAMP` eşanlamlı `DateTime` tür. [\#3390](https://github.com/ClickHouse/ClickHouse/pull/3390) +- Günlük satırı bir sorgu ile ilişkili olmasa bile, sunucu günlüklerinde query\_id için ayrılmış alan her zaman vardır. Bu, sunucu metin günlüklerini üçüncü taraf araçlarla Ayrıştırmayı kolaylaştırır. +- Gigabayt bir tamsayı sonraki düzeyini aştığında, bir sorgu tarafından bellek tüketimi kaydedilir. [\#3205](https://github.com/ClickHouse/ClickHouse/pull/3205) +- Yerel protokolü kullanan istemci kitaplığı sunucu ekleme sorgusu için beklediğinden daha az sütun yanlışlıkla gönderdiğinde durum için uyumluluk modu eklendi. Bu senaryo, clickhouse-cpp kitaplığını kullanırken mümkün oldu. Daha önce, bu senaryo sunucunun çökmesine neden oldu. [\#3171](https://github.com/ClickHouse/ClickHouse/pull/3171) +- Kullanıcı tanımlı bir ifade içinde `clickhouse-copier` şimdi kullanabilirsiniz `partition_key` alias (kaynak tablo bölümüne göre ek filtreleme için). Bu, bölümleme şeması kopyalama sırasında değişirse, ancak yalnızca biraz değişirse yararlıdır. [\#3166](https://github.com/ClickHouse/ClickHouse/pull/3166) +- Bu iş akışı `Kafka` motor, yüksek yüklerde veri okuma hızını otomatik olarak azaltmak için bir arka plan iş parçacığı havuzuna taşındı. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3215). +- Okuma desteği `Tuple` ve `Nested` gibi yapıların değerleri `struct` in the `Cap'n'Proto format`. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3216) +- İçin en üst düzey etki alanlarının listesi `firstSignificantSubdomain` işlev artık etki alanını içerir `biz`. [decaseal](https://github.com/ClickHouse/ClickHouse/pull/3219) +- Dış sözlüklerin yapılandırmasında, `null_value` varsayılan veri türü değeri olarak yorumlanır. [\#3330](https://github.com/ClickHouse/ClickHouse/pull/3330) +- İçin destek `intDiv` ve `intDivOrZero` fonksiyonlar için `Decimal`. [b48402e8](https://github.com/ClickHouse/ClickHouse/commit/b48402e8712e2b9b151e0eef8193811d433a1264) +- İçin destek `Date`, `DateTime`, `UUID`, ve `Decimal` anahtar olarak türleri `sumMap` toplama işlevi. [\#3281](https://github.com/ClickHouse/ClickHouse/pull/3281) +- İçin destek `Decimal` harici sözlüklerde veri türü. [\#3324](https://github.com/ClickHouse/ClickHouse/pull/3324) +- İçin destek `Decimal` veri türü `SummingMergeTree` Tablolar. [\#3348](https://github.com/ClickHouse/ClickHouse/pull/3348) +- İçin uzmanlık eklendi `UUID` içinde `if`. [\#3366](https://github.com/ClickHouse/ClickHouse/pull/3366) +- Sayısı azal thedı `open` ve `close` bir gelen okurken sistem çağrıları `MergeTree table`. [\#3283](https://github.com/ClickHouse/ClickHouse/pull/3283) +- A `TRUNCATE TABLE` sorgu (sorgu için lider yineleme geçirilir) herhangi bir yineleme üzerinde çalıştırılabilir. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/3375) + +#### Hata düzeltmeleri: {#bug-fixes-10} + +- İle bir sorun düzeltildi `Dictionary` için tablolar `range_hashed` sözlükler. Bu hata 18.12.17 sürümünde oluştu. [\#1702](https://github.com/ClickHouse/ClickHouse/pull/1702) +- Yüklerken bir hata düzeltildi `range_hashed` söz dictionarieslükler (mesaj `Unsupported type Nullable (...)`). Bu hata 18.12.17 sürümünde oluştu. [\#3362](https://github.com/ClickHouse/ClickHouse/pull/3362) +- Sabit hatalar `pointInPolygon` birbirine yakın bulunan çok sayıda köşe ile çokgenler için yanlış hesaplamaların birikmesi nedeniyle işlev. [\#3331](https://github.com/ClickHouse/ClickHouse/pull/3331) [\#3341](https://github.com/ClickHouse/ClickHouse/pull/3341) +- Veri parçalarını birleştirdikten sonra, elde edilen parçanın sağlama toplamı, başka bir kopyadaki aynı birleştirme sonucundan farklıysa, birleştirme sonucu silinir ve veri kısmı diğer kopyadan indirilir (bu doğru davranıştır). Ancak, veri bölümünü indirdikten sonra, parçanın zaten var olduğu bir hata nedeniyle çalışma kümesine eklenemedi(çünkü veri kısmı birleşmeden sonra bir miktar gecikmeyle silindi). Bu, aynı verileri indirmek için döngüsel girişimlere yol açtı. [\#3194](https://github.com/ClickHouse/ClickHouse/pull/3194) +- Sorgularla toplam bellek tüketiminin yanlış hesaplanması düzeltildi (yanlış hesaplama nedeniyle, `max_memory_usage_for_all_queries` ayar yanlış çalıştı ve `MemoryTracking` metrik yanlış bir değere sahipti). Bu hata, 18.12.13 sürümünde oluştu. [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3344) +- İşlevselliğini sabit `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` Bu hata, 18.12.13 sürümünde oluştu. [\#3247](https://github.com/ClickHouse/ClickHouse/pull/3247) +- İçin veri yapılarının sabit gereksiz hazırlanması `JOIN`eğer sorgu başlatan sunucuda s `JOIN` yalnızca uzak sunucularda gerçekleştirilir. [\#3340](https://github.com/ClickHouse/ClickHouse/pull/3340) +- Sabit hatalar `Kafka` motor: verileri okumaya başladığınızda istisnalardan sonra kilitlenmeler ve tamamlandıktan sonra kilitler [Marek Vavruša](https://github.com/ClickHouse/ClickHouse/pull/3215). +- İçin `Kafka` tablolar, isteğe bağlı `schema` parametre geç notilm (edi (şema `Cap'n'Proto` biçimli). [Vojtech Splichal](https://github.com/ClickHouse/ClickHouse/pull/3150) +- ZooKeeper sunucularının topluluğu, bağlantıyı kabul eden ancak el sıkışmaya yanıt vermek yerine hemen kapatan sunuculara sahipse, ClickHouse başka bir sunucuya bağlanmayı seçer. Daha önce, bu hata üretti `Cannot read all data. Bytes read: 0. Bytes expected: 4.` ve sunucu başlatılamadı. [8218cf3a](https://github.com/ClickHouse/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) +- Zookeeper sunucularının topluluğu, DNS sorgusunun bir hata döndürdüğü sunucular içeriyorsa, bu sunucular yoksayılır. [17b8e209](https://github.com/ClickHouse/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) +- Arasında sabit tip dönüşüm `Date` ve `DateTime` veri eklerken `VALUES` biçim (eğer `input_format_values_interpret_expressions = 1`). Daha önce, dönüşüm Unix dönem zamanındaki gün sayısının sayısal değeri ile Unix zaman damgası arasında gerçekleştirildi ve bu da beklenmedik sonuçlara yol açtı. [\#3229](https://github.com/ClickHouse/ClickHouse/pull/3229) +- Arasında düzeltilmiş tip dönüşümü `Decimal` ve tam sayı. [\#3211](https://github.com/ClickHouse/ClickHouse/pull/3211) +- Sabit hatalar `enable_optimize_predicate_expression` ayar. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3231) +- Varsayılan olmayan bir CSV ayırıcı kullanılıyorsa, kayan noktalı sayılarla CSV formatında bir ayrıştırma hatası düzeltildi `;` [\#3155](https://github.com/ClickHouse/ClickHouse/pull/3155) +- Sabit `arrayCumSumNonNegative` fonksiyon (akümülatör sıfırdan az ise negatif değerler biriktirmez). [Aleksey Studnev](https://github.com/ClickHouse/ClickHouse/pull/3163) +- Sabit nasıl `Merge` tablolar üstünde çalışır `Distributed` kullanırken tablolar `PREWHERE`. [\#3165](https://github.com/ClickHouse/ClickHouse/pull/3165) +- Hata düzeltmeleri `ALTER UPDATE` sorgu. +- Sabit hatalar `odbc` 18.12 sürümünde görünen tablo işlevi. [\#3197](https://github.com/ClickHouse/ClickHouse/pull/3197) +- İle toplama fonksiyon operationlarının çalışmasını sabit `StateArray` birleştiriciler. [\#3188](https://github.com/ClickHouse/ClickHouse/pull/3188) +- Bir bölme sırasında bir kilitlenme düzeltildi `Decimal` sıfır değeri. [69dd6609](https://github.com/ClickHouse/ClickHouse/commit/69dd6609193beb4e7acd3e6ad216eca0ccfb8179) +- Kullanarak işlemler için sabit çıktı türleri `Decimal` ve tamsayı argümanları. [\#3224](https://github.com/ClickHouse/ClickHouse/pull/3224) +- Sırasında segfault sabit `GROUP BY` üzerinde `Decimal128`. [3359ba06](https://github.com/ClickHouse/ClickHouse/commit/3359ba06c39fcd05bfdb87d6c64154819621e13a) +- Bu `log_query_threads` ayar (sorgu yürütme her iş parçacığı hakkında bilgi günlüğe kaydetme) şimdi yalnızca `log_queries` (sorgularla ilgili bilgileri günlüğe kaydetme) seçeneği 1 olarak ayarlanır. Sin thece the `log_query_threads` seçenek varsayılan olarak etkindir, sorgu günlüğü devre dışı bırakılmış olsa bile iş parçacıkları hakkında bilgi daha önce günlüğe kaydedildi. [\#3241](https://github.com/ClickHouse/ClickHouse/pull/3241) +- Quantiles toplama fonksiyonunun dağıtılmış işleminde bir hata düzeltildi (hata mesajı `Not found column quantile...`). [292a8855](https://github.com/ClickHouse/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) +- Aynı anda sürüm 18.12.17 sunucuları ve eski sunuculardan oluşan bir küme üzerinde çalışırken uyumluluk sorunu düzeltildi. Hem sabit hem de sabit olmayan uzunluktaki GROUP BY anahtarlarıyla dağıtılmış sorgular için, toplanacak büyük miktarda veri varsa, döndürülen veriler her zaman tam olarak toplanmadı (iki farklı satır aynı toplama anahtarlarını içeriyordu). [\#3254](https://github.com/ClickHouse/ClickHouse/pull/3254) +- Değiştirmelerin sabit kullanımı `clickhouse-performance-test`, sorgu testte bildirilen değiştirmelerin yalnızca bir kısmını içeriyorsa. [\#3263](https://github.com/ClickHouse/ClickHouse/pull/3263) +- Kullanırken bir hata düzeltildi `FINAL` ile `PREWHERE`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) +- Kullanırken bir hata düzeltildi `PREWHERE` sırasında eklenen sütun overların üzerinde `ALTER`. [\#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) +- Olmaması için bir çek eklendi `arrayJoin` için `DEFAULT` ve `MATERIALIZED` ifadeler. Önceden, `arrayJoin` veri eklerken bir hataya yol açtı. [\#3337](https://github.com/ClickHouse/ClickHouse/pull/3337) +- Olmaması için bir çek eklendi `arrayJoin` in a `PREWHERE` yan. Daha önce, bu gibi mesajlara yol açtı `Size ... doesn't match` veya `Unknown compression method` sorguları yürütürken. [\#3357](https://github.com/ClickHouse/ClickHouse/pull/3357) +- Değiştirilen optimizasyon ve eşitlik değerlendirmelerinden karşılık gelen ifadeyle zincirler sonra nadir durumlarda ortaya çıkabilecek sabit segfault. [liuyimin-bytedance](https://github.com/ClickHouse/ClickHouse/pull/3339) +- İçin küçük düzeltmeler `clickhouse-benchmark`: daha önce, istemci bilgileri sunucuya gönderilmedi; şimdi yürütülen sorguların sayısı kapatılırken ve yineleme sayısını sınırlamak için daha doğru hesaplanır. [\#3351](https://github.com/ClickHouse/ClickHouse/pull/3351) [\#3352](https://github.com/ClickHouse/ClickHouse/pull/3352) + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-1} + +- Kaldır theılan `allow_experimental_decimal_type` seçenek. Bu `Decimal` veri türü varsayılan kullanım için kullanılabilir. [\#3329](https://github.com/ClickHouse/ClickHouse/pull/3329) + +## ClickHouse sürümü 18.12 {#clickhouse-release-18-12} + +### ClickHouse yayın 18.12.17, 2018-09-16 {#clickhouse-release-18-12-17-2018-09-16} + +#### Yenilik: {#new-features-2} + +- `invalidate_query` (bir dış sözlük güncelleştirilmesi gerekip gerekmediğini denetlemek için bir sorgu belirtmek için yeteneği) `clickhouse` kaynaklı. [\#3126](https://github.com/ClickHouse/ClickHouse/pull/3126) +- Kullanma yeteneği eklendi `UInt*`, `Int*`, ve `DateTime` veri türleri (birlikte `Date` type) as a `range_hashed` aralıkların sınırlarını tanımlayan dış sözlük anahtarı. Şimdi `NULL` açık bir aralık belirlemek için kullanılabilir. [Vasily Nemkov](https://github.com/ClickHouse/ClickHouse/pull/3123) +- Bu `Decimal` şimdi yazın destekler `var*` ve `stddev*` toplama fonksiyonları. [\#3129](https://github.com/ClickHouse/ClickHouse/pull/3129) +- Bu `Decimal` tip artık matematiksel fonksiyonları destekliyor (`exp`, `sin` ve böyle devam eder.) [\#3129](https://github.com/ClickHouse/ClickHouse/pull/3129) +- Bu `system.part_log` tablo şimdi var `partition_id` sütun. [\#3089](https://github.com/ClickHouse/ClickHouse/pull/3089) + +#### Hata düzeltmeleri: {#bug-fixes-11} + +- `Merge` şimdi düzgün çalışıyor `Distributed` Tablolar. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3159) +- Sabit uyumsuzluk (gereksiz bağımlılık `glibc` sürüm) bu Clickhouse'u çalıştırmayı imkansız hale getirdi `Ubuntu Precise` ve eski sürümleri. Uyumsuzluk 18.12.13 sürümünde ortaya çıktı. [\#3130](https://github.com/ClickHouse/ClickHouse/pull/3130) +- Sabit hatalar `enable_optimize_predicate_expression` ayar. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) +- 18.12.13'ten önceki sürümlerde bir kopya kümesiyle çalışırken ve aynı anda daha yeni bir sürüme sahip bir sunucuda bir tablonun yeni bir kopyasını oluştururken ortaya çıkan geriye dönük uyumluluğa sahip küçük bir sorun düzeltildi (mesajda gösterilir `Can not clone replica, because the ... updated to new ClickHouse version`, mantıklı, ama olmamalı). [\#3122](https://github.com/ClickHouse/ClickHouse/pull/3122) + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-2} + +- Bu `enable_optimize_predicate_expression` seçenek varsayılan olarak etkindir (oldukça iyimser olan). Sütun adları için arama için ilgili sorgu çözümleme hataları oluşursa, ayarla `enable_optimize_predicate_expression` 0'a. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3107) + +### ClickHouse yayın 18.12.14, 2018-09-13 {#clickhouse-release-18-12-14-2018-09-13} + +#### Yenilik: {#new-features-3} + +- İçin destek eklendi `ALTER UPDATE` sorgular. [\#3035](https://github.com/ClickHouse/ClickHouse/pull/3035) +- Add theed the `allow_ddl` kullanıcının DDL sorgularına erişimini kısıtlayan seçenek. [\#3104](https://github.com/ClickHouse/ClickHouse/pull/3104) +- Add theed the `min_merge_bytes_to_use_direct_io` seçeneği için `MergeTree` birleştirmenin toplam boyutu için bir eşik ayarlamanıza izin veren motorlar(eşiğin üstünde olduğunda, veri parçası dosyaları o\_direct kullanılarak işlenecektir). [\#3117](https://github.com/ClickHouse/ClickHouse/pull/3117) +- Bu `system.merges` sistem tablosu şimdi içerir `partition_id` sütun. [\#3099](https://github.com/ClickHouse/ClickHouse/pull/3099) + +#### Geliştirmeler {#improvements-3} + +- Bir veri parçası mutasyon sırasında değişmeden kalırsa, yinelemeler tarafından indirilmez. [\#3103](https://github.com/ClickHouse/ClickHouse/pull/3103) +- Otomatik tamamlama ile çalışırken ayarların adları için kullanılabilir `clickhouse-client`. [\#3106](https://github.com/ClickHouse/ClickHouse/pull/3106) + +#### Hata düzeltmeleri: {#bug-fixes-12} + +- Elemanları olan dizilerin boyutları için bir kontrol eklendi `Nested` eklerken alanları yazın. [\#3118](https://github.com/ClickHouse/ClickHouse/pull/3118) +- İle harici sözlükler güncellenmesi bir hata düzeltildi `ODBC` kaynak ve `hashed` depolama. Bu hata, 18.12.13 sürümünde oluştu. +- Bir sorgudan geçici bir tablo oluştururken bir kilitlenme düzeltildi `IN` koşul. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3098) +- Olabilecek diziler için toplam işlevlerde bir hata düzeltildi `NULL` öğeler. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/3097) + +### ClickHouse yayın 18.12.13, 2018-09-10 {#clickhouse-release-18-12-13-2018-09-10} + +#### Yenilik: {#new-features-4} + +- Add theed the `DECIMAL(digits, scale)` veri türü (`Decimal32(scale)`, `Decimal64(scale)`, `Decimal128(scale)`). Etkinleştirmek için ayarı kullanın `allow_experimental_decimal_type`. [\#2846](https://github.com/ClickHouse/ClickHouse/pull/2846) [\#2970](https://github.com/ClickHouse/ClickHouse/pull/2970) [\#3008](https://github.com/ClickHouse/ClickHouse/pull/3008) [\#3047](https://github.com/ClickHouse/ClickHouse/pull/3047) +- Yeni `WITH ROLLUP` değiştirici için `GROUP BY` (alternatif sözdizimi: `GROUP BY ROLLUP(...)`). [\#2948](https://github.com/ClickHouse/ClickHouse/pull/2948) +- JOIN ile sorgularda, yıldız karakteri SQL standardına uygun olarak tüm tablolardaki sütunların bir listesine genişletir. Ayarlayarak eski davranışı geri yükleyebilirsiniz `asterisk_left_columns_only` kullanıcı yapılandırma düzeyinde 1'e. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2787) +- Tablo fonksiyonları ile katılmak için destek eklendi. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) +- Clickhouse-client sekmesine basarak otomatik tamamlama. [Sergey Shcherbin](https://github.com/ClickHouse/ClickHouse/pull/2447) +- CTRL + C clickhouse-client girilen bir sorguyu temizler. [\#2877](https://github.com/ClickHouse/ClickHouse/pull/2877) +- Add theed the `join_default_strictness` ayar (değerler: `"`, `'any'`, `'all'`). Bu belirtmemenizi sağlar `ANY` veya `ALL` için `JOIN`. [\#2982](https://github.com/ClickHouse/ClickHouse/pull/2982) +- Sorgu işleme ile ilgili sunucu günlüğünün her satırı sorgu kimliğini gösterir. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Artık clickhouse-client'da sorgu yürütme günlükleri alabilirsiniz ( `send_logs_level` ayar). Dağıtılmış sorgu işleme ile günlükleri tüm sunuculardan basamaklandırılır. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Bu `system.query_log` ve `system.processes` (`SHOW PROCESSLIST`) bir sorgu çalıştırdığınızda tablolar artık tüm değiştirilen ayarları hakkında bilgi var (iç içe geçmiş yapısı `Settings` veriler). Add theed the `log_query_settings` ayar. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Bu `system.query_log` ve `system.processes` tablolar artık sorgu yürütülmesine katılan iş parçacığı sayısı hakkında bilgi gösterir (bkz. `thread_numbers` sütun). [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Katma `ProfileEvents` ağ üzerinden okuma ve yazma ve diske okuma ve yazma için harcanan zamanı, ağ hatalarının sayısını ve ağ bant genişliği sınırlı olduğunda bekleyen harcanan zamanı ölçen sayaçlar. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Katma `ProfileEvents`rusage gelen ((bekle zaman G/Ç hakkında daha fazla bilgi edinmek için bu kullanım, CPU zamanı bekle ve veri miktarını okuma ve kaydedilen, sayfa önbelleği olan ve olmayan) kullanıcı alanı ve çekirdek, sayfa hataları ve bağlam anahtarlarının yanı sıra taskstats ölçülerine CPU kullanımı hakkında bilgi almak için onları kullanabilirsiniz) sistem ölçümlerini içeren sayaçları. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Bu `ProfileEvents` sayaçlar, genel olarak ve her sorgu için ve ayrıca sorguya göre kaynak tüketimini ayrıntılı olarak profillemenize izin veren her sorgu yürütme iş parçacığı için uygulanır. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Add theed the `system.query_thread_log` her sorgu yürütme iş parçacığı hakkında bilgi içeren tablo. Add theed the `log_query_threads` ayar. [\#2482](https://github.com/ClickHouse/ClickHouse/pull/2482) +- Bu `system.metrics` ve `system.events` tablolar artık yerleşik belgelere sahiptir. [\#3016](https://github.com/ClickHouse/ClickHouse/pull/3016) +- Add theed the `arrayEnumerateDense` işlev. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2975) +- Add theed the `arrayCumSumNonNegative` ve `arrayDifference` işlevler. [Aleksey Studnev](https://github.com/ClickHouse/ClickHouse/pull/2942) +- Add theed the `retention` toplama işlevi. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2887) +- Artık Plus operatörünü kullanarak toplama işlevlerinin durumlarını ekleyebilir (birleştirebilirsiniz) ve toplama işlevlerinin durumlarını negatif olmayan bir sabitle çarpabilirsiniz. [\#3062](https://github.com/ClickHouse/ClickHouse/pull/3062) [\#3034](https://github.com/ClickHouse/ClickHouse/pull/3034) +- Mergetree ailesindeki tablolar artık sanal sütuna sahip `_partition_id`. [\#3089](https://github.com/ClickHouse/ClickHouse/pull/3089) + +#### Deneysel özellikler: {#experimental-features-1} + +- Add theed the `LowCardinality(T)` veri türü. Bu veri türü otomatik olarak yerel bir değer sözlüğü oluşturur ve sözlüğü açmadan veri işlemeye izin verir. [\#2830](https://github.com/ClickHouse/ClickHouse/pull/2830) +- JIT derlenmiş işlevlerin bir önbellek ve derlemeden önce kullanım sayısı için bir sayaç eklendi. JIT derleme ifadeleri için `compile_expressions` ayar. [\#2990](https://github.com/ClickHouse/ClickHouse/pull/2990) [\#3077](https://github.com/ClickHouse/ClickHouse/pull/3077) + +#### Geliştirmeler: {#improvements-4} + +- Terkedilmiş yinelemeler olduğunda çoğaltma günlüğünün sınırsız birikimi ile ilgili sorun giderildi. Uzun bir gecikme ile kopyaları için etkili bir kurtarma modu eklendi. +- Geliştirilmiş performans `GROUP BY` bunlardan biri dize ve diğerleri sabit uzunlukta olduğunda birden fazla toplama alanı ile. +- Kullanırken geliştirilmiş performans `PREWHERE` ve ifad ofelerin örtülü olarak aktar ofılmasıyla `PREWHERE`. +- Metin formatları için geliştirilmiş ayrıştırma performansı (`CSV`, `TSV`). [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2977) [\#2980](https://github.com/ClickHouse/ClickHouse/pull/2980) +- İkili biçimlerde okuma dizeleri ve diziler geliştirilmiş performans. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2955) +- Sorgular için artan performans ve azaltılmış bellek tüketimi `system.tables` ve `system.columns` tek bir sunucuda çok sayıda tablo olduğunda. [\#2953](https://github.com/ClickHouse/ClickHouse/pull/2953) +- Bir hataya neden olan büyük bir sorgu akışı durumunda bir performans sorunu düzeltildi ( `_dl_addr` fonksiyon görünür `perf top`, ancak sunucu çok fazla CPU kullanmıyor). [\#2938](https://github.com/ClickHouse/ClickHouse/pull/2938) +- Koşullar Görünüm içine atılır (ne zaman `enable_optimize_predicate_expression` etkin) değildir. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2907) +- İçin işlevsellik geliştirmeleri `UUID` veri türü. [\#3074](https://github.com/ClickHouse/ClickHouse/pull/3074) [\#2985](https://github.com/ClickHouse/ClickHouse/pull/2985) +- Bu `UUID` veri türü-Simyacı sözlüklerde desteklenir. [\#2822](https://github.com/ClickHouse/ClickHouse/pull/2822) +- Bu `visitParamExtractRaw` işlev iç içe geçmiş yapılarla düzgün çalışır. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2974) +- Ne zaman `input_format_skip_unknown_fields` ayarı etkin, nesne alanları `JSONEachRow` biçim doğru atlanır. [BlahGeek](https://github.com/ClickHouse/ClickHouse/pull/2958) +- İçin... `CASE` koşullarla ifade, şimdi atlayabilirsiniz `ELSE` eşdeğer olan `ELSE NULL`. [\#2920](https://github.com/ClickHouse/ClickHouse/pull/2920) +- ZooKeeper ile çalışırken operasyon zaman aşımı artık yapılandırılabilir. [urykhy](https://github.com/ClickHouse/ClickHouse/pull/2971) +- İçin bir ofset belirtebilirsiniz `LIMIT n, m` olarak `LIMIT n OFFSET m`. [\#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) +- Kullanabilirsiniz `SELECT TOP n` için alternatif olarak sözdizimi `LIMIT`. [\#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) +- Sistem tablolarına yazmak için sıranın boyutunu arttırdı, böylece `SystemLog parameter queue is full` hata sık sık olmaz. +- Bu `windowFunnel` toplama işlevi artık birden çok koşulu karşılayan olayları destekler. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2801) +- Yinelenen sütunlar bir `USING` için fık forra `JOIN`. [\#3006](https://github.com/ClickHouse/ClickHouse/pull/3006) +- `Pretty` biçimlerin artık genişliğe göre sütun hizalaması üzerinde bir sınırı vardır. Kullan... `output_format_pretty_max_column_pad_width` ayar. Bir değer daha genişse, yine de bütünüyle görüntülenecektir, ancak tablodaki diğer hücreler çok geniş olmayacaktır. [\#3003](https://github.com/ClickHouse/ClickHouse/pull/3003) +- Bu `odbc` tablo işlevi artık veritabanı/şema adını belirtmenizi sağlar. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2885) +- Belirtilen bir kullanıcı adı kullanma yeteneği eklendi `clickhouse-client` yapılandırma dosyası. [Vladimir Kozbin](https://github.com/ClickHouse/ClickHouse/pull/2909) +- Bu `ZooKeeperExceptions` sayaç üç sayaçlara ayrılmıştır: `ZooKeeperUserExceptions`, `ZooKeeperHardwareExceptions`, ve `ZooKeeperOtherExceptions`. +- `ALTER DELETE` sorgular hayata görünümler için çalışır. +- Temizleme ipliğini periyodik olarak çalıştırırken randomizasyon eklendi `ReplicatedMergeTree` çok sayıda olduğunda periyodik yük ani önlemek için tablolar `ReplicatedMergeTree` Tablolar. +- İçin destek `ATTACH TABLE ... ON CLUSTER` sorgular. [\#3025](https://github.com/ClickHouse/ClickHouse/pull/3025) + +#### Hata düzeltmeleri: {#bug-fixes-13} + +- İle bir sorun düzeltildi `Dictionary` tablolar (atar `Size of offsets doesn't match size of column` veya `Unknown compression method` özel). Bu hata 18.10.3 sürümünde göründü. [\#2913](https://github.com/ClickHouse/ClickHouse/issues/2913) +- Birleştirirken bir hata düzeltildi `CollapsingMergeTree` veri parçalarından biri boşsa tablolar (bu parçalar birleştirme sırasında oluşturulur veya `ALTER DELETE` tüm veriler silindiyse) ve `vertical` birleştirme için algoritma kullanıldı. [\#3049](https://github.com/ClickHouse/ClickHouse/pull/3049) +- Sırasında bir yarış durumu düzeltildi `DROP` veya `TRUNCATE` için `Memory` eş zamanlı tablolar `SELECT`, sunucu çökmelerine neden olabilir. Bu hata 1.1.54388 sürümünde göründü. [\#3038](https://github.com/ClickHouse/ClickHouse/pull/3038) +- Takarken veri kaybı olasılığı düzeltildi `Replicated` tablolar ise `Session is expired` hata döndürülür (veri kaybı tarafından tespit edilebilir `ReplicatedDataLoss` ölçü). Bu hata 1.1.54378 sürümünde oluştu. [\#2939](https://github.com/ClickHouse/ClickHouse/pull/2939) [\#2949](https://github.com/ClickHouse/ClickHouse/pull/2949) [\#2964](https://github.com/ClickHouse/ClickHouse/pull/2964) +- Sırasında bir segfault sabit `JOIN ... ON`. [\#3000](https://github.com/ClickHouse/ClickHouse/pull/3000) +- Sütun adlarını ararken hata düzeltildi `WHERE` ifade tamamen nitelikli bir sütun adından oluşur, örneğin `WHERE table.column`. [\#2994](https://github.com/ClickHouse/ClickHouse/pull/2994) +- Sabit “Not found column” uzak bir sunucudan bir alt sorgu ile bir In ifadesinden oluşan tek bir sütun isteniyorsa, dağıtılmış sorgular yürütülürken oluşan hata. [\#3087](https://github.com/ClickHouse/ClickHouse/pull/3087) +- Sabit `Block structure mismatch in UNION stream: different number of columns` kırıklardan biri yerel ve diğeri değilse dağıtılmış sorgular için oluşan hata ve taşınmanın en iyi duruma getirilmesi `PREWHERE` tetik .lenir. [\#2226](https://github.com/ClickHouse/ClickHouse/pull/2226) [\#3037](https://github.com/ClickHouse/ClickHouse/pull/3037) [\#3055](https://github.com/ClickHouse/ClickHouse/pull/3055) [\#3065](https://github.com/ClickHouse/ClickHouse/pull/3065) [\#3073](https://github.com/ClickHouse/ClickHouse/pull/3073) [\#3090](https://github.com/ClickHouse/ClickHouse/pull/3090) [\#3093](https://github.com/ClickHouse/ClickHouse/pull/3093) +- Sabit `pointInPolygon` dışbükey olmayan poligonların belirli durumları için işlev. [\#2910](https://github.com/ClickHouse/ClickHouse/pull/2910) +- Karşılaştırırken yanlış sonuç düzeltildi `nan` tamsayılarla. [\#3024](https://github.com/ClickHouse/ClickHouse/pull/3024) +- Sabit bir hata `zlib-ng` nadir durumlarda segfault yol açabilir kütüphane. [\#2854](https://github.com/ClickHouse/ClickHouse/pull/2854) +- Bir tabloya eklerken bir bellek sızıntısı düzeltildi `AggregateFunction` sütunlar, toplama işlevinin durumu basit değilse (belleği ayrı olarak ayırır) ve tek bir ekleme isteği birden çok küçük blokla sonuçlanırsa. [\#3084](https://github.com/ClickHouse/ClickHouse/pull/3084) +- Aynı şeyi oluştururken ve silerken bir yarış durumu düzeltildi `Buffer` veya `MergeTree` aynı anda tablo. +- Tuples gibi bazı önemsiz olmayan türlerinden oluşan tuples karşılaştırırken bir segfault olasılığını düzeltildi. [\#2989](https://github.com/ClickHouse/ClickHouse/pull/2989) +- Belirli çalıştırırken bir segfault olasılığı düzeltildi `ON CLUSTER` sorgular. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2960) +- Sabit bir hata `arrayDistinct` fonksiyonu için `Nullable` dizi elemanları. [\#2845](https://github.com/ClickHouse/ClickHouse/pull/2845) [\#2937](https://github.com/ClickHouse/ClickHouse/pull/2937) +- Bu `enable_optimize_predicate_expression` seçenek şimdi doğru olan durumları destekler `SELECT *`. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2929) +- Zookeeper oturumunu yeniden başlatırken segfault düzeltildi. [\#2917](https://github.com/ClickHouse/ClickHouse/pull/2917) +- ZooKeeper ile çalışırken sabit potansiyel engelleme. +- Bir iç içe veri yapıları eklemek için sabit yanlış kod `SummingMergeTree`. +- Toplama işlevlerinin durumları için bellek ayırırken, hizalama doğru bir şekilde dikkate alınır, bu da toplama işlevlerinin durumlarını uygularken hizalama gerektiren işlemlerin kullanılmasını mümkün kılar. [chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2808) + +#### Güvenlik düzeltme: {#security-fix} + +- ODBC veri kaynaklarının güvenli kullanımı. ODBC sürücüleri ile etkileşim ayrı bir `clickhouse-odbc-bridge` işleyiş. Üçüncü taraf ODBC sürücülerindeki hatalar artık sunucu kararlılığı veya güvenlik açıklarıyla ilgili sorunlara neden olmaz. [\#2828](https://github.com/ClickHouse/ClickHouse/pull/2828) [\#2879](https://github.com/ClickHouse/ClickHouse/pull/2879) [\#2886](https://github.com/ClickHouse/ClickHouse/pull/2886) [\#2893](https://github.com/ClickHouse/ClickHouse/pull/2893) [\#2921](https://github.com/ClickHouse/ClickHouse/pull/2921) +- Dosya yolunun yanlış doğrulanması düzeltildi `catBoostPool` tablo işlevi. [\#2894](https://github.com/ClickHouse/ClickHouse/pull/2894) +- Sistem tablolarının içeriği (`tables`, `databases`, `parts`, `columns`, `parts_columns`, `merges`, `mutations`, `replicas`, ve `replication_queue`) kullanıcının veri tabanlarına yapılandırılmış erişimine göre filtrelenir (`allow_databases`). [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2856) + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-3} + +- JOIN ile sorgularda, yıldız karakteri SQL standardına uygun olarak tüm tablolardaki sütunların bir listesine genişletir. Ayarlayarak eski davranışı geri yükleyebilirsiniz `asterisk_left_columns_only` kullanıcı yapılandırma düzeyinde 1'e. + +#### Yapı değişiklikleri: {#build-changes-2} + +- Çoğu entegrasyon testleri artık commit tarafından çalıştırılabilir. +- Kod stili kontrolleri de commit tarafından çalıştırılabilir. +- Bu `memcpy` CentOS7/Fedora üzerinde inşa ederken uygulama doğru seçilir. [Etienne Champetier](https://github.com/ClickHouse/ClickHouse/pull/2912) +- Oluşturmak için clang kullanırken, bazı uyarılar `-Weverything` düzenli ek olarak, eklenmiştir `-Wall-Wextra -Werror`. [\#2957](https://github.com/ClickHouse/ClickHouse/pull/2957) +- Yapı hata ayıklama kullanır `jemalloc` hata ayıklama seçeneği. +- ZooKeeper ile etkileşim için kütüphanenin arayüzü soyut olarak ilan edilir. [\#2950](https://github.com/ClickHouse/ClickHouse/pull/2950) + +## ClickHouse sürümü 18.10 {#clickhouse-release-18-10} + +### ClickHouse yayın 18.10.3, 2018-08-13 {#clickhouse-release-18-10-3-2018-08-13} + +#### Yenilik: {#new-features-5} + +- HTTPS çoğaltma için kullanılabilir. [\#2760](https://github.com/ClickHouse/ClickHouse/pull/2760) +- Fonksiyonları eklendi `murmurHash2_64`, `murmurHash3_32`, `murmurHash3_64`, ve `murmurHash3_128` mevcut ek olarak `murmurHash2_32`. [\#2791](https://github.com/ClickHouse/ClickHouse/pull/2791) +- ClickHouse ODBC sürücüsündeki null türleri için destek (`ODBCDriver2` çıkış biçimi). [\#2834](https://github.com/ClickHouse/ClickHouse/pull/2834) +- İçin destek `UUID` anahtar sütunlarda. + +#### Geliştirmeler: {#improvements-5} + +- Kümeler, yapılandırma dosyalarından silindiğinde sunucuyu yeniden başlatmadan kaldırılabilir. [\#2777](https://github.com/ClickHouse/ClickHouse/pull/2777) +- Dış sözlükler, yapılandırma dosyalarından kaldırıldıklarında sunucuyu yeniden başlatmadan kaldırılabilir. [\#2779](https://github.com/ClickHouse/ClickHouse/pull/2779) +- Katma `SETTINGS` için destek `Kafka` masa motoru. [Alexander Marshalov](https://github.com/ClickHouse/ClickHouse/pull/2781) +- İçin iyileştirmeler `UUID` veri türü (henüz tamamlanmadı). [\#2618](https://github.com/ClickHouse/ClickHouse/pull/2618) +- Birleştikten sonra boş parçalar için destek `SummingMergeTree`, `CollapsingMergeTree` ve `VersionedCollapsingMergeTree` motorlar. [\#2815](https://github.com/ClickHouse/ClickHouse/pull/2815) +- Tamamlanan Mut recordsasyon recordsların eski kayıtları silinir (`ALTER DELETE`). [\#2784](https://github.com/ClickHouse/ClickHouse/pull/2784) +- Add theed the `system.merge_tree_settings` Tablo. [Kirill Shvakov](https://github.com/ClickHouse/ClickHouse/pull/2841) +- Bu `system.tables` tablo artık bağımlılık sütunları var: `dependencies_database` ve `dependencies_table`. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2851) +- Add theed the `max_partition_size_to_drop` yapılandırma seçeneği. [\#2782](https://github.com/ClickHouse/ClickHouse/pull/2782) +- Add theed the `output_format_json_escape_forward_slashes` seçenek. [Alexander Bocharov](https://github.com/ClickHouse/ClickHouse/pull/2812) +- Add theed the `max_fetch_partition_retries_count` ayar. [\#2831](https://github.com/ClickHouse/ClickHouse/pull/2831) +- Add theed the `prefer_localhost_replica` yerel bir yineleme tercihini devre dışı bırakmak ve işlemler arası etkileşim olmadan yerel bir yinelemeye gitmek için ayarlama. [\#2832](https://github.com/ClickHouse/ClickHouse/pull/2832) +- Bu `quantileExact` toplama işlevi döndürür `nan` boş bir toplama durumunda `Float32` veya `Float64` koymak. [Sundy Li](https://github.com/ClickHouse/ClickHouse/pull/2855) + +#### Hata düzeltmeleri: {#bug-fixes-14} + +- ODBC için bağlantı dizesi parametrelerinin gereksiz bir şekilde kaçması kaldırıldı, bu da bağlantı kurmayı imkansız hale getirdi. Bu hata 18.6.0 sürümünde oluştu. +- İşleme mantığı düzeltildi `REPLACE PARTITION` çoğaltma sırasındaki komutlar. İki tane varsa `REPLACE` aynı bölüm için komutlar, yanlış mantık, bunlardan birinin çoğaltma kuyruğunda kalmasına ve yürütülmesine neden olabilir. [\#2814](https://github.com/ClickHouse/ClickHouse/pull/2814) +- Tüm veri parçaları boş olduğunda bir birleştirme hatası düzeltildi (birleştirme veya `ALTER DELETE` tüm veriler silindiyse). Bu hata 18.1.0 sürümünde göründü. [\#2930](https://github.com/ClickHouse/ClickHouse/pull/2930) +- Eşzamanlı için bir hata düzeltildi `Set` veya `Join`. [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2823) +- Sabit `Block structure mismatch in UNION stream: different number of columns` için oluşan hata `UNION ALL` bir alt sorgu içindeki sorgular `SELECT` sorgular yinelenen sütun adları içerir. [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2094) +- Bir MySQL sunucusuna bağlanırken bir istisna oluşursa bir bellek sızıntısı düzeltildi. +- Bir sorgu hatası durumunda sabit yanlış clickhouse-istemci yanıt kodu. +- Farklı içeren hayata görünümleri sabit yanlış davranış. [\#2795](https://github.com/ClickHouse/ClickHouse/issues/2795) + +#### Geriye dönük uyumsuz değişiklikler {#backward-incompatible-changes-4} + +- Dağıtılmış tablolar için kontrol tablosu sorguları için destek kaldırıldı. + +#### Yapı değişiklikleri: {#build-changes-3} + +- Ayırıcı değiştirildi: `jemalloc` şimdi yerine kullanılır `tcmalloc`. Bazı senaryolarda, bu hız %20'ye kadar artar. Ancak, %20'ye kadar yavaşlamış sorgular vardır. Bellek tüketimi, bazı senaryolarda geliştirilmiş kararlılık ile yaklaşık %10 oranında azaltılmıştır. Son derece rekabetçi yüklerle, userspace ve sistemdeki CPU kullanımı sadece hafif bir artış gösterir. [\#2773](https://github.com/ClickHouse/ClickHouse/pull/2773) +- Bir alt modülden libressl kullanımı. [\#1983](https://github.com/ClickHouse/ClickHouse/pull/1983) [\#2807](https://github.com/ClickHouse/ClickHouse/pull/2807) +- Bir alt modülden unixodbc kullanımı. [\#2789](https://github.com/ClickHouse/ClickHouse/pull/2789) +- Bir alt modülden mariadb-connector-c kullanımı. [\#2785](https://github.com/ClickHouse/ClickHouse/pull/2785) +- Test verilerinin kullanılabilirliğine bağlı olan depoya işlevsel test dosyaları eklendi (şimdilik, test verilerinin kendisi olmadan). + +## ClickHouse sürümü 18.6 {#clickhouse-release-18-6} + +### ClickHouse yayın 18.6.0, 2018-08-02 {#clickhouse-release-18-6-0-2018-08-02} + +#### Yenilik: {#new-features-6} + +- Sözdizimi üzerinde katılmak için ifadeler için destek eklendi: + `JOIN ON Expr([table.]column ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` + İfade, and operatörü tarafından birleştirilen bir eşitlik zinciri olmalıdır. Eşitliğin her bir tarafı, tablolardan birinin sütunları üzerinde keyfi bir ifade olabilir. Tam sütun adlarının kullanımı desteklenir (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) doğru tablo için. [\#2742](https://github.com/ClickHouse/ClickHouse/pull/2742) +- HTTPS çoğaltma için etkinleştirilebilir. [\#2760](https://github.com/ClickHouse/ClickHouse/pull/2760) + +#### Geliştirmeler: {#improvements-6} + +- Sunucu, sürümünün düzeltme eki bileşenini istemciye geçirir. Yama sürümü bileşeni ile ilgili veriler `system.processes` ve `query_log`. [\#2646](https://github.com/ClickHouse/ClickHouse/pull/2646) + +## ClickHouse sürümü 18.5 {#clickhouse-release-18-5} + +### ClickHouse sürümü 18.5.1, 2018-07-31 {#clickhouse-release-18-5-1-2018-07-31} + +#### Yenilik: {#new-features-7} + +- Karma fonksiyonu eklendi `murmurHash2_32` [\#2756](https://github.com/ClickHouse/ClickHouse/pull/2756). + +#### Geliştirmeler: {#improvements-7} + +- Şimdi kullanabilirsiniz `from_env` [\#2741](https://github.com/ClickHouse/ClickHouse/pull/2741) ortam değişkenlerinden yapılandırma dosyalarındaki değerleri ayarlamak için öznitelik. +- Eklenen büyük / küçük harf duyarlı sürümleri `coalesce`, `ifNull`, ve `nullIf functions` [\#2752](https://github.com/ClickHouse/ClickHouse/pull/2752). + +#### Hata düzeltmeleri: {#bug-fixes-15} + +- Bir kopya başlatırken olası bir hata düzeltildi [\#2759](https://github.com/ClickHouse/ClickHouse/pull/2759). + +## ClickHouse sürümü 18.4 {#clickhouse-release-18-4} + +### ClickHouse yayın 18.4.0, 2018-07-28 {#clickhouse-release-18-4-0-2018-07-28} + +#### Yenilik: {#new-features-8} + +- Eklenen sistem tabloları: `formats`, `data_type_families`, `aggregate_function_combinators`, `table_functions`, `table_engines`, `collations` [\#2721](https://github.com/ClickHouse/ClickHouse/pull/2721). +- Bir argüman olarak bir tablo yerine bir tablo işlevini kullanma yeteneği eklendi `remote` veya `cluster table function` [\#2708](https://github.com/ClickHouse/ClickHouse/pull/2708). +- İçin destek `HTTP Basic` çoğaltma protokolünde kimlik doğrulama [\#2727](https://github.com/ClickHouse/ClickHouse/pull/2727). +- Bu `has` fonksiyon artık bir dizi sayısal bir değer için arama sağlar `Enum` değerler [Maxim Khrisanfov](https://github.com/ClickHouse/ClickHouse/pull/2699). +- Gelen okurken keyfi mesaj ayırıcılar eklemek için destek `Kafka` [Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2701). + +#### Geliştirmeler: {#improvements-8} + +- Bu `ALTER TABLE t DELETE WHERE` sorgu, where koşulundan etkilenmeyen veri bölümlerini yeniden yazmaz [\#2694](https://github.com/ClickHouse/ClickHouse/pull/2694). +- Bu `use_minimalistic_checksums_in_zookeeper` seçeneği için `ReplicatedMergeTree` tablolar varsayılan olarak etkindir. Bu ayar 1.1.54378, 2018-04-16 sürümüne eklendi. 1.1.54378'den eski sürümler artık yüklenemez. +- Koşu için destek `KILL` ve `OPTIMIZE` belirten sorgular `ON CLUSTER` [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2689). + +#### Hata düzeltmeleri: {#bug-fixes-16} + +- Hata düzeltildi `Column ... is not under an aggregate function and not in GROUP BY` bir in ifadesi ile toplama için. Bu hata 18.1.0 sürümünde göründü. ([bbdd780b](https://github.com/ClickHouse/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) +- Bu bir hata düzeltildi `windowFunnel aggregate function` [Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2735). +- Bu bir hata düzeltildi `anyHeavy` toplama fonksiyonu ([a2101df2](https://github.com/ClickHouse/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) +- Kullanırken sabit sunucu çökmesi `countArray()` toplama işlevi. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-5} + +- İçin parametreler `Kafka` motor değiştirildi `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` -e doğru `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. Tablolar kullanıyorsanız `kafka_schema` veya `kafka_num_consumers` parametreleri, elle meta dosyaları düzenlemek zorunda `path/metadata/database/table.sql` ve Ekle `kafka_row_delimiter` parametre ile `''` değer. + +## ClickHouse sürümü 18.1 {#clickhouse-release-18-1} + +### ClickHouse sürümü 18.1.0, 2018-07-23 {#clickhouse-release-18-1-0-2018-07-23} + +#### Yenilik: {#new-features-9} + +- İçin destek `ALTER TABLE t DELETE WHERE` çoğaltılmamış MergeTree tabloları için sorgu ([\#2634](https://github.com/ClickHouse/ClickHouse/pull/2634)). +- İçin keyfi türleri için destek `uniq*` agrega fonksiyonları ailesi ([\#2010](https://github.com/ClickHouse/ClickHouse/issues/2010)). +- Karşılaştırma operatörlerinde keyfi türler için destek ([\#2026](https://github.com/ClickHouse/ClickHouse/issues/2026)). +- Bu `users.xml` dosya biçiminde bir alt ağ maskesi ayarlama sağlar `10.0.0.1/255.255.255.0`. Bu, ortada sıfır olan IPv6 ağları için Maskeler kullanmak için gereklidir ([\#2637](https://github.com/ClickHouse/ClickHouse/pull/2637)). +- Add theed the `arrayDistinct` işlev ([\#2670](https://github.com/ClickHouse/ClickHouse/pull/2670)). +- SummingMergeTree altyapısı artık AggregateFunction türü sütunları ile çalışabilir ([Constantin S. Pan](https://github.com/ClickHouse/ClickHouse/pull/2566)). + +#### Geliştirmeler: {#improvements-9} + +- Sürüm sürümleri için numaralandırma düzenini değiştirdi. Şimdi ilk bölüm sürüm yılını içeriyor (A. D., Moskova saat dilimi, eksi 2000), ikinci bölüm büyük değişikliklerin sayısını içeriyor (çoğu sürüm için artar) ve üçüncü bölüm yama sürümüdür. Aksi changelog belirtilmediği sürece bültenleri hala geriye dönük uyumludur. +- Kayan noktalı sayıların bir dizeye daha hızlı dönüştürülmesi ([Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2664)). +- Ayrıştırma hataları nedeniyle bir ekleme sırasında bazı satırlar atlanmışsa (bu, `input_allow_errors_num` ve `input_allow_errors_ratio` ayarları etkin), atlanan satır sayısı şimdi sunucu günlüğüne yazılır ([Leonardo Cecchi](https://github.com/ClickHouse/ClickHouse/pull/2669)). + +#### Hata düzeltmeleri: {#bug-fixes-17} + +- Geçici tablolar için TRUNCATE komutu düzeltildi ([Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2624)). +- Yanıtı okurken bir ağ hatası olduğunda meydana gelen ZooKeeper istemci kütüphanesinde nadir bir kilitlenme düzeltildi ([c315200](https://github.com/ClickHouse/ClickHouse/commit/c315200e64b87e44bdf740707fc857d1fdf7e947)). +- Null türleri için bir döküm sırasında bir hata düzeltildi ([\#1322](https://github.com/ClickHouse/ClickHouse/issues/1322)). +- Yanlış sonuç düzeltildi `maxIntersection()` aralıkların sınırları çakıştığında işlev ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2657)). +- Bir işlev argümanında or ifade zincirinin yanlış dönüşümü düzeltildi ([chenxing-xc](https://github.com/ClickHouse/ClickHouse/pull/2663)). +- İçeren sorgular için sabit performans düşüşü `IN (subquery)` başka bir alt sorgu içindeki ifadeler ([\#2571](https://github.com/ClickHouse/ClickHouse/issues/2571)). +- Bir kullanan dağıtılmış sorgularda farklı sürümleri ile sunucular arasında sabit uyumsuzluk `CAST` büyük harflerle olmayan işlev ([fe8c4d6](https://github.com/ClickHouse/ClickHouse/commit/fe8c4d64e434cacd4ceef34faa9005129f2190a5)). +- Harici bir DBMS sorguları için tanımlayıcıları alıntı eksik eklendi ([\#2635](https://github.com/ClickHouse/ClickHouse/issues/2635)). + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-6} + +- DateTime için sıfır sayısını içeren bir dize dönüştürme çalışmaz. Örnek: `SELECT toDateTime('0')`. Bu da nedeni `DateTime DEFAULT '0'` tablolarda da çalışmıyor `0` sözlüklerde. Çözüm: değiştirin `0` ile `0000-00-00 00:00:00`. + +## ClickHouse sürüm 1.1 {#clickhouse-release-1-1} + +### ClickHouse sürümü 1.1.54394, 2018-07-12 {#clickhouse-release-1-1-54394-2018-07-12} + +#### Yenilik: {#new-features-10} + +- Add theed the `histogram` toplama fonksiyonu ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2521)). +- Şimdi `OPTIMIZE TABLE ... FINAL` için bölümler belirt specifyingmeden kullanılabilir `ReplicatedMergeTree` ([Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2600)). + +#### Hata düzeltmeleri: {#bug-fixes-18} + +- Çoğaltılmış verileri gönderirken ve indirirken okumak ve yazmak için soketler için çok küçük bir zaman aşımı (bir saniye) ile ilgili bir sorun düzeltildi, bu da ağ veya diskte bir yük varsa daha büyük parçaları indirmeyi imkansız hale getirdi (parçaları indirmek için döngüsel girişimlerle sonuçlandı). Bu hata 1.1.54388 sürümünde oluştu. +- Tabloda yinelenen veri blokları eklediyseniz zookeeper chroot kullanırken sorunlar giderildi. +- Bu `has` işlev artık boş öğelere sahip bir dizi için doğru çalışıyor ([\#2115](https://github.com/ClickHouse/ClickHouse/issues/2115)). +- Bu `system.tables` tablo şimdi dağıtılmış sorgularda kullanıldığında düzgün çalışır. Bu `metadata_modification_time` ve `engine_full` sütunlar artık sanal değil. Tablodan yalnızca bu sütunlar sorgulanırsa oluşan bir hata düzeltildi. +- Sabit nasıl boş `TinyLog` tablo boş bir veri bloğu ekledikten sonra çalışır ([\#2563](https://github.com/ClickHouse/ClickHouse/issues/2563)). +- Bu `system.zookeeper` zookeeper düğümün değeri NULL ise Tablo çalışır. + +### ClickHouse sürümü 1.1.54390, 2018-07-06 {#clickhouse-release-1-1-54390-2018-07-06} + +#### Yenilik: {#new-features-11} + +- Sorgular gönderilebilir `multipart/form-data` biçim (içinde `query` alan), sorgu işleme için harici veriler de gönderilirse yararlıdır ([Olga Hvostikova](https://github.com/ClickHouse/ClickHouse/pull/2490)). +- CSV formatında veri okurken tek veya çift tırnak işleme etkinleştirmek veya devre dışı bırakmak için yeteneği eklendi. Bunu şu şekilde yapılandırabilirsiniz: `format_csv_allow_single_quotes` ve `format_csv_allow_double_quotes` ayarlar ([Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2574)). +- Şimdi `OPTIMIZE TABLE ... FINAL` olmayan çoğaltılmış varyantları için bölüm belirtmeden kullanılabilir `MergeTree` ([Amos Kuşu](https://github.com/ClickHouse/ClickHouse/pull/2599)). + +#### Geliştirmeler: {#improvements-10} + +- Geliştirilmiş performans, azaltılmış bellek tüketimi, ve doğru bellek tüketimi izleme kullanımı ile In operatör zaman bir tablo Endeksi kullanılabilir ([\#2584](https://github.com/ClickHouse/ClickHouse/pull/2584)). +- Bir veri parçası eklerken sağlama toplamlarının gereksiz denetimi kaldırıldı. Bu, çok sayıda çoğaltma olduğunda önemlidir,çünkü bu durumlarda toplam kontrol sayısı n^2'ye eşittir. +- İçin destek eklendi `Array(Tuple(...))` arg theum argumentsents for the `arrayEnumerateUniq` işlev ([\#2573](https://github.com/ClickHouse/ClickHouse/pull/2573)). +- Katma `Nullable` için destek `runningDifference` işlev ([\#2594](https://github.com/ClickHouse/ClickHouse/pull/2594)). +- Çok sayıda ifade olduğunda geliştirilmiş sorgu analizi performansı ([\#2572](https://github.com/ClickHouse/ClickHouse/pull/2572)). +- Birleştirme için veri parçalarının daha hızlı seçimi `ReplicatedMergeTree` Tablolar. ZooKeeper oturumunun daha hızlı iyileşmesi ([\#2597](https://github.com/ClickHouse/ClickHouse/pull/2597)). +- Bu `format_version.txt` için dosya `MergeTree` eksikse tablolar yeniden oluşturulur, bu da dizin yapısını dosyalar olmadan kopyaladıktan sonra ClickHouse başlatılırsa mantıklı olur ([Ciprian Hacman](https://github.com/ClickHouse/ClickHouse/pull/2593)). + +#### Hata düzeltmeleri: {#bug-fixes-19} + +- ZooKeeper ile çalışırken, sunucuyu yeniden başlatmadan önce oturumu ve readonly tablo durumlarını kurtarmayı imkansız hale getirebilecek bir hata düzeltildi. +- Oturum kesilirse eski düğümlerin silinmemesine neden olabilecek ZooKeeper ile çalışırken bir hata düzeltildi. +- Sabit bir hata `quantileTDigest` Float argümanları için işlev (bu hata 1.1.54388 sürümünde tanıtıldı) ([Mikhail Surin](https://github.com/ClickHouse/ClickHouse/pull/2553)). +- Birincil anahtar sütunu, aynı boyuttaki imzalı ve imzasız tamsayılar arasındaki türleri dönüştürmek için işlevin içinde bulunuyorsa, mergetree tabloları için dizinde bir hata düzeltildi ([\#2603](https://github.com/ClickHouse/ClickHouse/pull/2603)). +- Sabit segfault eğer `macros` kullanılır ancak yapılandırma dosyasında değildir ([\#2570](https://github.com/ClickHouse/ClickHouse/pull/2570)). +- İstemci yeniden bağlanırken varsayılan veritabanına sabit anahtarlama ([\#2583](https://github.com/ClickHouse/ClickHouse/pull/2583)). +- Ne zaman meydana gelen bir hata düzeltildi `use_index_for_in_with_subqueries` ayar devre dışı bırakıldı. + +#### Güvenlik düzeltme: {#security-fix-1} + +- Mysql'e bağlandığında dosya göndermek artık mümkün değil (`LOAD DATA LOCAL INFILE`). + +### ClickHouse sürümü 1.1.54388, 2018-06-28 {#clickhouse-release-1-1-54388-2018-06-28} + +#### Yenilik: {#new-features-12} + +- İçin destek `ALTER TABLE t DELETE WHERE` çoğaltılmış tablolar için sorgu. Add theed the `system.mutations` bu tür sorguların ilerlemesini izlemek için tablo. +- İçin destek `ALTER TABLE t [REPLACE|ATTACH] PARTITION` \*MergeTree tabloları için sorgu. +- İçin destek `TRUNCATE TABLE` sorgu ([Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2260)) +- Birkaç yeni `SYSTEM` çoğaltılmış tablolar için sorgular (`RESTART REPLICAS`, `SYNC REPLICA`, `[STOP|START] [MERGES|FETCHES|SENDS REPLICATED|REPLICATION QUEUES]`). +- MySQL motoru ve ilgili tablo fonksiyonu ile bir tabloya yazma yeteneği eklendi ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2294)). +- Add theed the `url()` tablo fonksiyonu ve `URL` masa motoru ([Alexander Sapin](https://github.com/ClickHouse/ClickHouse/pull/2501)). +- Add theed the `windowFunnel` toplama fonksiyonu ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2352)). +- Yeni `startsWith` ve `endsWith` dizeler için işlevler ([Vadim Plakhtinsky](https://github.com/ClickHouse/ClickHouse/pull/2429)). +- Bu `numbers()` tablo işlevi artık ofset belirtmenizi sağlar ([Kış Zhang](https://github.com/ClickHouse/ClickHouse/pull/2535)). +- Şifre için `clickhouse-client` etkileşimli olarak girilebilir. +- Sunucu günlükleri artık syslog'a gönderilebilir ([Alexander Krasheninnikov](https://github.com/ClickHouse/ClickHouse/pull/2459)). +- Paylaşılan kitaplık kaynağı ile sözlüklerde oturum açma desteği ([Alexander Sapin](https://github.com/ClickHouse/ClickHouse/pull/2472)). +- Özel CSV sınırlayıcılar için destek ([Ivan Zhukov](https://github.com/ClickHouse/ClickHouse/pull/2263)) +- Add theed the `date_time_input_format` ayar. Bu ayarı şu şekilde değiştirirseniz `'best_effort'`, DateTime değerleri biçimleri geniş bir yelpazede okunacaktır. +- Add theed the `clickhouse-obfuscator` veri gizleme için yardımcı program. Kullanım örneği: performans testlerinde kullanılan verileri yayınlama. + +#### Deneysel özellikler: {#experimental-features-2} + +- Hesaplamak için yeteneği eklendi `and` argümanlar sadece ihtiyaç duydukları yerde ([Anastasia Tsarkova](https://github.com/ClickHouse/ClickHouse/pull/2272)) +- Yerel kod JIT derleme bazı ifadeler için artık kullanılabilir ([pyos](https://github.com/ClickHouse/ClickHouse/pull/2277)). + +#### Hata düzeltmeleri: {#bug-fixes-20} + +- Yinelemeler artık bir sorgu için görünmüyor `DISTINCT` ve `ORDER BY`. +- İle sorgular `ARRAY JOIN` ve `arrayFilter` artık yanlış bir sonuç döndürmez. +- İç içe geçmiş bir yapıdan bir dizi sütunu okurken bir hata düzeltildi ([\#2066](https://github.com/ClickHouse/ClickHouse/issues/2066)). +- Gibi bir having yan tümcesi ile sorguları analiz ederken bir hata düzeltildi `HAVING tuple IN (...)`. +- Özyinelemeli takma adlarla sorguları analiz ederken bir hata düzeltildi. +- Tüm satırları filtreleyen PREWHERE bir koşul ile ReplacingMergeTree okurken bir hata düzeltildi ([\#2525](https://github.com/ClickHouse/ClickHouse/issues/2525)). +- HTTP arabirimindeki oturumları kullanırken kullanıcı profili ayarları uygulanmadı. +- Clickhouse-local'deki komut satırı parametrelerinden ayarların nasıl uygulandığı düzeltildi. +- ZooKeeper istemci kitaplığı artık sunucudan alınan oturum zaman aşımını kullanır. +- İstemci zaman aşımı daha uzun sunucu yanıt bekledi ZooKeeper istemci kütüphanesinde bir hata düzeltildi. +- Bölüm anahtar sütunlarındaki koşullarla sorgular için parçaların sabit budaması ([\#2342](https://github.com/ClickHouse/ClickHouse/issues/2342)). +- Birleşmeler şimdi mümkün `CLEAR COLUMN IN PARTITION` ([\#2315](https://github.com/ClickHouse/ClickHouse/issues/2315)). +- ODBC tablo işlevinde tür eşleme düzeltildi ([sundy-li](https://github.com/ClickHouse/ClickHouse/pull/2268)). +- Tür karşılaştır formaları Düzelt forildi `DateTime` saat dilimi ile ve olmadan ([Alexander Bocharov](https://github.com/ClickHouse/ClickHouse/pull/2400)). +- Sabit sözdizimsel ayrıştırma ve biçimlendirme `CAST` operatör. +- Dağıtılmış tablo motoru için somutlaştırılmış bir görünüme sabit ekleme ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2411)). +- Veri yazarken bir yarış durumu düzeltildi `Kafka` motor için hayata görünümleri ([Yangkuan Liu](https://github.com/ClickHouse/ClickHouse/pull/2448)). +- Uzak () tablo işlevinde sabit SSRF. +- Sabit çıkış davranışı `clickhouse-client` çok satırlı modda ([\#2510](https://github.com/ClickHouse/ClickHouse/issues/2510)). + +#### Geliştirmeler: {#improvements-11} + +- Çoğaltılmış tablolardaki arka plan görevleri artık ayrı iş parçacıkları yerine bir iş parçacığı havuzunda gerçekleştirilir ([Silviu Caragea](https://github.com/ClickHouse/ClickHouse/pull/1722)). +- Geliştirilmiş lz4 sıkıştırma performansı. +- Çok sayıda birleştirme ve alt sorgu ile sorgular için daha hızlı analiz. +- Çok fazla ağ hatası olduğunda DNS önbelleği artık otomatik olarak güncellenir. +- Çok fazla parçaya sahip olduğundan, materialized görünümlerden birine ekleme mümkün değilse, tablo ekleri artık oluşmaz. +- Olay sayaçlarındaki tutarsızlık düzeltildi `Query`, `SelectQuery`, ve `InsertQuery`. +- Gibi ifadeler `tuple IN (SELECT tuple)` tuple türleri eşleşirse izin verilir. +- Çoğaltılmış tabloları olan bir sunucu, Zookeeper'ı yapılandırmasanız bile başlayabilir. +- Mevcut CPU çekirdeklerinin sayısını hesaplarken, C gruplarındaki sınırlar şimdi dikkate alınmaktadır ([Atri Sharma](https://github.com/ClickHouse/ClickHouse/pull/2325)). +- Systemd yapılandırma dosyasında yapılandırma dizinleri için chown eklendi ([Mikhail Shiryaev](https://github.com/ClickHouse/ClickHouse/pull/2421)). + +#### Yapı değişiklikleri: {#build-changes-4} + +- Gcc8 derleyicisi yapılar için kullanılabilir. +- Alt modülden llvm oluşturma yeteneği eklendi. +- Librdkafka kütüphanesinin sürümü v0.11.4 olarak güncellendi. +- Sistem libcpuid kütüphanesini kullanma yeteneği eklendi. Kütüphane sürümü 0.4.0 olarak güncellendi. +- Vectorclass kütüphanesini kullanarak yapı düzeltildi ([Babacar Diassé](https://github.com/ClickHouse/ClickHouse/pull/2274)). +- Cmake şimdi varsayılan olarak ninja için dosyalar üretir (kullanırken olduğu gibi `-G Ninja`). +- Libtermcap yerine libtınfo kütüphanesini kullanma yeteneği eklendi ([Georgy Kondratiev](https://github.com/ClickHouse/ClickHouse/pull/2519)). +- Fedora Rawhide bir başlık dosyası çakışması düzeltildi ([\#2520](https://github.com/ClickHouse/ClickHouse/issues/2520)). + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-7} + +- Çıkar escapingıldı `Vertical` ve `Pretty*` format Andlar ve silinen `VerticalRaw` biçimli. +- 1.1.54388 (veya daha yeni) sürümüne sahip sunucular ve daha eski bir sürüme sahip sunucular dağıtılmış bir sorguda aynı anda kullanılıyorsa ve sorgunun `cast(x, 'Type')` ifade olmadan `AS` anahtar kelime ve kelime yok `cast` büyük harfle, bir istisna gibi bir mesajla atılır `Not found column cast(0, 'UInt8') in block`. Çözüm: tüm kümedeki sunucuyu güncelleyin. + +### ClickHouse sürümü 1.1.54385, 2018-06-01 {#clickhouse-release-1-1-54385-2018-06-01} + +#### Hata düzeltmeleri: {#bug-fixes-21} + +- Bazı durumlarda zookeeper işlemlerinin engellenmesine neden olan bir hata düzeltildi. + +### ClickHouse sürümü 1.1.54383, 2018-05-22 {#clickhouse-release-1-1-54383-2018-05-22} + +#### Hata düzeltmeleri: {#bug-fixes-22} + +- Bir tablo birçok yinelemeler varsa çoğaltma kuyruğunun bir yavaşlama düzeltildi. + +### ClickHouse sürümü 1.1.54381, 2018-05-14 {#clickhouse-release-1-1-54381-2018-05-14} + +#### Hata düzeltmeleri: {#bug-fixes-23} + +- ClickHouse ZooKeeper sunucusuna bağlantı kaybettiğinde ZooKeeper bir düğüm sızıntısı düzeltildi. + +### ClickHouse sürüm 1.1.54380, 2018-04-21 {#clickhouse-release-1-1-54380-2018-04-21} + +#### Yenilik: {#new-features-13} + +- Tablo fonksiyonu eklendi `file(path, format, structure)`. Örnek okuma baytları `/dev/urandom`: ``` ln -s /dev/urandom /var/lib/clickhouse/user_files/random``clickhouse-client -q "SELECT * FROM file('random', 'RowBinary', 'd UInt8') LIMIT 10" ```. + +#### Geliştirmeler: {#improvements-12} + +- Alt sorgular sarılabilir `()` sorgu okunabilirliğini artırmak için parantez. Mesela: `(SELECT 1) UNION ALL (SELECT 1)`. +- Basit `SELECT` gelen sorgular `system.processes` tablo dahil değildir `max_concurrent_queries` sınır. + +#### Hata düzeltmeleri: {#bug-fixes-24} + +- Sabit yanlış davranış `IN` operatör ne zaman seçin `MATERIALIZED VIEW`. +- Gibi ifadelerde bölüm indeksi ile sabit yanlış filtreleme `partition_key_column IN (...)`. +- Sabit yetersizlik yürütmek için `OPTIMIZE` eğer lider olmayan çoğaltma üzerinde sorgu `REANAME` masaya yapıldı. +- Yürütülürken yetkilendirme hatası düzeltildi `OPTIMIZE` veya `ALTER` olmayan bir lider çoğaltma sorgular. +- Sabit donma `KILL QUERY`. +- Saatlerin kaybına yol açan ZooKeeper istemci kütüphanesinde bir hata, dağıtılmış DDL kuyruğunun dondurulması ve boş olmayan bir çoğaltma kuyruğundaki yavaşlamalar düzeltildi `chroot` önek ZooKeeper yapılandırmasında kullanılır. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-8} + +- Gibi ifadeler için destek kaldırıldı `(a, b) IN (SELECT (a, b))` (eşdeğer ifadeyi kullanabilirsiniz `(a, b) IN (SELECT a, b)`). Önceki sürümlerde, bu ifadeler belirsizliğe yol açtı `WHERE` filtreleme veya neden olan hatalar. + +### ClickHouse sürümü 1.1.54378, 2018-04-16 {#clickhouse-release-1-1-54378-2018-04-16} + +#### Yenilik: {#new-features-14} + +- Günlük düzeyi sunucuyu yeniden başlatmadan değiştirilebilir. +- Add theed the `SHOW CREATE DATABASE` sorgu. +- Bu `query_id` geç canilebilir `clickhouse-client` (elBroom). +- Yeni ayar: `max_network_bandwidth_for_all_users`. +- İçin destek eklendi `ALTER TABLE ... PARTITION ...` için `MATERIALIZED VIEW`. +- Sistem tablosunda sıkıştırılmamış formdaki veri parçalarının boyutu hakkında bilgi eklendi. +- Dağıtılmış tablolar için sunucudan sunucuya şifreleme desteği (`1` rep thelik thea config in ``). +- Tablo düzeyi için yapılandırma `ReplicatedMergeTree` Zookeeper saklanan veri miktarını en aza indirmek için aile: : `use_minimalistic_checksums_in_zookeeper = 1` +- Yapılandırma `clickhouse-client` istem. Varsayılan olarak, sunucu adları artık istemine çıktı. Sunucunun görünen adı değiştirilebilir. Ayrıca gönderilen `X-ClickHouse-Display-Name` HTTP Başlığı (Kirill Shvakov). +- Birden çok virgülle ayrılmış `topics` için belirt theilebilir `Kafka` motor (Tobias Adamson) +- Bir sorgu tarafından durdurulduğunda `KILL QUERY` veya `replace_running_query`, müşteri alır `Query was canceled` eksik bir sonuç yerine istisna. + +#### Geliştirmeler: {#improvements-13} + +- `ALTER TABLE ... DROP/DETACH PARTITION` sorgular, çoğaltma kuyruğunun önünde çalıştırılır. +- `SELECT ... FINAL` ve `OPTIMIZE ... FINAL` tablo tek bir veri parçası olduğunda bile kullanılabilir. +- A `query_log` el ile silindiyse tablo anında yeniden oluşturulur (Kirill Shvakov). +- Bu `lengthUTF8` fonksiyon daha hızlı çalışır (zhang2014). +- Senkron ekler geliştirilmiş performans `Distributed` Tablolar (`insert_distributed_sync = 1`) çok sayıda parça olduğunda. +- Sunucu kabul eder `send_timeout` ve `receive_timeout` istemciden ayarlar ve istemciye bağlanırken bunları uygular (bunlar ters sırada uygulanır: sunucu soketinin `send_timeout` için ayarlanır `receive_timeout` müşteriden alınan değer ve tersi). +- Asenkron ekleme için daha sağlam kilitlenme kurtarma `Distributed` Tablolar. +- Dönüş türü `countEqual` fonksiyon değiştirildi `UInt32` -e doğru `UInt64` (谢磊). + +#### Hata düzeltmeleri: {#bug-fixes-25} + +- Bir hata düzeltildi `IN` ifadenin sol tarafı olduğunda `Nullable`. +- Tuples ile birlikte kullanıldığında doğru sonuçlar şimdi döndürülür `IN` bazı tuple bileşenleri tablo dizininde olduğunda. +- Bu `max_execution_time` limit şimdi dağıtılmış sorgularla düzgün çalışır. +- Bileşik sütunların boyutunu hesaplarken sabit hatalar `system.columns` Tablo. +- Geçici bir tablo oluştururken bir hata düzeltildi `CREATE TEMPORARY TABLE IF NOT EXISTS.` +- Sabit hatalar `StorageKafka` (\#\#2075) +- Sabit sunucu, belirli toplam işlevlerin geçersiz argümanlarından çöker. +- Engellenen hata düzeltildi `DETACH DATABASE` için arka plan görevlerini durdurma sorgusu `ReplicatedMergeTree` Tablolar. +- `Too many parts` toplu materialized görünümler (\#\#2084) eklerken durum daha az olasıdır. +- Bir ikame aynı düzeyde başka bir ikame tarafından takip edilmesi gerekiyorsa, yapılandırmada değiştirmelerin düzeltilmiş özyinelemeli işleme. +- Bir oluştururken meta veri dosyasında sözdizimi düzeltildi `VIEW` bu bir sorgu ile kullanır `UNION ALL`. +- `SummingMergeTree` şimdi bir bileşik anahtar ile iç içe veri yapılarının toplamı için doğru çalışır. +- Lider seçerken bir yarış durumu olasılığı sabit `ReplicatedMergeTree` Tablolar. + +#### Yapı değişiklikleri: {#build-changes-5} + +- Yapı destekler `ninja` yerine `make` ve kullanımları `ninja` sürümleri oluşturmak için varsayılan olarak. +- Yeniden adlandırılan paketler: `clickhouse-server-base` içinde `clickhouse-common-static`; `clickhouse-server-common` içinde `clickhouse-server`; `clickhouse-common-dbg` içinde `clickhouse-common-static-dbg`. Yüklemek için kullanın `clickhouse-server clickhouse-client`. Eski adlara sahip paketler, geriye dönük uyumluluk için depolara yüklenmeye devam edecektir. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-9} + +- Sol tarafta bir dizi belirtilmişse, bir In ifadesinin özel yorumunu kaldırıldı. Daha önce, ifade `arr IN (set)` olarak yorum waslandı “at least one `arr` element belongs to the `set`”. Yeni sürümde aynı davranışı elde etmek için şunları yazın `arrayExists(x -> x IN (set), arr)`. +- Soket seçeneğinin yanlış kullanımı devre dışı bırakıldı `SO_REUSEPORT`, poco kitaplığında varsayılan olarak yanlış etkinleştirildi. Linux'ta artık adresleri aynı anda belirtmek için herhangi bir neden olmadığını unutmayın `::` ve `0.0.0.0` for listen – use just `::`(varsayılan çekirdek yapılandırma ayarları ile) IPv4 ve IPv6 üzerinden hem bağlantı dinleme sağlar. Belirterek önceki sürümlerden davranışa da geri dönebilirsiniz `1` config. + +### ClickHouse sürümü 1.1.54370, 2018-03-16 {#clickhouse-release-1-1-54370-2018-03-16} + +#### Yenilik: {#new-features-15} + +- Add theed the `system.macros` tablo ve yapılandırma dosyası değiştirildiğinde makroların otomatik güncellenmesi. +- Add theed the `SYSTEM RELOAD CONFIG` sorgu. +- Add theed the `maxIntersections(left_col, right_col)` aynı anda kesişen aralıkların maksimum sayısını döndüren toplama işlevi `[left; right]`. Bu `maxIntersectionsPosition(left, right)` fonksiyonun başlangıcını döndürür “maximum” aralıklı. ([Michael Furmur](https://github.com/ClickHouse/ClickHouse/pull/2012)). + +#### Geliştirmeler: {#improvements-14} + +- Bir veri eklerken `Replicated` tablo, daha az istek yapılır `ZooKeeper` (ve kullanıcı düzeyinde hataların çoğu `ZooKeeper` günlük). +- Veri kümeleri için takma ad oluşturma yeteneği eklendi. Örnek: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`. + +#### Hata düzeltmeleri: {#bug-fixes-26} + +- Sabit `Illegal PREWHERE` birleştirme tablolarından okunurken hata `Distributed`Tablolar. +- Eğer IPv4 sadece Docker kaplarda clickhouse-server başlatmak için izin eklendi düzeltmeler. +- Sistemden okurken bir yarış durumu düzeltildi `system.parts_columns tables.` +- Bir senkron ekleme sırasında çift tamponlama kaldırıldı `Distributed` zaman aşımı için bağlantı neden olabilir tablo. +- Başlamadan önce kullanılamayan bir kopya için aşırı uzun beklemelere neden olan bir hata düzeltildi `SELECT` sorgu. +- Sabit yanlış tarihler `system.parts` Tablo. +- İmkansız bir veri eklemek için yapılan bir hata düzeltildi `Replicated` tablo ise `chroot` yapılandırmada boş değildi `ZooKeeper` küme. +- Boş bir dikey birleştirme algoritması düzeltildi `ORDER BY` Tablo. +- Uzak tablo, bu sözlük istemcisi sunucu üzerinde yoksa bile sorgular sözlük kullanma olanağı sağlandı. Bu işlevsellik 1.1.54362 sürümünde kayboldu. +- Gibi sorgular için davranışı geri `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` ne zaman sağ tarafı `IN` bir uzaktan kumanda kullanmalı `default.table` yerel bir yerine. Bu davranış 1.1.54358 sürümünde bozuldu. +- Kaldırılan gereksiz hata düzeyi günlüğü `Not found column ... in block`. + +### ClickHouse Yayın 1.1.54362, 2018-03-11 {#clickhouse-release-1-1-54362-2018-03-11} + +#### Yenilik: {#new-features-16} + +- Olmadan toplama `GROUP BY` boş bir set için (örneğin `SELECT count(*) FROM table WHERE 0`) şimdi SQL standardına uygun olarak toplam işlevler için boş değerlere sahip bir satırla bir sonuç döndürür. Eski davranışı geri yüklemek için (boş bir sonuç döndürür), `empty_result_for_aggregation_by_empty_set` 1'e. +- İçin Tip dönüştürme eklendi `UNION ALL`. Farklı takma ad adlarına izin verilir `SELECT` pozisyon inlar `UNION ALL`, SQL standardına uygun olarak. +- Keyfi ifadeler desteklenir `LIMIT BY` yanlar. Daha önce, yalnızca aşağıdakilerden kaynaklanan sütunları kullanmak mümkündü `SELECT`. +- Dizini `MergeTree` tablolar şu durumlarda kullanılır `IN` birincil anahtarın sütunlarından bir ifade kümesine uygulanır. Örnek: `WHERE (UserID, EventDate) IN ((123, '2000-01-01'), ...)` (Anastasiya Tsarkova). +- Add theed the `clickhouse-copier` kümeleri ve resharding veri (beta) arasında kopyalama aracı. +- Tutarlı karma fonksiyonları eklendi: `yandexConsistentHash`, `jumpConsistentHash`, `sumburConsistentHash`. Onlar sonraki reshardings sırasında ağ trafiği miktarını azaltmak için bir sharding anahtar olarak kullanılabilir. +- Eklenen fonksiyonlar: `arrayAny`, `arrayAll`, `hasAny`, `hasAll`, `arrayIntersect`, `arrayResize`. +- Add theed the `arrayCumSum` fonksiyon (Javi Santana). +- Add theed the `parseDateTimeBestEffort`, `parseDateTimeBestEffortOrZero`, ve `parseDateTimeBestEffortOrNull` çok çeşitli Olası biçimlerde metin içeren bir dizeden datetime okumak için işlevler. +- Veriler, güncelleme sırasında harici sözlüklerden kısmen yeniden yüklenebilir (yalnızca belirtilen alanın değerinin önceki indirmeden daha büyük olduğu kayıtları yükleyin) (Arsen Hakobyan). +- Add theed the `cluster` tablo işlevi. Örnek: `cluster(cluster_name, db, table)`. Bu `remote` tablo işlevi, bir tanımlayıcı olarak belirtilirse, küme adını ilk bağımsız değişken olarak kabul edebilir. +- Bu `remote` ve `cluster` tablo fonksiyonları kullanılabilir `INSERT` sorgular. +- Add theed the `create_table_query` ve `engine_full` sanal sütunlar için `system.tables`Tablo . Bu `metadata_modification_time` sütun sanal. +- Add theed the `data_path` ve `metadata_path` sütunlar için `system.tables`ve`system.databases` tablolar ve eklenen `path` Col theum then to the `system.parts` ve `system.parts_columns` Tablolar. +- Birleştirme hakkında ek bilgi eklendi `system.part_log` Tablo. +- Keyfi bir bölümleme anahtarı için kullanılabilir `system.query_log` tablo (Kirill Shvakov). +- Bu `SHOW TABLES` sorgu şimdi de geçici tablolar gösterir. Geçici tablolar eklendi ve `is_temporary` sütun için `system.tables` (zhang2014). +- Katma `DROP TEMPORARY TABLE` ve `EXISTS TEMPORARY TABLE` sorgular (zhang2014). +- İçin destek `SHOW CREATE TABLE` geçici tablolar için (zhang2014). +- Add theed the `system_profile` iç işlemler tarafından kullanılan ayarlar için yapılandırma parametresi. +- Yükleme desteği `object_id` bir nitelik olarak `MongoDB` sözlükler (Pavel Litvinenko). +- Okuma `null` harici bir sözlük için veri yüklerken varsayılan değer olarak `MongoDB` kaynak (Pavel Litvinenko). +- Okuma `DateTime` değerleri `Values` tek tırnak işaretleri olmadan bir Unix zaman damgasından biçimlendirin. +- Yük devretme desteklenir `remote` tablo işlevleri bazı yinelemeler istenen tablo eksik olduğunda durumlar için. +- Çalıştırdığınızda yapılandırma ayarları komut satırında geçersiz kılınabilir `clickhouse-server`. Örnek: `clickhouse-server -- --logger.level=information`. +- Uygulanan `empty` fonksiyonu bir `FixedString` argüman: dize tamamen boş bayttan oluşuyorsa (zhang2014) işlev 1 döndürür. +- Add theed the `listen_try`bazı adresler dinlenemiyorsa (IPv4 veya IPv6 için engelli desteği olan sistemler için kullanışlıdır), dinleme adreslerinden en az birini bırakmadan dinlemek için yapılandırma parametresi. +- Add theed the `VersionedCollapsingMergeTree` masa motoru. +- Satır ve rasgele sayısal türleri için destek `library` sözlük kaynağı. +- `MergeTree` tablolar birincil anahtar olmadan kullanılabilir (belirtmeniz gerekir `ORDER BY tuple()`). +- A `Nullable` tipi olabilir `CAST` olmayan -`Nullable` bağımsız değişken değilse yazın `NULL`. +- `RENAME TABLE` için yapılabilir `VIEW`. +- Add theed the `throwIf` işlev. +- Add theed the `odbc_default_field_size` bir ODBC kaynağından yüklenen değerin en büyük boyutunu genişletmenizi sağlayan seçenek (varsayılan olarak, 1024'tür). +- Bu `system.processes` masa ve `SHOW PROCESSLIST` şimdi var `is_cancelled` ve `peak_memory_usage` sütun. + +#### Geliştirmeler: {#improvements-15} + +- Sonuç üzerindeki sınırlar ve kotalar artık Ara verilere uygulanmıyor `INSERT SELECT` sorgular veya `SELECT` alt sorgular. +- Daha az yanlış tetikleyici `force_restore_data` durumunu kontrol ederken `Replicated` sunucu başladığında tablolar. +- Add theed the `allow_distributed_ddl` seçenek. +- Nondeterministic işlevleri için ifadelerde izin verilmez `MergeTree` masa tuşları. +- Değiştirmeleri olan dosyalar `config.d` dizinler alfabetik sırayla yüklenir. +- Geliştirilmiş performans `arrayElement` elemanlardan biri olarak boş bir dizi ile sabit çok boyutlu bir dizi durumunda işlev. Örnek: `[[1], []][x]`. +- Sunucu, yapılandırma dosyalarını çok büyük ikamelerle (örneğin, çok büyük IP ağları listeleri) kullanırken daha hızlı başlar. +- Bir sorgu çalıştırırken, tablo değerli işlevleri bir kez çalıştırın. Önceden, `remote` ve `mysql` tablo değerli işlevler, tablo yapısını uzak bir sunucudan almak için aynı sorguyu iki kez gerçekleştirdi. +- Bu `MkDocs` belg .eleme üret .eci kullanılır. +- Bir tablo sütunu silmeye çalıştığınızda, `DEFAULT`/`MATERIALIZED` diğer sütunların ifadeleri bağlıdır, bir istisna atılır (zhang2014). +- İçin sayı 0 olarak metin biçimlerinde boş bir satır ayrıştırmak için yeteneği eklendi `Float` veri türleri. Bu özellik daha önce mevcuttu, ancak 1.1.54342 sürümünde kayboldu. +- `Enum` değerleri kullanılabilir `min`, `max`, `sum` ve diğer bazı fonksiyonlar. Bu durumlarda, karşılık gelen sayısal değerleri kullanır. Bu özellik daha önce mevcuttu, ancak 1.1.54337 sürümünde kayboldu. +- Katma `max_expanded_ast_elements` yinelemeli genişleyen takma adlar sonra AST boyutunu kısıtlamak için. + +#### Hata düzeltmeleri: {#bug-fixes-27} + +- Gereksiz sütunlar hatalı alt sorgulardan kaldırıldığında veya içeren alt sorgulardan kaldırılmadığında sabit durumlar `UNION ALL`. +- İçin birleştirir bir hata düzeltildi `ReplacingMergeTree` Tablolar. +- Sabit senkron eklemeler `Distributed` Tablolar (`insert_distributed_sync = 1`). +- Belirli kullanımlar için sabit segfault `FULL` ve `RIGHT JOIN` alt sorgularda yinelenen sütunlarla. +- Belirli kullanımlar için sabit segfault `replace_running_query` ve `KILL QUERY`. +- Sabit sipariş `source` ve `last_exception` Col theum thens in the `system.dictionaries` Tablo. +- Sabit hata zaman `DROP DATABASE` sorgu meta verileri ile dosyayı silmedi. +- Sabit `DROP DATABASE` sorgu için `Dictionary` veritabanılar. +- Sabit düşük hassasiyet `uniqHLL12` ve `uniqCombined` 100 milyondan fazla öğe için işlevler (Alex Bocharov). +- Aynı anda varsayılan açık ifadeleri hesaplamak için gerektiğinde örtülü varsayılan değerlerin hesaplanması düzeltildi `INSERT` sorgular (zhang2014). +- Bir sorgu için nadir bir durum düzeltildi `MergeTree` tablo bitiremedi (chenxing-xc). +- Bir çalışırken meydana gelen bir kilitlenme düzeltildi `CHECK` sorgu için `Distributed` tüm parçalar yerel ise tablolar (chenxing.xcc). +- Düzenli ifadeler kullanan işlevlerle hafif bir performans gerilemesi düzeltildi. +- Karmaşık ifadelerden çok boyutlu diziler oluştururken bir performans gerilemesi düzeltildi. +- Ekstra bir neden olabilecek bir hata düzeltildi `FORMAT` bir bölümde görün toecek bölüm `.sql` meta veriler içeren dosya. +- Neden bir hata düzeltildi `max_table_size_to_drop` s deleteilmeye çalış aılırken uygulanacak sınır `MATERIALIZED VIEW` açıkça belirtilen bir tabloya bakıyor. +- Eski istemcilerle sabit uyumsuzluk (eski istemciler bazen `DateTime('timezone')` anlamadıkları tür). +- Okurken bir hata düzeltildi `Nested` kullanılarak eklenen yapıların sütun elemanları `ALTER` ancak, bu sütunların koşulları taşındığında eski bölümler için boş `PREWHERE`. +- Tabloları sanal olarak filtrelerken bir hata düzeltildi `_table` sorgu columnslardaki sütunlar `Merge` Tablolar. +- Kullanırken bir hata düzeltildi `ALIAS` Col inum inns in `Distributed` Tablolar. +- Gelen toplam fonksiyonları ile sorgular için dinamik derleme imkansız hale bir hata düzeltildi `quantile` aile. +- Kullanırken çok nadir durumlarda meydana gelen sorgu yürütme boru hattında bir yarış durumu düzeltildi `Merge` çok sayıda tablo içeren tablolar ve kullanırken `GLOBAL` alt sorgular. +- Farklı boyutlarda dizileri geçerken bir kilitlenme düzeltildi `arrayReduce` birden çok bağımsız değişkenden toplam işlevleri kullanırken işlev. +- İle sorgu useların kullanılması yasaktır `UNION ALL` in a `MATERIALIZED VIEW`. +- Başlatma sırasında bir hata düzeltildi `part_log` sunucu başlatıldığında sistem tablosu (varsayılan olarak, `part_log` devre dışı) olur. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-10} + +- Kaldır theılan `distributed_ddl_allow_replicated_alter` seçenek. Bu davranış varsayılan olarak etkindir. +- Kaldır theılan `strict_insert_defaults` ayar. Bu işlevi kullanıyorsanız, `clickhouse-feedback@yandex-team.com`. +- Kaldır theılan `UnsortedMergeTree` motor. + +### ClickHouse Yayın 1.1.54343, 2018-02-05 {#clickhouse-release-1-1-54343-2018-02-05} + +- Dağıtılmış DDL sorgularında ve dağıtılmış tabloların oluşturucularında küme adlarını tanımlamak için makro desteği eklendi: `CREATE TABLE distr ON CLUSTER '{cluster}' (...) ENGINE = Distributed('{cluster}', 'db', 'table')`. +- Şimdi gibi sorgular `SELECT ... FROM table WHERE expr IN (subquery)` kullanılarak iş arelenir `table` dizin. +- Çoğaltılmış tablolara eklerken çoğaltmaların işlenmesi geliştirildi, böylece artık çoğaltma kuyruğunun yürütülmesini yavaşlatmazlar. + +### ClickHouse Sürümü 1.1.54342, 2018-01-22 {#clickhouse-release-1-1-54342-2018-01-22} + +Bu sürüm önceki sürüm 1.1.54337 için hata düzeltmeleri içerir: + +- 1.1.54337'de bir gerileme düzeltildi: varsayılan kullanıcı salt okunur erişime sahipse, sunucu mesajla başlamayı reddediyor `Cannot create database in readonly mode`. +- 1.1.54337 bir gerileme sabit: systemd ile sistemlerde, günlükleri her zaman ne olursa olsun yapılandırma syslog yazılır; watchdog komut dosyası hala init kullanır.d. +- Docker görüntüde yanlış varsayılan yapılandırma: 1.1.54337 bir gerileme düzeltildi. +- Graphıtemergetree sabit nondeterministic davranış (günlük mesajları görebilirsiniz `Data after merge is not byte-identical to the data on another replicas`). +- Çoğaltılmış tablolara sorguyu OPTİMİZE ettikten sonra tutarsız birleştirmelere yol açabilecek bir hata düzeltildi (günlük iletilerinde görebilirsiniz `Part ... intersects the previous part`). +- (ZHANG2014 tarafından) hedef tabloda MATERİALİZED sütunlar mevcut olduğunda arabellek tabloları şimdi düzgün çalışır. +- NULL uygulanmasında bir hata düzeltildi. + +### ClickHouse Sürümü 1.1.54337, 2018-01-18 {#clickhouse-release-1-1-54337-2018-01-18} + +#### Yenilik: {#new-features-17} + +- Çok boyutlu diziler ve tuples depolanması için destek eklendi (`Tuple` veri türü) tablolarda. +- Masa fonksiyonları için destek `DESCRIBE` ve `INSERT` sorgular. Alt sorgular için destek eklendi `DESCRIBE`. Örnekler: `DESC TABLE remote('host', default.hits)`; `DESC TABLE (SELECT 1)`; `INSERT INTO TABLE FUNCTION remote('host', default.hits)`. İçin destek `INSERT INTO TABLE` ayrıca `INSERT INTO`. +- Saat dilimleri için geliştirilmiş destek. Bu `DateTime` veri türü ayrıştırma ve metin biçimlerinde biçimlendirme için kullanılan saat dilimi ile açıklamalı. Örnek: `DateTime('Europe/Moscow')`. Zaman dilimleri için işlevlerde belirtildiğinde `DateTime` bağımsız değişkenler, dönüş türü saat dilimini izler ve değer beklendiği gibi görüntülenir. +- Fonksiyonları eklendi `toTimeZone`, `timeDiff`, `toQuarter`, `toRelativeQuarterNum`. Bu `toRelativeHour`/`Minute`/`Second` fonksiyonlar bir tür değeri alabilir `Date` bir argüman olarak. Bu `now` işlev adı büyük / küçük harf duyarlıdır. +- Add theed the `toStartOfFifteenMinutes` fonksiyon (Kirill Shvakov). +- Add theed the `clickhouse format` sorguları biçimlendirme aracı. +- Add theed the `format_schema_path` configuration parameter (Marek Vavruşa). It is used for specifying a schema in `Cap'n Proto` biçimli. Şema dosyaları yalnızca belirtilen dizinde bulunabilir. +- Yapılandırma değiştirmeleri için destek eklendi (`incl` ve `conf.d`) dış sözlüklerin ve modellerin konfigürasyonu için (Pavel Yakunin). +- İçin belgeler içeren bir sütun eklendi `system.settings` tablo (Kirill Shvakov). +- Add theed the `system.parts_columns` her veri bölümünde sütun boyutları hakkında bilgi içeren tablo `MergeTree` Tablolar. +- Add theed the `system.models` yüklenen hakkında bilgi içeren tablo `CatBoost` makine öğrenme modelleri. +- Add theed the `mysql` ve `odbc` tablo fonksiyonu ve karşılık gelen `MySQL` ve `ODBC` uzak veritabanlarına erişmek için tablo motorları. Bu işlevsellik beta aşamasındadır. +- Türünde bir argüman geçme imkanı eklendi `AggregateFunction` için `groupArray` toplama işlevi (böylece bazı toplama işlevinin bir dizi durumunu oluşturabilirsiniz). +- Agrega fonksiyon birleştiricilerinin çeşitli kombinasyonlarındaki kısıtlamalar kaldırıldı. Örneğin, kullanabilirsiniz `avgForEachIf` keza `avgIfForEach` farklı davranışlara sahip toplam fonksiyonlar. +- Bu `-ForEach` toplama işlevi Birleştiricisi, birden çok bağımsız değişkenin toplama işlevleri için genişletilir. +- Toplam fonksiyonları için destek eklendi `Nullable` işlevin bir non döndürdüğü durumlarda bile argümanlar-`Nullable` sonuç (Silviu Caragea'nın katkısıyla eklendi). Örnek: `groupArray`, `groupUniqArray`, `topK`. +- Add theed the `max_client_network_bandwidth` için `clickhouse-client` (Kirill Shvakov). +- İle kullanıcılar `readonly = 2` setting are allowed to work with TEMPORARY tables (CREATE, DROP, INSERT…) (Kirill Shvakov). +- İle birden fazla tüketici kullanmak için destek eklendi `Kafka` motor. İçin genişletilmiş yapılandırma seçenekleri `Kafka` (Marek Vavruša). +- Add theed the `intExp3` ve `intExp4` işlevler. +- Add theed the `sumKahan` toplama işlevi. +- To \* Number\* ornull işlevleri eklendi, burada \* Number \* sayısal bir türdür. +- İçin destek eklendi `WITH` CLA anus anes for an `INSERT SELECT` sorgu (yazar: zhang2014). +- Eklenen ayarlar: `http_connection_timeout`, `http_send_timeout`, `http_receive_timeout`. Özellikle, bu ayarlar çoğaltma için veri parçalarını indirmek için kullanılır. Bu ayarları değiştirmek, ağ aşırı yüklenmişse daha hızlı yük devretme olanağı sağlar. +- İçin destek eklendi `ALTER` tip tablolar için `Null` (Anastasiya Tsarkova). +- Bu `reinterpretAsString` işlev, bellekte bitişik olarak depolanan tüm veri türleri için genişletilir. +- Add theed the `--silent` seçeneği için `clickhouse-local` aracı. Bu stderr yazdırma sorgu yürütme bilgi bastırır. +- Tip değerlerini okumak için destek eklendi `Date` ay ve/veya Ayın gün iki basamak (Amos kuş) yerine tek bir basamak kullanılarak belirtilen bir biçimde metinden. + +#### Performans iyileştirmeleri: {#performance-optimizations} + +- Agrega fonksiyonlarının geliştirilmiş performansı `min`, `max`, `any`, `anyLast`, `anyHeavy`, `argMin`, `argMax` dize argümanlarından. +- Fonksiyonların geliştirilmiş performansı `isInfinite`, `isFinite`, `isNaN`, `roundToExp2`. +- Ayrıştırma ve biçimlendirmenin geliştirilmiş performansı `Date` ve `DateTime` değerleri metin biçiminde yazın. +- Kayan nokta sayılarının ayrıştırılmasında geliştirilmiş performans ve hassasiyet. +- İçin bellek kullanımını azalt fortı `JOIN` sol ve sağ parçaların, içinde bulunmayan aynı adlara sahip sütunlara sahip olması durumunda `USING` . +- Agrega fonksiyonlarının geliştirilmiş performansı `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr` hesaplama kararlılığını azaltarak. Eski fonksiyonlar isimler altında mevcuttur `varSampStable`, `varPopStable`, `stddevSampStable`, `stddevPopStable`, `covarSampStable`, `covarPopStable`, `corrStable`. + +#### Hata düzeltmeleri: {#bug-fixes-28} + +- Bir çalıştırdıktan sonra sabit veri tekilleştirme `DROP` veya `DETACH PARTITION` sorgu. Önceki sürümde, bir bölümü bırakmak ve aynı verileri tekrar eklemek işe yaramadı çünkü eklenen bloklar kopya olarak kabul edildi. +- Yanlış yorumlanmasına yol açabilecek bir hata düzeltildi `WHERE` için fık forra `CREATE MATERIALIZED VIEW` ile sorgular `POPULATE` . +- Kullanarak bir hata düzeltildi `root_path` param parametereter in the `zookeeper_servers` yapılandırma. +- Geçen sabit beklenmedik sonuçlar `Date` arg toum argumentent to `toStartOfDay` . +- Sabit `addMonths` ve `subtractMonths` fonksiyonlar ve aritmetik `INTERVAL n MONTH` sonuç bir önceki yıla sahip olduğu durumlarda. +- İçin eksik destek eklendi `UUID` için veri türü `DISTINCT` , `JOIN` , ve `uniq` agrega fonksiyonları ve dış sözlükler (Evgeniy Ivanov). İçin destek `UUID` hala eksik. +- Sabit `SummingMergeTree` satırların sıfıra toplandığı durumlarda davranış. +- İçin çeşitli düzeltmeler `Kafka` engine (Marek Vavruša). +- Sabit yanlış davranış `Join` masa motoru (Amos kuş). +- FreeBSD ve OS X altında yanlış ayırıcı davranışı düzeltildi. +- Bu `extractAll` fonksiyon artık boş eşleşmeleri destekliyor. +- Kullanımını engelleyen bir hata düzeltildi `libressl` yerine `openssl` . +- Sabit `CREATE TABLE AS SELECT` geçici tablolardan sorgu. +- Çoğaltma kuyruğunu güncelleme sabit olmayan atomicity. Bu, sunucu yeniden başlatılıncaya kadar eşlemelerin eşitlenmemesine neden olabilir. +- Sabit Olası taşma `gcd` , `lcm` ve `modulo` (`%` operatör) (Maks Skorokhod). +- `-preprocessed` dosyalar şimdi değiştirdikten sonra oluşturulur `umask` (`umask` config değişmiş) olabilir. +- Parçaların arka plan kontrol bir hata düzeltildi (`MergeTreePartChecker` ) özel bir bölüm anahtarı kullanırken. +- Tuples sabit ayrıştırma (değerleri `Tuple` veri türü) metin biçimlerinde. +- Uyumsuz türlerle ilgili geliştirilmiş hata mesajları `multiIf` , `array` ve diğer bazı fonksiyonlar. +- İçin yeniden tasarlanmış destek `Nullable` türler. Bir sunucu çökmesine yol açabilir sabit hatalar. İlgili hemen hemen tüm diğer hatalar düzeltildi `NULL` destek: INSERT SELECT yanlış tür dönüşümleri, sahip ve PREWHERE içinde Nullable için yetersiz destek, `join_use_nulls` mod, argümanlar olarak Nullable türleri `OR` operatör, vb. +- Veri türlerinin iç semantiği ile ilgili çeşitli hatalar düzeltildi. Örnekler: gereksiz toplama `Enum` type Fi fieldsel fieldsds in `SummingMergeTree` ; hizalama `Enum` yazmak `Pretty` format ,lar vb. +- Bileşik sütunların izin verilen kombinasyonları için daha sıkı kontroller. +- İçin çok büyük bir parametre belirlerken taşma düzeltildi `FixedString` veri türü. +- Bu bir hata düzeltildi `topK` genel bir durumda toplama işlevi. +- Toplam fonksiyonların n-ary varyantlarının argümanlarında dizi boyutlarının eşitliği için eksik kontrol eklendi `-Array` birleştirici. +- Bir hata düzeltildi `--pager` için `clickhouse-client` (yazar: ks1322). +- Sabit hassasiyet `exp10` işlev. +- Davranışı Düzelt theildi `visitParamExtract` belgelere daha iyi uyum için işlev. +- Yanlış veri türleri belirtildiğinde kilitlenme düzeltildi. +- Davranışı düzeltildi `DISTINCT` tüm sütunların sabitler olması durumunda. +- Kullanarak durumunda sabit sorgu biçimlendirme `tupleElement` tuple öğesi dizini olarak karmaşık bir sabit ifadeyle işlev. +- Bir hata düzeltildi `Dictionary` için tablolar `range_hashed` sözlükler. +- Sonucunda aşırı satırlara yol açan bir hata düzeltildi `FULL` ve `RIGHT JOIN` (Amos Kuşu). +- Geçici dosyaları oluştururken ve kaldırırken bir sunucu çökmesi düzeltildi `config.d` yapılandırma yeniden yükleme sırasında dizinler. +- Sabit `SYSTEM DROP DNS CACHE` sorgu: önbellek temizlendi, ancak küme düğümlerinin adresleri güncelleştirilmedi. +- Davranışı düzeltildi `MATERIALIZED VIEW` çalıştırdıktan sonra `DETACH TABLE` for the table under the view (Marek Vavruša). + +#### İyileştirmeler oluşturun: {#build-improvements-4} + +- Bu `pbuilder` aracı oluşturur için kullanılır. Yapı işlemi, yapı ana bilgisayarı ortamından neredeyse tamamen bağımsızdır. +- Farklı işletim sistemi sürümleri için tek bir yapı kullanılır. Paketler ve ikili Linux sistemleri geniş bir yelpazede ile uyumlu yapılmıştır. +- Add theed the `clickhouse-test` paket. Fonksiyonel testleri çalıştırmak için kullanılabilir. +- Kaynak tarball artık depoya yayınlanabilir. Github kullanmadan yapıyı yeniden oluşturmak için kullanılabilir. +- Travis CI ile sınırlı entegrasyon eklendi. Travis oluşturma süresi sınırları nedeniyle, yalnızca hata ayıklama yapı sınanır ve sınamaların sınırlı bir alt kümesi çalıştırılır. +- İçin destek eklendi `Cap'n'Proto` varsayılan yapı içinde. +- Dokümantasyon kaynaklarının biçimini değiştirdi `Restricted Text` -e doğru `Markdown`. +- İçin destek eklendi `systemd` (Vladimir Smirnov). Bazı işletim sistemi görüntüleri ile uyumsuzluk nedeniyle varsayılan olarak devre dışı bırakılır ve manuel olarak etkinleştirilebilir. +- Dinamik kod üretimi için, `clang` ve `lld` içine gömülü `clickhouse` ikilik. Olarak da çağr alsoıla invokedbilirler `clickhouse clang` ve `clickhouse lld` . +- GNU uzantılarının koddan kaldırılması. Etkin `-Wextra` seçenek. İle inşa ederken `clang` varsayılan değer `libc++` yerine `libstdc++`. +- Elde `clickhouse_parsers` ve `clickhouse_common_io` kütüphaneler hızlandırmak için çeşitli araçlar oluşturur. + +#### Geriye dönük uyumsuz değişiklikler: {#backward-incompatible-changes-11} + +- İşaret formatlerin biçimi `Log` içeren tabloları yazın `Nullable` sütunlar geriye dönük uyumsuz bir şekilde değiştirildi. Bu tablolar varsa, bunları dönüştürmek gerekir `TinyLog` yeni sunucu sürümünü başlatmadan önce yazın. Bunu yapmak için değiştirin `ENGINE = Log` ile `ENGINE = TinyLog` karşılık gelen `.sql` dosya içinde `metadata` dizin. Masanız yoksa `Nullable` sütun veya tablonuzun türü değilse `Log` o zaman hiçbir şey yapmanıza gerek yok. +- Kaldır theılan `experimental_allow_extended_storage_definition_syntax` ayar. Şimdi bu özellik varsayılan olarak etkindir. +- Bu `runningIncome` işlev olarak yeniden adlandırıldı `runningDifferenceStartingWithFirstvalue` karışıklığı önlemek için. +- Kaldır theılan `FROM ARRAY JOIN arr` array JOİN, tablo olmadan (Amos Bird) doğrudan FROM sonra belirtildiğinde sözdizimi. +- Kaldır theılan `BlockTabSeparated` sadece gösteri amaçlı kullanılan format. +- Toplu işlevler için durum biçimini değiştirdi `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Bu toplama işlevlerinin durumlarını tablolarda depoladıysanız ( `AggregateFunction` veri türü veya ilgili durumları ile hayata görünümleri), yazınız clickhouse-feedback@yandex-team.com. +- Önceki sunucu sürümlerinde belgelenmemiş bir özellik vardı: bir toplama işlevi parametrelere bağlıysa, yine de AggregateFunction veri türünde parametreler olmadan belirtebilirsiniz. Örnek: `AggregateFunction(quantiles, UInt64)` yerine `AggregateFunction(quantiles(0.5, 0.9), UInt64)`. Bu özellik kayboldu. Belgesiz olmasına rağmen, gelecek sürümlerde tekrar desteklemeyi planlıyoruz. +- Enum veri türleri min / max toplama işlevlerinde kullanılamaz. Bu yetenek bir sonraki sürümde iade edilecektir. + +#### Yükseltme yaparken lütfen unutmayın: {#please-note-when-upgrading} + +- Bir kümede, bazı yinelemelerin Clickhouse'un eski sürümünü çalıştırdığı ve bazılarının yeni sürümü çalıştırdığı noktada yuvarlanan bir güncelleme yaparken, çoğaltma geçici olarak durdurulur ve ileti `unknown parameter 'shard'` günlüğünde görünür. Kümenin tüm yinelemeleri güncelleştirildikten sonra çoğaltma devam eder. +- Clickhouse'un farklı sürümleri küme sunucularında çalışıyorsa, aşağıdaki işlevleri kullanarak dağıtılmış sorguların yanlış sonuçları olması olasıdır: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Tüm küme düğümlerini güncelleştirmelisiniz. + +## [2017 için Changelog](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2017.md) {#changelog-for-2017} diff --git a/docs/tr/whats_new/changelog/2019.md b/docs/tr/whats_new/changelog/2019.md new file mode 100644 index 00000000000..065cb822bc1 --- /dev/null +++ b/docs/tr/whats_new/changelog/2019.md @@ -0,0 +1,2074 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 77 +toc_title: '2019' +--- + +## ClickHouse sürüm v19. 17 {#clickhouse-release-v19-17} + +### ClickHouse sürümü v19.17.6. 36, 2019-12-27 {#clickhouse-release-v19-17-6-36-2019-12-27} + +#### Hata Düzeltme {#bug-fix} + +- Sıkıştırmada sabit potansiyel tampon taşması. Kötü niyetli kullanıcı, arabellekten sonra okunmasına neden olabilecek sıkıştırılmış verileri iletebilir. Bu sorun Yandex bilgi güvenliği ekibinden Eldar Zaitov tarafından bulundu. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit Olası sunucu çökmesi (`std::terminate`) sunucu, dize veri türü (UTF-8 doğrulaması gerektiren) değerleriyle json veya XML biçiminde veri gönderemez veya yazamaz veya sonuç verilerini Brotli algoritması ile sıkıştırırken veya diğer bazı nadir durumlarda. [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bir clickhouse kaynak ile sabit sözlükler `VIEW`, şimdi bu tür sözlükleri okumak hataya neden olmaz `There is no query`. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Bir istemci ana bilgisayar kullanıcıları belirtilen host\_regexp tarafından izin verilip verilmediğini kontrol sabit.xml. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241), [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) +- `RENAME TABLE` dağıtılmış bir tablo için artık parçalara göndermeden önce eklenen verileri içeren klasörü yeniden adlandırır. Bu, ardışık yeniden adlarla ilgili bir sorunu giderir `tableA->tableB`, `tableC->tableA`. [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) +- `range_hashed` DDL sorguları tarafından oluşturulan dış sözlükler artık rasgele sayısal tür aralıklarına izin verir. [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) +- Sabit `INSERT INTO table SELECT ... FROM mysql(...)` tablo işlevi. [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) +- Sabit segfault içinde `INSERT INTO TABLE FUNCTION file()` mevcut olmayan bir dosyaya eklerken. Şimdi bu durumda dosya oluşturulur ve daha sonra insert işlenir. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +- Birleştirilmiş bir bitmap ve bir skaler bitmap kesişen sabit bitmapAnd hatası. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) +- Sabit segfault zaman `EXISTS` sorgu olmadan kullanıldı `TABLE` veya `DICTIONARY` niteleyici, tıpkı `EXISTS t`. [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fonksiyonlar için sabit dönüş tipi `rand` ve `randConstant` geçersiz argüman durumunda. Şimdi fonksiyonlar her zaman geri döner `UInt32` ve asla `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Sabit `DROP DICTIONARY IF EXISTS db.dict`, şimdi Eğer istisna atmaz `db` yok. [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +- Sunucu çökmesi nedeniyle bir tablo tamamen düşmediyse, sunucu geri yüklemeyi ve yüklemeyi deneyecektir [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) +- İkiden fazla shard yerel tablo varsa, dağıtılmış bir tablo için önemsiz bir sayım sorgusu düzeltildi. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) +- DB::Blockstreamprofileınfo::calculateRowsBeforeLimit bir veri yarış yol Sabit hata() [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) +- Sabit `ALTER table MOVE part` belirtilen parçayı birleştirdikten hemen sonra yürütülür, bu da belirtilen parçanın birleştirildiği bir parçanın taşınmasına neden olabilir. Şimdi belirtilen kısmı doğru hareket ettirir. [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Sözlükler için ifadeler şimdi dizeler olarak belirtilebilir. Bu, ClickHouse olmayan kaynaklardan veri ayıklarken özniteliklerin hesaplanması için yararlıdır, çünkü bu ifadeler için ClickHouse olmayan sözdizimini kullanmanıza izin verir. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) +- Çok nadir bir yarış sabit `clickhouse-copier` zxıd'de bir taşma nedeniyle. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +- Sorgu başarısız olduktan sonra hata düzeltildi (nedeniyle “Too many simultaneous queries” örneğin) harici tablolar bilgisini okumaz ve + sonraki istek, bu bilgiyi bir sonraki sorgunun başlangıcı olarak yorumlayacak ve aşağıdaki gibi bir hataya neden olacaktır `Unknown packet from client`. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) +- Sonra null dereference kaçının “Unknown packet X from server” [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) +- Tüm YBÜ yerel destek Geri Yükleme, sabit ifadeler için harmanlama uygulamak ve sisteme dil adı eklemek için yeteneği ekleyin.harmanlama tablosu. [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) +- Okuma için akış sayısı `StorageFile` ve `StorageHDFS` bellek sınırını aşmamak için artık sınırlıdır. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) +- Sabit `CHECK TABLE` sorgu için `*MergeTree` anahtarsız tablolar. [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) +- Mutasyon olmaması durumunda bir parça adından mutasyon numarası kaldırıldı. Bu kaldırma eski sürümleri ile uyumluluk geliştirilmiş. [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) +- Mutasyonlar nedeniyle data\_version bazı ekli parçalar için atlanır hata tablo mutasyon sürümü daha büyüktür düzeltildi. [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) +- Başka bir cihaza taşıdıktan sonra yedek parça kopyalarıyla sunucuyu başlatmaya izin verin. [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Hata düzeltildi “Sizes of columns doesn’t match” bu, toplama işlevi sütunlarını kullanırken görünebilir. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +- Şimdi LİMİT BY yanında bağları ile kullanılması durumunda bir istisna atılır. Ve şimdi LİMİT ile üst kullanmak mümkündür. [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Varsa sözlük yeniden yükle `invalidate_query`, güncellemeleri durdurdu ve önceki güncelleme denemelerinde bazı istisnalar. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) + +### ClickHouse sürümü v19.17.4. 11, 2019-11-22 {#clickhouse-release-v19-17-4-11-2019-11-22} + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change} + +- Daha iyi performans için skaler alt sorgu sonuçlarını depolamak için AST yerine sütun kullanma. Ayar `enable_scalar_subquery_optimization` 19.17'de eklendi ve varsayılan olarak etkinleştirildi. Gibi hat toalara yol açar [bu](https://github.com/ClickHouse/ClickHouse/issues/7851) önceki sürümlerden 19.17.2 veya 19.17.3'e yükseltme sırasında. Bu ayar, 19.17.4'te, 19.16 ve daha eski sürümlerden hatasız yükseltme yapılmasını sağlamak için varsayılan olarak devre dışı bırakıldı. [\#7392](https://github.com/ClickHouse/ClickHouse/pull/7392) ([Amos Kuşu](https://github.com/amosbird)) + +#### Yenilik {#new-feature} + +- DDL sorguları ile sözlükler oluşturma yeteneği ekleyin. [\#7360](https://github.com/ClickHouse/ClickHouse/pull/7360) ([alesapin](https://github.com/alesapin)) +- Yapmak `bloom_filter` endeks destek türü `LowCardinality` ve `Nullable` [\#7363](https://github.com/ClickHouse/ClickHouse/issues/7363) [\#7561](https://github.com/ClickHouse/ClickHouse/pull/7561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fonksiyon Ekle `isValidJSON` geçirilen dizenin geçerli bir json olduğunu kontrol etmek için. [\#5910](https://github.com/ClickHouse/ClickHouse/issues/5910) [\#7293](https://github.com/ClickHouse/ClickHouse/pull/7293) ([Vdimir](https://github.com/Vdimir)) +- Uygulamak `arrayCompact` işlev [\#7328](https://github.com/ClickHouse/ClickHouse/pull/7328) ([Not](https://github.com/Joeywzr)) +- Oluşturulan işlev `hex` ondalık sayılar için. Gibi çalışır `hex(reinterpretAsString())`, ancak son sıfır bayt silmez. [\#7355](https://github.com/ClickHouse/ClickHouse/pull/7355) ([Mikhail Korotov](https://github.com/millb)) +- Eklemek `arrayFill` ve `arrayReverseFill` dizideki ön / arka öğelerdeki diğer öğelerle öğeleri değiştiren işlevler. [\#7380](https://github.com/ClickHouse/ClickHouse/pull/7380) ([hcz](https://github.com/hczhcz)) +- Eklemek `CRC32IEEE()`/`CRC64()` destek [\#7480](https://github.com/ClickHouse/ClickHouse/pull/7480) ([Azat Khuzhin](https://github.com/azat)) +- Uygulamak `char` fonksiyon birine benzer [mysql](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char) [\#7486](https://github.com/ClickHouse/ClickHouse/pull/7486) ([sundyli](https://github.com/sundy-li)) +- Eklemek `bitmapTransform` işlev. Değerler bir dizi için bir bit eşlem değerler dizisi dönüştüren, sonuçta yeni bir bit eşlem [\#7598](https://github.com/ClickHouse/ClickHouse/pull/7598) ([Zhichang Yu](https://github.com/yuzhichang)) +- Uyguluyordu `javaHashUTF16LE()` işlev [\#7651](https://github.com/ClickHouse/ClickHouse/pull/7651) ([achimbab](https://github.com/achimbab)) +- Eklemek `_shard_num` dağıtılmış motor için sanal sütun [\#7624](https://github.com/ClickHouse/ClickHouse/pull/7624) ([Azat Khuzhin](https://github.com/azat)) + +#### Deneysel Özellik {#experimental-feature} + +- İşlemciler için destek (yeni sorgu yürütme boru hattı) `MergeTree`. [\#7181](https://github.com/ClickHouse/ClickHouse/pull/7181) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Hata Düzeltme {#bug-fix-1} + +- Yanlış float ayrıştırma düzeltme `Values` [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817) [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) +- Trace\_log etkinleştirildiğinde oluşabilecek nadir kilitlenmeyi düzeltin. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) +- Kafka tablosu üretirken mesaj çoğaltmasını önle, ondan seçim yapan herhangi bir MVs var [\#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([I ivanvan](https://github.com/abyss7)) +- İçin destek `Array(LowCardinality(Nullable(String)))` içinde `IN`. Gideriyor [\#7364](https://github.com/ClickHouse/ClickHouse/issues/7364) [\#7366](https://github.com/ClickHouse/ClickHouse/pull/7366) ([achimbab](https://github.com/achimbab)) +- Add hand ofling of `SQL_TINYINT` ve `SQL_BIGINT`, ve düzeltme işleme `SQL_FLOAT` ODBC köprüsünde veri kaynağı türleri. [\#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([Denis Glazachev](https://github.com/traceon)) +- Toplama düzeltme (`avg` ve quantiles) boş ondalık sütunlar üzerinde [\#7431](https://github.com/ClickHouse/ClickHouse/pull/7431) ([Andrey Konyaev](https://github.com/akonyaev90)) +- Düzeltmek `INSERT` ile dağıtılan içine `MATERIALIZED` sütun [\#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) +- Yapmak `MOVE PARTITION` bölümün bazı bölümleri zaten hedef disk veya birimde ise çalışın [\#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([Vladimir Chebotarev](https://github.com/excitoon)) +- İçinde mutasyonlar sırasında oluşturulacak başarısız hardlinks ile Sabit hata `ReplicatedMergeTree` çoklu disk yapılandırmalarında. [\#7558](https://github.com/ClickHouse/ClickHouse/pull/7558) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Bütün parça değişmeden kalır ve en iyi alan başka bir diskte bulunurken bir MergeTree üzerinde bir mutasyon ile bir hata düzeltildi [\#7602](https://github.com/ClickHouse/ClickHouse/pull/7602) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Sabit hata ile `keep_free_space_ratio` disk yapılandırmasından okunmuyor [\#7645](https://github.com/ClickHouse/ClickHouse/pull/7645) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Tablo ile Fix hata sadece içerir `Tuple` karmaşık yolları olan sütunlar veya sütunlar. Düzeltiyor [7541](https://github.com/ClickHouse/ClickHouse/issues/7541). [\#7545](https://github.com/ClickHouse/ClickHouse/pull/7545) ([alesapin](https://github.com/alesapin)) +- Bellek Max\_memory\_usage sınırında arabellek altyapısı için hesap değil [\#7552](https://github.com/ClickHouse/ClickHouse/pull/7552) ([Azat Khuzhin](https://github.com/azat)) +- Son işaret kullanımını düzeltin `MergeTree` tarafından sipariş edilen tablolar `tuple()`. Nadir durumlarda yol açabilir `Can't adjust last granule` seçerken hata. [\#7639](https://github.com/ClickHouse/ClickHouse/pull/7639) ([Anton Popov](https://github.com/CurtizJ)) +- Çökmelere veya garip istisnalara yol açabilecek bağlam gerektiren eylemlerle (örneğin json için işlevler) yüklü olan mutasyonlardaki hatayı düzeltin. [\#7664](https://github.com/ClickHouse/ClickHouse/pull/7664) ([alesapin](https://github.com/alesapin)) +- Kaçan veritabanı ve tablo adlarının uyumsuzluğunu düzeltin `data/` ve `shadow/` dizinler [\#7575](https://github.com/ClickHouse/ClickHouse/pull/7575) ([Alexander Burmak](https://github.com/Alex-Burmak)) +- Support duplicated keys in RIGHT\|FULL JOINs, e.g. `ON t.x = u.x AND t.x = u.y`. Bu durumda kazayı düzeltin. [\#7586](https://github.com/ClickHouse/ClickHouse/pull/7586) ([Artem Zuikov](https://github.com/4ertus2)) +- Düzeltmek `Not found column in block` ifadeye doğru veya tam birleştirme ile katılırken. [\#7641](https://github.com/ClickHouse/ClickHouse/pull/7641) ([Artem Zuikov](https://github.com/4ertus2)) +- Sonsuz döngüyü düzeltmek için bir girişim daha `PrettySpace` biçimli [\#7591](https://github.com/ClickHouse/ClickHouse/pull/7591) ([Olga Khvostikova](https://github.com/stavrolia)) +- Hatayı düzeltin `concat` tüm argümanlar olduğunda işlev `FixedString` aynı boyutta. [\#7635](https://github.com/ClickHouse/ClickHouse/pull/7635) ([alesapin](https://github.com/alesapin)) +- S3, URL ve HDFS depolarını tanımlarken 1 argüman kullanılması durumunda sabit istisna. [\#7618](https://github.com/ClickHouse/ClickHouse/pull/7618) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Sorgu ile görünümler için Interpretersselectquery'nin kapsamını düzeltin [\#7601](https://github.com/ClickHouse/ClickHouse/pull/7601) ([Azat Khuzhin](https://github.com/azat)) + +#### Geliştirme {#improvement} + +- `Nullable` ODBC-bridge tarafından doğru işlenen sütunlar tanınan ve NULL değerleri [\#7402](https://github.com/ClickHouse/ClickHouse/pull/7402) ([Vasily Nemkov](https://github.com/Enmk)) +- Atomically dağıtılmış göndermek için geçerli toplu yazma [\#7600](https://github.com/ClickHouse/ClickHouse/pull/7600) ([Azat Khuzhin](https://github.com/azat)) +- Sorguda sütun adı için tablo algılayamazsak bir istisna atın. [\#7358](https://github.com/ClickHouse/ClickHouse/pull/7358) ([Artem Zuikov](https://github.com/4ertus2)) +- Eklemek `merge_max_block_size` ayarı için `MergeTreeSettings` [\#7412](https://github.com/ClickHouse/ClickHouse/pull/7412) ([Artem Zuikov](https://github.com/4ertus2)) +- İle sorgular `HAVING` ve olmadan `GROUP BY` sabit olarak grup varsayalım. Böyle, `SELECT 1 HAVING 1` şimdi bir sonuç döndürür. [\#7496](https://github.com/ClickHouse/ClickHouse/pull/7496) ([Amos Kuşu](https://github.com/amosbird)) +- Destek ayrıştırma `(X,)` python'a benzer tuple olarak. [\#7501](https://github.com/ClickHouse/ClickHouse/pull/7501), [\#7562](https://github.com/ClickHouse/ClickHouse/pull/7562) ([Amos Kuşu](https://github.com/amosbird)) +- Yapmak `range` işlev davranışları neredeyse pythonic gibi. [\#7518](https://github.com/ClickHouse/ClickHouse/pull/7518) ([sundyli](https://github.com/sundy-li)) +- Eklemek `constraints` tabloya sütunlar `system.settings` [\#7553](https://github.com/ClickHouse/ClickHouse/pull/7553) ([Vitaly Baranov](https://github.com/vitlibar)) +- Tcp işleyicisi için daha iyi boş biçim, böylece kullanmak mümkün `select ignore() from table format Null` clickhouse-client ile perf ölçümü için [\#7606](https://github.com/ClickHouse/ClickHouse/pull/7606) ([Amos Kuşu](https://github.com/amosbird)) +- Gibi sorgular `CREATE TABLE ... AS (SELECT (1, 2))` doğru ayrıştırılır [\#7542](https://github.com/ClickHouse/ClickHouse/pull/7542) ([hcz](https://github.com/hczhcz)) + +#### Performans İyileştirme {#performance-improvement} + +- Kısa dize anahtarları üzerinde toplama performansı artırıldı. [\#6243](https://github.com/ClickHouse/ClickHouse/pull/6243) ([Alexander Kuzmenkov](https://github.com/akuzm), [Amos Kuşu](https://github.com/amosbird)) +- Sabit yüklemler katlandıktan sonra olası optimizasyonları almak için sözdizimi/ifade analizinin başka bir geçişini çalıştırın. [\#7497](https://github.com/ClickHouse/ClickHouse/pull/7497) ([Amos Kuşu](https://github.com/amosbird)) +- Önemsiz değerlendirmek için depolama meta bilgisi kullanın `SELECT count() FROM table;` [\#7510](https://github.com/ClickHouse/ClickHouse/pull/7510) ([Amos Kuşu](https://github.com/amosbird), [alexey-milovidov](https://github.com/alexey-milovidov)) +- Vectorize işleme `arrayReduce` toplayıcı benzer `addBatch`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Kuşu](https://github.com/amosbird)) +- Performansında küçük iyileştirmeler `Kafka` tüketim [\#7475](https://github.com/ClickHouse/ClickHouse/pull/7475) ([I ivanvan](https://github.com/abyss7)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement} + +- AARCH64 CPU mimarisine çapraz derleme için destek ekleyin. Refactor packager komut. [\#7370](https://github.com/ClickHouse/ClickHouse/pull/7370) [\#7539](https://github.com/ClickHouse/ClickHouse/pull/7539) ([I ivanvan](https://github.com/abyss7)) +- Paketleri oluştururken darwin-x86\_64 ve linux-aarch64 araç zincirlerini monte edilmiş Docker hacmine açın [\#7534](https://github.com/ClickHouse/ClickHouse/pull/7534) ([I ivanvan](https://github.com/abyss7)) +- İkili Paketleyici için Docker görüntüsünü güncelle [\#7474](https://github.com/ClickHouse/ClickHouse/pull/7474) ([I ivanvan](https://github.com/abyss7)) +- MacOS Catalina'da sabit derleme hataları [\#7585](https://github.com/ClickHouse/ClickHouse/pull/7585) ([Ernest Poletaev](https://github.com/ernestp)) +- Sorgu analizi mantığında bazı yeniden düzenleme: karmaşık sınıfı birkaç basit sınıfa ayırın. [\#7454](https://github.com/ClickHouse/ClickHouse/pull/7454) ([Artem Zuikov](https://github.com/4ertus2)) +- Submodules olmadan yapı düzeltme [\#7295](https://github.com/ClickHouse/ClickHouse/pull/7295) ([proller](https://github.com/proller)) +- İyileştirmek `add_globs` cmake dosyalarında [\#7418](https://github.com/ClickHouse/ClickHouse/pull/7418) ([Amos Kuşu](https://github.com/amosbird)) +- Kodlanmış yolları Kaldır `unwind` hedef [\#7460](https://github.com/ClickHouse/ClickHouse/pull/7460) ([Konstantin Podshumok](https://github.com/podshumok)) +- Ssl olmadan mysql formatını kullanmasına izin ver [\#7524](https://github.com/ClickHouse/ClickHouse/pull/7524) ([proller](https://github.com/proller)) + +#### Diğer {#other} + +- ClickHouse SQL lehçesi için ANTLR4 dilbilgisi eklendi [\#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [\#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +## ClickHouse sürüm v19. 16 {#clickhouse-release-v19-16} + +#### ClickHouse yayın v19. 16. 14. 65, 2020-03-25 {#clickhouse-release-v19-16-14-65-2020-03-25} + +- Birden argümanlar (10'dan fazla) üçlü mantıksal OPs toplu hesaplamalarda bir hata düzeltildi. [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) Bu hata düzeltme Altınity özel bir istek ile sürüm 19.16 backported edildi. + +#### ClickHouse yayın v19. 16. 14. 65, 2020-03-05 {#clickhouse-release-v19-16-14-65-2020-03-05} + +- Dağıtılmış alt sorgular uyumsuzluğunu eski CH sürümleriyle düzeltin. Düzeltiyor [\#7851](https://github.com/ClickHouse/ClickHouse/issues/7851) + [(tabplubix)](https://github.com/tavplubix) +- Yürüt whenürken `CREATE` sorgu, depolama motoru argümanlarında sabit ifadeleri katlayın. Boş veritabanı adı geçerli veritabanı ile değiştirin. Düzeltiyor [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Ayrıca yerel adresi kontrol edin `ClickHouseDictionarySource`. + [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) +- Şimdi arka plan birleşir `*MergeTree` tablo motorları ailesi depolama ilkesi hacim sırasını daha doğru bir şekilde korur. + [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Veri kaybını önlemek `Kafka` nadir durumlarda istisna sonek okuduktan sonra ancak taahhütten önce gerçekleşir. Düzeltiyor [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). İlgili: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) + [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) +- Kullanmaya / bırakmaya çalışırken sunucu sonlandırmasına giden hatayı düzeltin `Kafka` tablo yanlış parametrelerle oluşturuldu. Düzeltiyor [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Birleşiyor [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). + [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) +- Kullanmaya izin ver `MaterializedView` yukarıdaki alt sorgularla `Kafka` Tablolar. + [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) + +#### Yenilik {#new-feature-1} + +- Eklemek `deduplicate_blocks_in_dependent_materialized_views` hayata manzaralı tablolara idempotent ekler davranışlarını kontrol etmek için Seçenek. Bu yeni özellik, altınity'den özel bir istek ile bugfix sürümüne eklendi. + [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) + +### ClickHouse sürümü v19.16.2. 2, 2019-10-30 {#clickhouse-release-v19-16-2-2-2019-10-30} + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-1} + +- Count/counİf için eksik arity doğrulama ekleyin. + [\#7095](https://github.com/ClickHouse/ClickHouse/issues/7095) + [\#7298](https://github.com/ClickHouse/ClickHouse/pull/7298) ([Vdimir](https://github.com/Vdimir)) +- Eski Kaldır `asterisk_left_columns_only` ayar (varsayılan olarak devre dışıdır). + [\#7335](https://github.com/ClickHouse/ClickHouse/pull/7335) ([Artem + Zuikov](https://github.com/4ertus2)) +- Şablon veri biçimi için Biçim dizeleri artık dosyalarda belirtilmiştir. + [\#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) + ([tavplubix](https://github.com/tavplubix)) + +#### Yenilik {#new-feature-2} + +- Uint\_max büyük önemlilik hesaplamak için uniqcombined64() tanıtmak. + [\#7213](https://github.com/ClickHouse/ClickHouse/pull/7213), + [\#7222](https://github.com/ClickHouse/ClickHouse/pull/7222) ([Azat + Khuzhin](https://github.com/azat)) +- Dizi sütunlarında Bloom filtre indekslerini destekleyin. + [\#6984](https://github.com/ClickHouse/ClickHouse/pull/6984) + ([achimbab](https://github.com/achimbab)) +- Bir işlev Ekle `getMacro(name)` bu karşılık gelen değeri ile dize döndürür `` + sunucu yapılandırmasından. [\#7240](https://github.com/ClickHouse/ClickHouse/pull/7240) + ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bir HTTP kaynağına dayalı bir sözlük için iki yapılandırma seçeneği belirleyin: `credentials` ve + `http-headers`. [\#7092](https://github.com/ClickHouse/ClickHouse/pull/7092) ([Guillaume + Tassery](https://github.com/YiuRULE)) +- Yeni bir ProfileEvent Ekle `Merge` bu, başlatılan arka plan birleşimlerinin sayısını sayar. + [\#7093](https://github.com/ClickHouse/ClickHouse/pull/7093) ([Mikhail + Korotov](https://github.com/millb)) +- Tam etki alanı adı döndüren fullHostName işlevi ekleyin. + [\#7263](https://github.com/ClickHouse/ClickHouse/issues/7263) + [\#7291](https://github.com/ClickHouse/ClickHouse/pull/7291) ([sundyli](https://github.com/sundy-li)) +- Fonksiyon Ekle `arraySplit` ve `arrayReverseSplit` bir diz byiyi böl anen “cut off” + şartlar. Zaman dizisinin işlenmesinde kullanışlıdırlar. + [\#7294](https://github.com/ClickHouse/ClickHouse/pull/7294) ([hcz](https://github.com/hczhcz)) +- Multimatch işlev ailesine eşleşen tüm dizinlerin dizisini döndüren yeni işlevler ekleyin. + [\#7299](https://github.com/ClickHouse/ClickHouse/pull/7299) ([Danila. + Kutenin](https://github.com/danlark1)) +- Yeni bir veritabanı altyapısı Ekle `Lazy` bu, çok sayıda küçük günlük depolamak için optimize edilmiştir + Tablolar. [\#7171](https://github.com/ClickHouse/ClickHouse/pull/7171) ([Nikita + Vasilev](https://github.com/nikvas0)) +- Bitmap sütunları için groupBitmapAnd, - veya-Xor toplam işlevleri ekleyin. [\#7109](https://github.com/ClickHouse/ClickHouse/pull/7109) ([Zhichang + Yu](https://github.com/yuzhichang)) +- Null döndüren toplam işlev birleştiricileri-OrNull ve-OrDefault ekleyin + veya toplamak için hiçbir şey olmadığında varsayılan değerler. + [\#7331](https://github.com/ClickHouse/ClickHouse/pull/7331) + ([hcz](https://github.com/hczhcz)) +- Özel kaçan destekleyen CustomSeparated veri biçimini tanıtmak ve + sınırlayıcı kuralları. [\#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) + ([tavplubix](https://github.com/tavplubix)) +- Destek Redis harici sözlük kaynağı olarak. [\#4361](https://github.com/ClickHouse/ClickHouse/pull/4361) [\#6962](https://github.com/ClickHouse/ClickHouse/pull/6962) ([comunodi](https://github.com/comunodi), [Anton + Popov](https://github.com/CurtizJ)) + +#### Hata Düzeltme {#bug-fix-2} + +- Varsa yanlış sorgu sonucunu düzeltin `WHERE IN (SELECT ...)` bölüm ve `optimize_read_in_order` oluyor + kullanılmış. [\#7371](https://github.com/ClickHouse/ClickHouse/pull/7371) ([Anton + Popov](https://github.com/CurtizJ)) +- Proje dışındaki dosyalara bağlı olarak engelli MariaDB kimlik doğrulama eklentisi. + [\#7140](https://github.com/ClickHouse/ClickHouse/pull/7140) ([Yuriy + Baranov](https://github.com/yurriy)) +- İstisnayı düzeltin `Cannot convert column ... because it is constant but values of constants are different in source and result` fonksiyonlar olduğunda nadiren olabilir `now()`, `today()`, + `yesterday()`, `randConstant()` kullanılır. + [\#7156](https://github.com/ClickHouse/ClickHouse/pull/7156) ([Nikolay + Kochetov](https://github.com/KochetovNicolai)) +- TCP yerine HTTP keep alive timeout kullanarak sabit sorunu alive timeout tutun. + [\#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily + Nemkov](https://github.com/Enmk)) +- Groupbitmapor'da bir segmentasyon hatası düzeltildi (sorun [\#7109](https://github.com/ClickHouse/ClickHouse/issues/7109)). + [\#7289](https://github.com/ClickHouse/ClickHouse/pull/7289) ([Zhichang + Yu](https://github.com/yuzhichang)) +- Materyalize edilmiş görünümler için, tüm veriler yazıldıktan sonra Kafka için taahhüt denir. + [\#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([I ivanvan](https://github.com/abyss7)) +- Sabit yanlış `duration_ms` değer içinde `system.part_log` Tablo. On kez kapalıydı. + [\#7172](https://github.com/ClickHouse/ClickHouse/pull/7172) ([Vladimir + Chebotarev](https://github.com/excitoon)) +- Canlı Görünüm tablosundaki çökmeyi çözmek ve tüm Canlı Görünüm testlerini yeniden etkinleştirmek için hızlı bir düzeltme. + [\#7201](https://github.com/ClickHouse/ClickHouse/pull/7201) + ([vzakaznikov](https://github.com/vzakaznikov)) +- MERGETREE parçalarının min/max dizinlerinde NULL değerleri doğru şekilde seri hale getirin. + [\#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([İskender + Kuzmenkov](https://github.com/akuzm)) +- Sanal sütunları koymayın .tablo olarak oluşturulduğunda sql meta verileri `CREATE TABLE AS`. + [\#7183](https://github.com/ClickHouse/ClickHouse/pull/7183) ([I ivanvan](https://github.com/abyss7)) +- Segmentasyon hatasını düzeltin `ATTACH PART` sorgu. + [\#7185](https://github.com/ClickHouse/ClickHouse/pull/7185) + ([alesapin](https://github.com/alesapin)) +- Alt sorgularda boş ve boş optimizasyonu ile verilen bazı sorgular için yanlış sonucu düzeltin + INNER/RIGHT JOIN. [\#7284](https://github.com/ClickHouse/ClickHouse/pull/7284) ([Nikolay + Kochetov](https://github.com/KochetovNicolai)) +- Live VİEW getHeader() yönteminde AddressSanitizer hatası düzeltildi. + [\#7271](https://github.com/ClickHouse/ClickHouse/pull/7271) + ([vzakaznikov](https://github.com/vzakaznikov)) + +#### Geliştirme {#improvement-1} + +- Queue\_wait\_max\_ms bekleme durumunda bir ileti ekleme gerçekleşir. + [\#7390](https://github.com/ClickHouse/ClickHouse/pull/7390) ([Azat + Khuzhin](https://github.com/azat)) +- Yapılan ayar `s3_min_upload_part_size` masa seviyesi. + [\#7059](https://github.com/ClickHouse/ClickHouse/pull/7059) ([Vladimir + Chebotarev](https://github.com/excitoon)) +- STORAGEFACTORY TTL kontrol edin. [\#7304](https://github.com/ClickHouse/ClickHouse/pull/7304) + ([sundyli](https://github.com/sundy-li)) +- Kısmi birleştirme birleşiminde Squash sol blokları (optimizasyon). + [\#7122](https://github.com/ClickHouse/ClickHouse/pull/7122) ([Artem + Zuikov](https://github.com/4ertus2)) +- Çoğaltılmış tablo motorlarının mutasyonlarında deterministik olmayan işlevlere izin vermeyin, çünkü bu + kopyaları arasındaki tutarsızlıkları tanıtabilir. + [\#7247](https://github.com/ClickHouse/ClickHouse/pull/7247) ([İskender + Kazakov](https://github.com/Akazz)) +- Özel durum yığını izini dizeye dönüştürürken bellek izleyicisini devre dışı bırakın. Bu kaybı önleyebilir + Tür hata mesaj oflarının `Memory limit exceeded` neden olan sunucuda `Attempt to read after eof` istemci üzerinde istisna. [\#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) + ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Çeşitli biçim geliştirmeleri. Gideriyor + [\#6033](https://github.com/ClickHouse/ClickHouse/issues/6033), + [\#2633](https://github.com/ClickHouse/ClickHouse/issues/2633), + [\#6611](https://github.com/ClickHouse/ClickHouse/issues/6611), + [\#6742](https://github.com/ClickHouse/ClickHouse/issues/6742) + [\#7215](https://github.com/ClickHouse/ClickHouse/pull/7215) + ([tavplubix](https://github.com/tavplubix)) +- ClickHouse sola dönüştürülebilir olmayan In işlecinin sağ tarafındaki değerleri yok sayar + side type. Make it work properly for compound types – Array and Tuple. + [\#7283](https://github.com/ClickHouse/ClickHouse/pull/7283) ([İskender + Kuzmenkov](https://github.com/akuzm)) +- ASOF JOİN için eksik eşitsizlikleri destekleyin. Daha az veya eşit varyant ve katı katılmak mümkündür + sözdiziminde asof sütunu için daha büyük ve daha az varyant. + [\#7282](https://github.com/ClickHouse/ClickHouse/pull/7282) ([Artem + Zuikov](https://github.com/4ertus2)) +- Kısmi birleştirme birleşimini Optimize edin. [\#7070](https://github.com/ClickHouse/ClickHouse/pull/7070) + ([Artem Zuikov](https://github.com/4ertus2)) +- UNİQCOMBİNED işlevlerinde 98 k'dan fazla bellek kullanmayın. + [\#7236](https://github.com/ClickHouse/ClickHouse/pull/7236), + [\#7270](https://github.com/ClickHouse/ClickHouse/pull/7270) ([Azat + Khuzhin](https://github.com/azat)) +- Partialmergejoin diskteki sağ birleştirme tablosunun parçalarını yıkayın (yeterli değilse + bellek). Gerektiğinde verileri geri yükleyin. [\#7186](https://github.com/ClickHouse/ClickHouse/pull/7186) + ([Artem Zuikov](https://github.com/4ertus2)) + +#### Performans İyileştirme {#performance-improvement-1} + +- Veri çoğaltma kaçınarak const argümanları ile joinget hızlandırın. + [\#7359](https://github.com/ClickHouse/ClickHouse/pull/7359) ([Amos + Kuş](https://github.com/amosbird)) +- Alt sorgu boşsa erken dönün. + [\#7007](https://github.com/ClickHouse/ClickHouse/pull/7007) ([小路](https://github.com/nicelulu)) +- Değerlerdeki SQL ifadesinin ayrıştırılmasını Optimize edin. + [\#6781](https://github.com/ClickHouse/ClickHouse/pull/6781) + ([tavplubix](https://github.com/tavplubix)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-1} + +- Mac OS için çapraz derleme için bazı katkılar devre dışı bırakın. + [\#7101](https://github.com/ClickHouse/ClickHouse/pull/7101) ([I ivanvan](https://github.com/abyss7)) +- Clickhouse\_common\_ıo için pocoxml ile eksik bağlantı ekleyin. + [\#7200](https://github.com/ClickHouse/ClickHouse/pull/7200) ([Azat + Khuzhin](https://github.com/azat)) +- Clickhouse-test'te birden fazla test filtresi argümanını kabul edin. + [\#7226](https://github.com/ClickHouse/ClickHouse/pull/7226) ([İskender + Kuzmenkov](https://github.com/akuzm)) +- Kol için musl ve jemalloc'u etkinleştirin. [\#7300](https://github.com/ClickHouse/ClickHouse/pull/7300) + ([Amos Kuşu](https://github.com/amosbird)) +- Katma `--client-option` parametre için `clickhouse-test` müşteriye ek parametreler aktarmak için. + [\#7277](https://github.com/ClickHouse/ClickHouse/pull/7277) ([Nikolay + Kochetov](https://github.com/KochetovNicolai)) +- Rpm paket yükseltmesinde mevcut yapılandırmaları koruyun. + [\#7103](https://github.com/ClickHouse/ClickHouse/pull/7103) + ([filimonov](https://github.com/filimonov)) +- PVS tarafından tespit edilen hataları düzeltin. [\#7153](https://github.com/ClickHouse/ClickHouse/pull/7153) ([Artem + Zuikov](https://github.com/4ertus2)) +- Darwin için yapı düzeltin. [\#7149](https://github.com/ClickHouse/ClickHouse/pull/7149) + ([I ivanvan](https://github.com/abyss7)) +- glibc 2.29 uyumluluk. [\#7142](https://github.com/ClickHouse/ClickHouse/pull/7142) ([Amos + Kuş](https://github.com/amosbird)) +- Dh\_clean potansiyel kaynak dosyaları dokunmaz emin olun. + [\#7205](https://github.com/ClickHouse/ClickHouse/pull/7205) ([Amos + Kuş](https://github.com/amosbird)) +- Altınity rpm'den güncellerken çakışmayı önlemeye çalışın-ayrı olarak paketlenmiş yapılandırma dosyası vardır + clickhouse-sunucu-ortak. [\#7073](https://github.com/ClickHouse/ClickHouse/pull/7073) + ([filimonov](https://github.com/filimonov)) +- Daha hızlı yeniden oluşturmak için bazı başlık dosyalarını Optimize edin. + [\#7212](https://github.com/ClickHouse/ClickHouse/pull/7212), + [\#7231](https://github.com/ClickHouse/ClickHouse/pull/7231) ([İskender + Kuzmenkov](https://github.com/akuzm)) +- Tarih ve DateTime için performans testleri ekleyin. [\#7332](https://github.com/ClickHouse/ClickHouse/pull/7332) ([Vasily + Nemkov](https://github.com/Enmk)) +- Deterministik olmayan mutasyonlar içeren bazı testleri düzeltin. + [\#7132](https://github.com/ClickHouse/ClickHouse/pull/7132) ([İskender + Kazakov](https://github.com/Akazz)) +- MemorySanitizer ile CI için yapı ekleyin. [\#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) + ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Metricstransmitter başlatılmamış değerleri kullanmaktan kaçının. + [\#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat + Khuzhin](https://github.com/azat)) +- MemorySanitizer tarafından bulunan alanlarda bazı sorunları düzeltin. + [\#7135](https://github.com/ClickHouse/ClickHouse/pull/7135), + [\#7179](https://github.com/ClickHouse/ClickHouse/pull/7179) ([İskender + Kuzmenkov](https://github.com/akuzm)), [\#7376](https://github.com/ClickHouse/ClickHouse/pull/7376) + ([Amos Kuşu](https://github.com/amosbird)) +- Murmurhash32'de tanımsız davranışı düzeltin. [\#7388](https://github.com/ClickHouse/ClickHouse/pull/7388) ([Amos + Kuş](https://github.com/amosbird)) +- Storagesınfostream tanımsız davranışı düzeltin. [\#7384](https://github.com/ClickHouse/ClickHouse/pull/7384) + ([tavplubix](https://github.com/tavplubix)) +- Harici veritabanı motorları (MySQL, ODBC, JDBC) için katlama sabit sabit ifadeler. Önceki + sürümler birden fazla sabit ifade için çalışmadı ve Tarih için hiç çalışmadı, + DateTime ve UUID. Bu düzeltmeler [\#7245](https://github.com/ClickHouse/ClickHouse/issues/7245) + [\#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) + ([alexey-milovidov](https://github.com/alexey-milovidov)) +- No\_users\_thread değişkenine erişirken canlı görünümde ThreadSanitizer veri yarışı hatası düzeltildi. + [\#7353](https://github.com/ClickHouse/ClickHouse/pull/7353) + ([vzakaznikov](https://github.com/vzakaznikov)) +- Libcommon'daki malloc sembollerinden kurtulun + [\#7134](https://github.com/ClickHouse/ClickHouse/pull/7134), + [\#7065](https://github.com/ClickHouse/ClickHouse/pull/7065) ([Amos + Kuş](https://github.com/amosbird)) +- Tüm kitaplıkları devre dışı bırakmak için genel bayrak ENABLE\_LİBRARİES ekleyin. + [\#7063](https://github.com/ClickHouse/ClickHouse/pull/7063) + ([proller](https://github.com/proller)) + +#### Kod temizleme {#code-cleanup} + +- Sözlükler için DDL'YE hazırlanmak için yapılandırma deposunu genelleştirin. [\#7155](https://github.com/ClickHouse/ClickHouse/pull/7155) + ([alesapin](https://github.com/alesapin)) +- Herhangi bir anlamsal olmadan sözlükler DDL için ayrıştırıcı. + [\#7209](https://github.com/ClickHouse/ClickHouse/pull/7209) + ([alesapin](https://github.com/alesapin)) +- Bölünmüş ParserCreateQuery farklı küçük ayrıştırıcılar içine. + [\#7253](https://github.com/ClickHouse/ClickHouse/pull/7253) + ([alesapin](https://github.com/alesapin)) +- Küçük refactoring ve dış sözlükler yakın yeniden adlandırma. + [\#7111](https://github.com/ClickHouse/ClickHouse/pull/7111) + ([alesapin](https://github.com/alesapin)) +- Refactor bazı kod rol tabanlı erişim kontrolü için hazırlamak. [\#7235](https://github.com/ClickHouse/ClickHouse/pull/7235) ([Vitaly. + Baranov](https://github.com/vitlibar)) +- Veritabanında bazı geliştirmelersıradan kod. + [\#7086](https://github.com/ClickHouse/ClickHouse/pull/7086) ([Nikita + Vasilev](https://github.com/nikvas0)) +- Karma tabloların find() ve emplace() yöntemlerinde yineleyiciler kullanmayın. + [\#7026](https://github.com/ClickHouse/ClickHouse/pull/7026) ([İskender + Kuzmenkov](https://github.com/akuzm)) +- Parametre kökü boş olmadığında getmultiplevaluesfromconfig'i düzeltin. [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) + ([Mikhail Korotov](https://github.com/millb)) +- Bazı kopyala yapıştır (TemporaryFile ve TemporaryFileStream) kaldırın) + [\#7166](https://github.com/ClickHouse/ClickHouse/pull/7166) ([Artem + Zuikov](https://github.com/4ertus2)) +- Geliştirilmiş kod okunabilirliği biraz (`MergeTreeData::getActiveContainingPart`). + [\#7361](https://github.com/ClickHouse/ClickHouse/pull/7361) ([Vladimir + Chebotarev](https://github.com/excitoon)) +- Yerel nesneler kullanan tüm zamanlanmış işleri bekleyin, eğer `ThreadPool::schedule(...)` atmalar + özel. Adlandırmak `ThreadPool::schedule(...)` -e doğru `ThreadPool::scheduleOrThrowOnError(...)` ve + atabileceği açık hale getirmek için yorumları düzeltin. + [\#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) + ([tavplubix](https://github.com/tavplubix)) + +## ClickHouse sürümü 19.15 {#clickhouse-release-19-15} + +### ClickHouse sürümü 19.15.4.10, 2019-10-31 {#clickhouse-release-19-15-4-10-2019-10-31} + +#### Hata Düzeltme {#bug-fix-3} + +- SQL\_TINYINT ve SQL\_BIGINT işleme eklendi ve ODBC Köprüsü SQL\_FLOAT veri kaynağı türlerinin işlenmesini düzeltin. + [\#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([Denis Glazachev](https://github.com/traceon)) +- Taşıma bölümünde hedef disk veya birim üzerinde bazı parçaların olmasına izin verilir. + [\#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([Vladimir Chebotarev](https://github.com/excitoon)) +- ODBC köprüsü aracılığıyla null sütunlarda sabit NULL değerleri. + [\#7402](https://github.com/ClickHouse/ClickHouse/pull/7402) ([Vasily Nemkov](https://github.com/Enmk)) +- SOMUTLAŞTIRILMIŞ sütunlarla dağıtılmış yerel olmayan düğüme sabit ekleme. + [\#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) +- Sabit fonksiyon getMultipleValuesFromConfig. + [\#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) ([Mikhail Korotov](https://github.com/millb)) +- TCP yerine HTTP keep alive timeout kullanarak sabit sorunu alive timeout tutun. + [\#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily Nemkov](https://github.com/Enmk)) +- Tüm işlerin istisnada bitmesini bekleyin (nadir segfault'ları düzeltir). + [\#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) ([tavplubix](https://github.com/tavplubix)) +- Kafka tabloya eklerken MVs için itmeyin. + [\#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([I ivanvan](https://github.com/abyss7)) +- İstisna yığını için bellek izleyicisini devre dışı bırakın. + [\#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Harici veritabanı için sorgu dönüştürme sabit kötü kod. + [\#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Metricstransmitter başlatılmamış değerleri kullanmaktan kaçının. + [\#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat Khuzhin](https://github.com/azat)) +- Testler için makrolarla örnek yapılandırma eklendi ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.15.3.6, 2019-10-09 {#clickhouse-release-19-15-3-6-2019-10-09} + +#### Hata Düzeltme {#bug-fix-4} + +- Karma sözlükte sabit bad\_variant. + ([alesapin](https://github.com/alesapin)) +- Bölüm sorgu Ekle segmentasyon hatası ile hata düzeltildi. + ([alesapin](https://github.com/alesapin)) +- Sabit zaman hesaplama `MergeTreeData`. + ([Vladimir Chebotarev](https://github.com/excitoon)) +- Yazma tamamlandıktan sonra açıkça Kafka'ya taahhüt edin. + [\#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([I ivanvan](https://github.com/abyss7)) +- MERGETREE parçalarının min/max dizinlerinde NULL değerleri doğru şekilde seri hale getirin. + [\#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([Alexander Kuzmenkov](https://github.com/akuzm)) + +### ClickHouse sürümü 19.15.2.2, 2019-10-01 {#clickhouse-release-19-15-2-2-2019-10-01} + +#### Yenilik {#new-feature-3} + +- Katmanlı depolama: MergeTree motoru ile tablolar için birden fazla depolama birimleri kullanmak için destek. Yeni verileri SSD'DE saklamak ve eski verileri otomatik olarak HDD'ye taşımak mümkündür. ([örnek](https://clickhouse.github.io/clickhouse-presentations/meetup30/new_features/#12)). [\#4918](https://github.com/ClickHouse/ClickHouse/pull/4918) ([Igr](https://github.com/ObjatieGroba)) [\#6489](https://github.com/ClickHouse/ClickHouse/pull/6489) ([alesapin](https://github.com/alesapin)) +- Tablo fonksiyonu Ekle `input` gelen verileri okumak için `INSERT SELECT` sorgu. [\#5450](https://github.com/ClickHouse/ClickHouse/pull/5450) ([palasonic1](https://github.com/palasonic1)) [\#6832](https://github.com/ClickHouse/ClickHouse/pull/6832) ([Anton Popov](https://github.com/CurtizJ)) +- Add a `sparse_hashed` sözlük düzeni, bu işlevsel olarak eşdeğerdir `hashed` düzen, ancak daha fazla bellek verimli. Daha yavaş değer alma pahasına yaklaşık iki kat daha az bellek kullanır. [\#6894](https://github.com/ClickHouse/ClickHouse/pull/6894) ([Azat Khuzhin](https://github.com/azat)) +- Sözlüklere erişim için kullanıcıların listesini tanımlama yeteneğini uygular. Sadece geçerli bağlı veritabanı kullanarak. [\#6907](https://github.com/ClickHouse/ClickHouse/pull/6907) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Eklemek `LIMIT` seçeneği `SHOW` sorgu. [\#6944](https://github.com/ClickHouse/ClickHouse/pull/6944) ([Philipp Malkovsky](https://github.com/malkfilipp)) +- Eklemek `bitmapSubsetLimit(bitmap, range_start, limit)` en küçük alt kümesini döndüren işlev `limit` küm valuesedeki değer smallerlerden daha küçük olmayan değerler `range_start`. [\#6957](https://github.com/ClickHouse/ClickHouse/pull/6957) ([Zhichang Yu](https://github.com/yuzhichang)) +- Eklemek `bitmapMin` ve `bitmapMax` işlevler. [\#6970](https://github.com/ClickHouse/ClickHouse/pull/6970) ([Zhichang Yu](https://github.com/yuzhichang)) +- Fonksiyon Ekle `repeat` ile ilgili [sayı-6648](https://github.com/ClickHouse/ClickHouse/issues/6648) [\#6999](https://github.com/ClickHouse/ClickHouse/pull/6999) ([flynn](https://github.com/ucasFL)) + +#### Deneysel Özellik {#experimental-feature-1} + +- Geçerli boru hattını değiştirmeyen birleştirme birleştirme varyantını (bellekte) uygulayın. Sonuç kısmen birleştirme anahtarına göre sıralanır. Koymak `partial_merge_join = 1` bu özelliği kullanmak için. Birleştirme birleştirme hala geliştirme aşamasındadır. [\#6940](https://github.com/ClickHouse/ClickHouse/pull/6940) ([Artem Zuikov](https://github.com/4ertus2)) +- Eklemek `S3` motor ve masa fonksiyonu. Hala geliştirme aşamasında (henüz kimlik doğrulama desteği yok). [\#5596](https://github.com/ClickHouse/ClickHouse/pull/5596) ([Vladimir Chebotarev](https://github.com/excitoon)) + +#### Geliştirme {#improvement-2} + +- Kafka'dan okunan her mesaj atomik olarak eklenir. Bu, Kafka engine ile bilinen hemen hemen tüm sorunları çözer. [\#6950](https://github.com/ClickHouse/ClickHouse/pull/6950) ([I ivanvan](https://github.com/abyss7)) +- Dağıtılmış sorguların yerine çalışma için iyileştirmeler. Kurtarma süresini kısaltın, ayrıca şimdi yapılandırılabilir ve görülebilir `system.clusters`. [\#6399](https://github.com/ClickHouse/ClickHouse/pull/6399) ([Vasily Nemkov](https://github.com/Enmk)) +- Doğrudan Enums için sayısal değerleri destekleyin `IN` bölme. \#6766 [\#6941](https://github.com/ClickHouse/ClickHouse/pull/6941) ([dimarub2000](https://github.com/dimarub2000)) +- Destek (isteğe bağlı, varsayılan olarak devre dışı) URL depolama yönlendirir. [\#6914](https://github.com/ClickHouse/ClickHouse/pull/6914) ([makroll](https://github.com/maqroll)) +- Eski bir sürümü olan istemci bir sunucuya bağlandığında bilgi iletisi ekleyin. [\#6893](https://github.com/ClickHouse/ClickHouse/pull/6893) ([Philipp Malkovsky](https://github.com/malkfilipp)) +- Dağıtılmış tablolarda veri göndermek için maksimum geri dönüş uyku süresi sınırını kaldırın [\#6895](https://github.com/ClickHouse/ClickHouse/pull/6895) ([Azat Khuzhin](https://github.com/azat)) +- Grafit kümülatif değerleri ile profil olayları (sayaçlar) göndermek için yeteneği ekleyin. Altında etkinleştir canilebilir `` server inda `config.xml`. [\#6969](https://github.com/ClickHouse/ClickHouse/pull/6969) ([Azat Khuzhin](https://github.com/azat)) +- Otomatik döküm türü Ekle `T` -e doğru `LowCardinality(T)` veri türü sütununa eklerken `LowCardinality(T)` HTTP üzerinden yerel formatta. [\#6891](https://github.com/ClickHouse/ClickHouse/pull/6891) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fonksiyonu kullanma yeteneği ekleyin `hex` kullanmadan `reinterpretAsString` için `Float32`, `Float64`. [\#7024](https://github.com/ClickHouse/ClickHouse/pull/7024) ([Mikhail Korotov](https://github.com/millb)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-2} + +- Hata ayıklama bilgisi ile clickhouse ikili için gdb-ındex ekleyin. Bu başlangıç süresini hızlandıracaktır `gdb`. [\#6947](https://github.com/ClickHouse/ClickHouse/pull/6947) ([alesapin](https://github.com/alesapin)) +- Kullanan yamalı dpkg-deb ile deb ambalajını hızlandırın `pigz`. [\#6960](https://github.com/ClickHouse/ClickHouse/pull/6960) ([alesapin](https://github.com/alesapin)) +- Koymak `enable_fuzzing = 1` tüm proje kodu libfuzzer enstrümantasyon etkinleştirmek için. [\#7042](https://github.com/ClickHouse/ClickHouse/pull/7042) ([kyprizel](https://github.com/kyprizel)) +- CI bölünmüş yapı duman testi ekleyin. [\#7061](https://github.com/ClickHouse/ClickHouse/pull/7061) ([alesapin](https://github.com/alesapin)) +- MemorySanitizer ile CI için yapı ekleyin. [\#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Değişmek `libsparsehash` ile `sparsehash-c11` [\#6965](https://github.com/ClickHouse/ClickHouse/pull/6965) ([Azat Khuzhin](https://github.com/azat)) + +#### Hata Düzeltme {#bug-fix-5} + +- Büyük tablolarda karmaşık anahtarlar üzerinde endeks analizi sabit performans düşüşü. Bu düzeltmeler \# 6924. [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kafka boş konudan seçerken segfaults'a neden olan mantıksal hatayı düzeltin. [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([I ivanvan](https://github.com/abyss7)) +- Çok erken MySQL bağlantısını düzeltin `MySQLBlockInputStream.cpp`. [\#6882](https://github.com/ClickHouse/ClickHouse/pull/6882) ([Clément Rodriguez](https://github.com/clemrodriguez)) +- Çok eski Linux çekirdekleri için geri destek (düzeltme [\#6841](https://github.com/ClickHouse/ClickHouse/issues/6841)) [\#6853](https://github.com/ClickHouse/ClickHouse/pull/6853) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Olası veri kaybını düzeltin `insert select` giriş akışında boş blok durumunda sorgu. \#6834 \#6862 [\#6911](https://github.com/ClickHouse/ClickHouse/pull/6911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fonksiyon için düzeltme `АrrayEnumerateUniqRanked` params boş diziler ile [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) +- Dizi birleşimleri ve genel alt sorgularla karmaşık sorguları düzeltin. [\#6934](https://github.com/ClickHouse/ClickHouse/pull/6934) ([I ivanvan](https://github.com/abyss7)) +- Düzeltmek `Unknown identifier` birden fazla birleşim ile ORDER BY ve GROUP BY hatası [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([Artem Zuikov](https://github.com/4ertus2)) +- Sabit `MSan` fonksiyonu çalıştırırken uyarı `LowCardinality` değişken. [\#7062](https://github.com/ClickHouse/ClickHouse/pull/7062) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-2} + +- Bitmap'in seri hale getirme biçimi değiştirildi \* performansı artırmak için toplama işlevi durumları. Önceki sürümlerden bitmap\* seri hale getirilmiş durumları okunamıyor. [\#6908](https://github.com/ClickHouse/ClickHouse/pull/6908) ([Zhichang Yu](https://github.com/yuzhichang)) + +## ClickHouse sürümü 19.14 {#clickhouse-release-19-14} + +### ClickHouse sürümü 19.14.7.15, 2019-10-02 {#clickhouse-release-19-14-7-15-2019-10-02} + +#### Hata Düzeltme {#bug-fix-6} + +- Bu sürüm aynı zamanda 19.11.12.69 tüm hata düzeltmeleri içerir. +- 19.14 ve önceki sürümleri arasında dağıtılmış sorgular için sabit uyumluluk. Bu düzeltmeler [\#7068](https://github.com/ClickHouse/ClickHouse/issues/7068). [\#7069](https://github.com/ClickHouse/ClickHouse/pull/7069) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.14.6.12, 2019-09-19 {#clickhouse-release-19-14-6-12-2019-09-19} + +#### Hata Düzeltme {#bug-fix-7} + +- Fonksiyon için düzeltme `АrrayEnumerateUniqRanked` params boş diziler ile. [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) +- Sorgularda sabit alt sorgu adı `ARRAY JOIN` ve `GLOBAL IN subquery` takma ad ile. Belirtilirse, dış tablo adı için alt sorgu takma adını kullanın. [\#6934](https://github.com/ClickHouse/ClickHouse/pull/6934) ([I ivanvan](https://github.com/abyss7)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-3} + +- Düzeltmek [çırpan](https://clickhouse-test-reports.s3.yandex.net/6944/aab95fd5175a513413c7395a73a82044bdafb906/functional_stateless_tests_(debug).html) test `00715_fetch_merged_or_mutated_part_zookeeper` mutasyonların uygulanmasını beklemek zorunda olduğu için bir kabuk komut dosyasına yeniden yazarak. [\#6977](https://github.com/ClickHouse/ClickHouse/pull/6977) ([Alexander Kazakov](https://github.com/Akazz)) +- Fonksiyon sabit UBSan ve MemSan hatası `groupUniqArray` emtpy dizi argümanı ile. Bu boş yerleştirerek neden oldu `PaddedPODArray` sıfır hücre değeri için yapıcı çağrılmadığı için karma tabloya sıfır hücre. [\#6937](https://github.com/ClickHouse/ClickHouse/pull/6937) ([Amos Kuşu](https://github.com/amosbird)) + +### ClickHouse sürümü 19.14.3.3, 2019-09-10 {#clickhouse-release-19-14-3-3-2019-09-10} + +#### Yenilik {#new-feature-4} + +- `WITH FILL` değiştirici için `ORDER BY`. (devam [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([Anton Popov](https://github.com/CurtizJ)) +- `WITH TIES` değiştirici için `LIMIT`. (devam [\#5069](https://github.com/ClickHouse/ClickHouse/issues/5069)) [\#6610](https://github.com/ClickHouse/ClickHouse/pull/6610) ([Anton Popov](https://github.com/CurtizJ)) +- Ayrıştırılmamış `NULL` NULL olarak değişmez (eğer ayar `format_csv_unquoted_null_literal_as_null=1`). Bu alanın veri türü null değilse, null alanları varsayılan değerlerle başlatın (eğer ayar `input_format_null_as_default=1`). [\#5990](https://github.com/ClickHouse/ClickHouse/issues/5990) [\#6055](https://github.com/ClickHouse/ClickHouse/pull/6055) ([tavplubix](https://github.com/tavplubix)) +- Tablo işlevlerinin yollarındaki joker karakterler için destek `file` ve `hdfs`. Yol joker karakterler içeriyorsa, tablo salt okunur olacaktır. Kullanım örneği: `select * from hdfs('hdfs://hdfs1:9000/some_dir/another_dir/*/file{0..9}{0..9}')` ve `select * from file('some_dir/{some_file,another_file,yet_another}.tsv', 'TSV', 'value UInt32')`. [\#6092](https://github.com/ClickHouse/ClickHouse/pull/6092) ([Olga Khvostikova](https://github.com/stavrolia)) +- Yeni `system.metric_log` değerleri depo tablelayan tablo `system.events` ve `system.metrics` belirtilen zaman aralığı ile. [\#6363](https://github.com/ClickHouse/ClickHouse/issues/6363) [\#6467](https://github.com/ClickHouse/ClickHouse/pull/6467) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6530](https://github.com/ClickHouse/ClickHouse/pull/6530) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ClickHouse metin günlükleri yazmak için izin ver `system.text_log` Tablo. [\#6037](https://github.com/ClickHouse/ClickHouse/issues/6037) [\#6103](https://github.com/ClickHouse/ClickHouse/pull/6103) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [\#6164](https://github.com/ClickHouse/ClickHouse/pull/6164) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yığın izlerinde özel sembolleri göster (bu, elf dosyalarının sembol tablolarını ayrıştırma yoluyla yapılır). Hata ayıklama bilgisi varsa yığın izlerinde dosya ve satır numarası hakkında bilgi eklendi. Programda mevcut indeksleme sembolleri ile Speedup sembol adı arama. İç gözlem için yeni SQL işlevleri eklendi: `demangle` ve `addressToLine`. Yeniden adlandırılan işlev `symbolizeAddress` -e doğru `addressToSymbol` tutarlılık için. İşlev `addressToSymbol` performans nedenleriyle karıştırılmış adını döndürür ve uygulamak zorunda `demangle`. Ayar eklendi `allow_introspection_functions` varsayılan olarak kapalı olan. [\#6201](https://github.com/ClickHouse/ClickHouse/pull/6201) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tablo fonksiyonu `values` (adı büyük / küçük harf duyarsız). Bu okumak için izin verir `VALUES` önerilen liste [\#5984](https://github.com/ClickHouse/ClickHouse/issues/5984). Örnek: `SELECT * FROM VALUES('a UInt64, s String', (1, 'one'), (2, 'two'), (3, 'three'))`. [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) +- Depolama ayarlarını değiştirmek için bir yetenek eklendi. Sözdizimi: `ALTER TABLE MODIFY SETTING = `. [\#6366](https://github.com/ClickHouse/ClickHouse/pull/6366) [\#6669](https://github.com/ClickHouse/ClickHouse/pull/6669) [\#6685](https://github.com/ClickHouse/ClickHouse/pull/6685) ([alesapin](https://github.com/alesapin)) +- Müstakil parçaların çıkarılması için destek. Sözdizimi: `ALTER TABLE DROP DETACHED PART ''`. [\#6158](https://github.com/ClickHouse/ClickHouse/pull/6158) ([tavplubix](https://github.com/tavplubix)) +- Tablo kısıtlamaları. Ekleme sırasında kontrol edilecek tablo tanımına kısıtlama eklemenize izin verir. [\#5273](https://github.com/ClickHouse/ClickHouse/pull/5273) ([Gleb Novikov](https://github.com/NanoBjorn)) [\#6652](https://github.com/ClickHouse/ClickHouse/pull/6652) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Basamaklı hayata görünümler için Suppport. [\#6324](https://github.com/ClickHouse/ClickHouse/pull/6324) ([Amos Kuşu](https://github.com/amosbird)) +- Her sorgu yürütme iş parçacığı saniyede bir örnek için varsayılan olarak sorgu profiler'ı açın. [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Giriş biçimi `ORC`. [\#6454](https://github.com/ClickHouse/ClickHouse/pull/6454) [\#6703](https://github.com/ClickHouse/ClickHouse/pull/6703) ([akonyaev90](https://github.com/akonyaev90)) +- İki yeni fonksiyon eklendi: `sigmoid` ve `tanh` (bu makine öğrenme uygulamaları için yararlıdır). [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İşlev `hasToken(haystack, token)`, `hasTokenCaseInsensitive(haystack, token)` verilen jetonun samanlıkta olup olmadığını kontrol etmek için. Token, iki alfasayısal olmayan ASCII karakteri (veya samanlığın sınırları) arasında maksimum uzunlukta bir alt dizedir. Token sabit bir dize olmalıdır. Tokenbf\_v1 Endeksi uzmanlaşma tarafından desteklenmektedir. [\#6596](https://github.com/ClickHouse/ClickHouse/pull/6596), [\#6662](https://github.com/ClickHouse/ClickHouse/pull/6662) ([Vasily Nemkov](https://github.com/Enmk)) +- Yeni fonksiyon `neighbor(value, offset[, default_value])`. Bir veri bloğunda sütun içindeki önceki / sonraki değere ulaşmayı sağlar. [\#5925](https://github.com/ClickHouse/ClickHouse/pull/5925) ([Alex Krash](https://github.com/alex-krash)) [6685365ab8c5b74f9650492c88a012596eb1b0c6](https://github.com/ClickHouse/ClickHouse/commit/6685365ab8c5b74f9650492c88a012596eb1b0c6) [341e2e4587a18065c2da1ca888c73389f48ce36c](https://github.com/ClickHouse/ClickHouse/commit/341e2e4587a18065c2da1ca888c73389f48ce36c) [Alexey Milovidov](https://github.com/alexey-milovidov) +- Bir işlev oluşturuldu `currentUser()`, yetkili kullanıcının giriş dönen. Add aliased al aliasias `user()` MySQL ile uyumluluk için. [\#6470](https://github.com/ClickHouse/ClickHouse/pull/6470) ([Alex Krash](https://github.com/alex-krash)) +- Yeni toplama fonksiyonları `quantilesExactInclusive` ve `quantilesExactExclusive` hangi teklif edildi [\#5885](https://github.com/ClickHouse/ClickHouse/issues/5885). [\#6477](https://github.com/ClickHouse/ClickHouse/pull/6477) ([dimarub2000](https://github.com/dimarub2000)) +- İşlev `bitmapRange(bitmap, range_begin, range_end)` hangi döner yeni set ile belirtilen aralığı (dahil değil `range_end`). [\#6314](https://github.com/ClickHouse/ClickHouse/pull/6314) ([Zhichang Yu](https://github.com/yuzhichang)) +- İşlev `geohashesInBox(longitude_min, latitude_min, longitude_max, latitude_max, precision)` hangi verilen alanı kapsayan geohash-kutuları hassas uzun dizeleri dizisi oluşturur. [\#6127](https://github.com/ClickHouse/ClickHouse/pull/6127) ([Vasily Nemkov](https://github.com/Enmk)) +- INSERT query ile destek uygulamak `Kafka` Tablolar. [\#6012](https://github.com/ClickHouse/ClickHouse/pull/6012) ([I ivanvan](https://github.com/abyss7)) +- İçin destek eklendi `_partition` ve `_timestamp` Kafka motoruna sanal sütunlar. [\#6400](https://github.com/ClickHouse/ClickHouse/pull/6400) ([I ivanvan](https://github.com/abyss7)) +- Hassas verileri kaldırma imkanı `query_log`, sunucu günlükleri, regexp tabanlı kurallar ile işlem listesi. [\#5710](https://github.com/ClickHouse/ClickHouse/pull/5710) ([filimonov](https://github.com/filimonov)) + +#### Deneysel Özellik {#experimental-feature-2} + +- Giriş ve çıkış Veri formatı `Template`. Bu giriş ve çıkış için özel biçim dizesi belirtmek için izin verir. [\#4354](https://github.com/ClickHouse/ClickHouse/issues/4354) [\#6727](https://github.com/ClickHouse/ClickHouse/pull/6727) ([tavplubix](https://github.com/tavplubix)) +- Uygulanması `LIVE VIEW` başlangıçta önerilen tablolar [\#2898](https://github.com/ClickHouse/ClickHouse/pull/2898) hazırlıklı olarak [\#3925](https://github.com/ClickHouse/ClickHouse/issues/3925), ve daha sonra güncellendi [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541). Görmek [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) detaylı açıklama için. [\#5541](https://github.com/ClickHouse/ClickHouse/issues/5541) ([vzakaznikov](https://github.com/vzakaznikov)) [\#6425](https://github.com/ClickHouse/ClickHouse/pull/6425) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#6656](https://github.com/ClickHouse/ClickHouse/pull/6656) ([vzakaznikov](https://github.com/vzakaznikov) Not bu `LIVE VIEW` özellik sonraki sürümlerinde kaldırılabilir. + +#### Hata Düzeltme {#bug-fix-8} + +- Bu sürüm aynı zamanda 19.13 ve 19.11 tüm hata düzeltmeleri içerir. +- Tablo atlama endeksleri ve dikey birleştirme gerçekleştiğinde segmentasyon hatasını düzeltin. [\#6723](https://github.com/ClickHouse/ClickHouse/pull/6723) ([alesapin](https://github.com/alesapin)) +- Önemsiz olmayan sütun varsayılanlarıyla sütun başına TTL'Yİ düzeltin. Daha önce kuvvet durumunda `OPTIMIZE ... FINAL` sorgu, süresi dolmuş değerler, kullanıcı tarafından belirtilen sütun varsayılanları yerine tür varsayılanları ile değiştirildi. [\#6796](https://github.com/ClickHouse/ClickHouse/pull/6796) ([Anton Popov](https://github.com/CurtizJ)) +- Normal sunucu yeniden başlatmada Kafka mesajları çoğaltma sorununu düzeltin. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([I ivanvan](https://github.com/abyss7)) +- Kafka mesajlarını okurken sabit sonsuz döngü. Tüketiciyi abonelikte hiç duraklatmayın/devam ettirmeyin - aksi takdirde bazı senaryolarda süresiz olarak duraklatılabilir. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([I ivanvan](https://github.com/abyss7)) +- Düzeltmek `Key expression contains comparison between inconvertible types` istisna içinde `bitmapContains` işlev. [\#6136](https://github.com/ClickHouse/ClickHouse/issues/6136) [\#6146](https://github.com/ClickHouse/ClickHouse/issues/6146) [\#6156](https://github.com/ClickHouse/ClickHouse/pull/6156) ([dimarub2000](https://github.com/dimarub2000)) +- Etkin olan segfault'u düzeltin `optimize_skip_unused_shards` ve kayıp sharding anahtarı. [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([Anton Popov](https://github.com/CurtizJ)) +- Bellek bozulmasına yol açabilecek mutasyonlarda yanlış kod düzeltildi. Adresin okunması ile sabit segfault `0x14c0` bu eşzamanlı nedeniyle happed olabilir `DROP TABLE` ve `SELECT` itibaren `system.parts` veya `system.parts_columns`. Mutasyon sorgularının hazırlanmasında sabit yarış durumu. Sabit kilitlenme neden `OPTIMIZE` çoğaltılmış tablolar ve değiştirir gibi eşzamanlı değişiklik işlemleri. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- MySQL arayüzü kaldırıldı ekstra ayrıntılı günlüğü [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Boolean ayarlarını ayrıştırma yeteneğini döndürür ‘true’ ve ‘false’ yapılandırma dosyasında. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) +- Çökmeyi düzeltin `quantile` ve `median` fonksiyonu üzerinde `Nullable(Decimal128)`. [\#6378](https://github.com/ClickHouse/ClickHouse/pull/6378) ([Artem Zuikov](https://github.com/4ertus2)) +- Tarafından iade edilen Olası eksik sonuç düzeltildi `SELECT` ile sorgu `WHERE` birincil anahtarda durum, Float türüne dönüşüm içeriyordu. Bu Monotonluk yanlış kontrol neden oldu `toFloat` işlev. [\#6248](https://github.com/ClickHouse/ClickHouse/issues/6248) [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) +- Kontrol `max_expanded_ast_elements` mutasyonlar için ayarlama. Sonra açık mutasyonlar `TRUNCATE TABLE`. [\#6205](https://github.com/ClickHouse/ClickHouse/pull/6205) ([Kış Zhang](https://github.com/zhang2014)) +- İle kullanıldığında anahtar sütunlar için birleştirme sonuçlarını düzeltin `join_use_nulls`. Sütun varsayılanları yerine Boşları ekleyin. [\#6249](https://github.com/ClickHouse/ClickHouse/pull/6249) ([Artem Zuikov](https://github.com/4ertus2)) +- Dikey birleştirme ve değiştirme ile atlama endeksleri için düzeltin. Fix for `Bad size of marks file` özel. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594) [\#6713](https://github.com/ClickHouse/ClickHouse/pull/6713) ([alesapin](https://github.com/alesapin)) +- Nadir kazayı düzeltin `ALTER MODIFY COLUMN` ve birleştirilmiş/değiştirilmiş parçalardan biri boş olduğunda dikey birleştirme (0 satır) [\#6746](https://github.com/ClickHouse/ClickHouse/issues/6746) [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) +- Dönüşüm Sabit hata `LowCardinality` yazmak `AggregateFunctionFactory`. Bu düzeltmeler [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Yanlış davranışı ve olası segfault'ları düzeltin `topK` ve `topKWeighted` toplu fonksiyonlar. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([Anton Popov](https://github.com/CurtizJ)) +- Sabit güvensiz kod etrafında `getIdentifier` işlev. [\#6401](https://github.com/ClickHouse/ClickHouse/issues/6401) [\#6409](https://github.com/ClickHouse/ClickHouse/pull/6409) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- MySQL Tel protokolünde Sabit hata (ClickHouse form MySQL istemcisine bağlanırken kullanılır). Yığın arabellek taşması nedeniyle `PacketPayloadWriteBuffer`. [\#6212](https://github.com/ClickHouse/ClickHouse/pull/6212) ([Yuriy Baranov](https://github.com/yurriy)) +- Sabit bellek sızıntısı `bitmapSubsetInRange` işlev. [\#6819](https://github.com/ClickHouse/ClickHouse/pull/6819) ([Zhichang Yu](https://github.com/yuzhichang)) +- Granülerlik değişiminden sonra mutasyon yürütüldüğünde nadir hatayı düzeltin. [\#6816](https://github.com/ClickHouse/ClickHouse/pull/6816) ([alesapin](https://github.com/alesapin)) +- Varsayılan olarak tüm alanlarla protobuf iletisine izin ver. [\#6132](https://github.com/ClickHouse/ClickHouse/pull/6132) ([Vitaly Baranov](https://github.com/vitlibar)) +- İle bir hatayı gidermek `nullIf` fonksiyonu zaman biz göndermek bir `NULL` ikinci argüman üzerinde argüman. [\#6446](https://github.com/ClickHouse/ClickHouse/pull/6446) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Sonsuz bellek tüketimine yol açan dize alanları ile karmaşık anahtar önbellek sözlüklerinde yanlış bellek ayırma/ayırma ile nadir hatayı düzeltin (bellek sızıntısı gibi görünüyor). Dize boyutu sekizden (8, 16, 32, vb.) başlayarak iki güç olduğunda hata çoğalır. [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alesapin](https://github.com/alesapin)) +- İstisnaya neden olan küçük dizilerde sabit goril kodlaması `Cannot write after end of buffer`. [\#6398](https://github.com/ClickHouse/ClickHouse/issues/6398) [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Vasily Nemkov](https://github.com/Enmk)) +- Birleşimlerde null olmayan türleri kullanmasına izin ver `join_use_nulls` etkin. [\#6705](https://github.com/ClickHouse/ClickHouse/pull/6705) ([Artem Zuikov](https://github.com/4ertus2)) +- Sakatlamak `Poco::AbstractConfiguration` sorguda yer değiştirmeler `clickhouse-client`. [\#6706](https://github.com/ClickHouse/ClickHouse/pull/6706) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kilitlenmeyi önlemek `REPLACE PARTITION`. [\#6677](https://github.com/ClickHouse/ClickHouse/pull/6677) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kullanım `arrayReduce` sabit argümanlar için segfault yol açabilir. [\#6242](https://github.com/ClickHouse/ClickHouse/issues/6242) [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Replica sonra geri yüklenmişse görünebilecek tutarsız parçaları düzeltin `DROP PARTITION`. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- Sabit asmak `JSONExtractRaw` işlev. [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yanlış atlama endeksleri seri hale getirme ve adaptif tanecikli toplama ile hatayı düzeltin. [\#6594](https://github.com/ClickHouse/ClickHouse/issues/6594). [\#6748](https://github.com/ClickHouse/ClickHouse/pull/6748) ([alesapin](https://github.com/alesapin)) +- Düzeltmek `WITH ROLLUP` ve `WITH CUBE` değiştiriciler `GROUP BY` iki seviyeli toplama ile. [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([Anton Popov](https://github.com/CurtizJ)) +- Adaptif tanecikli ikincil endeksler işaretleri yazma ile hatayı düzeltin. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alesapin](https://github.com/alesapin)) +- Sunucu başlatma sırasında başlatma sırasını düzeltin. Beri `StorageMergeTree::background_task_handle` içinde Başlat isılmıştır `startup()` bu `MergeTreeBlockOutputStream::write()` başlatmadan önce kullanmayı deneyebilir. Eğer başlatıldı ise sadece kontrol. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([I ivanvan](https://github.com/abyss7)) +- Bir hata ile tamamlanan önceki okuma işleminden veri arabelleğini temizleme. [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) +- Çoğaltılan \* MergeTree tablo için yeni bir çoğaltma oluştururken adaptif taneciklik sağlayan Fix hata. [\#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) +- İstisna durumunda sunucu başlatma sırasında Olası kilitlenme düzeltildi `libunwind` başlatılmamış erişim sırasında istisna sırasında `ThreadStatus` yapılı. [\#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Çökmeyi düzeltin `yandexConsistentHash` işlev. Fuzz testi ile bulundu. [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304) [\#6305](https://github.com/ClickHouse/ClickHouse/pull/6305) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sunucu aşırı ve küresel iş parçacığı havuzu tam yakın olduğunda sorguları asılı olasılığı düzeltildi. Dağıtılmış sorgular, her bir parçaya bağlantı başına bir iş parçacığı ayırdığından, çok sayıda parçaya (yüzlerce) sahip kümelerde daha yüksek şansa sahiptir. Örneğin, bir küme 330 kırıkları 30 eşzamanlı dağıtılmış sorgu işleme, bu sorun yeniden oluşturabilir. Bu sorun, 19.2'den başlayarak tüm sürümleri etkiler. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit mantık `arrayEnumerateUniqRanked` işlev. [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sembol tablosunu çözerken segfault'u düzeltin. [\#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([Amos Kuşu](https://github.com/amosbird)) +- Döküm sabit alakasız istisna `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Kaldırılan ekstra açıklamadan alıntı `system.settings` Tablo. [\#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [\#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Olası kilitlenmeyi önlemek `TRUNCATE` çoğaltılan tablonun. [\#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Anahtarı sıralama sırasına göre okumayı düzeltin. [\#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([Anton Popov](https://github.com/CurtizJ)) +- Düzeltmek `ALTER TABLE ... UPDATE` ile tablolar için sorgu `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +- Tarafından açılan hatayı düzeltin [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) (19.4.0'dan beri). Herhangi bir sütunu sorgulamadığımızda Mergetree tabloları üzerinden dağıtılmış tablolara sorgularda çoğaltır (`SELECT 1`). [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) +- İmzalı türün imzasız türe tamsayı bölümünde sabit taşma. Davranış tam olarak C veya C++ dilinde (tamsayı promosyon kuralları) olduğu gibi şaşırtıcı olabilir. Büyük imzalı numarayı büyük imzasız numaraya bölerken veya tam tersi durumda taşmanın hala mümkün olduğunu lütfen unutmayın (ancak bu durum daha az olağandır). Sorun tüm sunucu sürümlerinde mevcuttu. [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214) [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Limit maksimum uyku süresi için kısma zaman `max_execution_speed` veya `max_execution_speed_bytes` ayar .lanmıştır. Gibi sabit yanlış hatalar `Estimated query execution time (inf seconds) is too long`. [\#5547](https://github.com/ClickHouse/ClickHouse/issues/5547) [\#6232](https://github.com/ClickHouse/ClickHouse/pull/6232) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kullanma ile ilgili sabit sorunlar `MATERIALIZED` sütunlar ve Takma adlar `MaterializedView`. [\#448](https://github.com/ClickHouse/ClickHouse/issues/448) [\#3484](https://github.com/ClickHouse/ClickHouse/issues/3484) [\#3450](https://github.com/ClickHouse/ClickHouse/issues/3450) [\#2878](https://github.com/ClickHouse/ClickHouse/issues/2878) [\#2285](https://github.com/ClickHouse/ClickHouse/issues/2285) [\#3796](https://github.com/ClickHouse/ClickHouse/pull/3796) ([Amos Kuşu](https://github.com/amosbird)) [\#6316](https://github.com/ClickHouse/ClickHouse/pull/6316) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltmek `FormatFactory` işlemci olarak uygulanmayan giriş akışları için davranış. [\#6495](https://github.com/ClickHouse/ClickHouse/pull/6495) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Sabit yazım hatası. [\#6631](https://github.com/ClickHouse/ClickHouse/pull/6631) ([Alex Ryndin](https://github.com/alexryndin)) +- Hata iletisinde yazım hatası (is -\> are ). [\#6839](https://github.com/ClickHouse/ClickHouse/pull/6839) ([Denis Zhuravlev](https://github.com/den-crane)) +- Tür bir virgül içeriyorsa, sütun listesinin dizeden ayrıştırılması sırasında hata düzeltildi (bu sorun AŞAĞIDAKİLERLE ilgiliydi `File`, `URL`, `HDFS` depolamalar) [\#6217](https://github.com/ClickHouse/ClickHouse/issues/6217). [\#6209](https://github.com/ClickHouse/ClickHouse/pull/6209) ([dimarub2000](https://github.com/dimarub2000)) + +#### Güvenlik Düzeltme {#security-fix} + +- Bu sürüm aynı zamanda 19.13 ve 19.11 tüm hata güvenlik düzeltmeleri içerir. +- SQL ayrıştırıcısında yığın taşması nedeniyle sunucu çökmesine neden olmak için fabrikasyon bir sorgu olasılığı düzeltildi. Birleştirme ve dağıtılmış tablolarda yığın taşması olasılığı, alt sorguları içeren satır düzeyinde güvenlik için somutlaştırılmış görünümler ve koşullar düzeltildi. [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Geliştirme {#improvement-3} + +- İçin üçlü mantığın doğru uygulanması `AND/OR`. [\#6048](https://github.com/ClickHouse/ClickHouse/pull/6048) ([Alexander Kazakov](https://github.com/Akazz)) +- Şimdi değerler ve süresi dolmuş TTL ile satırlar sonra kaldırılacak `OPTIMIZE ... FINAL` query from old parts without TTL infos or with outdated TTL infos, e.g. after `ALTER ... MODIFY TTL` sorgu. Eklenen sorgular `SYSTEM STOP/START TTL MERGES` izin vermek/izin vermek için tüm birleştirmelerde TTL ve filtre süresi dolmuş değerleri ile birleşmeleri atayın. [\#6274](https://github.com/ClickHouse/ClickHouse/pull/6274) ([Anton Popov](https://github.com/CurtizJ)) +- Imkanı kullanarak istemci için ClickHouse geçmiş dosyasının konumunu değiştirmek için `CLICKHOUSE_HISTORY_FILE` en .v. [\#6840](https://github.com/ClickHouse/ClickHouse/pull/6840) ([filimonov](https://github.com/filimonov)) +- Kaldırmak `dry_run` 'dan bayrak `InterpreterSelectQuery`. … [\#6375](https://github.com/ClickHouse/ClickHouse/pull/6375) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Destek `ASOF JOIN` ile `ON` bölme. [\#6211](https://github.com/ClickHouse/ClickHouse/pull/6211) ([Artem Zuikov](https://github.com/4ertus2)) +- Mutasyonlar ve çoğaltma için Atlama dizinlerinin daha iyi desteği. İçin destek `MATERIALIZE/CLEAR INDEX ... IN PARTITION` sorgu. `UPDATE x = x` sütun kullanan tüm indeksleri yeniden hesaplar `x`. [\#5053](https://github.com/ClickHouse/ClickHouse/pull/5053) ([Nikita Vasilev](https://github.com/nikvas0)) +- İzin ver `ATTACH` canlı görünümler (örneğin, sunucu başlangıçta) ne olursa olsun `allow_experimental_live_view` ayar. [\#6754](https://github.com/ClickHouse/ClickHouse/pull/6754) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sorgu profiler tarafından toplanan yığın izlemeleri için sorgu profiler kendisi tarafından oluşturulan yığın çerçeveleri içermez. [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi tablo fonksiyonları `values`, `file`, `url`, `hdfs` ALİAS sütunları için destek var. [\#6255](https://github.com/ClickHouse/ClickHouse/pull/6255) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eğer bir istisna atmak `config.d` dosya, yapılandırma dosyası olarak ilgili kök öğeye sahip değildir. [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) +- İçin özel durum iletisinde ekstra bilgi Yazdır `no space left on device`. [\#6182](https://github.com/ClickHouse/ClickHouse/issues/6182), [\#6252](https://github.com/ClickHouse/ClickHouse/issues/6252) [\#6352](https://github.com/ClickHouse/ClickHouse/pull/6352) ([tavplubix](https://github.com/tavplubix)) +- Bir kırıkları belirlerken `Distributed` bir okuma sorgusu tarafından kapsanacak tablo (için `optimize_skip_unused_shards` = 1) ClickHouse şimdi her ikisinden de koşulları kontrol eder `prewhere` ve `where` select deyimi yan tümceleri. [\#6521](https://github.com/ClickHouse/ClickHouse/pull/6521) ([Alexander Kazakov](https://github.com/Akazz)) +- Etkin `SIMDJSON` avx2 olmayan ancak sse 4.2 ve PCLMUL komut seti ile makineler için. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285) [\#6320](https://github.com/ClickHouse/ClickHouse/pull/6320) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ClickHouse olmadan dosya sistemleri üzerinde çalışabilir `O_DIRECT` ek ayar yapmadan destek (ZFS ve BtrFS gibi). [\#4449](https://github.com/ClickHouse/ClickHouse/issues/4449) [\#6730](https://github.com/ClickHouse/ClickHouse/pull/6730) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Destek son alt sorgu için yüklemi aşağı itin. [\#6120](https://github.com/ClickHouse/ClickHouse/pull/6120) ([TCeason](https://github.com/TCeason)) [\#6162](https://github.com/ClickHouse/ClickHouse/pull/6162) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İyileştirmek `JOIN ON` anahtarlar çıkarma [\#6131](https://github.com/ClickHouse/ClickHouse/pull/6131) ([Artem Zuikov](https://github.com/4ertus2)) +- Güncel `SIMDJSON`. [\#6285](https://github.com/ClickHouse/ClickHouse/issues/6285). [\#6306](https://github.com/ClickHouse/ClickHouse/pull/6306) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin en küçük sütunun seçilmesini Optimize edin `SELECT count()` sorgu. [\#6344](https://github.com/ClickHouse/ClickHouse/pull/6344) ([Amos Kuşu](https://github.com/amosbird)) +- Katma `strict` param parametereter in `windowFunnel()`. Ne zaman `strict` ayarlanırsa, `windowFunnel()` yalnızca benzersiz değerler için koşulları uygular. [\#6548](https://github.com/ClickHouse/ClickHouse/pull/6548) ([achimbab](https://github.com/achimbab)) +- Daha güvenli arayüzü `mysqlxx::Pool`. [\#6150](https://github.com/ClickHouse/ClickHouse/pull/6150) ([avasiliev](https://github.com/avasiliev)) +- Seçenekler satır boyutu ile yürütülürken `--help` seçenek şimdi terminal boyutuna karşılık gelir. [\#6590](https://github.com/ClickHouse/ClickHouse/pull/6590) ([dimarub2000](https://github.com/dimarub2000)) +- Sakatlamak “read in order” anahtarsız toplama için optimizasyon. [\#6599](https://github.com/ClickHouse/ClickHouse/pull/6599) ([Anton Popov](https://github.com/CurtizJ)) +- İçin HTTP durum kodu `INCORRECT_DATA` ve `TYPE_MISMATCH` hata kodları varsayılan olarak değiştirildi `500 Internal Server Error` -e doğru `400 Bad Request`. [\#6271](https://github.com/ClickHouse/ClickHouse/pull/6271) ([Alexander Rodin](https://github.com/a-rodin)) +- Birleştirme nesnesini taşı `ExpressionAction` için `AnalyzedJoin`. `ExpressionAnalyzer` ve `ExpressionAction` hakkında bilmiyorum `Join` sınıf artık. Mantığı giz bylidir `AnalyzedJoin` ifface. [\#6801](https://github.com/ClickHouse/ClickHouse/pull/6801) ([Artem Zuikov](https://github.com/4ertus2)) +- Kırıklardan biri localhost olduğunda ancak sorgu ağ bağlantısı üzerinden gönderildiğinde dağıtılmış sorguların Olası kilitlenmesi düzeltildi. [\#6759](https://github.com/ClickHouse/ClickHouse/pull/6759) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Birden çok tablonun semantik değişti `RENAME` Olası kilitlenmeleri önlemek için. [\#6757](https://github.com/ClickHouse/ClickHouse/issues/6757). [\#6756](https://github.com/ClickHouse/ClickHouse/pull/6756) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bellekte tam paket yükü yüklenmesini önlemek için yeniden yazılan MySQL uyumluluk sunucusu. Her bağlantı için bellek tüketiminin yaklaşık olarak azalması `2 * DBMS_DEFAULT_BUFFER_SIZE` (okuma / yazma tamponları). [\#5811](https://github.com/ClickHouse/ClickHouse/pull/5811) ([Yuriy Baranov](https://github.com/yurriy)) +- Sorgu semantiği hakkında hiçbir şey bilmek zorunda olmayan ayrıştırıcıdan AST alias yorumlama mantığını taşıyın. [\#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([Artem Zuikov](https://github.com/4ertus2)) +- Biraz daha güvenli ayrıştırma `NamesAndTypesList`. [\#6408](https://github.com/ClickHouse/ClickHouse/issues/6408). [\#6410](https://github.com/ClickHouse/ClickHouse/pull/6410) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `clickhouse-copier`: Kullanımına izin ver `where_condition` config ile `partition_key` bölüm varlığını kontrol etmek için sorgudaki diğer ad (daha önce yalnızca veri sorgularını okumada kullanıldı). [\#6577](https://github.com/ClickHouse/ClickHouse/pull/6577) ([proller](https://github.com/proller)) +- İsteğe bağlı mesaj argümanı eklendi `throwIf`. ([\#5772](https://github.com/ClickHouse/ClickHouse/issues/5772)) [\#6329](https://github.com/ClickHouse/ClickHouse/pull/6329) ([Vdimir](https://github.com/Vdimir)) +- Ekleme verilerini gönderirken sunucu istisnası şimdi istemcide de işleniyor. [\#5891](https://github.com/ClickHouse/ClickHouse/issues/5891) [\#6711](https://github.com/ClickHouse/ClickHouse/pull/6711) ([dimarub2000](https://github.com/dimarub2000)) +- Bir metrik eklendi `DistributedFilesToInsert` bu, dağıtılmış tablolarla uzak sunuculara göndermek için seçilen dosya sistemindeki toplam dosya sayısını gösterir. Sayı tüm parçalar arasında toplanır. [\#6600](https://github.com/ClickHouse/ClickHouse/pull/6600) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Birleş ofimlerin çoğunu taşı `ExpressionAction/ExpressionAnalyzer` -e doğru `AnalyzedJoin`. [\#6785](https://github.com/ClickHouse/ClickHouse/pull/6785) ([Artem Zuikov](https://github.com/4ertus2)) +- Tsan'ı Düzeltin [uyarıcı](https://clickhouse-test-reports.s3.yandex.net/6399/c1c1d1daa98e199e620766f1bd06a5921050a00d/functional_stateful_tests_(thread).html) ‘lock-order-inversion’. [\#6740](https://github.com/ClickHouse/ClickHouse/pull/6740) ([Vasily Nemkov](https://github.com/Enmk)) +- Linux yetenekleri eksikliği hakkında daha iyi bilgi mesajları. İle önemli hataları günlüğe kaydetme “fatal” seviye, bu daha kolay bulmak için yapacak `system.text_log`. [\#6441](https://github.com/ClickHouse/ClickHouse/pull/6441) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sırasında bellek kullanımını kısıtlamak için diske geçici veri damping etkinleştirdiğinizde `GROUP BY`, `ORDER BY`, boş disk alanını kontrol etmedi. Düzeltme yeni bir ayar Ekle `min_free_disk_space`, boş disk alanı daha küçük olduğunda eşik, sorgu durur ve atar `ErrorCodes::NOT_ENOUGH_SPACE`. [\#6678](https://github.com/ClickHouse/ClickHouse/pull/6678) ([Weiqing Xu](https://github.com/weiqxu)) [\#6691](https://github.com/ClickHouse/ClickHouse/pull/6691) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İş parçacığı tarafından özyinelemeli rwlock kaldırıldı. Hiçbir anlam ifade etmiyor, çünkü iş parçacıkları sorgular arasında yeniden kullanılıyor. `SELECT` sorgu bir iş parçacığında bir kilit edinebilir, başka bir iş parçacığından bir kilit tutabilir ve ilk iş parçacığından çıkabilir. Aynı zamanda, ilk iş parçacığı tarafından yeniden kullanılabilir `DROP` sorgu. Bu yanlış yol açacaktır “Attempt to acquire exclusive lock recursively” iletiler. [\#6771](https://github.com/ClickHouse/ClickHouse/pull/6771) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bölme `ExpressionAnalyzer.appendJoin()`. Bir yer hazırlayın `ExpressionAnalyzer` için `MergeJoin`. [\#6524](https://github.com/ClickHouse/ClickHouse/pull/6524) ([Artem Zuikov](https://github.com/4ertus2)) +- Katma `mysql_native_password` MySQL uyumluluk sunucusuna kimlik doğrulama eklentisi. [\#6194](https://github.com/ClickHouse/ClickHouse/pull/6194) ([Yuriy Baranov](https://github.com/yurriy)) +- Daha az sayıda `clock_gettime` aramalar; hata ayıklama / sürüm arasındaki sabit ABİ uyumluluğu `Allocator` (önemsiz konu). [\#6197](https://github.com/ClickHouse/ClickHouse/pull/6197) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Taşınmak `collectUsedColumns` itibaren `ExpressionAnalyzer` -e doğru `SyntaxAnalyzer`. `SyntaxAnalyzer` yapıyor `required_source_columns` şimdi kendisi. [\#6416](https://github.com/ClickHouse/ClickHouse/pull/6416) ([Artem Zuikov](https://github.com/4ertus2)) +- Ayar Ekle `joined_subquery_requires_alias` alt seçimler ve tablo işlevleri için takma adlar istemek için `FROM` that more than one table is present (i.e. queries with JOINs). [\#6733](https://github.com/ClickHouse/ClickHouse/pull/6733) ([Artem Zuikov](https://github.com/4ertus2)) +- Ayıklamak `GetAggregatesVisitor` sınıf fromından `ExpressionAnalyzer`. [\#6458](https://github.com/ClickHouse/ClickHouse/pull/6458) ([Artem Zuikov](https://github.com/4ertus2)) +- `system.query_log`: veri türünü değiştir `type` sütun için `Enum`. [\#6265](https://github.com/ClickHouse/ClickHouse/pull/6265) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Statik bağlama `sha256_password` kimlik doğrulama eklentisi. [\#6512](https://github.com/ClickHouse/ClickHouse/pull/6512) ([Yuriy Baranov](https://github.com/yurriy)) +- Ayar için ekstra bağımlılıktan kaçının `compile` çalışmak. Önceki sürümlerde, kullanıcı gibi hata alabilirsiniz `cannot open crti.o`, `unable to find library -lc` vb. [\#6309](https://github.com/ClickHouse/ClickHouse/pull/6309) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kötü amaçlı kopyadan gelebilecek girdinin daha fazla doğrulaması. [\#6303](https://github.com/ClickHouse/ClickHouse/pull/6303) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi `clickhouse-obfuscator` dosya mevcuttur `clickhouse-client` paket. Önceki sürümlerde şu şekilde mevcuttu `clickhouse obfuscator` (boşluk ile). [\#5816](https://github.com/ClickHouse/ClickHouse/issues/5816) [\#6609](https://github.com/ClickHouse/ClickHouse/pull/6609) ([dimarub2000](https://github.com/dimarub2000)) +- Farklı sırayla en az iki tablo ve tablolardan birinde DDL işlemi gerçekleştiren başka bir sorgu okumak en az iki sorgu olduğunda sabit kilitlenme. Başka bir çok nadir kilitlenme düzeltildi. [\#6764](https://github.com/ClickHouse/ClickHouse/pull/6764) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Katma `os_thread_ids` sütun için `system.processes` ve `system.query_log` daha iyi hata ayıklama olanakları için. [\#6763](https://github.com/ClickHouse/ClickHouse/pull/6763) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- PHP mysqlnd uzantısı hataları için bir geçici çözüm `sha256_password` varsayılan kimlik doğrulama eklentisi olarak kullanılır ( [\#6031](https://github.com/ClickHouse/ClickHouse/issues/6031)). [\#6113](https://github.com/ClickHouse/ClickHouse/pull/6113) ([Yuriy Baranov](https://github.com/yurriy)) +- Değiştirilen nullability sütunlarıyla gereksiz yeri kaldırın. [\#6693](https://github.com/ClickHouse/ClickHouse/pull/6693) ([Artem Zuikov](https://github.com/4ertus2)) +- Set default value of `queue_max_wait_ms` sıfıra, çünkü mevcut değer (beş saniye) hiçbir anlam ifade etmiyor. Bu ayarların herhangi bir kullanımı olduğunda nadir durumlar vardır. Eklenen ayarlar `replace_running_query_max_wait_ms`, `kafka_max_wait_ms` ve `connection_pool_max_wait_ms` anlam ayrımı için. [\#6692](https://github.com/ClickHouse/ClickHouse/pull/6692) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Ayıklamak `SelectQueryExpressionAnalyzer` itibaren `ExpressionAnalyzer`. Seçilmemiş sorgular için sonuncusu tutun. [\#6499](https://github.com/ClickHouse/ClickHouse/pull/6499) ([Artem Zuikov](https://github.com/4ertus2)) +- Giriş ve çıkış formatlarını çoğaltarak kaldırıldı. [\#6239](https://github.com/ClickHouse/ClickHouse/pull/6239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Kullanıcının geçersiz kılmasına izin ver `poll_interval` ve `idle_connection_timeout` bağlantı ayarları. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `MergeTree` şimdi ek bir seçenek var `ttl_only_drop_parts` (varsayılan olarak devre dışı) parçaların kısmi budamasını önlemek için, böylece bir parçadaki tüm satırların süresi dolduğunda tamamen düştüler. [\#6191](https://github.com/ClickHouse/ClickHouse/pull/6191) ([Sergi Vladykin](https://github.com/svladykin)) +- Tür set dizin işlevleri için denetler. Fonksiyon yanlış bir türe sahipse istisna atın. Bu UBSan ile fuzz testini düzeltir. [\#6511](https://github.com/ClickHouse/ClickHouse/pull/6511) ([Nikita Vasilev](https://github.com/nikvas0)) + +#### Performans İyileştirme {#performance-improvement-2} + +- İle Optimize sorguları `ORDER BY expressions` fık ,ra, nerede `expressions` sıralama anahtarı ile çakışan önek var `MergeTree` Tablolar. Bu optimizasyon tarafından kontrol edilir `optimize_read_in_order` ayar. [\#6054](https://github.com/ClickHouse/ClickHouse/pull/6054) [\#6629](https://github.com/ClickHouse/ClickHouse/pull/6629) ([Anton Popov](https://github.com/CurtizJ)) +- Parça yükleme ve çıkarma sırasında birden fazla diş kullanmasına izin verin. [\#6372](https://github.com/ClickHouse/ClickHouse/issues/6372) [\#6074](https://github.com/ClickHouse/ClickHouse/issues/6074) [\#6438](https://github.com/ClickHouse/ClickHouse/pull/6438) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Toplu işlev durumlarının güncellenmesi için uygulanan toplu varyant. Performans avantaj .larına yol açabilir. [\#6435](https://github.com/ClickHouse/ClickHouse/pull/6435) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kullanım `FastOps` fonksiyonlar için kütüphane `exp`, `log`, `sigmoid`, `tanh`. FastOps Michael Parakhin (Yandex CTO) bir hızlı vektör matematik kütüphanesidir. Geliştirilmiş performans `exp` ve `log` fonksiyonları fazla 6 kez. İşlev `exp` ve `log` itibaren `Float32` argüman geri dönecek `Float32` (önceki sürümlerde her zaman geri dönerler `Float64`). Şimdi `exp(nan)` dön mayebilir `inf`. Sonucu `exp` ve `log` fonksiyonlar, gerçek cevaba en yakın makine temsil edilebilir numarası olmayabilir. [\#6254](https://github.com/ClickHouse/ClickHouse/pull/6254) ([alexey-milovidov](https://github.com/alexey-milovidov)) Fastops çalışmasını sağlamak için Danila Kutenin varyantını kullanma [\#6317](https://github.com/ClickHouse/ClickHouse/pull/6317) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Ardışık anahtar optimizasyonunu devre dışı bırak `UInt8/16`. [\#6298](https://github.com/ClickHouse/ClickHouse/pull/6298) [\#6701](https://github.com/ClickHouse/ClickHouse/pull/6701) ([akuzm](https://github.com/akuzm)) +- Geliştirilmiş performans `simdjson` içinde dinamik tahs ofis ridat ridtan Kurtul ofarak kütüphane `ParsedJson::Iterator`. [\#6479](https://github.com/ClickHouse/ClickHouse/pull/6479) ([Vitaly Baranov](https://github.com/vitlibar)) +- Bellek ayırırken ön hata sayfaları `mmap()`. [\#6667](https://github.com/ClickHouse/ClickHouse/pull/6667) ([akuzm](https://github.com/akuzm)) +- Performans hatasını düzeltin `Decimal` karşılaştırma. [\#6380](https://github.com/ClickHouse/ClickHouse/pull/6380) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-4} + +- Derleyici (çalışma zamanı şablonu örneği) kaldırın, çünkü performansı kazandık. [\#6646](https://github.com/ClickHouse/ClickHouse/pull/6646) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Gcc-9'daki performansın daha izole bir şekilde bozulmasını göstermek için performans testi eklendi. [\#6302](https://github.com/ClickHouse/ClickHouse/pull/6302) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eklenen tablo fonksiyonu `numbers_mt`, çok iş parçacıklı sürümü olan `numbers`. Karma fonksiyonları ile güncellenmiş performans testleri. [\#6554](https://github.com/ClickHouse/ClickHouse/pull/6554) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Karşılaştırma modu `clickhouse-benchmark` [\#6220](https://github.com/ClickHouse/ClickHouse/issues/6220) [\#6343](https://github.com/ClickHouse/ClickHouse/pull/6343) ([dimarub2000](https://github.com/dimarub2000)) +- Yığın izlerini yazdırmak için en iyi çaba. Ayrıca eklendi `SIGPROF` çalışan bir iş parçacığının yığın izini yazdırmak için hata ayıklama sinyali olarak. [\#6529](https://github.com/ClickHouse/ClickHouse/pull/6529) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Her fonksiyon kendi dosyasında, Bölüm 10. [\#6321](https://github.com/ClickHouse/ClickHouse/pull/6321) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İki katına çıkar const `TABLE_IS_READ_ONLY`. [\#6566](https://github.com/ClickHouse/ClickHouse/pull/6566) ([filimonov](https://github.com/filimonov)) +- İçin biçimlendirme değişiklikleri `StringHashMap` PR [\#5417](https://github.com/ClickHouse/ClickHouse/issues/5417). [\#6700](https://github.com/ClickHouse/ClickHouse/pull/6700) ([akuzm](https://github.com/akuzm)) +- Birleştirme oluşturma için daha iyi alt sorgu `ExpressionAnalyzer`. [\#6824](https://github.com/ClickHouse/ClickHouse/pull/6824) ([Artem Zuikov](https://github.com/4ertus2)) +- Gereksiz bir koşulu kaldırın (PVS Studio tarafından bulunur). [\#6775](https://github.com/ClickHouse/ClickHouse/pull/6775) ([akuzm](https://github.com/akuzm)) +- İçin karma tablo arayüzünü ayırın `ReverseIndex`. [\#6672](https://github.com/ClickHouse/ClickHouse/pull/6672) ([akuzm](https://github.com/akuzm)) +- Ayarların yeniden düzenlenmesi. [\#6689](https://github.com/ClickHouse/ClickHouse/pull/6689) ([alesapin](https://github.com/alesapin)) +- İçin yorum ekle `set` dizin fonksiyonları. [\#6319](https://github.com/ClickHouse/ClickHouse/pull/6319) ([Nikita Vasilev](https://github.com/nikvas0)) +- Linux'ta hata ayıklama sürümünde oom puanını artırın. [\#6152](https://github.com/ClickHouse/ClickHouse/pull/6152) ([akuzm](https://github.com/akuzm)) +- HDFS ha şimdi hata ayıklama yapısında çalışıyor. [\#6650](https://github.com/ClickHouse/ClickHouse/pull/6650) ([Weiqing Xu](https://github.com/weiqxu)) +- Bir test eklendi `transform_query_for_external_database`. [\#6388](https://github.com/ClickHouse/ClickHouse/pull/6388) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kafka tablosu için birden fazla somutlaştırılmış görünüm için test ekleyin. [\#6509](https://github.com/ClickHouse/ClickHouse/pull/6509) ([I ivanvan](https://github.com/abyss7)) +- Daha iyi bir yapı şeması yapın. [\#6500](https://github.com/ClickHouse/ClickHouse/pull/6500) ([I ivanvan](https://github.com/abyss7)) +- Sabit `test_external_dictionaries` kök olmayan kullanıcı altında çalıştırılması durumunda entegrasyon. [\#6507](https://github.com/ClickHouse/ClickHouse/pull/6507) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Yazılı paketlerin toplam boyutu aştığında hata çoğalır `DBMS_DEFAULT_BUFFER_SIZE`. [\#6204](https://github.com/ClickHouse/ClickHouse/pull/6204) ([Yuriy Baranov](https://github.com/yurriy)) +- İçin bir test eklendi `RENAME` tablo yarış durumu [\#6752](https://github.com/ClickHouse/ClickHouse/pull/6752) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Ayarlarda veri yarışından kaçının `KILL QUERY`. [\#6753](https://github.com/ClickHouse/ClickHouse/pull/6753) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bir önbellek sözlüğü ile hataları işlemek için entegrasyon testi ekleyin. [\#6755](https://github.com/ClickHouse/ClickHouse/pull/6755) ([Vitaly Baranov](https://github.com/vitlibar)) +- Mac OS'de elf nesne dosyalarının ayrıştırılmasını devre dışı bırakın, çünkü mantıklı değil. [\#6578](https://github.com/ClickHouse/ClickHouse/pull/6578) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Changelog jeneratörünü daha iyi hale getirmeye çalışın. [\#6327](https://github.com/ClickHouse/ClickHouse/pull/6327) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Ekleme `-Wshadow` GCC'YE geçin. [\#6325](https://github.com/ClickHouse/ClickHouse/pull/6325) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +- İçin eski kod kaldırıldı `mimalloc` destek. [\#6715](https://github.com/ClickHouse/ClickHouse/pull/6715) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `zlib-ng` x86 yeteneklerini belirler ve bu bilgiyi genel değişkenlere kaydeder. Bu, aynı anda farklı iş parçacıkları tarafından yapılabilen defalteınit çağrısında yapılır. Çok iş parçacıklı yazılardan kaçınmak için, kitaplık başlangıcında yapın. [\#6141](https://github.com/ClickHouse/ClickHouse/pull/6141) ([akuzm](https://github.com/akuzm)) +- İçinde gider wasilen bir hata için regresyon testi [\#5192](https://github.com/ClickHouse/ClickHouse/issues/5192). [\#6147](https://github.com/ClickHouse/ClickHouse/pull/6147) ([Bakhtiyor Ruziev](https://github.com/theruziev)) +- Sabit MSan raporu. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltme çırparak TTL testi. [\#6782](https://github.com/ClickHouse/ClickHouse/pull/6782) ([Anton Popov](https://github.com/CurtizJ)) +- Sabit yanlış veri yarışı `MergeTreeDataPart::is_frozen` alan. [\#6583](https://github.com/ClickHouse/ClickHouse/pull/6583) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fuzz testinde sabit zaman aşımları. Önceki sürümde, sorguda yanlış hangup bulmayı başardı `SELECT * FROM numbers_mt(gccMurmurHash(''))`. [\#6582](https://github.com/ClickHouse/ClickHouse/pull/6582) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Hata ayıklama kontrolleri eklendi `static_cast` sütunların. [\#6581](https://github.com/ClickHouse/ClickHouse/pull/6581) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Resmi RPM paketlerinde Oracle Linux için destek. [\#6356](https://github.com/ClickHouse/ClickHouse/issues/6356) [\#6585](https://github.com/ClickHouse/ClickHouse/pull/6585) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Değiştirilen json perftests `once` -e doğru `loop` tür. [\#6536](https://github.com/ClickHouse/ClickHouse/pull/6536) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- `odbc-bridge.cpp` tanımlıyor `main()` bu yüzden dahil edilmemelidir `clickhouse-lib`. [\#6538](https://github.com/ClickHouse/ClickHouse/pull/6538) ([Orivej Desh](https://github.com/orivej)) +- Çarpışma testi `FULL|RIGHT JOIN` sağ tablonun tuşlarında boş alanlar var. [\#6362](https://github.com/ClickHouse/ClickHouse/pull/6362) ([Artem Zuikov](https://github.com/4ertus2)) +- Her ihtimale karşı takma adların genişletilmesiyle ilgili sınır için bir test eklendi. [\#6442](https://github.com/ClickHouse/ClickHouse/pull/6442) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Anahtar fromlı `boost::filesystem` -e doğru `std::filesystem` uygun olan yerlerde. [\#6253](https://github.com/ClickHouse/ClickHouse/pull/6253) [\#6385](https://github.com/ClickHouse/ClickHouse/pull/6385) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Web sitesine RPM paketleri eklendi. [\#6251](https://github.com/ClickHouse/ClickHouse/pull/6251) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit için bir test Ekle `Unknown identifier` istisna içinde `IN` bölme. [\#6708](https://github.com/ClickHouse/ClickHouse/pull/6708) ([Artem Zuikov](https://github.com/4ertus2)) +- Basitleştirmek `shared_ptr_helper` çünkü insanlar bunu anlamakta zorluk çekiyorlar. [\#6675](https://github.com/ClickHouse/ClickHouse/pull/6675) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit Gorilla ve DoubleDelta codec için performans testleri eklendi. [\#6179](https://github.com/ClickHouse/ClickHouse/pull/6179) ([Vasily Nemkov](https://github.com/Enmk)) +- Entegrasyon testini bölün `test_dictionaries` 4 ayrı test içine. [\#6776](https://github.com/ClickHouse/ClickHouse/pull/6776) ([Vitaly Baranov](https://github.com/vitlibar)) +- Pvs-Studio uyarısını düzeltin `PipelineExecutor`. [\#6777](https://github.com/ClickHouse/ClickHouse/pull/6777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Kullanmasına izin ver `library` asan ile sözlük kaynağı. [\#6482](https://github.com/ClickHouse/ClickHouse/pull/6482) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- PRs listesinden changelog oluşturmak için seçenek eklendi. [\#6350](https://github.com/ClickHouse/ClickHouse/pull/6350) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Lock the `TinyLog` depolama okurken. [\#6226](https://github.com/ClickHouse/ClickHouse/pull/6226) ([akuzm](https://github.com/akuzm)) +- CI kırık symlinks için kontrol edin. [\#6634](https://github.com/ClickHouse/ClickHouse/pull/6634) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin zaman aşımını artırın “stack overflow” hata ayıklama yapı uzun zaman alabilir, çünkü sınayın. [\#6637](https://github.com/ClickHouse/ClickHouse/pull/6637) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Çift whitespaces için bir çek eklendi. [\#6643](https://github.com/ClickHouse/ClickHouse/pull/6643) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltmek `new/delete` bellek izleme zaman sanitizers ile inşa. İzleme net değil. Sadece testlerde bellek sınırı istisnalarını önler. [\#6450](https://github.com/ClickHouse/ClickHouse/pull/6450) ([Artem Zuikov](https://github.com/4ertus2)) +- Bağlantı sırasında tanımlanmamış sembollerin kontrolünü geri etkinleştirin. [\#6453](https://github.com/ClickHouse/ClickHouse/pull/6453) ([I ivanvan](https://github.com/abyss7)) +- Yeniden inşa etmekten kaçının `hyperscan` hergün. [\#6307](https://github.com/ClickHouse/ClickHouse/pull/6307) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit UBSan raporu `ProtobufWriter`. [\#6163](https://github.com/ClickHouse/ClickHouse/pull/6163) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Uyumlu olmadığı için sanitizers ile sorgu profiler kullanmak için izin vermeyin. [\#6769](https://github.com/ClickHouse/ClickHouse/pull/6769) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Zamanlayıcı tarafından başarısız olduktan sonra bir sözlüğü yeniden yüklemek için test ekleyin. [\#6114](https://github.com/ClickHouse/ClickHouse/pull/6114) ([Vitaly Baranov](https://github.com/vitlibar)) +- Tutarsızlığı düzeltin `PipelineExecutor::prepareProcessor` argüman türü. [\#6494](https://github.com/ClickHouse/ClickHouse/pull/6494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Kötü Urı'ler için bir test eklendi. [\#6493](https://github.com/ClickHouse/ClickHouse/pull/6493) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Daha fazla kontrol eklendi `CAST` işlev. Bu, bulanık testte segmentasyon hatası hakkında daha fazla bilgi almalıdır. [\#6346](https://github.com/ClickHouse/ClickHouse/pull/6346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Katma `gcc-9` güçlendirmek `docker/builder` yerel olarak görüntü oluşturan konteyner. [\#6333](https://github.com/ClickHouse/ClickHouse/pull/6333) ([Gleb Novikov](https://github.com/NanoBjorn)) +- Birincil anahtar için Test `LowCardinality(String)`. [\#5044](https://github.com/ClickHouse/ClickHouse/issues/5044) [\#6219](https://github.com/ClickHouse/ClickHouse/pull/6219) ([dimarub2000](https://github.com/dimarub2000)) +- Yavaş yığın izleri yazdırma etkilenen sabit testler. [\#6315](https://github.com/ClickHouse/ClickHouse/pull/6315) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kilitlenme için bir test durumu ekleyin `groupUniqArray` sabit içinde [\#6029](https://github.com/ClickHouse/ClickHouse/pull/6029). [\#4402](https://github.com/ClickHouse/ClickHouse/issues/4402) [\#6129](https://github.com/ClickHouse/ClickHouse/pull/6129) ([akuzm](https://github.com/akuzm)) +- Sabit endeks mutasyonlar testleri. [\#6645](https://github.com/ClickHouse/ClickHouse/pull/6645) ([Nikita Vasilev](https://github.com/nikvas0)) +- Performans testinde, çalıştırmadığımız sorgular için sorgu günlüğünü okuma. [\#6427](https://github.com/ClickHouse/ClickHouse/pull/6427) ([akuzm](https://github.com/akuzm)) +- Materialized görünüm şimdi şüpheli düşük kardinalite türleri hakkında ayar ne olursa olsun herhangi bir düşük kardinalite türleri ile oluşturulabilir. [\#6428](https://github.com/ClickHouse/ClickHouse/pull/6428) ([Olga Khvostikova](https://github.com/stavrolia)) +- İçin güncel testslenmiş testler `send_logs_level` ayar. [\#6207](https://github.com/ClickHouse/ClickHouse/pull/6207) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Gcc-8.2 altında yapı düzeltin. [\#6196](https://github.com/ClickHouse/ClickHouse/pull/6196) ([Max Akhmedov](https://github.com/zlobober)) +- Dahili libc++ile yapı düzeltin. [\#6724](https://github.com/ClickHouse/ClickHouse/pull/6724) ([I ivanvan](https://github.com/abyss7)) +- Paylaşılan yapıyı düzeltin `rdkafka` kitaplık [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([I ivanvan](https://github.com/abyss7)) +- Mac OS build için düzeltmeler (eksik). [\#6390](https://github.com/ClickHouse/ClickHouse/pull/6390) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#6429](https://github.com/ClickHouse/ClickHouse/pull/6429) ([alex-zaitsev](https://github.com/alex-zaitsev)) +- Düzeltmek “splitted” yapmak. [\#6618](https://github.com/ClickHouse/ClickHouse/pull/6618) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Diğer yapı düzeltmeleri: [\#6186](https://github.com/ClickHouse/ClickHouse/pull/6186) ([Amos Kuşu](https://github.com/amosbird)) [\#6486](https://github.com/ClickHouse/ClickHouse/pull/6486) [\#6348](https://github.com/ClickHouse/ClickHouse/pull/6348) ([vxider](https://github.com/Vxider)) [\#6744](https://github.com/ClickHouse/ClickHouse/pull/6744) ([I ivanvan](https://github.com/abyss7)) [\#6016](https://github.com/ClickHouse/ClickHouse/pull/6016) [\#6421](https://github.com/ClickHouse/ClickHouse/pull/6421) [\#6491](https://github.com/ClickHouse/ClickHouse/pull/6491) ([proller](https://github.com/proller)) + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-3} + +- Kaldırıldı nadiren kullanılan tablo fonksiyonu `catBoostPool` ve depolama `CatBoostPool`. Bu tablo işlevini kullandıysanız, lütfen e-posta yazınız `clickhouse-feedback@yandex-team.com`. CatBoost entegrasyonunun devam ettiğini ve destekleneceğini unutmayın. [\#6279](https://github.com/ClickHouse/ClickHouse/pull/6279) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sakatlamak `ANY RIGHT JOIN` ve `ANY FULL JOIN` varsayılan olarak. Koymak `any_join_distinct_right_table_keys` bunları etkinleştirmek için ayarlama. [\#5126](https://github.com/ClickHouse/ClickHouse/issues/5126) [\#6351](https://github.com/ClickHouse/ClickHouse/pull/6351) ([Artem Zuikov](https://github.com/4ertus2)) + +## ClickHouse sürümü 19.13 {#clickhouse-release-19-13} + +### ClickHouse sürümü 19.13.6.51, 2019-10-02 {#clickhouse-release-19-13-6-51-2019-10-02} + +#### Hata Düzeltme {#bug-fix-9} + +- Bu sürüm aynı zamanda 19.11.12.69 tüm hata düzeltmeleri içerir. + +### ClickHouse sürümü 19.13.5.44, 2019-09-20 {#clickhouse-release-19-13-5-44-2019-09-20} + +#### Hata Düzeltme {#bug-fix-10} + +- Bu sürüm aynı zamanda 19.14.6.12 tüm hata düzeltmeleri içerir. +- Yürütülürken tablonun Olası tutarsız durumu düzeltildi `DROP` zookeeper erişilebilir değilken çoğaltılmış tablo için sorgu. [\#6045](https://github.com/ClickHouse/ClickHouse/issues/6045) [\#6413](https://github.com/ClickHouse/ClickHouse/pull/6413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Storagemerge'de veri yarışı için düzeltme [\#6717](https://github.com/ClickHouse/ClickHouse/pull/6717) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Soketten sonsuz recv yol açar sorgu profiler tanıtılan Fix hata. [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) ([alesapin](https://github.com/alesapin)) +- Çalışırken aşırı CPU kullanımını düzeltin `JSONExtractRaw` bir Boole değeri üzerinde işlev. [\#6208](https://github.com/ClickHouse/ClickHouse/pull/6208) ([Vitaly Baranov](https://github.com/vitlibar)) +- Gerçekleşmiş görünüme iterken gerilemeyi düzeltir. [\#6415](https://github.com/ClickHouse/ClickHouse/pull/6415) ([I ivanvan](https://github.com/abyss7)) +- Tablo fonksiyonu `url` güvenlik açığı saldırganın istekte rasgele HTTP üstbilgileri enjekte etmesine izin vermişti. Bu konu tarafından bulundu [Nikita Tikhomirov](https://github.com/NSTikhomirov). [\#6466](https://github.com/ClickHouse/ClickHouse/pull/6466) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İşe yaramaz düzeltmek `AST` set dizini kontrol edin. [\#6510](https://github.com/ClickHouse/ClickHouse/issues/6510) [\#6651](https://github.com/ClickHouse/ClickHouse/pull/6651) ([Nikita Vasilev](https://github.com/nikvas0)) +- Sabit ayrıştırma `AggregateFunction` sorguya gömülü değerler. [\#6575](https://github.com/ClickHouse/ClickHouse/issues/6575) [\#6773](https://github.com/ClickHouse/ClickHouse/pull/6773) ([Zhichang Yu](https://github.com/yuzhichang)) +- Sabit yanlış davranış `trim` fonksiyonlar aile. [\#6647](https://github.com/ClickHouse/ClickHouse/pull/6647) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.13.4.32, 2019-09-10 {#clickhouse-release-19-13-4-32-2019-09-10} + +#### Hata Düzeltme {#bug-fix-11} + +- Bu sürüm aynı zamanda 19.11.9.52 ve 19.11.10.54 tüm hata güvenlik düzeltmeleri içerir. +- Sabit veri yarışı `system.parts` masa ve `ALTER` sorgu. [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245) [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Akışlarda sabit eşleşmeyen başlık, örnek ve prewhere ile boş dağıtılmış tablodan okuma durumunda oldu. [\#6167](https://github.com/ClickHouse/ClickHouse/issues/6167) ([Lixiang Qian](https://github.com/fancyqlx)) [\#6823](https://github.com/ClickHouse/ClickHouse/pull/6823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Kullanırken sabit çökme `IN` bir tuple ile bir alt sorgu ile yan tümce. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +- Aynı sütun adlarıyla durumu düzeltin `GLOBAL JOIN ON` bölme. [\#6181](https://github.com/ClickHouse/ClickHouse/pull/6181) ([Artem Zuikov](https://github.com/4ertus2)) +- Türleri döküm yaparken çökmeyi düzeltin `Decimal` bu onu desteklemiyor. Bunun yerine istisna atın. [\#6297](https://github.com/ClickHouse/ClickHouse/pull/6297) ([Artem Zuikov](https://github.com/4ertus2)) +- Sabit çökme `extractAll()` işlev. [\#6644](https://github.com/ClickHouse/ClickHouse/pull/6644) ([Artem Zuikov](https://github.com/4ertus2)) +- İçin sorgu dönüşümü `MySQL`, `ODBC`, `JDBC` tablo işlevleri artık düzgün çalışıyor `SELECT WHERE` birden fazla sorgu `AND` ifadeler. [\#6381](https://github.com/ClickHouse/ClickHouse/issues/6381) [\#6676](https://github.com/ClickHouse/ClickHouse/pull/6676) ([dimarub2000](https://github.com/dimarub2000)) +- MySQL 8 entegrasyonu için önceki beyan kontrolleri eklendi. [\#6569](https://github.com/ClickHouse/ClickHouse/pull/6569) ([Rafael David Tinoco](https://github.com/rafaeldtinoco)) + +#### Güvenlik Düzeltme {#security-fix-1} + +- Dekompresyon aşamasında kodeklerdeki iki güvenlik açığını düzeltin (kötü niyetli kullanıcı, dekompresyonda tampon taşmasına yol açacak sıkıştırılmış verileri üretebilir). [\#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([Artem Zuikov](https://github.com/4ertus2)) + +### ClickHouse sürümü 19.13.3.26, 2019-08-22 {#clickhouse-release-19-13-3-26-2019-08-22} + +#### Hata Düzeltme {#bug-fix-12} + +- Düzeltmek `ALTER TABLE ... UPDATE` ile tablolar için sorgu `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +- Bir tuple ile bir alt sorgu ile yan tümcesinde kullanırken npe düzeltin. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +- Eski bir kopya canlı hale gelirse, yine de açılan bölüm tarafından kaldırılan veri parçalarına sahip olabileceği sorunu düzeltildi. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- CSV ayrıştırma ile ilgili sorun giderildi [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +- Sistemde sabit veri yarışı.parçalar tablo ve alter sorgu. Bu düzeltmeler [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bellek bozulmasına yol açabilecek mutasyonlarda yanlış kod düzeltildi. Adresin okunması ile sabit segfault `0x14c0` bu eşzamanlı nedeniyle happed olabilir `DROP TABLE` ve `SELECT` itibaren `system.parts` veya `system.parts_columns`. Mutasyon sorgularının hazırlanmasında sabit yarış durumu. Sabit kilitlenme neden `OPTIMIZE` çoğaltılmış tablolar ve değiştirir gibi eşzamanlı değişiklik işlemleri. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sonra sabit olası veri kaybı `ALTER DELETE` tabloda dizin atlama ile sorgu. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) + +#### Güvenlik Düzeltme {#security-fix-2} + +- Saldırganın Zookeeper'a yazma erişimi varsa ve Clickhouse'un çalıştığı ağdan kullanılabilen özel sunucuyu çalıştırabiliyorsa, ClickHouse replica olarak hareket edecek ve Zookeeper'a kaydedecek özel olarak oluşturulmuş kötü amaçlı sunucu oluşturabilir. Başka bir çoğaltma, kötü amaçlı kopyadan veri bölümünü getirdiğinde, clickhouse-Server'ı dosya sistemindeki keyfi yola yazmaya zorlayabilir. Yandex'teki bilgi güvenliği ekibi Eldar Zaitov tarafından bulundu. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.13.2.19, 2019-08-14 {#clickhouse-release-19-13-2-19-2019-08-14} + +#### Yenilik {#new-feature-5} + +- Sorgu düzeyinde örnekleme profiler. [Örnek](https://gist.github.com/alexey-milovidov/92758583dd41c24c360fdb8d6a4da194). [\#4247](https://github.com/ClickHouse/ClickHouse/issues/4247) ([laplab](https://github.com/laplab)) [\#6124](https://github.com/ClickHouse/ClickHouse/pull/6124) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) [\#6283](https://github.com/ClickHouse/ClickHouse/pull/6283) [\#6386](https://github.com/ClickHouse/ClickHouse/pull/6386) +- Sütunların bir listesini belirtmek için izin ver `COLUMNS('regexp')` daha sofistike bir varyantı gibi çalışan ifade `*` Yıldız işareti. [\#5951](https://github.com/ClickHouse/ClickHouse/pull/5951) ([mfridental](https://github.com/mfridental)), ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `CREATE TABLE AS table_function()` artık mümkün [\#6057](https://github.com/ClickHouse/ClickHouse/pull/6057) ([dimarub2000](https://github.com/dimarub2000)) +- Stokastik Gradyan iniş için Adam optimizer varsayılan olarak kullanılır `stochasticLinearRegression()` ve `stochasticLogisticRegression()` agrega fonksiyonları, çünkü neredeyse herhangi bir ayar yapmadan iyi kalite gösterir. [\#6000](https://github.com/ClickHouse/ClickHouse/pull/6000) ([Quid37](https://github.com/Quid37)) +- Added functions for working with the сustom week number [\#5212](https://github.com/ClickHouse/ClickHouse/pull/5212) ([Andy Yang](https://github.com/andyyzh)) +- `RENAME` sorgular artık tüm depolarla çalışır. [\#5953](https://github.com/ClickHouse/ClickHouse/pull/5953) ([I ivanvan](https://github.com/abyss7)) +- Şimdi istemci ayarlayarak istenilen düzeyde sunucudan günlükleri almak `send_logs_level` sunucu ayarlarında belirtilen günlük düzeyine bakılmaksızın. [\#5964](https://github.com/ClickHouse/ClickHouse/pull/5964) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-4} + +- Ayar `input_format_defaults_for_omitted_fields` varsayılan olarak etkinleştirilir. Dağıtılmış tablolardaki ekler bu ayarın kümede aynı olması gerekir (güncellemeyi çalıştırmadan önce ayarlamanız gerekir). Atlanmış alanlar için karmaşık varsayılan ifadelerin hesaplanmasını sağlar `JSONEachRow` ve `CSV*` biçimliler. Beklenen davranış olmalı, ancak ihmal edilebilir performans farkına yol açabilir. [\#6043](https://github.com/ClickHouse/ClickHouse/pull/6043) ([Artem Zuikov](https://github.com/4ertus2)), [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) + +#### Deneysel özellikler {#experimental-features} + +- Yeni sorgu işleme boru hattı. Kullanmak `experimental_use_processors=1` etkinleştirmek için Seçenek. Kendi sorun için kullanın. [\#4914](https://github.com/ClickHouse/ClickHouse/pull/4914) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Hata Düzeltme {#bug-fix-13} + +- Kafka entegrasyonu bu sürümde düzeltildi. +- Sabit `DoubleDelta` kodlama `Int64` büyük için `DoubleDelta` değerler, geliştirilmiş `DoubleDelta` rastgele veri için kodlama `Int32`. [\#5998](https://github.com/ClickHouse/ClickHouse/pull/5998) ([Vasily Nemkov](https://github.com/Enmk)) +- Sabit aşırı tahmin `max_rows_to_read` eğer ayar `merge_tree_uniform_read_distribution` 0 olarak ayarlanır. [\#6019](https://github.com/ClickHouse/ClickHouse/pull/6019) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Geliştirme {#improvement-4} + +- Eğer bir istisna atar `config.d` dosya, yapılandırma dosyası olarak ilgili kök öğeye sahip değil [\#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) + +#### Performans İyileştirme {#performance-improvement-3} + +- Getirmek `count()`. Şimdi en küçük sütunu kullanır (mümkünse). [\#6028](https://github.com/ClickHouse/ClickHouse/pull/6028) ([Amos Kuşu](https://github.com/amosbird)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-5} + +- Performans testlerinde bellek kullanımını bildirin. [\#5899](https://github.com/ClickHouse/ClickHouse/pull/5899) ([akuzm](https://github.com/akuzm)) +- Harici yapı ile Düzeltme `libcxx` [\#6010](https://github.com/ClickHouse/ClickHouse/pull/6010) ([I ivanvan](https://github.com/abyss7)) +- Paylaşılan yapıyı düzeltin `rdkafka` kitaplık [\#6101](https://github.com/ClickHouse/ClickHouse/pull/6101) ([I ivanvan](https://github.com/abyss7)) + +## ClickHouse yayın 19.11 {#clickhouse-release-19-11} + +### ClickHouse sürümü 19.11.13.74, 2019-11-01 {#clickhouse-release-19-11-13-74-2019-11-01} + +#### Hata Düzeltme {#bug-fix-14} + +- Sabit nadir kazasında `ALTER MODIFY COLUMN` ve birleştirilmiş/değiştirilmiş parçalardan biri boş olduğunda dikey birleştirme (0 satır). [\#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) +- Manuel güncelleme `SIMDJSON`. Bu sahte json teşhis mesajları ile stderr dosyalarının Olası sel giderir. [\#7548](https://github.com/ClickHouse/ClickHouse/pull/7548) ([Alexander Kazakov](https://github.com/Akazz)) +- Sabit hata ile `mrk` Mut fileasyonlar için dosya uzantısı ([alesapin](https://github.com/alesapin)) + +### ClickHouse yayın 19.11.12.69, 2019-10-02 {#clickhouse-release-19-11-12-69-2019-10-02} + +#### Hata Düzeltme {#bug-fix-15} + +- Büyük tablolarda karmaşık anahtarlar üzerinde endeks analizi sabit performans düşüşü. Bu düzeltmeler [\#6924](https://github.com/ClickHouse/ClickHouse/issues/6924). [\#7075](https://github.com/ClickHouse/ClickHouse/pull/7075) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Dağıtılmış motorlu tablolarda veri gönderirken nadir SIGSEGV'DEN kaçının (`Failed to send batch: file with index XXXXX is absent`). [\#7032](https://github.com/ClickHouse/ClickHouse/pull/7032) ([Azat Khuzhin](https://github.com/azat)) +- Düzeltmek `Unknown identifier` birden fazla birleşim ile. Bu düzeltmeler [\#5254](https://github.com/ClickHouse/ClickHouse/issues/5254). [\#7022](https://github.com/ClickHouse/ClickHouse/pull/7022) ([Artem Zuikov](https://github.com/4ertus2)) + +### ClickHouse sürümü 19.11.11.57, 2019-09-13 {#clickhouse-release-19-11-11-57-2019-09-13} + +- Kafka boş konudan seçerken segfaults'a neden olan mantıksal hatayı düzeltin. [\#6902](https://github.com/ClickHouse/ClickHouse/issues/6902) [\#6909](https://github.com/ClickHouse/ClickHouse/pull/6909) ([I ivanvan](https://github.com/abyss7)) +- Fonksiyon için düzeltme `АrrayEnumerateUniqRanked` params boş diziler ile. [\#6928](https://github.com/ClickHouse/ClickHouse/pull/6928) ([proller](https://github.com/proller)) + +### ClickHouse sürümü 19.11.10.54, 2019-09-10 {#clickhouse-release-19-11-10-54-2019-09-10} + +#### Hata Düzeltme {#bug-fix-16} + +- Tüm bölümler için hepsini bir kerede işlemek için Kafka mesajları için uzaklıkları manuel olarak saklayın. Potansiyel çoğaltmayı giderir “one consumer - many partitions” senaryo. [\#6872](https://github.com/ClickHouse/ClickHouse/pull/6872) ([I ivanvan](https://github.com/abyss7)) + +### ClickHouse yayın 19.11.9.52, 2019-09-6 {#clickhouse-release-19-11-9-52-2019-09-6} + +- Önbellek sözlüklerde hata işleme geliştirin. [\#6737](https://github.com/ClickHouse/ClickHouse/pull/6737) ([Vitaly Baranov](https://github.com/vitlibar)) +- Fonksiyonu Sabit hata `arrayEnumerateUniqRanked`. [\#6779](https://github.com/ClickHouse/ClickHouse/pull/6779) ([proller](https://github.com/proller)) +- Düzeltmek `JSONExtract` bir ayıklarken işlev `Tuple` JSON dan. [\#6718](https://github.com/ClickHouse/ClickHouse/pull/6718) ([Vitaly Baranov](https://github.com/vitlibar)) +- Sonra sabit olası veri kaybı `ALTER DELETE` tabloda dizin atlama ile sorgu. [\#6224](https://github.com/ClickHouse/ClickHouse/issues/6224) [\#6282](https://github.com/ClickHouse/ClickHouse/pull/6282) ([Nikita Vasilev](https://github.com/nikvas0)) +- Sabit performans testi. [\#6392](https://github.com/ClickHouse/ClickHouse/pull/6392) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Parke: Boolean sütunlarını okumayı düzeltin. [\#6579](https://github.com/ClickHouse/ClickHouse/pull/6579) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit yanlış davranış `nullIf` sabit argümanlar için işlev. [\#6518](https://github.com/ClickHouse/ClickHouse/pull/6518) ([Guillaume Tassery](https://github.com/YiuRULE)) [\#6580](https://github.com/ClickHouse/ClickHouse/pull/6580) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Normal sunucu yeniden başlatmada Kafka mesajları çoğaltma sorununu düzeltin. [\#6597](https://github.com/ClickHouse/ClickHouse/pull/6597) ([I ivanvan](https://github.com/abyss7)) +- Uzun zaman bir sorun düzeltildi `ALTER UPDATE` veya `ALTER DELETE` düzenli birleştirmelerin çalışmasını engelleyebilir. Yeterli boş iş parçacığı yoksa mutasyonların yürütülmesini önleyin. [\#6502](https://github.com/ClickHouse/ClickHouse/issues/6502) [\#6617](https://github.com/ClickHouse/ClickHouse/pull/6617) ([tavplubix](https://github.com/tavplubix)) +- İşleme ile Sabit hata “timezone” sunucu yapılandırma dosyasında. [\#6709](https://github.com/ClickHouse/ClickHouse/pull/6709) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kafka testlerini düzeltin. [\#6805](https://github.com/ClickHouse/ClickHouse/pull/6805) ([I ivanvan](https://github.com/abyss7)) + +#### Güvenlik Düzeltme {#security-fix-3} + +- Saldırganın Zookeeper'a yazma erişimi varsa ve Clickhouse'un çalıştığı ağdan kullanılabilen özel sunucuyu çalıştırabiliyorsa, ClickHouse replica olarak hareket edecek ve Zookeeper'a kaydedecek özel olarak oluşturulmuş kötü amaçlı sunucu oluşturabilir. Başka bir çoğaltma, kötü amaçlı kopyadan veri bölümünü getirdiğinde, clickhouse-Server'ı dosya sistemindeki keyfi yola yazmaya zorlayabilir. Yandex'teki bilgi güvenliği ekibi Eldar Zaitov tarafından bulundu. [\#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.11.8.46, 2019-08-22 {#clickhouse-release-19-11-8-46-2019-08-22} + +#### Hata Düzeltme {#bug-fix-17} + +- Düzeltmek `ALTER TABLE ... UPDATE` ile tablolar için sorgu `enable_mixed_granularity_parts=1`. [\#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +- Bir tuple ile bir alt sorgu ile yan tümcesinde kullanırken npe düzeltin. [\#6125](https://github.com/ClickHouse/ClickHouse/issues/6125) [\#6550](https://github.com/ClickHouse/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +- Eski bir kopya canlı hale gelirse, yine de açılan bölüm tarafından kaldırılan veri parçalarına sahip olabileceği sorunu düzeltildi. [\#6522](https://github.com/ClickHouse/ClickHouse/issues/6522) [\#6523](https://github.com/ClickHouse/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +- CSV ayrıştırma ile ilgili sorun giderildi [\#6426](https://github.com/ClickHouse/ClickHouse/issues/6426) [\#6559](https://github.com/ClickHouse/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +- Sistemde sabit veri yarışı.parçalar tablo ve alter sorgu. Bu düzeltmeler [\#6245](https://github.com/ClickHouse/ClickHouse/issues/6245). [\#6513](https://github.com/ClickHouse/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bellek bozulmasına yol açabilecek mutasyonlarda yanlış kod düzeltildi. Adresin okunması ile sabit segfault `0x14c0` bu eşzamanlı nedeniyle happed olabilir `DROP TABLE` ve `SELECT` itibaren `system.parts` veya `system.parts_columns`. Mutasyon sorgularının hazırlanmasında sabit yarış durumu. Sabit kilitlenme neden `OPTIMIZE` çoğaltılmış tablolar ve değiştirir gibi eşzamanlı değişiklik işlemleri. [\#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.11.7.40, 2019-08-14 {#clickhouse-release-19-11-7-40-2019-08-14} + +#### Hata düzeltme {#bug-fix-18} + +- Kafka entegrasyonu bu sürümde düzeltildi. +- Kullanırken segfault'u düzeltin `arrayReduce` sürekli argümanlar için. [\#6326](https://github.com/ClickHouse/ClickHouse/pull/6326) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit `toFloat()` Monotonluk. [\#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) +- Etkin olan segfault'u düzeltin `optimize_skip_unused_shards` ve kayıp sharding anahtarı. [\#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([CurtizJ](https://github.com/CurtizJ)) +- Sabit mantık `arrayEnumerateUniqRanked` işlev. [\#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- MySQL işleyicisinden ekstra ayrıntılı günlük kaydı kaldırıldı. [\#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yanlış davranışı ve olası segfault'ları düzeltin `topK` ve `topKWeighted` toplu fonksiyonlar. [\#6404](https://github.com/ClickHouse/ClickHouse/pull/6404) ([CurtizJ](https://github.com/CurtizJ)) +- Sanal sütunları göstermeyin `system.columns` Tablo. Bu geriye dönük uyumluluk için gereklidir. [\#6406](https://github.com/ClickHouse/ClickHouse/pull/6406) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Karmaşık anahtar önbellek sözlüğünde dize alanları için bellek tahsisi ile hatayı düzeltin. [\#6447](https://github.com/ClickHouse/ClickHouse/pull/6447) ([alesapin](https://github.com/alesapin)) +- İçin yeni bir çoğaltma oluştururken uyarlamalı ayrıntıyı etkinleştirerek hatayı düzeltin `Replicated*MergeTree` Tablo. [\#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) +- Kafka mesajlarını okurken sonsuz döngüyü düzeltin. [\#6354](https://github.com/ClickHouse/ClickHouse/pull/6354) ([abyss7](https://github.com/abyss7)) +- SQL ayrıştırıcısında yığın taşması ve yığın taşması olasılığı nedeniyle sunucu çökmesine neden olmak için fabrikasyon bir sorgu olasılığı düzeltildi `Merge` ve `Distributed` Tablolar [\#6433](https://github.com/ClickHouse/ClickHouse/pull/6433) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Küçük dizilerde sabit goril kodlama hatası. [\#6444](https://github.com/ClickHouse/ClickHouse/pull/6444) ([Enmk](https://github.com/Enmk)) + +#### Geliştirme {#improvement-5} + +- Kullanıcının geçersiz kılmasına izin ver `poll_interval` ve `idle_connection_timeout` bağlantı ayarları. [\#6230](https://github.com/ClickHouse/ClickHouse/pull/6230) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.11.5.28, 2019-08-05 {#clickhouse-release-19-11-5-28-2019-08-05} + +#### Hata düzeltme {#bug-fix-19} + +- Sunucu aşırı yüklendiğinde sorguları asılı olasılığı düzeltildi. [\#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- YANDEXCONSİSTENTHASH işlevinde FPE'Yİ düzeltin. Bu düzeltmeler [\#6304](https://github.com/ClickHouse/ClickHouse/issues/6304). [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Dönüşüm Sabit hata `LowCardinality` yazmak `AggregateFunctionFactory`. Bu düzeltmeler [\#6257](https://github.com/ClickHouse/ClickHouse/issues/6257). [\#6281](https://github.com/ClickHouse/ClickHouse/pull/6281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Ayrıştırmayı düzeltin `bool` ayarlar `true` ve `false` yapılandırma dosyalarındaki dizeler. [\#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) +- Sorgularda uyumsuz akış başlıklarıyla nadir hatayı düzeltin `Distributed` masa üstü `MergeTree` tablo parça `WHERE` MOV toes to `PREWHERE`. [\#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) +- İmzalı türün imzasız türe tamsayı bölümünde sabit taşma. Bu düzeltmeler [\#6214](https://github.com/ClickHouse/ClickHouse/issues/6214). [\#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-5} + +- `Kafka` hala kırık. + +### ClickHouse sürümü 19.11.4.24, 2019-08-01 {#clickhouse-release-19-11-4-24-2019-08-01} + +#### Hata Düzeltme {#bug-fix-20} + +- Adaptif tanecikli ikincil endeksler işaretleri yazma ile hatayı düzeltin. [\#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alesapin](https://github.com/alesapin)) +- Düzeltmek `WITH ROLLUP` ve `WITH CUBE` değiştiriciler `GROUP BY` iki seviyeli toplama ile. [\#6225](https://github.com/ClickHouse/ClickHouse/pull/6225) ([Anton Popov](https://github.com/CurtizJ)) +- Sabit asmak `JSONExtractRaw` işlev. Sabit [\#6195](https://github.com/ClickHouse/ClickHouse/issues/6195) [\#6198](https://github.com/ClickHouse/ClickHouse/pull/6198) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ExternalLoader::reloadOutdated () içinde segfault Fix. [\#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([Vitaly Baranov](https://github.com/vitlibar)) +- Sunucu dinleme soketlerini kapatabilir, ancak kapanmaz ve kalan sorguları sunmaya devam edebilir. İki çalışan clickhouse-server süreçleri ile sona erebilir. Bazen, sunucu bir hata döndürebilir `bad_function_call` kalan sorgular için. [\#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ODBC, MySQL, ClickHouse ve HTTP üzerinden harici sözlüklerin ilk yüklenmesi için güncelleme alanında işe yaramaz ve yanlış durum düzeltildi. Bu düzeltmeler [\#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [\#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Döküm sabit alakasız istisna `LowCardinality(Nullable)` to not-Nullable column in case if it doesn't contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [\#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [\#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Deterministik olmayan sonucu düzeltin “uniq” aşırı nadir durumlarda toplama fonksiyonu. Hata tüm ClickHouse sürümlerinde mevcuttu. [\#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Segfault biz fonksiyonu üzerinde biraz fazla yüksek CIDR ayarladığınızda `IPv6CIDRToRange`. [\#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Sunucu birçok farklı bağlamdan birçok istisna attığında sabit küçük bellek sızıntısı. [\#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tüketici abonelikten önce duraklatıldığında ve daha sonra devam etmediğinde durumu düzeltin. [\#6075](https://github.com/ClickHouse/ClickHouse/pull/6075) ([I ivanvan](https://github.com/abyss7)) Kafka'nın bu sürümde bozulduğunu unutmayın. +- Bir hatayla tamamlanan önceki okuma işleminden Kafka veri arabelleğini temizleme [\#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) Kafka'nın bu sürümde bozulduğunu unutmayın. +- Beri `StorageMergeTree::background_task_handle` içinde Başlat isılmıştır `startup()` bu `MergeTreeBlockOutputStream::write()` başlatmadan önce kullanmayı deneyebilir. Eğer başlatıldı ise sadece kontrol. [\#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([I ivanvan](https://github.com/abyss7)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-6} + +- Resmi eklendi `rpm` paketler. [\#5740](https://github.com/ClickHouse/ClickHouse/pull/5740) ([proller](https://github.com/proller)) ([alesapin](https://github.com/alesapin)) +- Oluşturmak için bir yetenek ekleyin `.rpm` ve `.tgz` paketleri ile `packager` komut. [\#5769](https://github.com/ClickHouse/ClickHouse/pull/5769) ([alesapin](https://github.com/alesapin)) +- İçin düzeltmeler “Arcadia” sistemi oluşturmak. [\#6223](https://github.com/ClickHouse/ClickHouse/pull/6223) ([proller](https://github.com/proller)) + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-6} + +- `Kafka` bu sürümde bozuldu. + +### ClickHouse sürümü 19.11.3.11, 2019-07-18 {#clickhouse-release-19-11-3-11-2019-07-18} + +#### Yenilik {#new-feature-6} + +- Hazırlanan ifadeler için destek eklendi. [\#5331](https://github.com/ClickHouse/ClickHouse/pull/5331/) ([İskender](https://github.com/sanych73)) [\#5630](https://github.com/ClickHouse/ClickHouse/pull/5630) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `DoubleDelta` ve `Gorilla` sütun kodekleri [\#5600](https://github.com/ClickHouse/ClickHouse/pull/5600) ([Vasily Nemkov](https://github.com/Enmk)) +- Katma `os_thread_priority` kontrol etmeyi sağlayan ayar “nice” dinamik zamanlama önceliğini ayarlamak için OS tarafından kullanılan sorgu işleme iş parçacıklarının değeri. Bu gerektirir `CAP_SYS_NICE` yetenekleri çalışmak. Bu uygular [\#5858](https://github.com/ClickHouse/ClickHouse/issues/5858) [\#5909](https://github.com/ClickHouse/ClickHouse/pull/5909) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Uygulamak `_topic`, `_offset`, `_key` Kafka motoru için sütunlar [\#5382](https://github.com/ClickHouse/ClickHouse/pull/5382) ([I ivanvan](https://github.com/abyss7)) Kafka'nın bu sürümde bozulduğunu unutmayın. +- Toplama fonksiyonu birleştirici Ekle `-Resample` [\#5590](https://github.com/ClickHouse/ClickHouse/pull/5590) ([hcz](https://github.com/hczhcz)) +- Toplama fonksiyonları `groupArrayMovingSum(win_size)(x)` ve `groupArrayMovingAvg(win_size)(x)`, pencere boyutu sınırlaması olan veya olmayan hareketli toplamı/AVG'yi hesaplar. [\#5595](https://github.com/ClickHouse/ClickHouse/pull/5595) ([ınv2004](https://github.com/inv2004)) +- Synonim Ekle `arrayFlatten` \<-\> `flatten` [\#5764](https://github.com/ClickHouse/ClickHouse/pull/5764) ([hcz](https://github.com/hczhcz)) +- Intergate H3 fonksiyonu `geoToH3` Uber gelen. [\#4724](https://github.com/ClickHouse/ClickHouse/pull/4724) ([Remen Ivan](https://github.com/BHYCHIK)) [\#5805](https://github.com/ClickHouse/ClickHouse/pull/5805) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Hata Düzeltme {#bug-fix-21} + +- Zaman uyumsuz güncelleştirme ile DNS önbelleğini uygulayın. Ayrı iş parçacığı tüm ana bilgisayarları çözer ve DNS önbelleğini nokta (ayar) ile günceller `dns_cache_update_period`). Ana bilgisayarların IP'si sık sık değiştiğinde yardımcı olmalıdır. [\#5857](https://github.com/ClickHouse/ClickHouse/pull/5857) ([Anton Popov](https://github.com/CurtizJ)) +- Segfault'u düzeltin `Delta` 32 bit boyutundan daha küçük değerlere sahip sütunları etkileyen codec bileşeni. Hata rastgele bellek bozulmasına yol açtı. [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) +- TTL fix segfault blok fiziksel olmayan sütunlarla birleştirme. [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) +- Parçanın kontrol edilmesinde nadir hatayı düzeltin `LowCardinality` sütun. Önceden `checkDataPart` her zaman ile parçası için başarısız `LowCardinality` sütun. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) +- Sunucu iş parçacığı havuzu dolduğunda bağlantıları asılı kaçının. Bağlantılar için önemlidir `remote` uzun bağlantı zaman aşımı olduğunda, tablo işlevi veya kopyaları olmayan bir parçaya bağlantılar. Bu düzeltmeler [\#5878](https://github.com/ClickHouse/ClickHouse/issues/5878) [\#5881](https://github.com/ClickHouse/ClickHouse/pull/5881) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit argümanlar için destek `evalMLModel` işlev. Bu düzeltmeler [\#5817](https://github.com/ClickHouse/ClickHouse/issues/5817) [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ClickHouse varsayılan saat dilimini belirlediğinde sorun giderildi `UCT` yerine `UTC`. Bu düzeltmeler [\#5804](https://github.com/ClickHouse/ClickHouse/issues/5804). [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit tampon underflow içinde `visitParamExtractRaw`. Bu düzeltmeler [\#5901](https://github.com/ClickHouse/ClickHouse/issues/5901) [\#5902](https://github.com/ClickHouse/ClickHouse/pull/5902) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi dağıtılmış `DROP/ALTER/TRUNCATE/OPTIMIZE ON CLUSTER` sorgular doğrudan lider çoğaltma üzerinde yürütülür. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) +- Düzeltmek `coalesce` için `ColumnConst` ile `ColumnNullable` + ilgili değişiklikler. [\#5755](https://github.com/ClickHouse/ClickHouse/pull/5755) ([Artem Zuikov](https://github.com/4ertus2)) +- Fix the `ReadBufferFromKafkaConsumer` böylece sonra yeni mesajlar okumaya devam ediyor `commit()` daha önce durmuş olsa bile [\#5852](https://github.com/ClickHouse/ClickHouse/pull/5852) ([I ivanvan](https://github.com/abyss7)) +- Düzeltmek `FULL` ve `RIGHT` Üzerinde katılırken sonuçları katılın `Nullable` sağ tablodaki tuşlar. [\#5859](https://github.com/ClickHouse/ClickHouse/pull/5859) ([Artem Zuikov](https://github.com/4ertus2)) +- Düşük öncelikli sorguların sonsuz uyku Olası düzeltme. [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bazı sorguların query\_log'da görünmemesine neden olan Yarış durumunu düzeltin `SYSTEM FLUSH LOGS` sorgu. [\#5456](https://github.com/ClickHouse/ClickHouse/issues/5456) [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) +- Sabit `heap-use-after-free` Clustercopier'de asan uyarısı, zaten kaldırılmış fotokopi nesnesini kullanmaya çalışan saatin neden olduğu. [\#5871](https://github.com/ClickHouse/ClickHouse/pull/5871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Sabit yanlış `StringRef` bazı uygulamalar tarafından döndürülen işaretçi `IColumn::deserializeAndInsertFromArena`. Bu hata sadece birim testlerini etkiledi. [\#5973](https://github.com/ClickHouse/ClickHouse/pull/5973) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Aynı ad sütunlarını maskeleme kaynak ve ara dizi birleştirme sütunlarını önleyin. [\#5941](https://github.com/ClickHouse/ClickHouse/pull/5941) ([Artem Zuikov](https://github.com/4ertus2)) +- Insert Fix ve MySQL stil tanımlayıcı Alıntı ile MySQL motoruna sorgu seçin. [\#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([Kış Zhang](https://github.com/zhang2014)) +- Şimdi `CHECK TABLE` sorgu MergeTree engine ailesi ile çalışabilir. Her bir parça için (veya simplier motorları durumunda dosya) varsa kontrol durumunu ve mesajı döndürür. Ayrıca, kırık bir parçanın getirilmesinde hatayı düzeltin. [\#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([alesapin](https://github.com/alesapin)) +- Split\_shared\_libraries çalışma zamanını düzeltme [\#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([Danila Kutenin](https://github.com/danlark1)) +- Sabit zaman dilimi başlatma `/etc/localtime` göreceli bir sembolik bağ gibi mi `../usr/share/zoneinfo/Europe/Moscow` [\#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- clickhouse-fotokopi makinesi: kullanımı düzeltin-kapatmadan sonra ücretsiz [\#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([proller](https://github.com/proller)) +- Güncel `simdjson`. Sıfır bayt ile bazı geçersiz JSONs başarıyla ayrıştırmak sorunu düzeltildi. [\#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sistem günlüklerinin kapatılmasını düzeltin [\#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([Anton Popov](https://github.com/CurtizJ)) +- Invalidate\_query'deki koşul bir sözlüğe bağlı olduğunda askıda kalmayı düzeltin. [\#6011](https://github.com/ClickHouse/ClickHouse/pull/6011) ([Vitaly Baranov](https://github.com/vitlibar)) + +#### Geliştirme {#improvement-6} + +- Küme yapılandırmasında çözümlenemeyen adreslere izin ver. Onlar kullanılamaz kabul ve her bağlantı girişimi çözmek için çalışılacaktır. Bu özellikle Kubernetes için yararlıdır. Bu düzeltmeler [\#5714](https://github.com/ClickHouse/ClickHouse/issues/5714) [\#5924](https://github.com/ClickHouse/ClickHouse/pull/5924) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Boş TCP bağlantılarını kapatın (varsayılan olarak bir saat zaman aşımı ile). Bu, her sunucuda birden çok dağıtılmış tablo bulunan büyük kümeler için özellikle önemlidir, çünkü her sunucu bir bağlantı havuzunu diğer her sunucuya saklayabilir ve en yüksek sorgu eşzamanlılığından sonra bağlantılar duracaktır. Bu düzeltmeler [\#5879](https://github.com/ClickHouse/ClickHouse/issues/5879) [\#5880](https://github.com/ClickHouse/ClickHouse/pull/5880) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Daha kaliteli `topK` işlev. Yeni öğenin daha büyük bir ağırlığı varsa, son öğeyi kaldırmak için SavingSpace set davranışını değiştirdi. [\#5833](https://github.com/ClickHouse/ClickHouse/issues/5833) [\#5850](https://github.com/ClickHouse/ClickHouse/pull/5850) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Etki alanlarıyla çalışmak için URL işlevleri artık şema olmadan tamamlanmamış URL'ler için çalışabilir [\#5725](https://github.com/ClickHouse/ClickHouse/pull/5725) ([alesapin](https://github.com/alesapin)) +- Checksums eklendi `system.parts_columns` Tablo. [\#5874](https://github.com/ClickHouse/ClickHouse/pull/5874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Katma `Enum` veri türü için bir synonim olarak `Enum8` veya `Enum16`. [\#5886](https://github.com/ClickHouse/ClickHouse/pull/5886) ([dimarub2000](https://github.com/dimarub2000)) +- Tam bit transpose varyantı için `T64` codec. İle daha iyi sıkıştırma yol açabilir `zstd`. [\#5742](https://github.com/ClickHouse/ClickHouse/pull/5742) ([Artem Zuikov](https://github.com/4ertus2)) +- Cond conditionition on `startsWith` fonksiyon şimdi birincil anahtar kullanır. Bu düzeltmeler [\#5310](https://github.com/ClickHouse/ClickHouse/issues/5310) ve [\#5882](https://github.com/ClickHouse/ClickHouse/issues/5882) [\#5919](https://github.com/ClickHouse/ClickHouse/pull/5919) ([dimarub2000](https://github.com/dimarub2000)) +- Kullanmasına izin ver `clickhouse-copier` boş veritabanı adı izin vererek çapraz çoğaltma küme topolojisi ile. [\#5745](https://github.com/ClickHouse/ClickHouse/pull/5745) ([nvartolomei](https://github.com/nvartolomei)) +- Kullanmak `UTC` olmadan bir sistemde varsayılan zaman dilimi olarak `tzdata` (e.g. bare Docker container). Before this patch, error message `Could not determine local time zone` yazdırıldı ve sunucu veya istemci başlatmayı reddetti. [\#5827](https://github.com/ClickHouse/ClickHouse/pull/5827) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fonksiyonda kayan nokta argümanı için geri destek döndü `quantileTiming` geriye dönük uyumluluk için. [\#5911](https://github.com/ClickHouse/ClickHouse/pull/5911) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Hata iletilerinde hangi tablonun eksik olduğunu göster. [\#5768](https://github.com/ClickHouse/ClickHouse/pull/5768) ([I ivanvan](https://github.com/abyss7)) +- Çeşitli kullanıcılar tarafından aynı query\_id ile sorgu çalıştırmasına izin verme [\#5430](https://github.com/ClickHouse/ClickHouse/pull/5430) ([proller](https://github.com/proller)) +- Grafite metrikleri göndermek için daha sağlam kod. Uzun çoklu sırasında bile çalışacaktır `RENAME TABLE` işleyiş. [\#5875](https://github.com/ClickHouse/ClickHouse/pull/5875) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ThreadPool yürütme için bir görev zamanlayamıyor, daha bilgilendirici hata iletileri görüntülenir. Bu düzeltmeler [\#5305](https://github.com/ClickHouse/ClickHouse/issues/5305) [\#5801](https://github.com/ClickHouse/ClickHouse/pull/5801) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Ngramsearch'i daha sezgisel olarak tersine çevirmek [\#5807](https://github.com/ClickHouse/ClickHouse/pull/5807) ([Danila Kutenin](https://github.com/danlark1)) +- HDFS engine builder'da kullanıcı ayrıştırma Ekle [\#5946](https://github.com/ClickHouse/ClickHouse/pull/5946) ([akonyaev90](https://github.com/akonyaev90)) +- Güncelleme varsayılan değeri `max_ast_elements parameter` [\#5933](https://github.com/ClickHouse/ClickHouse/pull/5933) ([Artem Konovalov](https://github.com/izebit)) +- Eski ayarlar kavramı eklendi. Eski ayar `allow_experimental_low_cardinality_type` hiçbir etkisi ile kullanılabilir. [0f15c01c6802f7ce1a1494c12c846be8c98944cd](https://github.com/ClickHouse/ClickHouse/commit/0f15c01c6802f7ce1a1494c12c846be8c98944cd) [Alexey Milovidov](https://github.com/alexey-milovidov) + +#### Performans İyileştirme {#performance-improvement-4} + +- İş parçacıklarının daha düzgün dağılımı için birleştirme tablosundan seçilecek akış sayısını artırın. Ayar eklendi `max_streams_multiplier_for_merge_tables`. Bu düzeltmeler [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5915](https://github.com/ClickHouse/ClickHouse/pull/5915) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-7} + +- Clickhouse'un farklı sürümleriyle istemci-sunucu etkileşimi için geriye dönük uyumluluk testi ekleyin. [\#5868](https://github.com/ClickHouse/ClickHouse/pull/5868) ([alesapin](https://github.com/alesapin)) +- Her taahhüt ve çekme isteğinde Test kapsamı bilgileri. [\#5896](https://github.com/ClickHouse/ClickHouse/pull/5896) ([alesapin](https://github.com/alesapin)) +- Özel ayırıcılarımızı desteklemek için adres dezenfektanı ile işbirliği yapın (`Arena` ve `ArenaWithFreeLists`) daha iyi hata ayıklama için “use-after-free” hatasızlar. [\#5728](https://github.com/ClickHouse/ClickHouse/pull/5728) ([akuzm](https://github.com/akuzm)) +- Değiştirmek [Llvm libunwind uygulaması](https://github.com/llvm-mirror/libunwind) C++ özel durum işleme ve yığın izleri yazdırma için [\#4828](https://github.com/ClickHouse/ClickHouse/pull/4828) ([Nikita Lapkov](https://github.com/laplab)) +- \- Weverything'den iki uyarı daha ekleyin [\#5923](https://github.com/ClickHouse/ClickHouse/pull/5923) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bellek dezenfektanı ile ClickHouse oluşturmak için izin verin. [\#3949](https://github.com/ClickHouse/ClickHouse/pull/3949) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit ubsan raporu hakkında `bitTest` fuzz testinde işlev. [\#5943](https://github.com/ClickHouse/ClickHouse/pull/5943) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Docker: kimlik doğrulaması gerektiren bir ClickHouse örneğini başlatmak için olasılık eklendi. [\#5727](https://github.com/ClickHouse/ClickHouse/pull/5727) ([Korviakov Andrey](https://github.com/shurshun)) +- Librdkafka'yı 1.1.0 sürümüne güncelleyin [\#5872](https://github.com/ClickHouse/ClickHouse/pull/5872) ([I ivanvan](https://github.com/abyss7)) +- Entegrasyon testleri için genel zaman aşımı ekleyin ve test kodunda bazılarını devre dışı bırakın. [\#5741](https://github.com/ClickHouse/ClickHouse/pull/5741) ([alesapin](https://github.com/alesapin)) +- Bazı iş parçacıklarını düzeltinnitizer arızaları. [\#5854](https://github.com/ClickHouse/ClickHouse/pull/5854) ([akuzm](https://github.com/akuzm)) +- Bu `--no-undefined` seçenek, bağlayıcıyı, bağlantı sırasında varoluş için tüm dış adları denetlemeye zorlar. Bölünmüş yapı modunda kütüphaneler arasındaki gerçek bağımlılıkları izlemek çok yararlıdır. [\#5855](https://github.com/ClickHouse/ClickHouse/pull/5855) ([I ivanvan](https://github.com/abyss7)) +- İçin performans testi eklendi [\#5797](https://github.com/ClickHouse/ClickHouse/issues/5797) [\#5914](https://github.com/ClickHouse/ClickHouse/pull/5914) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Gcc-7 ile sabit uyumluluk. [\#5840](https://github.com/ClickHouse/ClickHouse/pull/5840) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Gcc-9 için destek eklendi. Bu düzeltmeler [\#5717](https://github.com/ClickHouse/ClickHouse/issues/5717) [\#5774](https://github.com/ClickHouse/ClickHouse/pull/5774) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Libunwind yanlış bağlanabilir Sabit hata. [\#5948](https://github.com/ClickHouse/ClickHouse/pull/5948) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- PVS-Studio tarafından bulunan birkaç uyarı düzeltildi. [\#5921](https://github.com/ClickHouse/ClickHouse/pull/5921) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin ilk destek eklendi `clang-tidy` statik analiz cihazı. [\#5806](https://github.com/ClickHouse/ClickHouse/pull/5806) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- BSD/Linux endian makrolarını dönüştürme( ‘be64toh’ ve ‘htobe64’) Mac OS X eşdeğentser thelerine [\#5785](https://github.com/ClickHouse/ClickHouse/pull/5785) ([Fu Ch Chenen](https://github.com/fredchenbj)) +- Geliştirilmiş entegrasyon testleri kılavuzu. [\#5796](https://github.com/ClickHouse/ClickHouse/pull/5796) ([Vladimir Chebotarev](https://github.com/excitoon)) +- MacOSX + gcc9'da yapı sabitleme [\#5822](https://github.com/ClickHouse/ClickHouse/pull/5822) ([filimonov](https://github.com/filimonov)) +- Sabit bir nokta yazım hatası düzeltin: aggreAGte - \> agrega. [\#5753](https://github.com/ClickHouse/ClickHouse/pull/5753) ([akuzm](https://github.com/akuzm)) +- Freebsd yapısını düzeltin [\#5760](https://github.com/ClickHouse/ClickHouse/pull/5760) ([proller](https://github.com/proller)) +- Web sitesine deneysel YouTube kanalına bağlantı Ekle [\#5845](https://github.com/ClickHouse/ClickHouse/pull/5845) ([Ivan Blinkov](https://github.com/blinkov)) +- Cmake: kapsama bayrakları için Seçenek Ekle: WİTH\_COVERAGE [\#5776](https://github.com/ClickHouse/ClickHouse/pull/5776) ([proller](https://github.com/proller)) +- Bazı satır içi Podarray'ın başlangıç boyutunu düzeltin. [\#5787](https://github.com/ClickHouse/ClickHouse/pull/5787) ([akuzm](https://github.com/akuzm)) +- clickhouse-sunucu.postınst: centos 6 için işletim sistemi algılamasını düzeltin [\#5788](https://github.com/ClickHouse/ClickHouse/pull/5788) ([proller](https://github.com/proller)) +- Arch linux paketi üretimi eklendi. [\#5719](https://github.com/ClickHouse/ClickHouse/pull/5719) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Split Common/config.libs tarafından h (dbms) [\#5715](https://github.com/ClickHouse/ClickHouse/pull/5715) ([proller](https://github.com/proller)) +- İçin düzeltmeler “Arcadia” yapı platformu [\#5795](https://github.com/ClickHouse/ClickHouse/pull/5795) ([proller](https://github.com/proller)) +- Alışılmamış yapı için düzeltmeler (gcc9, alt modül yok) [\#5792](https://github.com/ClickHouse/ClickHouse/pull/5792) ([proller](https://github.com/proller)) +- Hata eğilimli olduğu kanıtlanmış olduğu için unalignedstore'da açık tip gerektirir [\#5791](https://github.com/ClickHouse/ClickHouse/pull/5791) ([akuzm](https://github.com/akuzm)) +- Düzeltmeler MacOS yapı [\#5830](https://github.com/ClickHouse/ClickHouse/pull/5830) ([filimonov](https://github.com/filimonov)) +- Burada istendiği gibi, daha büyük veri kümesine sahip yeni JIT özelliği ile ilgili performans testi [\#5263](https://github.com/ClickHouse/ClickHouse/issues/5263) [\#5887](https://github.com/ClickHouse/ClickHouse/pull/5887) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Stres testinde durumsal testler çalıştırın [12693e568722f11e19859742f56428455501fd2a](https://github.com/ClickHouse/ClickHouse/commit/12693e568722f11e19859742f56428455501fd2a) ([alesapin](https://github.com/alesapin)) + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-7} + +- `Kafka` bu sürümde bozuldu. +- Etkinleştirmek `adaptive_index_granularity` = Yeni için varsayılan olarak 10MB `MergeTree` Tablolar. 19.11 + sürümünde yeni MergeTree tabloları oluşturduysanız, 19.6'dan önceki sürümlere düşürme imkansız olacaktır. [\#5628](https://github.com/ClickHouse/ClickHouse/pull/5628) ([alesapin](https://github.com/alesapin)) +- Yandex tarafından kullanılan eski belgesiz gömülü sözlükler kaldırıldı.Metrica. İşlev `OSIn`, `SEIn`, `OSToRoot`, `SEToRoot`, `OSHierarchy`, `SEHierarchy` artık kullanılamaz. Bu işlevleri kullanıyorsanız, e-posta yazın clickhouse-feedback@yandex-team.com. not: son anda bu işlevleri bir süre tutmaya karar verdik. [\#5780](https://github.com/ClickHouse/ClickHouse/pull/5780) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +## ClickHouse yayın 19.10 {#clickhouse-release-19-10} + +### ClickHouse sürümü 19.10.1.5, 2019-07-12 {#clickhouse-release-19-10-1-5-2019-07-12} + +#### Yenilik {#new-feature-7} + +- Yeni sütun codec Ekle: `T64`. (U)IntX/EnumX/Data(Time)/DecimalX sütunları için yapılmıştır. Sabit veya küçük Aralık değerlerine sahip sütunlar için iyi olmalıdır. Codec kendisi büyütmek veya yeniden sıkıştırma olmadan veri türünü küçültmek sağlar. [\#5557](https://github.com/ClickHouse/ClickHouse/pull/5557) ([Artem Zuikov](https://github.com/4ertus2)) +- Veritabanı Altyapısı Ekle `MySQL` uzak MySQL sunucusundaki tüm tabloları görüntülemenize izin veren [\#5599](https://github.com/ClickHouse/ClickHouse/pull/5599) ([Kış Zhang](https://github.com/zhang2014)) +- `bitmapContains` uygulanış. 2x daha hızlı `bitmapHasAny` ikinci bit eşlem bir öğe içeriyorsa. [\#5535](https://github.com/ClickHouse/ClickHouse/pull/5535) ([Zhichang Yu](https://github.com/yuzhichang)) +- İçin destek `crc32` işlev (tam olarak MySQL veya PHP'DE olduğu gibi davranışla). Bir karma işlevine ihtiyacınız varsa kullanmayın. [\#5661](https://github.com/ClickHouse/ClickHouse/pull/5661) ([Remen Ivan](https://github.com/BHYCHIK)) +- Uyguluyordu `SYSTEM START/STOP DISTRIBUTED SENDS` zaman uyumsuz ekler içine denetlemek için sorgular `Distributed` Tablolar. [\#4935](https://github.com/ClickHouse/ClickHouse/pull/4935) ([Kış Zhang](https://github.com/zhang2014)) + +#### Hata Düzeltme {#bug-fix-22} + +- Mutasyonları yürütürken birleştirme sınırları için sorgu yürütme sınırlarını ve en fazla parça boyutunu yoksayın. [\#5659](https://github.com/ClickHouse/ClickHouse/pull/5659) ([Anton Popov](https://github.com/CurtizJ)) +- Normal blokların tekilleştirilmesine (son derece nadir) ve yinelenen blokların eklenmesine (daha sık) yol açabilecek hatayı düzeltin. [\#5549](https://github.com/ClickHouse/ClickHouse/pull/5549) ([alesapin](https://github.com/alesapin)) +- Fonksiyonun düzeltilmesi `arrayEnumerateUniqRanked` boş dizilere sahip argümanlar için [\#5559](https://github.com/ClickHouse/ClickHouse/pull/5559) ([proller](https://github.com/proller)) +- Herhangi bir mesaj yoklamak niyetiyle olmadan Kafka konulara abone etmeyin. [\#5698](https://github.com/ClickHouse/ClickHouse/pull/5698) ([I ivanvan](https://github.com/abyss7)) +- Ayar yap `join_use_nulls` Nullable içinde olamaz türleri için hiçbir etkisi olsun [\#5700](https://github.com/ClickHouse/ClickHouse/pull/5700) ([Olga Khvostikova](https://github.com/stavrolia)) +- Sabit `Incorrect size of index granularity` hatasızlar [\#5720](https://github.com/ClickHouse/ClickHouse/pull/5720) ([coraxster](https://github.com/coraxster)) +- Float'ı ondalık dönüştürme taşmasına düzeltin [\#5607](https://github.com/ClickHouse/ClickHouse/pull/5607) ([coraxster](https://github.com/coraxster)) +- Floş tampon zaman `WriteBufferFromHDFS`'In yıkıcı denir. Bu düzeltmeler içine yazma `HDFS`. [\#5684](https://github.com/ClickHouse/ClickHouse/pull/5684) ([Xindong Peng](https://github.com/eejoin)) + +#### Geliştirme {#improvement-7} + +- Boş hücreleri tedavi edin `CSV` varsayılan değerler olarak zaman ayarı `input_format_defaults_for_omitted_fields` etkindir. [\#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) +- Harici sözlüklerin engellenmeyen yüklenmesi. [\#5567](https://github.com/ClickHouse/ClickHouse/pull/5567) ([Vitaly Baranov](https://github.com/vitlibar)) +- Ayarlara göre önceden kurulmuş bağlantılar için Ağ zaman aşımları dinamik olarak değiştirilebilir. [\#4558](https://github.com/ClickHouse/ClickHouse/pull/4558) ([Konstantin Podshumok](https://github.com/podshumok)) +- Kullanım “public\_suffix\_list” fonksiyonlar için `firstSignificantSubdomain`, `cutToFirstSignificantSubdomain`. Tarafından oluşturulan mükemmel bir karma tablo kullanıyor `gperf` dosyadan oluşturulan bir liste ile: https://publicsuffix.org/list/public\_suffix\_list.dat. (örneğin, şimdi etki alanını tanıyoruz `ac.uk` olmayan önemli). [\#5030](https://github.com/ClickHouse/ClickHouse/pull/5030) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Kabul edilen `IPv6` sistem tablolarında veri türü; Birleşik istemci bilgi sütunları `system.processes` ve `system.query_log` [\#5640](https://github.com/ClickHouse/ClickHouse/pull/5640) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- MySQL uyumluluk protokolü ile bağlantılar için oturumları kullanma. \#5476 [\#5646](https://github.com/ClickHouse/ClickHouse/pull/5646) ([Yuriy Baranov](https://github.com/yurriy)) +- Destek daha fazla `ALTER` sorgular `ON CLUSTER`. [\#5593](https://github.com/ClickHouse/ClickHouse/pull/5593) [\#5613](https://github.com/ClickHouse/ClickHouse/pull/5613) ([sundyli](https://github.com/sundy-li)) +- Destek `` bölümünde `clickhouse-local` yapılandırma dosyası. [\#5540](https://github.com/ClickHouse/ClickHouse/pull/5540) ([proller](https://github.com/proller)) +- Sorgu çalıştırmasına izin ver `remote` tablo fonksiyonu `clickhouse-local` [\#5627](https://github.com/ClickHouse/ClickHouse/pull/5627) ([proller](https://github.com/proller)) + +#### Performans İyileştirme {#performance-improvement-5} + +- Mergetree sütunların sonunda son işareti yazma imkanı ekleyin. İşe yaramaz önlemek için tablo veri aralığı anahtarları için okuma sağlar. Yalnızca uyarlamalı dizin parçalı yapı kullanımda ise etkinleştirilir. [\#5624](https://github.com/ClickHouse/ClickHouse/pull/5624) ([alesapin](https://github.com/alesapin)) +- Sayısını azaltarak çok yavaş dosya sistemlerinde mergetree tabloları geliştirilmiş performans `stat` syscalls. [\#5648](https://github.com/ClickHouse/ClickHouse/pull/5648) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sürüm 19.6 tanıtıldı MergeTree tablolardan okuma sabit performans düşüşü. Düzeltmeler \# 5631. [\#5633](https://github.com/ClickHouse/ClickHouse/pull/5633) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-8} + +- Uyguluyordu `TestKeeper` test için kullanılan ZooKeeper arayüzünün bir uygulaması olarak [\#5643](https://github.com/ClickHouse/ClickHouse/pull/5643) ([alexey-milovidov](https://github.com/alexey-milovidov)) ([levushkin aleksej](https://github.com/alexey-milovidov)) +- Bundan sonra `.sql` testler rasgele veritabanı ile paralel olarak, sunucu tarafından izole çalıştırılabilir. Onları daha hızlı çalıştırmanıza, özel sunucu yapılandırmaları ile yeni testler eklemenize ve farklı testlerin birbirini etkilemediğinden emin olmanızı sağlar. [\#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([I ivanvan](https://github.com/abyss7)) +- Kaldırmak `` ve `` performans test fromlerinden [\#5672](https://github.com/ClickHouse/ClickHouse/pull/5672) ([Olga Khvostikova](https://github.com/stavrolia)) +- Sabit “select\_format” performans testi için `Pretty` biçimliler [\#5642](https://github.com/ClickHouse/ClickHouse/pull/5642) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +## ClickHouse sürüm 19.9 {#clickhouse-release-19-9} + +### ClickHouse yayın 19.9.3.31, 2019-07-05 {#clickhouse-release-19-9-3-31-2019-07-05} + +#### Hata Düzeltme {#bug-fix-23} + +- 32 bit boyutundan daha küçük değerlere sahip sütunları etkileyen Delta codec'te segfault'u düzeltin. Hata rastgele bellek bozulmasına yol açtı. [\#5786](https://github.com/ClickHouse/ClickHouse/pull/5786) ([alesapin](https://github.com/alesapin)) +- LowCardinality sütunu ile parçanın kontrol nadir hata düzeltin. [\#5832](https://github.com/ClickHouse/ClickHouse/pull/5832) ([alesapin](https://github.com/alesapin)) +- TTL fix segfault blok fiziksel olmayan sütunlarla birleştirme. [\#5819](https://github.com/ClickHouse/ClickHouse/pull/5819) ([Anton Popov](https://github.com/CurtizJ)) +- Düşük öncelikli sorguların potansiyel sonsuz uykusunu düzeltin. [\#5842](https://github.com/ClickHouse/ClickHouse/pull/5842) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Clickhouse'un varsayılan saat dilimini UTC yerine UCT olarak nasıl belirlediğini düzeltin. [\#5828](https://github.com/ClickHouse/ClickHouse/pull/5828) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Lider çoğaltma önce takipçi çoğaltma üzerinde küme sorguları üzerinde dağıtılmış damla/ALTER/TRUNCATE/OPTİMİZE yürütme hakkında Fix hata. Şimdi doğrudan lider kopya üzerinde idam edilecektir. [\#5757](https://github.com/ClickHouse/ClickHouse/pull/5757) ([alesapin](https://github.com/alesapin)) +- Bazı sorgular anında sistem floş günlükleri sorgu sonra query\_log görünmeyebilir neden yarış durumu, düzeltin. [\#5685](https://github.com/ClickHouse/ClickHouse/pull/5685) ([Anton Popov](https://github.com/CurtizJ)) +- Sabit argümanlar için eksik destek eklendi `evalMLModel` işlev. [\#5820](https://github.com/ClickHouse/ClickHouse/pull/5820) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.9.2.4, 2019-06-24 {#clickhouse-release-19-9-2-4-2019-06-24} + +#### Yenilik {#new-feature-8} + +- Dondurulmuş parçalar hakkında bilgi Yazdır `system.parts` Tablo. [\#5471](https://github.com/ClickHouse/ClickHouse/pull/5471) ([proller](https://github.com/proller)) +- Clickhouse'da istemci şifresini sor-argümanlarda ayarlanmamışsa tty üzerinde istemci Başlat [\#5092](https://github.com/ClickHouse/ClickHouse/pull/5092) ([proller](https://github.com/proller)) +- Uygulamak `dictGet` ve `dictGetOrDefault` ondalık türleri için fonksiyonlar. [\#5394](https://github.com/ClickHouse/ClickHouse/pull/5394) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Geliştirme {#improvement-8} + +- Debian init: servis durdurma zaman aşımı Ekle [\#5522](https://github.com/ClickHouse/ClickHouse/pull/5522) ([proller](https://github.com/proller)) +- Lowcardinality için şüpheli türlerle tablo oluşturmak için varsayılan olarak yasaklanan ayar Ekle [\#5448](https://github.com/ClickHouse/ClickHouse/pull/5448) ([Olga Khvostikova](https://github.com/stavrolia)) +- Regresyon işlevleri, işlevde durum olarak kullanılmadığında model ağırlıklarını döndürür `evalMLMethod`. [\#5411](https://github.com/ClickHouse/ClickHouse/pull/5411) ([Quid37](https://github.com/Quid37)) +- Regresyon yöntemlerini yeniden adlandırın ve geliştirin. [\#5492](https://github.com/ClickHouse/ClickHouse/pull/5492) ([Quid37](https://github.com/Quid37)) +- Dize arayanların daha net arayüzleri. [\#5586](https://github.com/ClickHouse/ClickHouse/pull/5586) ([Danila Kutenin](https://github.com/danlark1)) + +#### Hata Düzeltme {#bug-fix-24} + +- Kafka'daki potansiyel veri kaybını düzeltin [\#5445](https://github.com/ClickHouse/ClickHouse/pull/5445) ([I ivanvan](https://github.com/abyss7)) +- Potansiyel sonsuz döngüyü düzeltin `PrettySpace` sıfır sütun ile çağrıldığında Biçimlendir [\#5560](https://github.com/ClickHouse/ClickHouse/pull/5560) ([Olga Khvostikova](https://github.com/stavrolia)) +- Doğrusal modellerde sabit uint32 taşma hatası. Const olmayan model argümanı için eval ML modeline izin verin. [\#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- `ALTER TABLE ... DROP INDEX IF EXISTS ...` sağlanan dizin yoksa bir özel durum yükseltmek değil [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) +- Segfault ile Düzeltme `bitmapHasAny` skaler alt sorguda [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) +- Sabit hata çoğaltma bağlantı havuzu DNS önbellek düştü bile, ana bilgisayarı çözmek için yeniden deneme değil. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) +- Sabit `ALTER ... MODIFY TTL` ReplicatedMergeTree üzerinde. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539) ([Anton Popov](https://github.com/CurtizJ)) +- SOMUTLAŞTIRILMIŞ sütun ile dağıtılmış tabloya ekleme düzeltin [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) +- Birleştirme depolama alanını keserken hatalı ayırma düzeltildi [\#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([TCeason](https://github.com/TCeason)) +- Tzdata paketinin son sürümlerinde bazı dosyalar artık sembolik bağlardır. Varsayılan zaman dilimini tespit etmek için geçerli mekanizma bozulur ve bazı zaman dilimleri için yanlış isimler verir. Şimdi en azından saat dilimi adını, sağlanırsa TZ içeriğine zorlarız. [\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([I ivanvan](https://github.com/abyss7)) +- Toplamı sabit iğneler en az 16KB uzun olduğunda MultiVolnitsky searcher ile bazı son derece nadir durumlarda düzeltin. Algoritma cevapsız veya yanlış sonuca yol açabilir önceki sonuçları overwrote `multiSearchAny`. [\#5588](https://github.com/ClickHouse/ClickHouse/pull/5588) ([Danila Kutenin](https://github.com/danlark1)) +- ExternalData istekleri için ayarlar ClickHouse ayarlarını kullanamadığında sorunu düzeltin. Ayrıca, şimdilik, ayarlar `date_time_input_format` ve `low_cardinality_allow_in_native_format` adların belirsizliği nedeniyle kullanılamaz (dış verilerde tablo biçimi olarak yorumlanabilir ve sorguda bir ayar olabilir). [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila Kutenin](https://github.com/danlark1)) +- Parçaları Zookeeper onları düşürmeden sadece FS kaldırıldı hata düzeltin. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alesapin](https://github.com/alesapin)) +- MySQL protokolünden hata ayıklama günlüğü kaldırma [\#5478](https://github.com/ClickHouse/ClickHouse/pull/5478) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- DDL sorgu işleme sırasında znonode atla [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) +- Fix mix `UNION ALL` sonuç sütun türü. Tutarsız veri ve sonuç sütunlarının sütun türleri olan durumlar vardı. [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) +- Yanlış tamsayılar üzerinde bir istisna atmak `dictGetT` çökme yerine işlevler. [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) +- Karma sözlük için yanlış element\_count ve load\_factor Fix `system.dictionaries` Tablo. [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-9} + +- Olmadan sabit yapı `Brotli` HTTP sıkıştırma desteği (`ENABLE_BROTLI=OFF` cmake değişkeni). [\#5521](https://github.com/ClickHouse/ClickHouse/pull/5521) ([Anton Yuzhaninov](https://github.com/citrin)) +- Kükreyen dahil.kükreyen/kükreyen olarak h.sa [\#5523](https://github.com/ClickHouse/ClickHouse/pull/5523) ([Orivej Desh](https://github.com/orivej)) +- Hyperscan'daki gcc9 uyarılarını düzeltin (\#line yönergesi kötüdür!) [\#5546](https://github.com/ClickHouse/ClickHouse/pull/5546) ([Danila Kutenin](https://github.com/danlark1)) +- Gcc-9 ile derlerken tüm uyarıları düzeltin. Bazı contrib sorunları düzeltin. Gcc9 ICE'Yİ düzeltin ve bugzilla'ya gönderin. [\#5498](https://github.com/ClickHouse/ClickHouse/pull/5498) ([Danila Kutenin](https://github.com/danlark1)) +- Lld ile sabit bağlantı [\#5477](https://github.com/ClickHouse/ClickHouse/pull/5477) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sözlüklerde kullanılmayan uzmanlıkları kaldırma [\#5452](https://github.com/ClickHouse/ClickHouse/pull/5452) ([Artem Zuikov](https://github.com/4ertus2)) +- Farklı dosya türleri için tabloları biçimlendirme ve ayrıştırma için iyileştirme performans testleri [\#5497](https://github.com/ClickHouse/ClickHouse/pull/5497) ([Olga Khvostikova](https://github.com/stavrolia)) +- Paralel test çalışması için düzeltmeler [\#5506](https://github.com/ClickHouse/ClickHouse/pull/5506) ([proller](https://github.com/proller)) +- Docker: clickhouse-test'ten yapılandırmaları kullanın [\#5531](https://github.com/ClickHouse/ClickHouse/pull/5531) ([proller](https://github.com/proller)) +- FreeBSD için derlemeyi düzeltin [\#5447](https://github.com/ClickHouse/ClickHouse/pull/5447) ([proller](https://github.com/proller)) +- Yükseltme boost için 1.70 [\#5570](https://github.com/ClickHouse/ClickHouse/pull/5570) ([proller](https://github.com/proller)) +- Yapı clickhouse'u submodule olarak düzeltin [\#5574](https://github.com/ClickHouse/ClickHouse/pull/5574) ([proller](https://github.com/proller)) +- Jsonextract performans testlerini geliştirin [\#5444](https://github.com/ClickHouse/ClickHouse/pull/5444) ([Vitaly Baranov](https://github.com/vitlibar)) + +## ClickHouse yayın 19.8 {#clickhouse-release-19-8} + +### ClickHouse yayın 19.8.3.8, 2019-06-11 {#clickhouse-release-19-8-3-8-2019-06-11} + +#### Yenilik {#new-features} + +- Json ile çalışmak için fonksiyonlar eklendi [\#4686](https://github.com/ClickHouse/ClickHouse/pull/4686) ([hcz](https://github.com/hczhcz)) [\#5124](https://github.com/ClickHouse/ClickHouse/pull/5124). ([Vitaly Baranov](https://github.com/vitlibar)) +- Bir çok dilde var olan bir basename işlevine benzer bir davranışa sahip bir işlev basename ekleyin (`os.path.basename` PY inth inon'da, `basename` in PHP, etc…). Work with both an UNIX-like path or a Windows path. [\#5136](https://github.com/ClickHouse/ClickHouse/pull/5136) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Katma `LIMIT n, m BY` veya `LIMIT m OFFSET n BY` limit BY yan tümcesi için n ofset ayarlamak için sözdizimi. [\#5138](https://github.com/ClickHouse/ClickHouse/pull/5138) ([Anton Popov](https://github.com/CurtizJ)) +- Yeni veri türü eklendi `SimpleAggregateFunction`, bir ışık toplama ile sütunlara sahip olmasını sağlar `AggregatingMergeTree`. Bu sadece aşağıdaki gibi basit işlevlerle kullanılabilir `any`, `anyLast`, `sum`, `min`, `max`. [\#4629](https://github.com/ClickHouse/ClickHouse/pull/4629) ([Boris Granveaud](https://github.com/bgranvea)) +- Fonksiyonda sabit olmayan argümanlar için destek eklendi `ngramDistance` [\#5198](https://github.com/ClickHouse/ClickHouse/pull/5198) ([Danila Kutenin](https://github.com/danlark1)) +- Eklenen fonksiyonlar `skewPop`, `skewSamp`, `kurtPop` ve `kurtSamp` sıra eğriliği, örnek eğrilik, kurtozis ve örnek kurtozis sırasıyla hesaplamak için. [\#5200](https://github.com/ClickHouse/ClickHouse/pull/5200) ([hcz](https://github.com/hczhcz)) +- Destek yeniden adlandırma işlemi için `MaterializeView` depolama. [\#5209](https://github.com/ClickHouse/ClickHouse/pull/5209) ([Guillaume Tassery](https://github.com/YiuRULE)) +- MySQL istemcisini kullanarak Clickhouse'a bağlanmayı sağlayan sunucu eklendi. [\#4715](https://github.com/ClickHouse/ClickHouse/pull/4715) ([Yuriy Baranov](https://github.com/yurriy)) +- Eklemek `toDecimal*OrZero` ve `toDecimal*OrNull` işlevler. [\#5291](https://github.com/ClickHouse/ClickHouse/pull/5291) ([Artem Zuikov](https://github.com/4ertus2)) +- Destek ondalık türleri fonksiyonları: `quantile`, `quantiles`, `median`, `quantileExactWeighted`, `quantilesExactWeighted`, medianExactWeighted. [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([Artem Zuikov](https://github.com/4ertus2)) +- Katma `toValidUTF8` function, which replaces all invalid UTF-8 characters by replacement character � (U+FFFD). [\#5322](https://github.com/ClickHouse/ClickHouse/pull/5322) ([Danila Kutenin](https://github.com/danlark1)) +- Katma `format` işlev. Argümanlarda listelenen dizelerle sabit desen (basitleştirilmiş Python biçim deseni) biçimlendirme. [\#5330](https://github.com/ClickHouse/ClickHouse/pull/5330) ([Danila Kutenin](https://github.com/danlark1)) +- Katma `system.detached_parts` ayrılmış bölümleri hakkında bilgi içeren tablo `MergeTree` Tablolar. [\#5353](https://github.com/ClickHouse/ClickHouse/pull/5353) ([akuzm](https://github.com/akuzm)) +- Katma `ngramSearch` iğne ve Samanlık arasındaki simetrik olmayan farkı hesaplamak için işlev. [\#5418](https://github.com/ClickHouse/ClickHouse/pull/5418)[\#5422](https://github.com/ClickHouse/ClickHouse/pull/5422) ([Danila Kutenin](https://github.com/danlark1)) +- Temel makine öğrenimi yöntemlerinin (stokastik doğrusal regresyon ve lojistik regresyon) agrega fonksiyonları arayüzünü kullanarak uygulanması. Model ağırlıklarını güncellemek için farklı stratejilere sahiptir (basit Gradyan iniş, momentum yöntemi, Nesterov yöntemi). Ayrıca özel boyuttaki mini partileri de destekler. [\#4943](https://github.com/ClickHouse/ClickHouse/pull/4943) ([Quid37](https://github.com/Quid37)) +- Uygulanması `geohashEncode` ve `geohashDecode` işlevler. [\#5003](https://github.com/ClickHouse/ClickHouse/pull/5003) ([Vasily Nemkov](https://github.com/Enmk)) +- Toplama fonksiyonu eklendi `timeSeriesGroupSum`, hangi zaman damgası hizalama değil örnek farklı zaman serileri toplayabilir. İki örnek zaman damgası arasında doğrusal enterpolasyon kullanacak ve daha sonra zaman serilerini birlikte toplayacaktır. Toplama fonksiyonu eklendi `timeSeriesGroupRateSum`, zaman serisi ve daha sonra toplam oranları birlikte oranını hesaplar. [\#4542](https://github.com/ClickHouse/ClickHouse/pull/4542) ([Yangkuan Liu](https://github.com/LiuYangkuan)) +- Eklenen fonksiyonlar `IPv4CIDRtoIPv4Range` ve `IPv6CIDRtoIPv6Range` bir CIDR kullanarak alt ağdaki bir IP için alt ve üst sınırları hesaplamak için. [\#5095](https://github.com/ClickHouse/ClickHouse/pull/5095) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Etkin ayar ile HTTP kullanarak bir sorgu gönderdiğimizde bir X-ClickHouse-Summary Başlığı ekleyin `send_progress_in_http_headers`. Sorguda kaç satır ve bayt eklendiği gibi ek bilgilerle birlikte, x-ClickHouse-Progress'in olağan bilgilerini döndürün. [\#5116](https://github.com/ClickHouse/ClickHouse/pull/5116) ([Guillaume Tassery](https://github.com/YiuRULE)) + +#### Geliştirmeler {#improvements} + +- Katma `max_parts_in_total` bölüm anahtarı \#5166 güvenli olmayan belirtimini engelleyen mergetree ailesi tabloları (varsayılan: 100 000) için ayarlama. [\#5171](https://github.com/ClickHouse/ClickHouse/pull/5171) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `clickhouse-obfuscator`: ilk tohum sütun adı, sütun konumu ile birleştirerek tek tek sütunlar için tohum türetmek. Bu, veri kümelerini birden çok ilgili tablolarla dönüştürmek için tasarlanmıştır, böylece tablolar dönüşümden sonra birleştirilebilir kalır. [\#5178](https://github.com/ClickHouse/ClickHouse/pull/5178) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eklenen fonksiyonlar `JSONExtractRaw`, `JSONExtractKeyAndValues`. Yeniden adlandırılan işlevler `jsonExtract` -e doğru `JSONExtract`. Bir şeyler ters gittiğinde bu işlevler muhabir değerleri döndürür, değil `NULL`. Modifiye fonksiyonu `JSONExtract`, şimdi son parametresinden dönüş türünü alır ve nullables enjekte etmez. Avx2 talimatlarının mevcut olmaması durumunda Rapidjson'a geri dönüş uygulandı. Simdjson KÜTÜPHANESİ yeni bir sürüme güncellendi. [\#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([Vitaly Baranov](https://github.com/vitlibar)) +- Şimdi `if` ve `multiIf` fonksiyonlar koşulun güvenmeyin `Nullable`, ancak SQL uyumluluğu için şubelere güvenin. [\#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([Jian Wu.](https://github.com/janplus)) +- `In` yüklemi şimdi üretir `Null` sonucu `Null` gibi girdi `Equal` işlev. [\#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([Jian Wu.](https://github.com/janplus)) +- Kafka'dan her (flush\_interval / poll\_timeout) satır sayısını zaman sınırını kontrol edin. Bu, Kafka tüketicisinden okumayı daha sık kırmaya ve üst düzey akışların zaman sınırlarını kontrol etmeye izin verir [\#5249](https://github.com/ClickHouse/ClickHouse/pull/5249) ([I ivanvan](https://github.com/abyss7)) +- Paketlenmiş SASL ile bağlantı RDKAFKA. SASL SCRAM kimlik doğrulamasını kullanmasına izin vermelidir [\#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([I ivanvan](https://github.com/abyss7)) +- Tüm birleşimler için rowreflist'in toplu sürümü. [\#5267](https://github.com/ClickHouse/ClickHouse/pull/5267) ([Artem Zuikov](https://github.com/4ertus2)) +- clickhouse-sunucu: daha bilgilendirici dinleme hata mesajları. [\#5268](https://github.com/ClickHouse/ClickHouse/pull/5268) ([proller](https://github.com/proller)) +- Clickhouse destek sözlükler-fonksiyonlar için fotokopi `` [\#5270](https://github.com/ClickHouse/ClickHouse/pull/5270) ([proller](https://github.com/proller)) +- Yeni ayar Ekle `kafka_commit_every_batch` Kafka'nın taahhüt politikasını düzenlemek. + Taahhüt modunu ayarlamanıza izin verir: her mesaj Partisi işlendikten sonra veya tüm blok depolamaya yazıldıktan sonra. Bazı mesajları kaybetmek veya bazı aşırı durumlarda iki kez okumak arasında bir takas. [\#5308](https://github.com/ClickHouse/ClickHouse/pull/5308) ([I ivanvan](https://github.com/abyss7)) +- Yapmak `windowFunnel` diğer imzasız tamsayı türlerini destekleyin. [\#5320](https://github.com/ClickHouse/ClickHouse/pull/5320) ([sundyli](https://github.com/sundy-li)) +- Sanal sütunun gölgesine izin ver `_table` birleştirme motorunda. [\#5325](https://github.com/ClickHouse/ClickHouse/pull/5325) ([I ivanvan](https://github.com/abyss7)) +- Yapmak `sequenceMatch` toplu işlevler diğer imzasız tamsayı türlerini destekler [\#5339](https://github.com/ClickHouse/ClickHouse/pull/5339) ([sundyli](https://github.com/sundy-li)) +- Sağlama toplamı uyuşmazlığı büyük olasılıkla donanım hatalarından kaynaklanıyorsa daha iyi hata iletileri. [\#5355](https://github.com/ClickHouse/ClickHouse/pull/5355) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Altta yatan tabloların örneklemeyi desteklediğini kontrol edin `StorageMerge` [\#5366](https://github.com/ClickHouse/ClickHouse/pull/5366) ([I ivanvan](https://github.com/abyss7)) +- Сlose MySQL connections after their usage in external dictionaries. It is related to issue \#893. [\#5395](https://github.com/ClickHouse/ClickHouse/pull/5395) ([Clément Rodriguez](https://github.com/clemrodriguez)) +- MySQL Wire protokolünün iyileştirmeleri. MySQLWire biçiminin adını değiştirdi. Rsa\_free çağırmak için RAII kullanma. Bağlam oluşturulamıyorsa SSL'Yİ devre dışı bırakma. [\#5419](https://github.com/ClickHouse/ClickHouse/pull/5419) ([Yuriy Baranov](https://github.com/yurriy)) +- clickhouse-client: allow to run with unaccessable history file (read-only, no disk space, file is directory, …). [\#5431](https://github.com/ClickHouse/ClickHouse/pull/5431) ([proller](https://github.com/proller)) +- Dağıtılmış tablolara zaman uyumsuz ekler sorgu ayarlarını saygı. [\#4936](https://github.com/ClickHouse/ClickHouse/pull/4936) ([TCeason](https://github.com/TCeason)) +- Yeniden adlandırılan işlevler `leastSqr` -e doğru `simpleLinearRegression`, `LinearRegression` -e doğru `linearRegression`, `LogisticRegression` -e doğru `logisticRegression`. [\#5391](https://github.com/ClickHouse/ClickHouse/pull/5391) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Performans İyileştirmeleri {#performance-improvements} + +- Parallelize işleme parçaları olmayan çoğaltılmış MergeTree tabloları alter değiştirme sorgu. [\#4639](https://github.com/ClickHouse/ClickHouse/pull/4639) ([Ivan Kush](https://github.com/IvanKush)) +- Düzenli ifadeler çıkarma optimizasyonlar. [\#5193](https://github.com/ClickHouse/ClickHouse/pull/5193) [\#5191](https://github.com/ClickHouse/ClickHouse/pull/5191) ([Danila Kutenin](https://github.com/danlark1)) +- Yalnızca joın on bölümünde kullanılıyorsa, sonucu birleştirmek için doğru joın anahtar sütunu eklemeyin. [\#5260](https://github.com/ClickHouse/ClickHouse/pull/5260) ([Artem Zuikov](https://github.com/4ertus2)) +- İlk boş yanıttan sonra Kafka tamponunu dondurun. Bu birden invokations önler `ReadBuffer::next()` bazı satır ayrıştırma akışlarında boş sonuç için. [\#5283](https://github.com/ClickHouse/ClickHouse/pull/5283) ([I ivanvan](https://github.com/abyss7)) +- `concat` birden çok argüman için işlev optimizasyonu. [\#5357](https://github.com/ClickHouse/ClickHouse/pull/5357) ([Danila Kutenin](https://github.com/danlark1)) +- Query optimisation. Allow push down IN statement while rewriting commа/cross join into inner one. [\#5396](https://github.com/ClickHouse/ClickHouse/pull/5396) ([Artem Zuikov](https://github.com/4ertus2)) +- Daha hızlı dekompresyona sahip olmak için lz4 uygulamamızı referansla yükseltin. [\#5070](https://github.com/ClickHouse/ClickHouse/pull/5070) ([Danila Kutenin](https://github.com/danlark1)) +- Uygulanan MSD radix sıralama (kxsort dayalı) ve kısmi sıralama. [\#5129](https://github.com/ClickHouse/ClickHouse/pull/5129) ([Evgenii Pravda](https://github.com/kvinty)) + +#### Hata Düzeltmeleri {#bug-fixes} + +- Fix itme birleştirme ile sütunlar gerektirir [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([Kış Zhang](https://github.com/zhang2014)) +- ClickHouse systemd tarafından çalıştırıldığında Sabit hata, komut `sudo service clickhouse-server forcerestart` beklendiği gibi çalışmadı. [\#5204](https://github.com/ClickHouse/ClickHouse/pull/5204) ([proller](https://github.com/proller)) +- Datapartsexchange'deki http hata kodlarını düzeltin (9009 bağlantı noktasındaki ınterserver http Sunucusu, hatalarda bile her zaman 200 kodunu döndürdü). [\#5216](https://github.com/ClickHouse/ClickHouse/pull/5216) ([proller](https://github.com/proller)) +- Max\_small\_string\_size daha uzun dize için SimpleAggregateFunction Fix [\#5311](https://github.com/ClickHouse/ClickHouse/pull/5311) ([Azat Khuzhin](https://github.com/azat)) +- İçin hatayı Düzelt fixin `Decimal` -e doğru `Nullable(Decimal)` içinde dönüşüm. (Farklı ölçekler dahil) ondalık dönüşümler diğer ondalık destekleyin. [\#5350](https://github.com/ClickHouse/ClickHouse/pull/5350) ([Artem Zuikov](https://github.com/4ertus2)) +- Yanlış hesaplama yol simdjson kütüphanesinde sabit FPU clobbering `uniqHLL` ve `uniqCombined` toplama fonksiyonu ve matematik fonksiyonları gibi `log`. [\#5354](https://github.com/ClickHouse/ClickHouse/pull/5354) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Json işlevlerinde karışık const / nonconst durumlarda sabit taşıma. [\#5435](https://github.com/ClickHouse/ClickHouse/pull/5435) ([Vitaly Baranov](https://github.com/vitlibar)) +- Düzeltmek `retention` işlev. Şimdi bir veri satırında tatmin eden tüm koşullar veri durumuna eklenir. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) +- Sonuç türünü düzeltme `quantileExact` ondalık sayılarla. [\#5304](https://github.com/ClickHouse/ClickHouse/pull/5304) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Belge {#documentation} + +- İçin belgeleri tercüme `CollapsingMergeTree` Çin. [\#5168](https://github.com/ClickHouse/ClickHouse/pull/5168) ([张风啸](https://github.com/AlexZFX)) +- Tablo motorları ile ilgili bazı belgeleri çince'ye çevirin. + [\#5134](https://github.com/ClickHouse/ClickHouse/pull/5134) + [\#5328](https://github.com/ClickHouse/ClickHouse/pull/5328) + ([Nev leeer lee](https://github.com/neverlee)) + +#### Yapı / Test / Ambalaj Geliştirmeleri {#buildtestingpackaging-improvements} + +- Olası kullanım sonrası ücretsiz gösteren bazı dezenfektan raporlarını düzeltin.[\#5139](https://github.com/ClickHouse/ClickHouse/pull/5139) [\#5143](https://github.com/ClickHouse/ClickHouse/pull/5143) [\#5393](https://github.com/ClickHouse/ClickHouse/pull/5393) ([I ivanvan](https://github.com/abyss7)) +- Kolaylık sağlamak için performans testlerini ayrı dizinlerden çıkarın. [\#5158](https://github.com/ClickHouse/ClickHouse/pull/5158) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yanlış performans testlerini düzeltin. [\#5255](https://github.com/ClickHouse/ClickHouse/pull/5255) ([alesapin](https://github.com/alesapin)) +- Donanım Sorunlarını ayıklamak için bit flips'in neden olduğu sağlama toplamlarını hesaplamak için bir araç eklendi. [\#5334](https://github.com/ClickHouse/ClickHouse/pull/5334) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Runner betiğini daha kullanışlı hale getirin. [\#5340](https://github.com/ClickHouse/ClickHouse/pull/5340)[\#5360](https://github.com/ClickHouse/ClickHouse/pull/5360) ([filimonov](https://github.com/filimonov)) +- Performans testlerinin nasıl yazılacağı konusunda küçük talimatlar ekleyin. [\#5408](https://github.com/ClickHouse/ClickHouse/pull/5408) ([alesapin](https://github.com/alesapin)) +- Performans testlerinde create, fill ve drop sorgularında değişiklik yapma yeteneği ekleme [\#5367](https://github.com/ClickHouse/ClickHouse/pull/5367) ([Olga Khvostikova](https://github.com/stavrolia)) + +## ClickHouse yayın 19.7 {#clickhouse-release-19-7} + +### ClickHouse yayın 19.7.5.29, 2019-07-05 {#clickhouse-release-19-7-5-29-2019-07-05} + +#### Hata Düzeltme {#bug-fix-25} + +- JOIN ile bazı sorgularda performans gerilemesini düzeltin. [\#5192](https://github.com/ClickHouse/ClickHouse/pull/5192) ([Kış Zhang](https://github.com/zhang2014)) + +### ClickHouse yayın 19.7.5.27, 2019-06-09 {#clickhouse-release-19-7-5-27-2019-06-09} + +#### Yenilik {#new-features-1} + +- Bitmap ile ilgili işlevler eklendi `bitmapHasAny` ve `bitmapHasAll` benzer `hasAny` ve `hasAll` diziler için fonksiyonlar. [\#5279](https://github.com/ClickHouse/ClickHouse/pull/5279) ([Sergi Vladykin](https://github.com/svladykin)) + +#### Hata Düzeltmeleri {#bug-fixes-1} + +- Segfault'u düzeltin `minmax` Null değeri ile dizin. [\#5246](https://github.com/ClickHouse/ClickHouse/pull/5246) ([Nikita Vasilev](https://github.com/nikvas0)) +- Tüm giriş sütunlarını LİMİT olarak gerekli çıktı olarak işaretleyin. Düzelt itiyor ‘Not found column’ bazı dağıtılmış sorgularda hata. [\#5407](https://github.com/ClickHouse/ClickHouse/pull/5407) ([Constantin S. Pan](https://github.com/kvap)) +- Düzeltmek “Column ‘0’ already exists” er errorror in `SELECT .. PREWHERE` ÖNTAN columnımlı süt onunda [\#5397](https://github.com/ClickHouse/ClickHouse/pull/5397) ([proller](https://github.com/proller)) +- Düzeltmek `ALTER MODIFY TTL` sorgu üzerinde `ReplicatedMergeTree`. [\#5539](https://github.com/ClickHouse/ClickHouse/pull/5539/commits) ([Anton Popov](https://github.com/CurtizJ)) +- Kafka tüketiciler başlatmak için başarısız olduğunda sunucu çökmesine etmeyin. [\#5285](https://github.com/ClickHouse/ClickHouse/pull/5285) ([I ivanvan](https://github.com/abyss7)) +- Sabit bitmap işlevleri yanlış sonuç üretir. [\#5359](https://github.com/ClickHouse/ClickHouse/pull/5359) ([Andy Yang](https://github.com/andyyzh)) +- Karma sözlük için element\_count'u düzeltin (çiftleri dahil etmeyin) [\#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) +- Ortam değişkeni TZ içeriğini saat dilimi adı olarak kullanın. Bazı durumlarda varsayılan zaman dilimini doğru bir şekilde tespit etmeye yardımcı olur.[\#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([I ivanvan](https://github.com/abyss7)) +- Tamsayıları dönüştürmeye çalışmayın `dictGetT` fonksiyonlar, çünkü düzgün çalışmıyor. Bunun yerine bir istisna atın. [\#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) +- Harici ayarları düzeltinveri HTTP isteği. [\#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila. + Kutenin](https://github.com/danlark1)) +- Parçaları Zookeeper onları düşürmeden sadece FS kaldırıldı hata düzeltin. [\#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alesapin](https://github.com/alesapin)) +- Segmentasyon hatasını düzeltin `bitmapHasAny` işlev. [\#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) +- Sabit hata çoğaltma bağlantı havuzu DNS önbellek düştü bile, ana bilgisayarı çözmek için yeniden deneme değil. [\#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) +- Sabit `DROP INDEX IF EXISTS` sorgu. Şimdi `ALTER TABLE ... DROP INDEX IF EXISTS ...` sağlanan dizin yoksa, sorgu bir özel durum oluşturmaz. [\#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) +- Birliği tüm supertype sütununu düzeltin. Tutarsız veri ve sonuç sütunlarının sütun türleri olan durumlar vardı. [\#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) +- DDL sorgu işleme sırasında znonode atlayın. Başka bir düğüm, görev sırasındaki znode'u kaldırmadan önce, + bunu işlemedi, ancak zaten çocukların listesini al, DDLWorker iş parçacığını sonlandıracak. [\#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) +- Somutlaştırılmış sütun ile dağıtılmış() tabloya Ekle düzeltin. [\#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) + +### ClickHouse yayın 19.7.3.9, 2019-05-30 {#clickhouse-release-19-7-3-9-2019-05-30} + +#### Yenilik {#new-features-2} + +- Kullanıcı tarafından belirtilebilecek bir ayar aralığını sınırlamaya izin verin. + Bu kısıtlamalar kullanıcı ayarları profilinde ayarlanabilir. + [\#4931](https://github.com/ClickHouse/ClickHouse/pull/4931) ([Vitaly. + Baranov](https://github.com/vitlibar)) +- İşlevin ikinci bir sürümünü ekleyin `groupUniqArray` isteğe bağlı + `max_size` elde edilen dizinin boyutunu sınırlayan parametre. Bu + davranış benzer `groupArray(max_size)(x)` işlev. + [\#5026](https://github.com/ClickHouse/ClickHouse/pull/5026) ([Guillaume + Tassery](https://github.com/YiuRULE)) +- Tsvwithnames / CSVWithNames giriş dosya formatları için, sütun sırası şimdi olabilir + dosya başlığından belirlenir. Bu tarafından kontrol edilir + `input_format_with_names_use_header` parametre. + [\#5081](https://github.com/ClickHouse/ClickHouse/pull/5081) + ([İskender](https://github.com/Akazz)) + +#### Hata Düzeltmeleri {#bug-fixes-2} + +- Birleştirme sırasında uncompressed\_cache + JOİN ile kilitlenme (\#5197) + [\#5133](https://github.com/ClickHouse/ClickHouse/pull/5133) ([Danila. + Kutenin](https://github.com/danlark1)) +- Sistem tablolarına bir clickhouse-istemci sorgusunda segmentasyon hatası. \#5066 + [\#5127](https://github.com/ClickHouse/ClickHouse/pull/5127) + ([I ivanvan](https://github.com/abyss7)) +- KafkaEngine üzerinden ağır yükte veri kaybı (\#4736) + [\#5080](https://github.com/ClickHouse/ClickHouse/pull/5080) + ([I ivanvan](https://github.com/abyss7)) +- Sistemden en az iki seçmeyi içeren UNION ile bir sorgu yürütülürken gerçekleşebilecek çok nadir veri yarışı durumu düzeltildi.sütunlar, sistem.tablolar, sistem.parçalar, sistem.parts\_tables veya birleştirme ailesinin tabloları ve ilgili tabloların sütunlarının aynı anda değiştirilmesi. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Performans İyileştirmeleri {#performance-improvements-1} + +- Tek bir sayısal sütuna göre sıralama için radix sıralamasını kullanın `ORDER BY` olarak + `LIMIT`. [\#5106](https://github.com/ClickHouse/ClickHouse/pull/5106), + [\#4439](https://github.com/ClickHouse/ClickHouse/pull/4439) + ([Evgenii Pravda](https://github.com/kvinty), + [alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Belge {#documentation-1} + +- Bazı tablo motorları için belgeleri Çince'ye çevirin. + [\#5107](https://github.com/ClickHouse/ClickHouse/pull/5107), + [\#5094](https://github.com/ClickHouse/ClickHouse/pull/5094), + [\#5087](https://github.com/ClickHouse/ClickHouse/pull/5087) + ([张风啸](https://github.com/AlexZFX)), + [\#5068](https://github.com/ClickHouse/ClickHouse/pull/5068) ([asla + lee](https://github.com/neverlee)) + +#### Yapı / Test / Ambalaj Geliştirmeleri {#buildtestingpackaging-improvements-1} + +- UTF-8 karakterlerini düzgün bir şekilde yazdırın `clickhouse-test`. + [\#5084](https://github.com/ClickHouse/ClickHouse/pull/5084) + ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Her zaman öneri yüklemek için clickhouse-client için komut satırı parametresi ekleyin + veriler. [\#5102](https://github.com/ClickHouse/ClickHouse/pull/5102) + ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bazı PVS-Studio uyarılarını çözün. + [\#5082](https://github.com/ClickHouse/ClickHouse/pull/5082) + ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Güncelleme LZ4 [\#5040](https://github.com/ClickHouse/ClickHouse/pull/5040) ([Danila. + Kutenin](https://github.com/danlark1)) +- Yaklaşan çekme isteği \#5030 için gereksinimleri oluşturmak için gperf ekleyin. + [\#5110](https://github.com/ClickHouse/ClickHouse/pull/5110) + ([proller](https://github.com/proller)) + +## ClickHouse yayın 19.6 {#clickhouse-release-19-6} + +### ClickHouse yayın 19.6.3.18, 2019-06-13 {#clickhouse-release-19-6-3-18-2019-06-13} + +#### Hata Düzeltmeleri {#bug-fixes-3} + +- Tablo işlevlerinden sorgular için koşul pushdown sabit `mysql` ve `odbc` ve ilgili masa motorları. Bu düzeltmeler \# 3540 ve \# 2384. [\#5313](https://github.com/ClickHouse/ClickHouse/pull/5313) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Zookeeper kilitlenme Fix. [\#5297](https://github.com/ClickHouse/ClickHouse/pull/5297) ([github1youlc](https://github.com/github1youlc)) +- CSV'DE alıntı ondalık sayılara izin ver. [\#5284](https://github.com/ClickHouse/ClickHouse/pull/5284) ([Artem Zuikov](https://github.com/4ertus2) +- Float Inf/Nan'dan ondalık sayılara dönüştürmeye izin verme (istisna atma). [\#5282](https://github.com/ClickHouse/ClickHouse/pull/5282) ([Artem Zuikov](https://github.com/4ertus2)) +- Yeniden adlandırma sorgusunda veri yarışını düzeltin. [\#5247](https://github.com/ClickHouse/ClickHouse/pull/5247) ([Kış Zhang](https://github.com/zhang2014)) +- Lfalloc'u geçici olarak devre dışı bırakın. Lfalloc kullanımı, UncompressedCache tahsis edilmesinde ve yüksek yüklü sunuculardaki sorguların çökmesine neden olan bir çok MAP\_FAİLED'E yol açabilir. [cfdba93](https://github.com/ClickHouse/ClickHouse/commit/cfdba938ce22f16efeec504f7f90206a515b1280)([Danila Kutenin](https://github.com/danlark1)) + +### ClickHouse yayın 19.6.2.11, 2019-05-13 {#clickhouse-release-19-6-2-11-2019-05-13} + +#### Yenilik {#new-features-3} + +- Sütunlar ve tablolar için TTL ifadeleri. [\#4212](https://github.com/ClickHouse/ClickHouse/pull/4212) ([Anton Popov](https://github.com/CurtizJ)) +- İçin destek eklendi `brotli` http yanıtları için sıkıştırma (Accept-Encoding: br) [\#4388](https://github.com/ClickHouse/ClickHouse/pull/4388) ([Mikhail](https://github.com/fandyushin)) +- Yeni fonksiyon eklendi `isValidUTF8` bir bayt kümesinin doğru utf-8 kodlanmış olup olmadığını kontrol etmek için. [\#4934](https://github.com/ClickHouse/ClickHouse/pull/4934) ([Danila Kutenin](https://github.com/danlark1)) +- Yeni Yük Dengeleme ilkesi Ekle `first_or_random` bu, belirtilen ilk ana bilgisayara sorgular gönderir ve erişilemezse, rasgele ana bilgisayarlara sorgular gönderir. Çapraz çoğaltma topoloji kurulumları için kullanışlıdır. [\#5012](https://github.com/ClickHouse/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei)) + +#### Deneysel Özellikler {#experimental-features-1} + +- Ayar Ekle `index_granularity_bytes` (adaptive ındex granularity) MergeTree\* tablolar ailesi için. [\#4826](https://github.com/ClickHouse/ClickHouse/pull/4826) ([alesapin](https://github.com/alesapin)) + +#### Geliştirmeler {#improvements-1} + +- Fonksiyon için sabit olmayan ve negatif boyut ve uzunluk argümanları için destek eklendi `substringUTF8`. [\#4989](https://github.com/ClickHouse/ClickHouse/pull/4989) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sol joın sağ tablo, sol tablo sağ joın ve her iki tablo tam joın push-aşağı devre dışı bırakın. Bu, bazı durumlarda yanlış birleştirme sonuçlarını düzeltir. [\#4846](https://github.com/ClickHouse/ClickHouse/pull/4846) ([I ivanvan](https://github.com/abyss7)) +- `clickhouse-copier`: otomatik yükleme görev yapılandırması `--task-file` seçenek [\#4876](https://github.com/ClickHouse/ClickHouse/pull/4876) ([proller](https://github.com/proller)) +- Depolama Fabrikası ve masa fonksiyonları Fabrikası için yazım hataları işleyicisi eklendi. [\#4891](https://github.com/ClickHouse/ClickHouse/pull/4891) ([Danila Kutenin](https://github.com/danlark1)) +- Alt sorgular olmadan birden fazla birleşim için yıldız işaretlerini ve nitelikli yıldız işaretlerini destekleyin [\#4898](https://github.com/ClickHouse/ClickHouse/pull/4898) ([Artem Zuikov](https://github.com/4ertus2)) +- Eksik sütun hata mesajını daha kullanıcı dostu hale getirin. [\#4915](https://github.com/ClickHouse/ClickHouse/pull/4915) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Performans İyileştirmeleri {#performance-improvements-2} + +- ASOF JOİN önemli hızlanma [\#4924](https://github.com/ClickHouse/ClickHouse/pull/4924) ([Martijn Bakker](https://github.com/Gladdy)) + +#### Geriye Dönük Uyumsuz Değişiklikler {#backward-incompatible-changes} + +- HTTP Başlığı `Query-Id` olarak değiştirildi `X-ClickHouse-Query-Id` tutarlılık için. [\#4972](https://github.com/ClickHouse/ClickHouse/pull/4972) ([Mikhail](https://github.com/fandyushin)) + +#### Hata Düzeltmeleri {#bug-fixes-4} + +- Sabit potansiyel boş işaretçi dereference `clickhouse-copier`. [\#4900](https://github.com/ClickHouse/ClickHouse/pull/4900) ([proller](https://github.com/proller)) +- JOİN + ARRAY JOİN ile sorguda Sabit hata [\#4938](https://github.com/ClickHouse/ClickHouse/pull/4938) ([Artem Zuikov](https://github.com/4ertus2)) +- Bir sözlük engine=Dictionary ile bir veritabanı üzerinden başka bir sözlüğe bağlı olduğunda sunucunun başlangıcında asılı sabit. [\#4962](https://github.com/ClickHouse/ClickHouse/pull/4962) ([Vitaly Baranov](https://github.com/vitlibar)) +- Partially fix distributed\_product\_mode = local. It's possible to allow columns of local tables in where/having/order by/… via table aliases. Throw exception if table does not have alias. There's not possible to access to the columns without table aliases yet. [\#4986](https://github.com/ClickHouse/ClickHouse/pull/4986) ([Artem Zuikov](https://github.com/4ertus2)) +- Potansiyel olarak yanlış sonucu düzeltin `SELECT DISTINCT` ile `JOIN` [\#5001](https://github.com/ClickHouse/ClickHouse/pull/5001) ([Artem Zuikov](https://github.com/4ertus2)) +- Sistemden en az iki seçmeyi içeren UNION ile bir sorgu yürütülürken gerçekleşebilecek çok nadir veri yarışı durumu düzeltildi.sütunlar, sistem.tablolar, sistem.parçalar, sistem.parts\_tables veya birleştirme ailesinin tabloları ve ilgili tabloların sütunlarının aynı anda değiştirilmesi. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Yapı / Test / Ambalaj Geliştirmeleri {#buildtestingpackaging-improvements-2} + +- Farklı ana bilgisayarda clickhouse-server çalıştırırken sabit test hataları [\#4713](https://github.com/ClickHouse/ClickHouse/pull/4713) ([Vasily Nemkov](https://github.com/Enmk)) +- clickhouse-test: tty olmayan ortamda renk kontrol dizilerini devre dışı bırakın. [\#4937](https://github.com/ClickHouse/ClickHouse/pull/4937) ([alesapin](https://github.com/alesapin)) +- clickhouse-test: herhangi bir test veritabanını kullanmasına izin ver (Kaldır `test.` mümkün olduğu yerde yeterlilik) [\#5008](https://github.com/ClickHouse/ClickHouse/pull/5008) ([proller](https://github.com/proller)) +- Ubsan hatalarını düzeltin [\#5037](https://github.com/ClickHouse/ClickHouse/pull/5037) ([Vitaly Baranov](https://github.com/vitlibar)) +- Yandex LFAlloc, Segfault'ları daha güvenilir yakalamak için MarkCache ve UncompressedCache verilerini farklı şekillerde ayırmak için Clickhouse'a eklendi [\#4995](https://github.com/ClickHouse/ClickHouse/pull/4995) ([Danila Kutenin](https://github.com/danlark1)) +- Python util backports ve changelogs ile yardımcı olmak için. [\#4949](https://github.com/ClickHouse/ClickHouse/pull/4949) ([I ivanvan](https://github.com/abyss7)) + +## ClickHouse sürümü 19.5 {#clickhouse-release-19-5} + +### ClickHouse yayın 19.5.4.22, 2019-05-13 {#clickhouse-release-19-5-4-22-2019-05-13} + +#### Hata düzeltmeleri {#bug-fixes-5} + +- Bitmap sabit Olası kazasında \* fonksiyonlar [\#5220](https://github.com/ClickHouse/ClickHouse/pull/5220) [\#5228](https://github.com/ClickHouse/ClickHouse/pull/5228) ([Andy Yang](https://github.com/andyyzh)) +- Sistemden en az iki seçmeyi içeren UNION ile bir sorgu yürütülürken gerçekleşebilecek çok nadir veri yarışı durumu düzeltildi.sütunlar, sistem.tablolar, sistem.parçalar, sistem.parts\_tables veya birleştirme ailesinin tabloları ve ilgili tabloların sütunlarının aynı anda değiştirilmesi. [\#5189](https://github.com/ClickHouse/ClickHouse/pull/5189) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit hata `Set for IN is not created yet in case of using single LowCardinality column in the left part of IN`. Bu hata, lowcardinality sütunu birincil anahtarın bir parçasıysa oldu. \#5031 [\#5154](https://github.com/ClickHouse/ClickHouse/pull/5154) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Tutma işlevinin değiştirilmesi: bir satır hem ilk hem de n. koşulu yerine getirirse, veri durumuna yalnızca ilk tatmin edilen koşul eklenir. Şimdi bir veri satırında tatmin eden tüm koşullar veri durumuna eklenir. [\#5119](https://github.com/ClickHouse/ClickHouse/pull/5119) ([小路](https://github.com/nicelulu)) + +### ClickHouse yayın 19.5.3.8, 2019-04-18 {#clickhouse-release-19-5-3-8-2019-04-18} + +#### Hata düzeltmeleri {#bug-fixes-6} + +- Sabit ayar türü `max_partitions_per_insert_block` boolean'dan Uint64'e. [\#5028](https://github.com/ClickHouse/ClickHouse/pull/5028) ([Muhammed Hüseyin Sekhavat](https://github.com/mhsekhavat)) + +### ClickHouse yayın 19.5.2.6, 2019-04-15 {#clickhouse-release-19-5-2-6-2019-04-15} + +#### Yenilik {#new-features-4} + +- [Hyperscan](https://github.com/intel/hyperscan) çoklu düzenli ifade eşleştirme eklendi (fonksiyonlar `multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780), [\#4841](https://github.com/ClickHouse/ClickHouse/pull/4841) ([Danila Kutenin](https://github.com/danlark1)) +- `multiSearchFirstPosition` işlevi eklendi. [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) +- Tablolar için satır başına önceden tanımlanmış ifade filtresini uygulayın. [\#4792](https://github.com/ClickHouse/ClickHouse/pull/4792) ([I ivanvan](https://github.com/abyss7)) +- Bloom filtrelerine dayalı endeksleri atlama veri yeni bir tür (için kullanılabilir `equal`, `in` ve `like` işlevler). [\#4499](https://github.com/ClickHouse/ClickHouse/pull/4499) ([Nikita Vasilev](https://github.com/nikvas0)) +- Katma `ASOF JOIN` bilinen en son değere katılan sorguları çalıştırmaya izin verir. [\#4774](https://github.com/ClickHouse/ClickHouse/pull/4774) [\#4867](https://github.com/ClickHouse/ClickHouse/pull/4867) [\#4863](https://github.com/ClickHouse/ClickHouse/pull/4863) [\#4875](https://github.com/ClickHouse/ClickHouse/pull/4875) ([Martijn Bakker](https://github.com/Gladdy), [Artem Zuikov](https://github.com/4ertus2)) +- Birden fazla yeniden yaz `COMMA JOIN` -e doğru `CROSS JOIN`. Sonra onları yeniden yaz `INNER JOIN` mümkünse. [\#4661](https://github.com/ClickHouse/ClickHouse/pull/4661) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Geliştirme {#improvement-9} + +- `topK` ve `topKWeighted` Şimdi özel destekler `loadFactor` (düzeltmeler sorunu [\#4252](https://github.com/ClickHouse/ClickHouse/issues/4252)). [\#4634](https://github.com/ClickHouse/ClickHouse/pull/4634) ([Kirill Danshin](https://github.com/kirillDanshin)) +- Kullanmasına izin ver `parallel_replicas_count > 1` örnekleme olmadan tablolar için bile (ayar sadece onlar için göz ardı edilir). Önceki sürümlerde istisna yol oldu. [\#4637](https://github.com/ClickHouse/ClickHouse/pull/4637) ([Alexey Elymanov](https://github.com/digitalist)) +- İçin destek `CREATE OR REPLACE VIEW`. Bir görünüm oluşturmak veya tek bir deyimde yeni bir tanım ayarlamak için izin ver. [\#4654](https://github.com/ClickHouse/ClickHouse/pull/4654) ([Boris Granveaud](https://github.com/bgranvea)) +- `Buffer` tablo motoru şimdi destekler `PREWHERE`. [\#4671](https://github.com/ClickHouse/ClickHouse/pull/4671) ([Yangkuan Liu](https://github.com/LiuYangkuan)) +- Zookeeper içinde meta veri olmadan çoğaltılmış tablo başlatmak için yeteneği ekleyin `readonly` modu. [\#4691](https://github.com/ClickHouse/ClickHouse/pull/4691) ([alesapin](https://github.com/alesapin)) +- Clickhouse-client ilerleme çubuğunun sabit titreme. Kullanırken sorun en dikkat çekiciydi `FORMAT Null` akış sorguları ile. [\#4811](https://github.com/ClickHouse/ClickHouse/pull/4811) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İle işlevleri devre dışı bırakmak için izin ver `hyperscan` potansiyel olarak aşırı ve kontrolsüz kaynak kullanımını sınırlamak için kullanıcı bazında kütüphane. [\#4816](https://github.com/ClickHouse/ClickHouse/pull/4816) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tüm hatalarda sürüm numarası günlüğü ekleyin. [\#4824](https://github.com/ClickHouse/ClickHouse/pull/4824) ([proller](https://github.com/proller)) +- Kısıtlama eklendi `multiMatch` içine sığacak şekilde dize boyutu gerektiren işlevler `unsigned int`. Ayrıca bağımsız değişken sayısı sınırı eklendi `multiSearch` işlevler. [\#4834](https://github.com/ClickHouse/ClickHouse/pull/4834) ([Danila Kutenin](https://github.com/danlark1)) +- Hyperscan çizik alanı ve hata işleme geliştirilmiş kullanımı. [\#4866](https://github.com/ClickHouse/ClickHouse/pull/4866) ([Danila Kutenin](https://github.com/danlark1)) +- Doldurma `system.graphite_detentions` bir tablo yapılandırmasından `*GraphiteMergeTree` motor tabloları. [\#4584](https://github.com/ClickHouse/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Adlandırmak `trigramDistance` fonksiyonu için `ngramDistance` ve daha fazla fonksiyon ekleyin `CaseInsensitive` ve `UTF`. [\#4602](https://github.com/ClickHouse/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1)) +- Geliştirilmiş veri endeksleri hesaplama atlama. [\#4640](https://github.com/ClickHouse/ClickHouse/pull/4640) ([Nikita Vasilev](https://github.com/nikvas0)) +- Sıradan tutmak, `DEFAULT`, `MATERIALIZED` ve `ALIAS` tek bir listedeki sütunlar (düzeltmeler sorunu [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn)) + +#### Hata Düzeltme {#bug-fix-26} + +- Önlemek `std::terminate` bellek ayırma hatası durumunda. Şimdi `std::bad_alloc` istisna beklendiği gibi atılır. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tampondan capnproto okumasını düzeltir. Bazen dosyalar HTTP tarafından başarıyla yüklenmedi. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) +- Hatayı Düzelt `Unknown log entry type: 0` sonra `OPTIMIZE TABLE FINAL` sorgu. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Kuşu](https://github.com/amosbird)) +- Yanlış argümanlar `hasAny` veya `hasAll` fonksiyonlar segfault yol açabilir. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yürütme sırasında kilitlenme oluşabilir `DROP DATABASE dictionary` sorgu. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tanımsız davranışı düzeltin `median` ve `quantile` işlevler. [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) +- Sıkıştırma seviyesi algılamasını ne zaman düzeltin `network_compression_method` küçük harfle. V19. 1'de kırık. [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) +- Sabit ceh ofalet `UTC` ayar (düzeltmeler sorunu [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) +- Düzeltmek `histogram` fonksiyon davranışı ile `Distributed` Tablolar. [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) +- Sabit tsan raporu `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Nedeniyle sistem günlükleri kullanımında yarış durumuna kapatma sabit TSan raporu. Part\_log etkinleştirildiğinde kapatma sırasında sabit potansiyel kullanım sonrası serbest. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Parçaları tekrar kontrol edin `ReplicatedMergeTreeAlterThread` hata durumunda. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Ara toplama işlev durumlarındaki aritmetik işlemler sabit argümanlar (alt sorgu sonuçları gibi) için çalışmadı. [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Her zaman meta veri sütun adlarını backquote. Aksi takdirde, adlandırılmış sütunlu bir tablo oluşturmak imkansızdır `index` (sunucu hatalı biçimlendirilmiş nedeniyle yeniden başlatılmaz `ATTACH` metadata sorgu). [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Çökmeyi düzeltin `ALTER ... MODIFY ORDER BY` üzerinde `Distributed` Tablo. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) +- Segfault'u düzeltin `JOIN ON` ile etkin `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([Kış Zhang](https://github.com/zhang2014)) +- Kafka'dan bir protobuf mesajı tükettikten sonra yabancı bir satır ekleyerek hatayı düzeltin. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) +- 'Nin çökmesini düzeltin `JOIN` null olmayan vs null sütun üzerinde. Düzeltmek `NULLs` sağ tuş inlarda `ANY JOIN` + `join_use_nulls`. [\#4815](https://github.com/ClickHouse/ClickHouse/pull/4815) ([Artem Zuikov](https://github.com/4ertus2)) +- Segmentasyon hatasını düzeltin `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- Sabit yarış durumu `SELECT` itibaren `system.tables` tablo aynı anda yeniden adlandırılırsa veya değiştirilirse. [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Zaten eskimiş olan veri bölümünü getirirken sabit veri yarışı. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sırasında meydana gelebilecek sabit nadir veri yarışı `RENAME` MergeTree ailesinin tablo. [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fonksiyonda sabit segmentasyon hatası `arrayIntersect`. Fonksiyon karışık sabit ve sıradan argümanlarla çağrılırsa segmentasyon hatası olabilir. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) +- Sabit okuma `Array(LowCardinality)` sütun nadir durumda, sütun uzun bir boş diziler dizisi içerdiğinde. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Çökmeyi düzeltin `FULL/RIGHT JOIN` biz nullable vs değil nullable üzerinde katılırken. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) +- Düzeltmek `No message received` kopyaları arasındaki parçaları getirirken istisna. [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin)) +- Sabit `arrayIntersect` tek dizide birkaç tekrarlanan değerler durumunda işlev yanlış sonuç. [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Eşzamanlı sırasında bir yarış durumunu düzeltin `ALTER COLUMN` bir sunucu çökmesine neden olabilecek sorgular (düzeltmeler sorunu [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) +- Yanlış sonucu düzeltin `FULL/RIGHT JOIN` const sütunu ile. [\#4723](https://github.com/ClickHouse/ClickHouse/pull/4723) ([Artem Zuikov](https://github.com/4ertus2)) +- Çiftleri düzeltin `GLOBAL JOIN` yıldız işareti ile. [\#4705](https://github.com/ClickHouse/ClickHouse/pull/4705) ([Artem Zuikov](https://github.com/4ertus2)) +- Parametre kesintisini düzeltin `ALTER MODIFY` of Col ofum ofn `CODEC` sütun türü belirtilmediğinde. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin)) +- İşlevler `cutQueryStringAndFragment()` ve `queryStringAndFragment()` şimdi ne zaman doğru çalışıyor `URL` bir parça ve hiçbir sorgu içerir. [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) +- Ayarlarken nadir hatayı düzeltin `min_bytes_to_use_direct_io` iş parçacığı sütun dosyasında geriye aramak zorunda olduğunda oluşan sıfırdan büyüktür. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin)) +- Toplama işlevleri için yanlış argüman türlerini düzeltin `LowCardinality` argümanlar (düzeltmeler sorunu [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Yanlış isim kalifikasyonunu düzeltin `GLOBAL JOIN`. [\#4969](https://github.com/ClickHouse/ClickHouse/pull/4969) ([Artem Zuikov](https://github.com/4ertus2)) +- Düzeltme fonksiyonu `toISOWeek` 1970 yılı için sonuç. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltmek `DROP`, `TRUNCATE` ve `OPTIMIZE` sorgular, çoğaltma, dosya üzerinde `ON CLUSTER` için `ReplicatedMergeTree*` tablolar aile. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin)) + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-8} + +- Ayarı yeniden adlandır `insert_sample_with_metadata` ayarlamak `input_format_defaults_for_omitted_fields`. [\#4771](https://github.com/ClickHouse/ClickHouse/pull/4771) ([Artem Zuikov](https://github.com/4ertus2)) +- Ayar eklendi `max_partitions_per_insert_block` (varsayılan değer 100 ile). Eklenen blok daha fazla sayıda bölüm içeriyorsa, bir özel durum atılır. Sınırı kaldırmak istiyorsanız 0 olarak ayarlayın (önerilmez). [\#4845](https://github.com/ClickHouse/ClickHouse/pull/4845) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Çoklu arama işlevleri yeniden adlandırıldı (`multiPosition` -e doğru `multiSearchAllPositions`, `multiSearch` -e doğru `multiSearchAny`, `firstMatch` -e doğru `multiSearchFirstIndex`). [\#4780](https://github.com/ClickHouse/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1)) + +#### Performans İyileştirme {#performance-improvement-6} + +- Birçok İğne veya birçok benzer bigrams ile sorgular için yaklaşık %5-10 arama iyileştirme vererek, inlining tarafından volnitsky searcher Optimize edin. [\#4862](https://github.com/ClickHouse/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1)) +- Ayarlarken performans sorununu düzeltin `use_uncompressed_cache` önbellekte bulunan tüm okuma verileri göründüğünde ortaya çıkan sıfırdan büyüktür. [\#4913](https://github.com/ClickHouse/ClickHouse/pull/4913) ([alesapin](https://github.com/alesapin)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-10} + +- Sertleştirme hata ayıklama yapı: daha ayrıntılı bellek eşlemeleri ve ASLR; ışareti önbellek ve dizin için bellek koruması ekleyin. Bu ASan ve MSan bunu yapamaz durumda daha fazla bellek stomping hata bulmanızı sağlar. [\#4632](https://github.com/ClickHouse/ClickHouse/pull/4632) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Cmake değişkenleri için destek Ekle `ENABLE_PROTOBUF`, `ENABLE_PARQUET` ve `ENABLE_BROTLI` yukarıdaki özellikleri etkinleştirmeye / devre dışı bırakmaya izin verir (librdkafka, mysql, vb.için yapabileceğimiz gibi). [\#4669](https://github.com/ClickHouse/ClickHouse/pull/4669) ([Silviu Caragea](https://github.com/silviucpp)) +- Bazı sorgular test çalıştırıldıktan sonra asılırsa, işlem listesi ve tüm iş parçacıklarının stacktraces yazdırmak için yeteneği ekleyin. [\#4675](https://github.com/ClickHouse/ClickHouse/pull/4675) ([alesapin](https://github.com/alesapin)) +- Yeniden deneme Ekle `Connection loss` er errorror in `clickhouse-test`. [\#4682](https://github.com/ClickHouse/ClickHouse/pull/4682) ([alesapin](https://github.com/alesapin)) +- Vagrant ile freebsd build ekleyin ve packager komut dosyasına iş parçacığı sanitizer ile oluşturun. [\#4712](https://github.com/ClickHouse/ClickHouse/pull/4712) [\#4748](https://github.com/ClickHouse/ClickHouse/pull/4748) ([alesapin](https://github.com/alesapin)) +- Şimdi kullanıcı kullanıcı için şifre istedi `'default'` kurulum sırasında. [\#4725](https://github.com/ClickHouse/ClickHouse/pull/4725) ([proller](https://github.com/proller)) +- İçinde uyarı bastır warningmak `rdkafka` kitaplık. [\#4740](https://github.com/ClickHouse/ClickHouse/pull/4740) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Ssl olmadan oluşturma yeteneğine izin ver. [\#4750](https://github.com/ClickHouse/ClickHouse/pull/4750) ([proller](https://github.com/proller)) +- Clickhouse-sunucu görüntüsünü özel bir kullanıcıdan başlatmanın bir yolunu ekleyin. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Contrib Boost'u 1.69'a yükseltin. [\#4793](https://github.com/ClickHouse/ClickHouse/pull/4793) ([proller](https://github.com/proller)) +- Kullanımını devre dışı bırak `mremap` iplik dezenfektanı ile derlendiğinde. Şaşırtıcı bir şekilde, TSan kesişmiyor `mremap` (kes doesmesine rağmen `mmap`, `munmap`) bu yanlış pozitiflere yol açar. Durumsal testlerde sabit TSan raporu. [\#4859](https://github.com/ClickHouse/ClickHouse/pull/4859) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- HTTP arayüzü üzerinden biçim şemasını kullanarak test denetimi ekleyin. [\#4864](https://github.com/ClickHouse/ClickHouse/pull/4864) ([Vitaly Baranov](https://github.com/vitlibar)) + +## ClickHouse sürüm 19.4 {#clickhouse-release-19-4} + +### ClickHouse yayın 19.4.4.33, 2019-04-17 {#clickhouse-release-19-4-4-33-2019-04-17} + +#### Hata Düzeltmeleri {#bug-fixes-7} + +- Önlemek `std::terminate` bellek ayırma hatası durumunda. Şimdi `std::bad_alloc` istisna beklendiği gibi atılır. [\#4665](https://github.com/ClickHouse/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tampondan capnproto okumasını düzeltir. Bazen dosyalar HTTP tarafından başarıyla yüklenmedi. [\#4674](https://github.com/ClickHouse/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs)) +- Hatayı Düzelt `Unknown log entry type: 0` sonra `OPTIMIZE TABLE FINAL` sorgu. [\#4683](https://github.com/ClickHouse/ClickHouse/pull/4683) ([Amos Kuşu](https://github.com/amosbird)) +- Yanlış argümanlar `hasAny` veya `hasAll` fonksiyonlar segfault yol açabilir. [\#4698](https://github.com/ClickHouse/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yürütme sırasında kilitlenme oluşabilir `DROP DATABASE dictionary` sorgu. [\#4701](https://github.com/ClickHouse/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tanımsız davranışı düzeltin `median` ve `quantile` işlevler. [\#4702](https://github.com/ClickHouse/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz)) +- Sıkıştırma seviyesi algılamasını ne zaman düzeltin `network_compression_method` küçük harfle. V19. 1'de kırık. [\#4706](https://github.com/ClickHouse/ClickHouse/pull/4706) ([proller](https://github.com/proller)) +- Sabit ceh ofalet `UTC` ayar (düzeltmeler sorunu [\#4658](https://github.com/ClickHouse/ClickHouse/issues/4658)). [\#4718](https://github.com/ClickHouse/ClickHouse/pull/4718) ([proller](https://github.com/proller)) +- Düzeltmek `histogram` fonksiyon davranışı ile `Distributed` Tablolar. [\#4741](https://github.com/ClickHouse/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv)) +- Sabit tsan raporu `destroy of a locked mutex`. [\#4742](https://github.com/ClickHouse/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Nedeniyle sistem günlükleri kullanımında yarış durumuna kapatma sabit TSan raporu. Part\_log etkinleştirildiğinde kapatma sırasında sabit potansiyel kullanım sonrası serbest. [\#4758](https://github.com/ClickHouse/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Parçaları tekrar kontrol edin `ReplicatedMergeTreeAlterThread` hata durumunda. [\#4772](https://github.com/ClickHouse/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Ara toplama işlev durumlarındaki aritmetik işlemler sabit argümanlar (alt sorgu sonuçları gibi) için çalışmadı. [\#4776](https://github.com/ClickHouse/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Her zaman meta veri sütun adlarını backquote. Aksi takdirde, adlandırılmış sütunlu bir tablo oluşturmak imkansızdır `index` (sunucu hatalı biçimlendirilmiş nedeniyle yeniden başlatılmaz `ATTACH` metadata sorgu). [\#4782](https://github.com/ClickHouse/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Çökmeyi düzeltin `ALTER ... MODIFY ORDER BY` üzerinde `Distributed` Tablo. [\#4790](https://github.com/ClickHouse/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason)) +- Segfault'u düzeltin `JOIN ON` ile etkin `enable_optimize_predicate_expression`. [\#4794](https://github.com/ClickHouse/ClickHouse/pull/4794) ([Kış Zhang](https://github.com/zhang2014)) +- Kafka'dan bir protobuf mesajı tükettikten sonra yabancı bir satır ekleyerek hatayı düzeltin. [\#4808](https://github.com/ClickHouse/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar)) +- Segmentasyon hatasını düzeltin `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- Sabit yarış durumu `SELECT` itibaren `system.tables` tablo aynı anda yeniden adlandırılırsa veya değiştirilirse. [\#4836](https://github.com/ClickHouse/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Zaten eskimiş olan veri bölümünü getirirken sabit veri yarışı. [\#4839](https://github.com/ClickHouse/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sırasında meydana gelebilecek sabit nadir veri yarışı `RENAME` MergeTree ailesinin tablo. [\#4844](https://github.com/ClickHouse/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fonksiyonda sabit segmentasyon hatası `arrayIntersect`. Fonksiyon karışık sabit ve sıradan argümanlarla çağrılırsa segmentasyon hatası olabilir. [\#4847](https://github.com/ClickHouse/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx)) +- Sabit okuma `Array(LowCardinality)` sütun nadir durumda, sütun uzun bir boş diziler dizisi içerdiğinde. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Düzeltmek `No message received` kopyaları arasındaki parçaları getirirken istisna. [\#4856](https://github.com/ClickHouse/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin)) +- Sabit `arrayIntersect` tek dizide birkaç tekrarlanan değerler durumunda işlev yanlış sonuç. [\#4871](https://github.com/ClickHouse/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Eşzamanlı sırasında bir yarış durumunu düzeltin `ALTER COLUMN` bir sunucu çökmesine neden olabilecek sorgular (düzeltmeler sorunu [\#3421](https://github.com/ClickHouse/ClickHouse/issues/3421)). [\#4592](https://github.com/ClickHouse/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn)) +- Parametre kesintisini düzeltin `ALTER MODIFY` of Col ofum ofn `CODEC` sütun türü belirtilmediğinde. [\#4883](https://github.com/ClickHouse/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin)) +- İşlevler `cutQueryStringAndFragment()` ve `queryStringAndFragment()` şimdi ne zaman doğru çalışıyor `URL` bir parça ve hiçbir sorgu içerir. [\#4894](https://github.com/ClickHouse/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar)) +- Ayarlarken nadir hatayı düzeltin `min_bytes_to_use_direct_io` iş parçacığı sütun dosyasında geriye aramak zorunda olduğunda oluşan sıfırdan büyüktür. [\#4897](https://github.com/ClickHouse/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin)) +- Toplama işlevleri için yanlış argüman türlerini düzeltin `LowCardinality` argümanlar (düzeltmeler sorunu [\#4919](https://github.com/ClickHouse/ClickHouse/issues/4919)). [\#4922](https://github.com/ClickHouse/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Düzeltme fonksiyonu `toISOWeek` 1970 yılı için sonuç. [\#4988](https://github.com/ClickHouse/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltmek `DROP`, `TRUNCATE` ve `OPTIMIZE` sorgular, çoğaltma, dosya üzerinde `ON CLUSTER` için `ReplicatedMergeTree*` tablolar aile. [\#4991](https://github.com/ClickHouse/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin)) + +#### Geliştirmeler {#improvements-2} + +- Sıradan tutmak, `DEFAULT`, `MATERIALIZED` ve `ALIAS` tek bir listedeki sütunlar (düzeltmeler sorunu [\#2867](https://github.com/ClickHouse/ClickHouse/issues/2867)). [\#4707](https://github.com/ClickHouse/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn)) + +### ClickHouse yayın 19.4.3.11, 2019-04-02 {#clickhouse-release-19-4-3-11-2019-04-02} + +#### Hata Düzeltmeleri {#bug-fixes-8} + +- Çökmeyi düzeltin `FULL/RIGHT JOIN` biz nullable vs değil nullable üzerinde katılırken. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) +- Segmentasyon hatasını düzeltin `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-11} + +- Clickhouse-sunucu görüntüsünü özel bir kullanıcıdan başlatmanın bir yolunu ekleyin. [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) + +### ClickHouse sürümü 19.4.2.7, 2019-03-30 {#clickhouse-release-19-4-2-7-2019-03-30} + +#### Hata Düzeltmeleri {#bug-fixes-9} + +- Sabit okuma `Array(LowCardinality)` sütun nadir durumda, sütun uzun bir boş diziler dizisi içerdiğinde. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +### ClickHouse yayın 19.4.1.3, 2019-03-19 {#clickhouse-release-19-4-1-3-2019-03-19} + +#### Hata Düzeltmeleri {#bug-fixes-10} + +- Her ikisini de içeren sabit uzak sorgular `LIMIT BY` ve `LIMIT`. Daha önce `LIMIT BY` ve `LIMIT` uzak sorgu için kullanıldı, `LIMIT` daha önce olabilirdi `LIMIT BY`, çok filtrelenmiş sonuca yol açtı. [\#4708](https://github.com/ClickHouse/ClickHouse/pull/4708) ([Constantin S. Pan](https://github.com/kvap)) + +### ClickHouse yayın 19.4.0.49, 2019-03-09 {#clickhouse-release-19-4-0-49-2019-03-09} + +#### Yenilik {#new-features-5} + +- İçin tam destek eklendi `Protobuf` biçim (giriş ve çıkış, iç içe veri yapıları). [\#4174](https://github.com/ClickHouse/ClickHouse/pull/4174) [\#4493](https://github.com/ClickHouse/ClickHouse/pull/4493) ([Vitaly Baranov](https://github.com/vitlibar)) +- Kükreyen bitmapler ile bitmap fonksiyonları eklendi. [\#4207](https://github.com/ClickHouse/ClickHouse/pull/4207) ([Andy Yang](https://github.com/andyyzh)) [\#4568](https://github.com/ClickHouse/ClickHouse/pull/4568) ([Vitaly Baranov](https://github.com/vitlibar)) +- Parke formatı desteği. [\#4448](https://github.com/ClickHouse/ClickHouse/pull/4448) ([proller](https://github.com/proller)) +- Bulanık dize karşılaştırması için N-gram mesafesi eklendi. R dilinde q-gram ölçümlerine benzer. [\#4466](https://github.com/ClickHouse/ClickHouse/pull/4466) ([Danila Kutenin](https://github.com/danlark1)) +- Özel toplama ve tutma kalıplarından grafit toplaması için kuralları birleştirin. [\#4426](https://github.com/ClickHouse/ClickHouse/pull/4426) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Katma `max_execution_speed` ve `max_execution_speed_bytes` kaynak kullanımını sınırlamak için. Katma `min_execution_speed_bytes` Tamam tolayacak ayar `min_execution_speed`. [\#4430](https://github.com/ClickHouse/ClickHouse/pull/4430) ([Kış Zhang](https://github.com/zhang2014)) +- Uygulanan işlev `flatten`. [\#4555](https://github.com/ClickHouse/ClickHouse/pull/4555) [\#4409](https://github.com/ClickHouse/ClickHouse/pull/4409) ([alexey-milovidov](https://github.com/alexey-milovidov), [kzon](https://github.com/kzon)) +- Eklenen fonksiyonlar `arrayEnumerateDenseRanked` ve `arrayEnumerateUniqRanked` (sanki `arrayEnumerateUniq` ancak, çok boyutlu dizilerin içine bakmak için dizi derinliğini ince ayarlamaya izin verir). [\#4475](https://github.com/ClickHouse/ClickHouse/pull/4475) ([proller](https://github.com/proller)) [\#4601](https://github.com/ClickHouse/ClickHouse/pull/4601) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Multiple JOINS with some restrictions: no asterisks, no complex aliases in ON/WHERE/GROUP BY/… [\#4462](https://github.com/ClickHouse/ClickHouse/pull/4462) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Hata Düzeltmeleri {#bug-fixes-11} + +- Bu sürüm aynı zamanda 19.3 ve 19.1 tüm hata düzeltmeleri içerir. +- Veri atlama endekslerinde Sabit hata: İNSERTTEN sonra granüllerin sırası yanlıştı. [\#4407](https://github.com/ClickHouse/ClickHouse/pull/4407) ([Nikita Vasilev](https://github.com/nikvas0)) +- Sabit `set` Ind Forex for `Nullable` ve `LowCardinality` sütun. Ondan önce, `set` ile ind withex `Nullable` veya `LowCardinality` sütun hataya yol açtı `Data type must be deserialized with multiple streams` seçerken. [\#4594](https://github.com/ClickHouse/ClickHouse/pull/4594) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Update\_time'ı tam olarak doğru şekilde ayarlayın `executable` sözlük güncelleme. [\#4551](https://github.com/ClickHouse/ClickHouse/pull/4551) ([Tema Novikov](https://github.com/temoon)) +- 19.3'te kırık ilerleme çubuğunu düzeltin. [\#4627](https://github.com/ClickHouse/ClickHouse/pull/4627) ([filimonov](https://github.com/filimonov)) +- Belirli durumlarda bellek bölgesi küçüldüğünde Memorytracker'ın tutarsız değerleri düzeltildi. [\#4619](https://github.com/ClickHouse/ClickHouse/pull/4619) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ThreadPool sabit tanımsız davranış. [\#4612](https://github.com/ClickHouse/ClickHouse/pull/4612) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Mesajla çok nadir bir kaza düzeltildi `mutex lock failed: Invalid argument` bu, bir MERGETREE tablosu bir SELECT ile aynı anda bırakıldığında gerçekleşebilir. [\#4608](https://github.com/ClickHouse/ClickHouse/pull/4608) ([Alex Zatelepin](https://github.com/ztlpn)) +- ODBC sürücüsü ile uyumluluk `LowCardinality` veri türü. [\#4381](https://github.com/ClickHouse/ClickHouse/pull/4381) ([proller](https://github.com/proller)) +- FreeBSD: için düzeltme `AIOcontextPool: Found io_event with unknown id 0` hatasız. [\#4438](https://github.com/ClickHouse/ClickHouse/pull/4438) ([urgordeadbeef](https://github.com/urgordeadbeef)) +- `system.part_log` tablo yapılandırmaya bakılmaksızın oluşturuldu. [\#4483](https://github.com/ClickHouse/ClickHouse/pull/4483) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tanımsız davranışı düzeltin `dictIsIn` önbellek sözlükler için işlev. [\#4515](https://github.com/ClickHouse/ClickHouse/pull/4515) ([alesapin](https://github.com/alesapin)) +- Fixed a deadlock when a SELECT query locks the same table multiple times (e.g. from different threads or when executing multiple subqueries) and there is a concurrent DDL query. [\#4535](https://github.com/ClickHouse/ClickHouse/pull/4535) ([Alex Zatelepin](https://github.com/ztlpn)) +- Biz kendi elde edene kadar varsayılan olarak compile\_expressions devre dışı bırakın `llvm` contrib ve ile test edebilirsiniz `clang` ve `asan`. [\#4579](https://github.com/ClickHouse/ClickHouse/pull/4579) ([alesapin](https://github.com/alesapin)) +- Önlemek `std::terminate` ne zaman `invalidate_query` için `clickhouse` dış sözlük kaynağı yanlış resultset döndürdü (boş veya birden fazla satır veya birden fazla sütun). Sabit sorun ne zaman `invalidate_query` ne olursa olsun her beş saniyede yapıldı `lifetime`. [\#4583](https://github.com/ClickHouse/ClickHouse/pull/4583) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kilitlenme önlemek zaman `invalidate_query` ile bir sözlük için `clickhouse` kaynak içeren oldu `system.dictionaries` tablo veya `Dictionaries` veritabanı (nadir durum). [\#4599](https://github.com/ClickHouse/ClickHouse/pull/4599) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Çapraz için düzeltmeler boş nerede ile katılmak. [\#4598](https://github.com/ClickHouse/ClickHouse/pull/4598) ([Artem Zuikov](https://github.com/4ertus2)) +- Fonksiyon sabit segfault “replicate” sabit argüman geçirildiğinde. [\#4603](https://github.com/ClickHouse/ClickHouse/pull/4603) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yükleme doktoru ile Lambda işlevini düzeltin. [\#4408](https://github.com/ClickHouse/ClickHouse/pull/4408) ([Kış Zhang](https://github.com/zhang2014)) +- Birden çok düzeltmeleri katıldı. [\#4595](https://github.com/ClickHouse/ClickHouse/pull/4595) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Geliştirmeler {#improvements-3} + +- Sağ tablo sütunları için JOİN on bölümündeki takma adları destekleyin. [\#4412](https://github.com/ClickHouse/ClickHouse/pull/4412) ([Artem Zuikov](https://github.com/4ertus2)) +- Birden fazla sonuç subselects kullanılacak doğru sonucu ismi Katıldı. Düz takma adları kaynak adları ile sonuç olarak değiştirin. [\#4474](https://github.com/ClickHouse/ClickHouse/pull/4474) ([Artem Zuikov](https://github.com/4ertus2)) +- Birleştirilmiş ifadeler için aşağı itme mantığını geliştirin. [\#4387](https://github.com/ClickHouse/ClickHouse/pull/4387) ([I ivanvan](https://github.com/abyss7)) + +#### Performans İyileştirmeleri {#performance-improvements-3} + +- Geliştirilmiş sezgisel “move to PREWHERE” optimizasyon. [\#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 8-bit ve 16-bit anahtarlar için HashTable API kullanan uygun arama tabloları kullanın. [\#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([Amos Kuşu](https://github.com/amosbird)) +- Dize karşılaştırma geliştirilmiş performans. [\#4564](https://github.com/ClickHouse/ClickHouse/pull/4564) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Ayrı bir iş parçacığında dağıtılmış DDL kuyruğunu temizleme, böylece dağıtılmış DDL görevlerini işleyen ana döngüyü yavaşlatmaz. [\#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([Alex Zatelepin](https://github.com/ztlpn)) +- Ne zaman `min_bytes_to_use_direct_io` 1 olarak ayarlanır, her dosya O\_DİRECT modu ile açılmamıştır, çünkü okunacak veri boyutu bazen sıkıştırılmış bir bloğun boyutuna göre hafife alınmıştır. [\#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-12} + +- Clang-9 için destek eklendi [\#4604](https://github.com/ClickHouse/ClickHouse/pull/4604) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yanlış Düzelt `__asm__` talimatlar (tekrar) [\#4621](https://github.com/ClickHouse/ClickHouse/pull/4621) ([Konstantin Podshumok](https://github.com/podshumok)) +- Ayarları belirtmek için yeteneği ekleyin `clickhouse-performance-test` komut satırından. [\#4437](https://github.com/ClickHouse/ClickHouse/pull/4437) ([alesapin](https://github.com/alesapin)) +- Entegrasyon testlerine sözlükler testleri ekleyin. [\#4477](https://github.com/ClickHouse/ClickHouse/pull/4477) ([alesapin](https://github.com/alesapin)) +- Otomatik performans testlerine web sitesinde kriter sorguları eklendi. [\#4496](https://github.com/ClickHouse/ClickHouse/pull/4496) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `xxhash.h` harici lz4'te mevcut değildir, çünkü bir uygulama ayrıntısıdır ve sembolleri ile adlandır `XXH_NAMESPACE` makro. Lz4 harici olduğunda, xxHash da harici olmalı ve bağımlıların buna bağlanması gerekir. [\#4495](https://github.com/ClickHouse/ClickHouse/pull/4495) ([Orivej Desh](https://github.com/orivej)) +- Sabit bir durumda zaman `quantileTiming` toplama işlevi, negatif veya kayan nokta argümanı ile çağrılabilir (bu, tanımsız davranış dezenfektanı ile fuzz testini düzeltir). [\#4506](https://github.com/ClickHouse/ClickHouse/pull/4506) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yazım hatası düzeltme. [\#4531](https://github.com/ClickHouse/ClickHouse/pull/4531) ([sdk2](https://github.com/sdk2)) +- Mac'te derlemeyi düzeltin. [\#4371](https://github.com/ClickHouse/ClickHouse/pull/4371) ([Vitaly Baranov](https://github.com/vitlibar)) +- FreeBSD ve çeşitli sıradışı yapı yapılandırmaları için düzeltmeler oluşturun. [\#4444](https://github.com/ClickHouse/ClickHouse/pull/4444) ([proller](https://github.com/proller)) + +## ClickHouse sürümü 19.3 {#clickhouse-release-19-3} + +### ClickHouse yayın 19.3.9.1, 2019-04-02 {#clickhouse-release-19-3-9-1-2019-04-02} + +#### Hata Düzeltmeleri {#bug-fixes-12} + +- Çökmeyi düzeltin `FULL/RIGHT JOIN` biz nullable vs değil nullable üzerinde katılırken. [\#4855](https://github.com/ClickHouse/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2)) +- Segmentasyon hatasını düzeltin `clickhouse-copier`. [\#4835](https://github.com/ClickHouse/ClickHouse/pull/4835) ([proller](https://github.com/proller)) +- Sabit okuma `Array(LowCardinality)` sütun nadir durumda, sütun uzun bir boş diziler dizisi içerdiğinde. [\#4850](https://github.com/ClickHouse/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-13} + +- Özel bir kullanıcıdan clickhouse-server görüntüsünü başlatmak için bir yol ekleyin [\#4753](https://github.com/ClickHouse/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) + +### ClickHouse yayın 19.3.7, 2019-03-12 {#clickhouse-release-19-3-7-2019-03-12} + +#### Hata düzeltmeleri {#bug-fixes-13} + +- \#3920'de Sabit hata. Bu hata kendisini rasgele önbellek bozulması (mesajlar) olarak gösterir `Unknown codec family code`, `Cannot seek through file`) ve segfaults. Bu hata ilk olarak 19.1 sürümünde ortaya çıktı ve 19.1.10 ve 19.3.6'ya kadar olan sürümlerde mevcut. [\#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse yayın 19.3.6, 2019-03-02 {#clickhouse-release-19-3-6-2019-03-02} + +#### Hata düzeltmeleri {#bug-fixes-14} + +- Bir iş parçacığı havuzunda 1000'den fazla iş parçacığı olduğunda, `std::terminate` iş parçacığı çıkışında meydana gelebilir. [Azat Khuzhin](https://github.com/azat) [\#4485](https://github.com/ClickHouse/ClickHouse/pull/4485) [\#4505](https://github.com/ClickHouse/ClickHouse/pull/4505) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi oluşturmak mümkün `ReplicatedMergeTree*` varsayılanları olmayan sütunlarda yorumlar içeren tablolar ve yorum ve varsayılanları olmayan sütun kodekleri içeren tablolar. Ayrıca codec karşılaştırmasını düzeltin. [\#4523](https://github.com/ClickHouse/ClickHouse/pull/4523) ([alesapin](https://github.com/alesapin)) +- Dizi veya tuple ile katılmak sabit kazasında. [\#4552](https://github.com/ClickHouse/ClickHouse/pull/4552) ([Artem Zuikov](https://github.com/4ertus2)) +- Clickhouse sabit kazasında-mesaj ile fotokopi `ThreadStatus not created`. [\#4540](https://github.com/ClickHouse/ClickHouse/pull/4540) ([Artem Zuikov](https://github.com/4ertus2)) +- Dağıtılmış DDL kullanılmışsa, sunucu kapanmasında sabit kapatma. [\#4472](https://github.com/ClickHouse/ClickHouse/pull/4472) ([Alex Zatelepin](https://github.com/ztlpn)) +- Yanlış sütun numaraları, 10'dan büyük sayı ile sütunlar için ayrıştırma metin biçimi hakkında hata iletisinde yazdırıldı. [\#4484](https://github.com/ClickHouse/ClickHouse/pull/4484) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Yapı / Test / Ambalaj Geliştirmeleri {#buildtestingpackaging-improvements-3} + +- Avx etkin sabit yapı. [\#4527](https://github.com/ClickHouse/ClickHouse/pull/4527) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Genişletilmiş muhasebe ve IO muhasebesini, derlendiği çekirdek yerine iyi bilinen sürüme göre etkinleştirin. [\#4541](https://github.com/ClickHouse/ClickHouse/pull/4541) ([nvartolomei](https://github.com/nvartolomei)) +- Core\_dump ayarını atlamaya izin ver.size\_limit, limit set başarısız olursa atmak yerine uyarı. [\#4473](https://github.com/ClickHouse/ClickHouse/pull/4473) ([proller](https://github.com/proller)) +- Kaldır theılan `inline` tags of `void readBinary(...)` içinde `Field.cpp`. Ayrıca birleştirilmiş gereksiz `namespace DB` Bloklar. [\#4530](https://github.com/ClickHouse/ClickHouse/pull/4530) ([hcz](https://github.com/hczhcz)) + +### ClickHouse yayın 19.3.5, 2019-02-21 {#clickhouse-release-19-3-5-2019-02-21} + +#### Hata düzeltmeleri {#bug-fixes-15} + +- Büyük http ekleme sorguları işleme ile Sabit hata. [\#4454](https://github.com/ClickHouse/ClickHouse/pull/4454) ([alesapin](https://github.com/alesapin)) +- Yanlış uygulama nedeniyle eski sürümlerle geriye dönük uyumsuzluk düzeltildi `send_logs_level` ayar. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tablo fonksiyonunun geriye dönük uyumsuzluğu düzeltildi `remote` sütun yorumları ile tanıtıldı. [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse yayın 19.3.4, 2019-02-16 {#clickhouse-release-19-3-4-2019-02-16} + +#### Geliştirmeler {#improvements-4} + +- Yaparken tablo dizin boyutu bellek sınırları için hesaplandı değil `ATTACH TABLE` sorgu. Bir tablo müstakil sonra eklenemez olasılığını Kaçınılması. [\#4396](https://github.com/ClickHouse/ClickHouse/pull/4396) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Zookeeper'dan alınan maksimum dize ve dizi boyutu üzerindeki sınırı hafifçe yükseltti. Bu artan boyutu ile çalışmaya devam sağlar `CLIENT_JVMFLAGS=-Djute.maxbuffer=...` ZooKeeper üzerinde. [\#4398](https://github.com/ClickHouse/ClickHouse/pull/4398) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Zaten kendi kuyruğunda düğümlerin çok sayıda olsa bile terk edilmiş çoğaltma onarmak için izin verin. [\#4399](https://github.com/ClickHouse/ClickHouse/pull/4399) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bir gerekli argüman Ekle `SET` dizin (en fazla saklanan satır numarası). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) + +#### Hata Düzeltmeleri {#bug-fixes-16} + +- Sabit `WITH ROLLUP` tek grup için sonuç `LowCardinality` anahtar. [\#4384](https://github.com/ClickHouse/ClickHouse/pull/4384) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Set dizinindeki Sabit hata (daha fazlasını içeriyorsa bir granül bırakarak `max_rows` satırlar). [\#4386](https://github.com/ClickHouse/ClickHouse/pull/4386) ([Nikita Vasilev](https://github.com/nikvas0)) +- Bir sürü FreeBSD yapı düzeltmesi. [\#4397](https://github.com/ClickHouse/ClickHouse/pull/4397) ([proller](https://github.com/proller)) +- Aynı takma ad içeren alt sorgularla sorgularda sabit takma ad değiştirme (sorun [\#4110](https://github.com/ClickHouse/ClickHouse/issues/4110)). [\#4351](https://github.com/ClickHouse/ClickHouse/pull/4351) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Yapı / Test / Ambalaj Geliştirmeleri {#buildtestingpackaging-improvements-4} + +- Çalıştırmak için yeteneği ekleyin `clickhouse-server` docker ımage vatansız testler için. [\#4347](https://github.com/ClickHouse/ClickHouse/pull/4347) ([Vasily Nemkov](https://github.com/Enmk)) + +### ClickHouse yayın 19.3.3, 2019-02-13 {#clickhouse-release-19-3-3-2019-02-13} + +#### Yenilik {#new-features-6} + +- Add theed the `KILL MUTATION` bazı nedenlerle sıkışmış olan mutasyonların giderilmesine izin veren ifade. Katma `latest_failed_part`, `latest_fail_time`, `latest_fail_reason` alanlar için `system.mutations` daha kolay sorun giderme için tablo. [\#4287](https://github.com/ClickHouse/ClickHouse/pull/4287) ([Alex Zatelepin](https://github.com/ztlpn)) +- Toplama fonksiyonu eklendi `entropy` Shannon entropisini hesaplar. [\#4238](https://github.com/ClickHouse/ClickHouse/pull/4238) ([Quid37](https://github.com/Quid37)) +- Sorguları göndermek için yeteneği eklendi `INSERT INTO tbl VALUES (....` yarat withoutmadan sunucuya `query` ve `data` parçalar. [\#4301](https://github.com/ClickHouse/ClickHouse/pull/4301) ([alesapin](https://github.com/alesapin)) +- Genel uygulama `arrayWithConstant` işlevi eklendi. [\#4322](https://github.com/ClickHouse/ClickHouse/pull/4322) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Uyguluyordu `NOT BETWEEN` karşılaştırma operatörü. [\#4228](https://github.com/ClickHouse/ClickHouse/pull/4228) ([Dmitry Naumov](https://github.com/nezed)) +- Uygulamak `sumMapFiltered` değerlerin toplanacağı anahtar sayısını sınırlayabilmek için `sumMap`. [\#4129](https://github.com/ClickHouse/ClickHouse/pull/4129) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) +- Desteği eklendi `Nullable` yazmak `mysql` tablo işlevi. [\#4198](https://github.com/ClickHouse/ClickHouse/pull/4198) ([Emmanuel Donin de Rosière](https://github.com/edonin)) +- İçinde keyfi sabit ifadeler için destek `LIMIT` yan. [\#4246](https://github.com/ClickHouse/ClickHouse/pull/4246) ([k3box](https://github.com/k3box)) +- Katma `topKWeighted` (imzasız tamsayı) ağırlığı ile ek argüman alır toplama işlevi. [\#4245](https://github.com/ClickHouse/ClickHouse/pull/4245) ([Andrew Golman](https://github.com/andrewgolman)) +- `StorageJoin` şimdi destekler `join_any_take_last_row` aynı anahtarın varolan değerlerinin üzerine yazmayı sağlayan ayar. [\#3973](https://github.com/ClickHouse/ClickHouse/pull/3973) ([Amos Kuşu](https://github.com/amosbird) +- Eklendi fonksiyonu `toStartOfInterval`. [\#4304](https://github.com/ClickHouse/ClickHouse/pull/4304) ([Vitaly Baranov](https://github.com/vitlibar)) +- Katma `RowBinaryWithNamesAndTypes` biçimli. [\#4200](https://github.com/ClickHouse/ClickHouse/pull/4200) ([Oleg V. Kozlyuk](https://github.com/DarkWanderer)) +- Katma `IPv4` ve `IPv6` veri türleri. Daha etkili uygulamalar `IPv*` işlevler. [\#3669](https://github.com/ClickHouse/ClickHouse/pull/3669) ([Vasily Nemkov](https://github.com/Enmk)) +- Eklendi fonksiyonu `toStartOfTenMinutes()`. [\#4298](https://github.com/ClickHouse/ClickHouse/pull/4298) ([Vitaly Baranov](https://github.com/vitlibar)) +- Katma `Protobuf` çıkış biçimi. [\#4005](https://github.com/ClickHouse/ClickHouse/pull/4005) [\#4158](https://github.com/ClickHouse/ClickHouse/pull/4158) ([Vitaly Baranov](https://github.com/vitlibar)) +- Veri içe aktarma (ekler) için HTTP arayüzü için brotli desteği eklendi. [\#4235](https://github.com/ClickHouse/ClickHouse/pull/4235) ([Mikhail](https://github.com/fandyushin)) +- Kullanıcı işlev adına yazım hatası yaparken veya komut satırı istemcisinde yazarken ipuçları eklendi. [\#4239](https://github.com/ClickHouse/ClickHouse/pull/4239) ([Danila Kutenin](https://github.com/danlark1)) +- Katma `Query-Id` sunucunun HTTP yanıt başlığına. [\#4231](https://github.com/ClickHouse/ClickHouse/pull/4231) ([Mikhail](https://github.com/fandyushin)) + +#### Deneysel özellikler {#experimental-features-2} + +- Katma `minmax` ve `set` MergeTree tablo motorları ailesi için veri atlama endeksleri. [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([Nikita Vasilev](https://github.com/nikvas0)) +- Eklenen dönüşüm `CROSS JOIN` -e doğru `INNER JOIN` mümkünse. [\#4221](https://github.com/ClickHouse/ClickHouse/pull/4221) [\#4266](https://github.com/ClickHouse/ClickHouse/pull/4266) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Hata Düzeltmeleri {#bug-fixes-17} + +- Sabit `Not found column` yinelenen sütunlar için `JOIN ON` bölme. [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) +- Yapmak `START REPLICATED SENDS` komut Başlat çoğaltılmış gönderir. [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) +- Sabit toplam fonksiyonları yürütme ile `Array(LowCardinality)` değişkenler. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Yaparken yanlış davranış düzeltildi `INSERT ... SELECT ... FROM file(...)` sorgu ve dosya var `CSVWithNames` veya `TSVWIthNames` biçim ve ilk veri satırı eksik. [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sözlük mevcut değilse sözlük yeniden sabit kazasında. Bu hata 19.1.6'da ortaya çıktı. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) +- Sabit `ALL JOIN` sağ tabloda çiftleri ile. [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) +- Sabit segmentasyon hatası `use_uncompressed_cache=1` ve yanlış sıkıştırılmamış boyutta istisna. Bu hata 19.1.6'da ortaya çıktı. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alesapin](https://github.com/alesapin)) +- Sabit `compile_expressions` büyük (ınt16'dan daha fazla) tarihlerin karşılaştırılması ile hata. [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alesapin](https://github.com/alesapin)) +- Tablo işlevinden seçerken sabit sonsuz döngü `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yüklem optimizasyonunu geçici olarak devre dışı bırak `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([Kış Zhang](https://github.com/zhang2014)) +- Sabit `Illegal instruction` eski CPU'larda base64 işlevlerini kullanırken hata. Bu hata yalnızca ClickHouse gcc-8 ile derlendiğinde yeniden üretildi. [\#4275](https://github.com/ClickHouse/ClickHouse/pull/4275) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit `No message received` TLS bağlantısı üzerinden PostgreSQL ODBC sürücüsü ile etkileşimde bulunurken hata. MySQL ODBC sürücüsü kullanırken de segfault giderir. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit yanlış sonuç ne zaman `Date` ve `DateTime` argümanlar koşullu operatörün dallarında kullanılır (işlev `if`). Fonksiyon için genel durum eklendi `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ClickHouse sözlükler şimdi içinde yük `clickhouse` işleyiş. [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit kilitlenme zaman `SELECT` ile bir tablo fromdan `File` motor sonra yeniden denendi `No such file or directory` hatasız. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Seçerken sabit yarış durumu `system.tables` verebilir `table doesn't exist` hatasız. [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `clickhouse-client` etkileşimli modda çalıştırıldıysa, komut satırı önerileri için veri yüklerken çıkışta segfault yapabilir. [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçeren mutasyonların yürütülmesi bir hata düzeltildi `IN` operatörler yanlış sonuçlar üretiyordu. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) +- Sabit hata: bir veritabanı varsa `Dictionary` motor, tüm sözlükler sunucu başlangıçta yüklemeye zorlanır ve localhost'tan ClickHouse kaynağı olan bir sözlük varsa, sözlük yüklenemez. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sistem günlükleri sunucu kapatma yeniden oluşturmak için denendiğinde hata düzeltildi. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Doğru türü doğru şekilde döndürün ve kilitleri düzgün şekilde tutun `joinGet` işlev. [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([Amos Kuşu](https://github.com/amosbird)) +- Katma `sumMapWithOverflow` işlev. [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) +- Sabit segfault ile `allow_experimental_multiple_joins_emulation`. [52de2c](https://github.com/ClickHouse/ClickHouse/commit/52de2cd927f7b5257dd67e175f0a5560a48840d0) ([Artem Zuikov](https://github.com/4ertus2)) +- Yanlış ile Sabit hata `Date` ve `DateTime` karşılaştırma. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) +- Tanımsız davranış sanitizer altında sabit fuzz testi: eklendi parametre tipi kontrol için `quantile*Weighted` fonksiyonlar ailesi. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eski veri parçalarının çıkarılması ile başarısız olabilir sabit nadir yarış durumu `File not found` hatasız. [\#4378](https://github.com/ClickHouse/ClickHouse/pull/4378) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eksik /etc/clickhouse-server/config ile paketi yükleyin.xml. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) + +#### Yapı / Test / Ambalaj Geliştirmeleri {#buildtestingpackaging-improvements-5} + +- Debian paketi: yapılandırmaya göre /etc/clickhouse-server/preprocessed bağlantısını düzeltin. [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) +- FreeBSD için çeşitli yapı düzeltmeleri. [\#4225](https://github.com/ClickHouse/ClickHouse/pull/4225) ([proller](https://github.com/proller)) +- Perftest'te tablolar oluşturma, doldurma ve bırakma yeteneği eklendi. [\#4220](https://github.com/ClickHouse/ClickHouse/pull/4220) ([alesapin](https://github.com/alesapin)) +- Yinelenen kontrol etmek için bir komut dosyası eklendi içerir. [\#4326](https://github.com/ClickHouse/ClickHouse/pull/4326) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Performans testinde dizin sorguları çalıştırmak için yeteneği eklendi. [\#4264](https://github.com/ClickHouse/ClickHouse/pull/4264) ([alesapin](https://github.com/alesapin)) +- Hata ayıklama sembolleri ile paket yüklü olması önerilir. [\#4274](https://github.com/ClickHouse/ClickHouse/pull/4274) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Performansın yeniden düzenlenmesi-test. Daha iyi günlüğü ve sinyalleri işleme. [\#4171](https://github.com/ClickHouse/ClickHouse/pull/4171) ([alesapin](https://github.com/alesapin)) +- Anonimleştirilmiş yandex'e dokümanlar eklendi.Metrika veri setleri. [\#4164](https://github.com/ClickHouse/ClickHouse/pull/4164) ([alesapin](https://github.com/alesapin)) +- Аdded tool for converting an old month-partitioned part to the custom-partitioned format. [\#4195](https://github.com/ClickHouse/ClickHouse/pull/4195) ([Alex Zatelepin](https://github.com/ztlpn)) +- S3'te iki veri kümesi hakkında dokümanlar eklendi. [\#4144](https://github.com/ClickHouse/ClickHouse/pull/4144) ([alesapin](https://github.com/alesapin)) +- Çekme istekleri açıklamasından changelog oluşturan komut dosyası eklendi. [\#4169](https://github.com/ClickHouse/ClickHouse/pull/4169) [\#4173](https://github.com/ClickHouse/ClickHouse/pull/4173) ([KochetovNicolai](https://github.com/KochetovNicolai)) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- ClickHouse için kukla modülü eklendi. [\#4182](https://github.com/ClickHouse/ClickHouse/pull/4182) ([Maxim Fedotov](https://github.com/MaxFedotov)) +- Bir grup belgesiz işlev için dokümanlar eklendi. [\#4168](https://github.com/ClickHouse/ClickHouse/pull/4168) ([Kış Zhang](https://github.com/zhang2014)) +- Arm yapı düzeltmeleri. [\#4210](https://github.com/ClickHouse/ClickHouse/pull/4210)[\#4306](https://github.com/ClickHouse/ClickHouse/pull/4306) [\#4291](https://github.com/ClickHouse/ClickHouse/pull/4291) ([proller](https://github.com/proller)) ([proller](https://github.com/proller)) +- Sözlük testleri artık çalıştırmak mümkün `ctest`. [\#4189](https://github.com/ClickHouse/ClickHouse/pull/4189) ([proller](https://github.com/proller)) +- Şimdi `/etc/ssl` SSL sertifikaları ile varsayılan dizin olarak kullanılır. [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Başlangıçta SSE ve AVX talimat kontrol eklendi. [\#4234](https://github.com/ClickHouse/ClickHouse/pull/4234) ([Igr](https://github.com/igron99)) +- İnit betiği, başlayana kadar sunucuyu bekleyecektir. [\#4281](https://github.com/ClickHouse/ClickHouse/pull/4281) ([proller](https://github.com/proller)) + +#### Geriye Dönük Uyumsuz Değişiklikler {#backward-incompatible-changes-1} + +- Kaldırıyordu `allow_experimental_low_cardinality_type` ayar. `LowCardinality` veri türleri üretime hazırdır. [\#4323](https://github.com/ClickHouse/ClickHouse/pull/4323) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kullanılabilir bellek miktarına göre işaretle önbellek boyutunu ve sıkıştırılmamış önbellek boyutunu azaltın. [\#4240](https://github.com/ClickHouse/ClickHouse/pull/4240) ([Lopatin Konstantin](https://github.com/k-lopatin) +- Eklenen anahtar kelime `INDEX` içinde `CREATE TABLE` sorgu. Adı olan bir sütun `index` backticks veya çift tırnak ile alıntı olmalıdır: `` `index` ``. [\#4143](https://github.com/ClickHouse/ClickHouse/pull/4143) ([Nikita Vasilev](https://github.com/nikvas0)) +- `sumMap` şimdi taşma yerine sonuç türünü tanıtın. Eskiler `sumMap` davranış kullanılarak elde edilebilir `sumMapWithOverflow` işlev. [\#4151](https://github.com/ClickHouse/ClickHouse/pull/4151) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) + +#### Performans İyileştirmeleri {#performance-improvements-4} + +- `std::sort` yerine göre `pdqsort` olmadan sorgular için `LIMIT`. [\#4236](https://github.com/ClickHouse/ClickHouse/pull/4236) ([Evgenii Pravda](https://github.com/kvinty)) +- Şimdi sunucu, genel iş parçacığı havuzundan iş parçacıklarını yeniden kullanır. Bu, bazı köşe durumlarda performansı etkiler. [\#4150](https://github.com/ClickHouse/ClickHouse/pull/4150) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Geliştirmeler {#improvements-5} + +- FreeBSD için AIO desteği uygulandı. [\#4305](https://github.com/ClickHouse/ClickHouse/pull/4305) ([urgordeadbeef](https://github.com/urgordeadbeef)) +- `SELECT * FROM a JOIN b USING a, b` şimdi geri dön `a` ve `b` sadece sol tablodan sütunlar. [\#4141](https://github.com/ClickHouse/ClickHouse/pull/4141) ([Artem Zuikov](https://github.com/4ertus2)) +- Vermek `-C` olarak çalışmak için müşterinin seçeneği `-c` seçenek. [\#4232](https://github.com/ClickHouse/ClickHouse/pull/4232) ([syominsergey](https://github.com/syominsergey)) +- Şimdi seçenek `--password` değer olmadan kullanılan stdın'den şifre gerektirir. [\#4230](https://github.com/ClickHouse/ClickHouse/pull/4230) ([BSD\_Conqueror](https://github.com/bsd-conqueror)) +- İçeren dize değişmezleri unescaped metacharacters eklendi vurgulama `LIKE` ifadeler veya regexps. [\#4327](https://github.com/ClickHouse/ClickHouse/pull/4327) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İstemci soketi kaybolursa HTTP salt okunur sorgularının iptal edilmesi eklendi. [\#4213](https://github.com/ClickHouse/ClickHouse/pull/4213) ([nvartolomei](https://github.com/nvartolomei)) +- Şimdi sunucu, istemci bağlantılarını canlı tutmak için ilerlemeyi bildiriyor. [\#4215](https://github.com/ClickHouse/ClickHouse/pull/4215) ([I ivanvan](https://github.com/abyss7)) +- Sorgu ile OPTİMİZE etmek için biraz daha iyi mesaj `optimize_throw_if_noop` ayar Etkin. [\#4294](https://github.com/ClickHouse/ClickHouse/pull/4294) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Desteği eklendi `--version` clickhouse sunucusu için Seçenek. [\#4251](https://github.com/ClickHouse/ClickHouse/pull/4251) ([Lopatin Konstantin](https://github.com/k-lopatin)) +- Katma `--help/-h` seçeneği `clickhouse-server`. [\#4233](https://github.com/ClickHouse/ClickHouse/pull/4233) ([Yuriy Baranov](https://github.com/yurriy)) +- Toplam işlev durumu sonucu ile skaler alt sorgular için destek eklendi. [\#4348](https://github.com/ClickHouse/ClickHouse/pull/4348) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Geliştirilmiş sunucu kapatma süresi ve bekleme süresini değiştirir. [\#4372](https://github.com/ClickHouse/ClickHouse/pull/4372) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sisteme replicated\_can\_become\_leader ayarı hakkında bilgi eklendi.yinelemeler ve çoğaltma lider olmaya çalışmayacaksa günlüğü ekleyin. [\#4379](https://github.com/ClickHouse/ClickHouse/pull/4379) ([Alex Zatelepin](https://github.com/ztlpn)) + +## ClickHouse sürümü 19.1 {#clickhouse-release-19-1} + +### ClickHouse sürümü 19.1.14, 2019-03-14 {#clickhouse-release-19-1-14-2019-03-14} + +- Sabit hata `Column ... queried more than once` bu ayar eğer gerçekleşebilir `asterisk_left_columns_only` kullanılması durumunda 1 olarak ayarlanır `GLOBAL JOIN` ile `SELECT *` (nadir bir durum). Sorun 19.3 ve daha yeni sürümlerde mevcut değil. [6bac7d8d](https://github.com/ClickHouse/ClickHouse/pull/4692/commits/6bac7d8d11a9b0d6de0b32b53c47eb2f6f8e7062) ([Artem Zuikov](https://github.com/4ertus2)) + +### ClickHouse sürümü 19.1.13, 2019-03-12 {#clickhouse-release-19-1-13-2019-03-12} + +Bu sürüm, 19.3.7 ile tam olarak aynı yamalar kümesini içerir. + +### ClickHouse sürümü 19.1.10, 2019-03-03 {#clickhouse-release-19-1-10-2019-03-03} + +Bu sürüm, 19.3.6 ile tam olarak aynı yamalar kümesini içerir. + +## ClickHouse sürümü 19.1 {#clickhouse-release-19-1-1} + +### ClickHouse sürümü 19.1.9, 2019-02-21 {#clickhouse-release-19-1-9-2019-02-21} + +#### Hata düzeltmeleri {#bug-fixes-18} + +- Yanlış uygulama nedeniyle eski sürümlerle geriye dönük uyumsuzluk düzeltildi `send_logs_level` ayar. [\#4445](https://github.com/ClickHouse/ClickHouse/pull/4445) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tablo fonksiyonunun geriye dönük uyumsuzluğu düzeltildi `remote` sütun yorumları ile tanıtıldı. [\#4446](https://github.com/ClickHouse/ClickHouse/pull/4446) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.1.8, 2019-02-16 {#clickhouse-release-19-1-8-2019-02-16} + +#### Hata Düzeltmeleri {#bug-fixes-19} + +- Eksik /etc/clickhouse-server/config ile paketi yükleyin.xml. [\#4343](https://github.com/ClickHouse/ClickHouse/pull/4343) ([proller](https://github.com/proller)) + +## ClickHouse sürümü 19.1 {#clickhouse-release-19-1-2} + +### ClickHouse sürümü 19.1.7, 2019-02-15 {#clickhouse-release-19-1-7-2019-02-15} + +#### Hata Düzeltmeleri {#bug-fixes-20} + +- Doğru türü doğru şekilde döndürün ve kilitleri düzgün şekilde tutun `joinGet` işlev. [\#4153](https://github.com/ClickHouse/ClickHouse/pull/4153) ([Amos Kuşu](https://github.com/amosbird)) +- Sistem günlükleri sunucu kapatma yeniden oluşturmak için denendiğinde hata düzeltildi. [\#4254](https://github.com/ClickHouse/ClickHouse/pull/4254) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit hata: bir veritabanı varsa `Dictionary` motor, tüm sözlükler sunucu başlangıçta yüklemeye zorlanır ve localhost'tan ClickHouse kaynağı olan bir sözlük varsa, sözlük yüklenemez. [\#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçeren mutasyonların yürütülmesi bir hata düzeltildi `IN` operatörler yanlış sonuçlar üretiyordu. [\#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) +- `clickhouse-client` etkileşimli modda çalıştırıldıysa, komut satırı önerileri için veri yüklerken çıkışta segfault yapabilir. [\#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Seçerken sabit yarış durumu `system.tables` verebilir `table doesn't exist` hatasız. [\#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit kilitlenme zaman `SELECT` ile bir tablo fromdan `File` motor sonra yeniden denendi `No such file or directory` hatasız. [\#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bir sorun düzeltildi: yerel ClickHouse sözlükleri TCP üzerinden yüklenir, ancak işlem içinde yüklenmelidir. [\#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit `No message received` TLS bağlantısı üzerinden PostgreSQL ODBC sürücüsü ile etkileşimde bulunurken hata. MySQL ODBC sürücüsü kullanırken de segfault giderir. [\#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yüklem optimizasyonunu geçici olarak devre dışı bırak `ORDER BY`. [\#3890](https://github.com/ClickHouse/ClickHouse/pull/3890) ([Kış Zhang](https://github.com/zhang2014)) +- Tablo işlevinden seçerken sabit sonsuz döngü `numbers(0)`. [\#4280](https://github.com/ClickHouse/ClickHouse/pull/4280) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit `compile_expressions` büyük (ınt16'dan daha fazla) tarihlerin karşılaştırılması ile hata. [\#4341](https://github.com/ClickHouse/ClickHouse/pull/4341) ([alesapin](https://github.com/alesapin)) +- Sabit segmentasyon hatası `uncompressed_cache=1` ve yanlış sıkıştırılmamış boyutta istisna. [\#4186](https://github.com/ClickHouse/ClickHouse/pull/4186) ([alesapin](https://github.com/alesapin)) +- Sabit `ALL JOIN` sağ tabloda çiftleri ile. [\#4184](https://github.com/ClickHouse/ClickHouse/pull/4184) ([Artem Zuikov](https://github.com/4ertus2)) +- Yaparken yanlış davranış düzeltildi `INSERT ... SELECT ... FROM file(...)` sorgu ve dosya var `CSVWithNames` veya `TSVWIthNames` biçim ve ilk veri satırı eksik. [\#4297](https://github.com/ClickHouse/ClickHouse/pull/4297) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit toplam fonksiyonları yürütme ile `Array(LowCardinality)` değişkenler. [\#4055](https://github.com/ClickHouse/ClickHouse/pull/4055) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Debian paketi: yapılandırmaya göre /etc/clickhouse-server/preprocessed bağlantısını düzeltin. [\#4205](https://github.com/ClickHouse/ClickHouse/pull/4205) ([proller](https://github.com/proller)) +- Tanımsız davranış sanitizer altında sabit fuzz testi: eklendi parametre tipi kontrol için `quantile*Weighted` fonksiyonlar ailesi. [\#4145](https://github.com/ClickHouse/ClickHouse/pull/4145) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yapmak `START REPLICATED SENDS` komut Başlat çoğaltılmış gönderir. [\#4229](https://github.com/ClickHouse/ClickHouse/pull/4229) ([nvartolomei](https://github.com/nvartolomei)) +- Sabit `Not found column` join on bölümündeki yinelenen sütunlar için. [\#4279](https://github.com/ClickHouse/ClickHouse/pull/4279) ([Artem Zuikov](https://github.com/4ertus2)) +- Şimdi `/etc/ssl` SSL sertifikaları ile varsayılan dizin olarak kullanılır. [\#4167](https://github.com/ClickHouse/ClickHouse/pull/4167) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sözlük mevcut değilse sözlük yeniden sabit kazasında. [\#4188](https://github.com/ClickHouse/ClickHouse/pull/4188) ([proller](https://github.com/proller)) +- Yanlış ile Sabit hata `Date` ve `DateTime` karşılaştırma. [\#4237](https://github.com/ClickHouse/ClickHouse/pull/4237) ([valexey](https://github.com/valexey)) +- Sabit yanlış sonuç ne zaman `Date` ve `DateTime` argümanlar koşullu operatörün dallarında kullanılır (işlev `if`). Fonksiyon için genel durum eklendi `if`. [\#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### ClickHouse sürümü 19.1.6, 2019-01-24 {#clickhouse-release-19-1-6-2019-01-24} + +#### Yenilik {#new-features-7} + +- Tablolar için sütun sıkıştırma codec başına özel. [\#3899](https://github.com/ClickHouse/ClickHouse/pull/3899) [\#4111](https://github.com/ClickHouse/ClickHouse/pull/4111) ([alesapin](https://github.com/alesapin), [Kış Zhang](https://github.com/zhang2014), [Anatoly](https://github.com/Sindbag)) +- Eklendi sıkıştırma codec `Delta`. [\#4052](https://github.com/ClickHouse/ClickHouse/pull/4052) ([alesapin](https://github.com/alesapin)) +- İzin ver `ALTER` sıkıştırma codec. [\#4054](https://github.com/ClickHouse/ClickHouse/pull/4054) ([alesapin](https://github.com/alesapin)) +- Eklenen fonksiyonlar `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` SQL standart uyumluluk için. [\#3826](https://github.com/ClickHouse/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov)) +- Yazma desteği `HDFS` tablolar ve `hdfs` tablo işlevi. [\#4084](https://github.com/ClickHouse/ClickHouse/pull/4084) ([alesapin](https://github.com/alesapin)) +- Büyük samanlıkta birden fazla sabit dizeleri aramak için fonksiyonlar eklendi: `multiPosition`, `multiSearch` ,`firstMatch` ayrıca ile `-UTF8`, `-CaseInsensitive`, ve `-CaseInsensitiveUTF8` varyantlar. [\#4053](https://github.com/ClickHouse/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1)) +- Kullanılmayan parçaların budaması `SELECT` sharding anahtarına göre sorgu filtreleri (ayar `optimize_skip_unused_shards`). [\#3851](https://github.com/ClickHouse/ClickHouse/pull/3851) ([Gleb Kanterov](https://github.com/kanterov), [I ivanvan](https://github.com/abyss7)) +- Vermek `Kafka` blok başına bazı ayrıştırma hatalarını görmezden gelmek için motor. [\#4094](https://github.com/ClickHouse/ClickHouse/pull/4094) ([I ivanvan](https://github.com/abyss7)) +- İçin destek eklendi `CatBoost` multiclass modelleri değerlendirme. İşlev `modelEvaluate` multiclass modelleri için sınıf başına ham tahminleri ile tuple döndürür. `libcatboostmodel.so` ile inşa edilmelidir [\#607](https://github.com/catboost/catboost/pull/607). [\#3959](https://github.com/ClickHouse/ClickHouse/pull/3959) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Eklenen fonksiyonlar `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [\#4097](https://github.com/ClickHouse/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea)) +- Karma fonksiyonları eklendi `xxHash64` ve `xxHash32`. [\#3905](https://github.com/ClickHouse/ClickHouse/pull/3905) ([filimonov](https://github.com/filimonov)) +- Katma `gccMurmurHash` aynı karma tohumu kullanan karma işlevi (GCC aromalı üfürüm hash) [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191) [\#4000](https://github.com/ClickHouse/ClickHouse/pull/4000) ([sundyli](https://github.com/sundy-li)) +- Karma fonksiyonları eklendi `javaHash`, `hiveHash`. [\#3811](https://github.com/ClickHouse/ClickHouse/pull/3811) ([shangshujie365](https://github.com/shangshujie365)) +- Eklenen tablo fonksiyonu `remoteSecure`. Fonksiyonu olarak çalışır `remote`, ancak güvenli bağlantı kullanır. [\#4088](https://github.com/ClickHouse/ClickHouse/pull/4088) ([proller](https://github.com/proller)) + +#### Deneysel özellikler {#experimental-features-3} + +- Birden fazla Katıl emımlı em emülasyon eklendi (`allow_experimental_multiple_joins_emulation` ayar). [\#3946](https://github.com/ClickHouse/ClickHouse/pull/3946) ([Artem Zuikov](https://github.com/4ertus2)) + +#### Hata Düzeltmeleri {#bug-fixes-21} + +- Yapmak `compiled_expression_cache_size` bellek tüketimini azaltmak için varsayılan olarak sınırlı ayarı. [\#4041](https://github.com/ClickHouse/ClickHouse/pull/4041) ([alesapin](https://github.com/alesapin)) +- Çoğaltılmış tabloların değiştirmelerini gerçekleştiren iş parçacıklarında ve zookeeper'dan yapılandırmayı güncelleyen iş parçacıklarında hangup'lara yol açan bir hatayı düzeltin. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3891](https://github.com/ClickHouse/ClickHouse/issues/3891) [\#3934](https://github.com/ClickHouse/ClickHouse/pull/3934) ([Alex Zatelepin](https://github.com/ztlpn)) +- Dağıtılmış bir ALTER görevi yürütürken bir yarış durumu düzeltildi. Yarış durumu, görevi yürütmeye çalışan birden fazla kopyaya ve bir ZooKeeper hatasıyla başarısız olan tüm kopyalara yol açtı. [\#3904](https://github.com/ClickHouse/ClickHouse/pull/3904) ([Alex Zatelepin](https://github.com/ztlpn)) +- Bir hatayı düzeltin `from_zk` zookeeper zaman aşımına uğradıktan sonra yapılandırma öğeleri yenilenmedi. [\#2947](https://github.com/ClickHouse/ClickHouse/issues/2947) [\#3947](https://github.com/ClickHouse/ClickHouse/pull/3947) ([Alex Zatelepin](https://github.com/ztlpn)) +- IPv4 alt ağ maskeleri için yanlış önek ile hatayı düzeltin. [\#3945](https://github.com/ClickHouse/ClickHouse/pull/3945) ([alesapin](https://github.com/alesapin)) +- Sabit kaza (`std::terminate`) nadir durumlarda, tükenmiş kaynaklar nedeniyle yeni bir iş parçacığı oluşturulamazsa. [\#3956](https://github.com/ClickHouse/ClickHouse/pull/3956) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fix hata ne zaman `remote` In için yanlış kısıtlamalar kullanıldığında tablo işlevi yürütme `getStructureOfRemoteTable`. [\#4009](https://github.com/ClickHouse/ClickHouse/pull/4009) ([alesapin](https://github.com/alesapin)) +- Netlink soketleri sızıntısını düzeltin. Hiçbir zaman silinmedikleri bir havuza yerleştirildiler ve tüm geçerli soketler kullanımda olduğunda yeni bir iş parçacığının başlangıcında yeni soketler oluşturuldu. [\#4017](https://github.com/ClickHouse/ClickHouse/pull/4017) ([Alex Zatelepin](https://github.com/ztlpn)) +- Kapanış ile hatayı düzeltin `/proc/self/fd` tüm fds'den önceki dizin okundu `/proc` çatal sonra `odbc-bridge` alt işlem. [\#4120](https://github.com/ClickHouse/ClickHouse/pull/4120) ([alesapin](https://github.com/alesapin)) +- Birincil anahtarda kullanım dizesi durumunda uint monotonik dönüşüm için sabit dize. [\#3870](https://github.com/ClickHouse/ClickHouse/pull/3870) ([Kış Zhang](https://github.com/zhang2014)) +- Tamsayı dönüşüm fonksiyonu Monotonluk hesaplanmasında Sabit hata. [\#3921](https://github.com/ClickHouse/ClickHouse/pull/3921) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit segfault içinde `arrayEnumerateUniq`, `arrayEnumerateDense` bazı geçersiz argümanlar durumunda işlevler. [\#3909](https://github.com/ClickHouse/ClickHouse/pull/3909) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- STORAGEMERGE'DE ub'yi düzeltin. [\#3910](https://github.com/ClickHouse/ClickHouse/pull/3910) ([Amos Kuşu](https://github.com/amosbird)) +- Fonksiyonlarda sabit segfault `addDays`, `subtractDays`. [\#3913](https://github.com/ClickHouse/ClickHouse/pull/3913) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit hata: fonksiyonlar `round`, `floor`, `trunc`, `ceil` tamsayı argümanı ve büyük negatif ölçekte yürütüldüğünde sahte sonuç döndürebilir. [\#3914](https://github.com/ClickHouse/ClickHouse/pull/3914) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tarafından uyarılan bir hata düzeltildi ‘kill query sync’ bu da bir çekirdek çöplüğüne yol açar. [\#3916](https://github.com/ClickHouse/ClickHouse/pull/3916) ([muVulDeePecker](https://github.com/fancyqlx)) +- Boş çoğaltma kuyruğundan sonra uzun gecikmeyle hatayı düzeltin. [\#3928](https://github.com/ClickHouse/ClickHouse/pull/3928) [\#3932](https://github.com/ClickHouse/ClickHouse/pull/3932) ([alesapin](https://github.com/alesapin)) +- İle tabloya ekleme durumunda sabit aşırı bellek kullanımı `LowCardinality` birincil anahtar. [\#3955](https://github.com/ClickHouse/ClickHouse/pull/3955) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Sabit `LowCardinality` için serileştirme `Native` boş diziler durumunda format. [\#3907](https://github.com/ClickHouse/ClickHouse/issues/3907) [\#4011](https://github.com/ClickHouse/ClickHouse/pull/4011) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Tek LowCardinality sayısal sütun tarafından dıstınct kullanırken yanlış sonuç düzeltildi. [\#3895](https://github.com/ClickHouse/ClickHouse/issues/3895) [\#4012](https://github.com/ClickHouse/ClickHouse/pull/4012) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- LowCardinality anahtarı ile sabit özel toplama (durumunda `compile` ayar etkinse) ' dir. [\#3886](https://github.com/ClickHouse/ClickHouse/pull/3886) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Çoğaltılmış tablo sorguları için kullanıcı ve şifre iletimini düzeltin. [\#3957](https://github.com/ClickHouse/ClickHouse/pull/3957) ([alesapin](https://github.com/alesapin)) ([小路](https://github.com/nicelulu)) +- Sözlükleri yeniden yüklerken sözlük veritabanındaki tabloları listelerken oluşabilecek çok nadir yarış durumu düzeltildi. [\#3970](https://github.com/ClickHouse/ClickHouse/pull/3970) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Toplaması veya küp ile kullanıldığında yanlış sonuç düzeltildi. [\#3756](https://github.com/ClickHouse/ClickHouse/issues/3756) [\#3837](https://github.com/ClickHouse/ClickHouse/pull/3837) ([Sam Chou](https://github.com/reflection)) +- Sorgu için sabit sütun takma adları `JOIN ON` sözdizimi ve dağıtılmış tablolar. [\#3980](https://github.com/ClickHouse/ClickHouse/pull/3980) ([Kış Zhang](https://github.com/zhang2014)) +- Dahili uygulamada Sabit hata `quantileTDigest` (Artem Vakhrushev tarafından bulundu). Bu hata hiçbir zaman Clickhouse'da olmaz ve yalnızca ClickHouse kod tabanını doğrudan bir kütüphane olarak kullananlar için geçerlidir. [\#3935](https://github.com/ClickHouse/ClickHouse/pull/3935) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Geliştirmeler {#improvements-6} + +- İçin destek `IF NOT EXISTS` içinde `ALTER TABLE ADD COLUMN` ile birlikte ifadeler `IF EXISTS` içinde `DROP/MODIFY/CLEAR/COMMENT COLUMN`. [\#3900](https://github.com/ClickHouse/ClickHouse/pull/3900) ([Boris Granveaud](https://github.com/bgranvea)) +- İşlev `parseDateTimeBestEffort`: format desteği `DD.MM.YYYY`, `DD.MM.YY`, `DD-MM-YYYY`, `DD-Mon-YYYY`, `DD/Month/YYYY` ve benzeri. [\#3922](https://github.com/ClickHouse/ClickHouse/pull/3922) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- `CapnProtoInputStream` şimdi pürüzlü yapıları destekleyin. [\#4063](https://github.com/ClickHouse/ClickHouse/pull/4063) ([Odin Hultgren Van Der Horst](https://github.com/Miniwoffer)) +- Kullanılabilirlik iyileştirme: sunucu işleminin veri dizininin sahibinden başlatıldığı bir kontrol eklendi. Veriler kök olmayan kullanıcıya aitse, sunucunun kökünden başlatılmasına izin vermeyin. [\#3785](https://github.com/ClickHouse/ClickHouse/pull/3785) ([sergey-v-galtsev](https://github.com/sergey-v-galtsev)) +- Birleştirme ile sorguların analizi sırasında gerekli sütunları kontrol etmenin daha iyi mantığı. [\#3930](https://github.com/ClickHouse/ClickHouse/pull/3930) ([Artem Zuikov](https://github.com/4ertus2)) +- Tek bir sunucuda çok sayıda dağıtılmış tablo durumunda bağlantı sayısını azalttı. [\#3726](https://github.com/ClickHouse/ClickHouse/pull/3726) ([Kış Zhang](https://github.com/zhang2014)) +- Desteklenen toplamlar için satır `WITH TOTALS` ODBC sürücüsü için sorgu. [\#3836](https://github.com/ClickHouse/ClickHouse/pull/3836) ([Maksim Koritckiy](https://github.com/nightweb)) +- Kullanmasına izin `Enum`eğer fonksiyon içinde tamsayılar olarak s. [\#3875](https://github.com/ClickHouse/ClickHouse/pull/3875) ([I ivanvan](https://github.com/abyss7)) +- Katma `low_cardinality_allow_in_native_format` ayar. Devre dışı bırakılırsa, kullanmayın `LowCadrinality` yazmak `Native` biçimli. [\#3879](https://github.com/ClickHouse/ClickHouse/pull/3879) ([KochetovNicolai](https://github.com/KochetovNicolai)) +- Daha düşük bellek kullanımı için derlenmiş ifadeler önbelleğinden bazı gereksiz nesneleri kaldırıldı. [\#4042](https://github.com/ClickHouse/ClickHouse/pull/4042) ([alesapin](https://github.com/alesapin)) +- Add check that `SET send_logs_level = 'value'` sorgu uygun değeri kabul eder. [\#3873](https://github.com/ClickHouse/ClickHouse/pull/3873) ([Sabyanin Maxim](https://github.com/s-mx)) +- Sabit veri türü türü dönüştürme işlevleri kontrol edin. [\#3896](https://github.com/ClickHouse/ClickHouse/pull/3896) ([Kış Zhang](https://github.com/zhang2014)) + +#### Performans İyileştirmeleri {#performance-improvements-5} + +- MergeTree ayarı ekleme `use_minimalistic_part_header_in_zookeeper`. Etkinse, çoğaltılmış tablolar kompakt parça meta verilerini tek parça znode depolar. Bu, ZooKeeper anlık görüntü boyutunu önemli ölçüde azaltabilir (özellikle tablolarda çok fazla sütun varsa). Bu ayarı etkinleştirdikten sonra, desteklemeyen bir sürüme indiremeyeceğinizi unutmayın. [\#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) +- İşlevler için DFA tabanlı bir uygulama ekleyin `sequenceMatch` ve `sequenceCount` durumda desen zaman içermez. [\#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) +- Tam sayı serileştirme için performans iyileştirme. [\#3968](https://github.com/ClickHouse/ClickHouse/pull/3968) ([Amos Kuşu](https://github.com/amosbird)) +- Sıfır sol dolgu PODArray böylece -1 öğesi her zaman geçerli ve sıfırlanır. Ofsetlerin dalsız hesaplanması için kullanılır. [\#3920](https://github.com/ClickHouse/ClickHouse/pull/3920) ([Amos Kuşu](https://github.com/amosbird)) +- Döndürüyordu `jemalloc` performans düşüşüne yol sürüm. [\#4018](https://github.com/ClickHouse/ClickHouse/pull/4018) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Geriye Dönük Uyumsuz Değişiklikler {#backward-incompatible-changes-2} + +- Kaldırılan belgesiz özellik `ALTER MODIFY PRIMARY KEY` çünkü onun yerini aldı. `ALTER MODIFY ORDER BY` komut. [\#3887](https://github.com/ClickHouse/ClickHouse/pull/3887) ([Alex Zatelepin](https://github.com/ztlpn)) +- Kaldırılan işlev `shardByHash`. [\#3833](https://github.com/ClickHouse/ClickHouse/pull/3833) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Türün sonucu ile skaler alt sorguları kullanmayı yasaklayın `AggregateFunction`. [\#3865](https://github.com/ClickHouse/ClickHouse/pull/3865) ([I ivanvan](https://github.com/abyss7)) + +#### Yapı / Test / Ambalaj Geliştirmeleri {#buildtestingpackaging-improvements-6} + +- PowerPC için destek eklendi (`ppc64le`) yapmak. [\#4132](https://github.com/ClickHouse/ClickHouse/pull/4132) ([Danila Kutenin](https://github.com/danlark1)) +- Duruma göre fonksiyonel testlerin ortak kullanılabilir veri kümesi üzerinde çalışır. [\#3969](https://github.com/ClickHouse/ClickHouse/pull/3969) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sunucu ile başlatılamıyor Sabit hata `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` Docker veya systemd-nspawn içindeki mesaj. [\#4136](https://github.com/ClickHouse/ClickHouse/pull/4136) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Güncel `rdkafka` kütüphane v1.0.0-RC5 için. Ham c arayüzü yerine cppkafka kullanılır. [\#4025](https://github.com/ClickHouse/ClickHouse/pull/4025) ([I ivanvan](https://github.com/abyss7)) +- Güncel `mariadb-client` kitaplık. UBSan tarafından bulunan sorunlardan biri düzeltildi. [\#3924](https://github.com/ClickHouse/ClickHouse/pull/3924) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- UBSan için bazı düzeltmeler oluşturur. [\#3926](https://github.com/ClickHouse/ClickHouse/pull/3926) [\#3021](https://github.com/ClickHouse/ClickHouse/pull/3021) [\#3948](https://github.com/ClickHouse/ClickHouse/pull/3948) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- UBSan yapı ile testlerin başına taahhüt ishal eklendi. +- PVS-Studio statik analyzer başına taahhüt ishal eklendi. +- PVS-Studio tarafından bulunan sabit hatalar. [\#4013](https://github.com/ClickHouse/ClickHouse/pull/4013) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit glibc uyumluluk sorunları. [\#4100](https://github.com/ClickHouse/ClickHouse/pull/4100) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Docker resimlerini 18.10'a taşıyın ve glibc \>= 2.28 için uyumluluk dosyası ekleyin [\#3965](https://github.com/ClickHouse/ClickHouse/pull/3965) ([alesapin](https://github.com/alesapin)) +- Kullanıcı sunucu Docker görüntü dizinleri chown istemiyorsanız env değişken ekleyin. [\#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([alesapin](https://github.com/alesapin)) +- Gelen uyar theıların en etkin `-Weverything` çınlama içinde. Etkin `-Wpedantic`. [\#3986](https://github.com/ClickHouse/ClickHouse/pull/3986) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sadece clang 8'de bulunan birkaç uyarı daha eklendi. [\#3993](https://github.com/ClickHouse/ClickHouse/pull/3993) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bağlanmak `libLLVM` paylaşılan bağlantı kullanırken bireysel LLVM libs yerine. [\#3989](https://github.com/ClickHouse/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej)) +- Test görüntüleri için sanitizer değişkenleri eklendi. [\#4072](https://github.com/ClickHouse/ClickHouse/pull/4072) ([alesapin](https://github.com/alesapin)) +- `clickhouse-server` debian paketi tavsiye edecek `libcap2-bin` kullanmak için paket `setcap` yetenekleri ayarlamak için bir araç. Bu isteğe bağlıdır. [\#4093](https://github.com/ClickHouse/ClickHouse/pull/4093) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Geliştirilmiş derleme süresi, sabit içerir. [\#3898](https://github.com/ClickHouse/ClickHouse/pull/3898) ([proller](https://github.com/proller)) +- Karma fonksiyonlar için performans testleri eklendi. [\#3918](https://github.com/ClickHouse/ClickHouse/pull/3918) ([filimonov](https://github.com/filimonov)) +- Sabit döngüsel kütüphane bağımlılıkları. [\#3958](https://github.com/ClickHouse/ClickHouse/pull/3958) ([proller](https://github.com/proller)) +- Düşük kullanılabilir bellek ile geliştirilmiş derleme. [\#4030](https://github.com/ClickHouse/ClickHouse/pull/4030) ([proller](https://github.com/proller)) +- Performans düşüşünü yeniden oluşturmak için test betiği eklendi `jemalloc`. [\#4036](https://github.com/ClickHouse/ClickHouse/pull/4036) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yorumlarda ve dize değişmezlerinde sabit yazım hataları `dbms`. [\#4122](https://github.com/ClickHouse/ClickHouse/pull/4122) ([maiha](https://github.com/maiha)) +- Yorumlarda sabit yazım hataları. [\#4089](https://github.com/ClickHouse/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty)) + +## [2018 için Changelog](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2018.md) {#changelog-for-2018} diff --git a/docs/tr/whats_new/changelog/index.md b/docs/tr/whats_new/changelog/index.md new file mode 100644 index 00000000000..5f446b526c7 --- /dev/null +++ b/docs/tr/whats_new/changelog/index.md @@ -0,0 +1,668 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Changelog +toc_priority: 74 +toc_title: '2020' +--- + +## ClickHouse sürüm v20. 3 {#clickhouse-release-v20-3} + +### ClickHouse yayın v20.3. 4. 10, 2020-03-20 {#clickhouse-release-v20-3-4-10-2020-03-20} + +#### Hata Düzeltme {#bug-fix} + +- Bu sürüm ayrıca 20.1.8.41 tüm hata düzeltmeleri içerir +- Eksik düzeltme `rows_before_limit_at_least` http üzerinden sorgular için (işlemciler boru hattı ile). Bu düzeltmeler [\#9730](https://github.com/ClickHouse/ClickHouse/issues/9730). [\#9757](https://github.com/ClickHouse/ClickHouse/pull/9757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +### ClickHouse yayın v20.3. 3. 6, 2020-03-17 {#clickhouse-release-v20-3-3-6-2020-03-17} + +#### Hata Düzeltme {#bug-fix-1} + +- Bu sürüm ayrıca 20.1.7.38 tüm hata düzeltmeleri içerir +- Kullanıcı önceki sürümde mutasyonlar yürüttüyse, çoğaltmanın çalışmasına izin vermeyen bir çoğaltmada hatayı düzeltin. Bu düzeltmeler [\#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [\#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)). 20.3 sürümünü tekrar geriye dönük uyumlu hale getirir. +- Ayar Ekle `use_compact_format_in_distributed_parts_names` hangi dosyaları yazmak için izin verir `INSERT` içine sorgular `Distributed` daha kompakt formatlı tablo. Bu düzeltmeler [\#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [\#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)). 20.3 sürümünü tekrar geriye dönük uyumlu hale getirir. + +### ClickHouse yayın v20.3. 2. 1, 2020-03-12 {#clickhouse-release-v20-3-2-1-2020-03-12} + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change} + +- Sorunu düzelt fixedildi `file name too long` için veri gönderirken `Distributed` çok sayıda çoğaltma için tablolar. Çoğaltma kimlik bilgileri sunucu günlüğüne maruz sorunu düzeltildi. Diskteki dizin adı biçimi değiştirildi `[shard{shard_index}[_replica{replica_index}]]`. [\#8911](https://github.com/ClickHouse/ClickHouse/pull/8911) ([Mikhail Korotov](https://github.com/millb)) Yeni sürüme yükselttikten sonra, eski sunucu sürümü yeni dizin biçimini tanımadığı için manuel müdahale olmadan düşüremezsiniz. Downgrade yapmak istiyorsanız, ilgili dizinleri eski biçime el ile yeniden adlandırmanız gerekir. Bu değişiklik yalnızca zaman uyumsuz kullandıysanız geçerlidir `INSERT`s to `Distributed` Tablolar. 20.3.3 sürümünde, yeni formatı kademeli olarak etkinleştirmenize izin verecek bir ayar sunacağız. +- Mutasyon komutları için çoğaltma günlük girişlerinin biçimini değiştirdi. Yeni sürümü yüklemeden önce eski mutasyonların işlenmesini beklemeniz gerekir. +- Stacktraces döker basit bellek profiler uygulamak `system.trace_log` her n bayt yumuşak ayırma sınırı üzerinde [\#8765](https://github.com/ClickHouse/ClickHouse/pull/8765) ([I ivanvan](https://github.com/abyss7)) [\#9472](https://github.com/ClickHouse/ClickHouse/pull/9472) ([alexey-milovidov](https://github.com/alexey-milovidov) Olan ) sütun `system.trace_log` 'den değiştirildi `timer_type` -e doğru `trace_type`. Bu, üçüncü taraf performans analizi ve flamegraph işleme araçlarında değişiklikler gerektirecektir. +- İç iş parçacığı numarası yerine her yerde OS iş parçacığı kimliğini kullanın. Bu düzeltmeler [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) Yaş `clickhouse-client` sunucu ne zaman ayarı gönderme günlükleri alınamıyor `send_logs_level` yapılandırılmış günlük iletilerinin adları ve türleri değiştirildiğinden etkindir. Öte yandan, farklı sunucu sürümleri birbirine farklı türlerde günlükleri gönderebilir. Kullan whenmay theınca `send_logs_level` ayar, umursamamalısın. [\#8954](https://github.com/ClickHouse/ClickHouse/pull/8954) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kaldırmak `indexHint` işlev [\#9542](https://github.com/ClickHouse/ClickHouse/pull/9542) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kaldırmak `findClusterIndex`, `findClusterValue` işlevler. Bu düzeltmeler [\#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). Bu işlevleri kullanıyorsanız, bir e-posta gönderin `clickhouse-feedback@yandex-team.com` [\#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi sütun oluşturmak veya sütun eklemek için izin verilmez `SELECT` varsayılan ifade olarak alt sorgu. [\#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([alesapin](https://github.com/alesapin)) +- JOİN alt sorgular için takma adlar gerektirir. [\#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) +- Gelişmiş `ALTER MODIFY/ADD` sorgu mantığı. Şimdi yapamazsın `ADD` türü olmayan sütun, `MODIFY` varsayılan ifade sütun türünü değiştirmez ve `MODIFY` tür varsayılan ifade değerini kaybetmez. Düzeltiyor [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alesapin](https://github.com/alesapin)) +- Günlük yapılandırma değişiklikleri uygulamak için yeniden başlatılması için sunucu gerektirir. Bu, sunucunun silinmiş bir günlük dosyasına oturum açtığı hatayı önlemek için geçici bir geçici çözümdür (bkz. [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Ayar `experimental_use_processors` varsayılan olarak etkinleştirilir. Bu ayar, yeni sorgu potansiyel kullanımını sağlar. Bu dahili refactoring ve görünür bir değişiklik beklemiyoruz. Eğer herhangi bir sorun göreceksiniz, sıfır geri ayarlayın. [\#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Yenilik {#new-feature} + +- Eklemek `Avro` ve `AvroConfluent` giriş / çıkış biçimleri [\#8571](https://github.com/ClickHouse/ClickHouse/pull/8571) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8957](https://github.com/ClickHouse/ClickHouse/pull/8957) ([Andrew Onyshchuk](https://github.com/oandrew)) [\#8717](https://github.com/ClickHouse/ClickHouse/pull/8717) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Süresi dolmuş anahtarların çok iş parçacıklı ve engellenmeyen güncellemeleri `cache` sözlükler (eskileri okumak için isteğe bağlı izin ile). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Sorgu Ekle `ALTER ... MATERIALIZE TTL`. Süresi dolmuş verileri TTL ile kaldırmaya zorlayan mutasyonu çalıştırır ve TTL ile ilgili meta bilgileri tüm kısımlarda yeniden hesaplar. [\#8775](https://github.com/ClickHouse/ClickHouse/pull/8775) ([Anton Popov](https://github.com/CurtizJ)) +- Gerekirse Hashjoin'den Mergejoin'e (diskte) geçin [\#9082](https://github.com/ClickHouse/ClickHouse/pull/9082) ([Artem Zuikov](https://github.com/4ertus2)) +- Katma `MOVE PARTITION` için komut `ALTER TABLE` [\#4729](https://github.com/ClickHouse/ClickHouse/issues/4729) [\#6168](https://github.com/ClickHouse/ClickHouse/pull/6168) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Anında yapılandırma dosyasından depolama yapılandırmasını yeniden yükleme. [\#8594](https://github.com/ClickHouse/ClickHouse/pull/8594) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Değiştirmek için izin `storage_policy` daha az zengin olana. [\#8107](https://github.com/ClickHouse/ClickHouse/pull/8107) ([Vladimir Chebotarev](https://github.com/excitoon)) +- S3 depolama ve masa fonksiyonu için globs/joker karakterler için destek eklendi. [\#8851](https://github.com/ClickHouse/ClickHouse/pull/8851) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Uygulamak `bitAnd`, `bitOr`, `bitXor`, `bitNot` için `FixedString(N)` datatype. [\#9091](https://github.com/ClickHouse/ClickHouse/pull/9091) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Eklendi fonksiyonu `bitCount`. Bu düzeltmeler [\#8702](https://github.com/ClickHouse/ClickHouse/issues/8702). [\#8708](https://github.com/ClickHouse/ClickHouse/pull/8708) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#8749](https://github.com/ClickHouse/ClickHouse/pull/8749) ([ıkopylov](https://github.com/ikopylov)) +- Eklemek `generateRandom` verilen şema ile rastgele satırlar oluşturmak için tablo işlevi. Rasgele test tablosunu verilerle doldurmaya izin verir. [\#8994](https://github.com/ClickHouse/ClickHouse/pull/8994) ([Ilya Yatsishin](https://github.com/qoega)) +- `JSONEachRowFormat`: destek özel durumda zaman nesneleri üst düzey dizi içine. [\#8860](https://github.com/ClickHouse/ClickHouse/pull/8860) ([Kruglov Pavel](https://github.com/Avogar)) +- Şimdi bir sütun oluşturmak mümkün `DEFAULT` varsayılan bir sütuna bağlı olan ifade `ALIAS` ifade. [\#9489](https://github.com/ClickHouse/ClickHouse/pull/9489) ([alesapin](https://github.com/alesapin)) +- Belirtmek için izin ver `--limit` kaynak veri boyutundan daha fazla `clickhouse-obfuscator`. Veri farklı rastgele tohum ile kendini tekrar edecektir. [\#9155](https://github.com/ClickHouse/ClickHouse/pull/9155) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Katma `groupArraySample` fonksiyonu (benzer `groupArray`) rezervuar örnekleme algoritması ile. [\#8286](https://github.com/ClickHouse/ClickHouse/pull/8286) ([Amos Kuşu](https://github.com/amosbird)) +- Şimdi güncelleme kuyruğunun boyutunu izleyebilirsiniz `cache`/`complex_key_cache` sistem metrikleri aracılığıyla sözlükler. [\#9413](https://github.com/ClickHouse/ClickHouse/pull/9413) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Crlf'yi CSV çıkış biçiminde bir hat ayırıcı olarak ayarlamaya izin ver `output_format_csv_crlf_end_of_line` 1 set [\#8934](https://github.com/ClickHouse/ClickHouse/pull/8934) [\#8935](https://github.com/ClickHouse/ClickHouse/pull/8935) [\#8963](https://github.com/ClickHouse/ClickHouse/pull/8963) ([Mikhail Korotov](https://github.com/millb)) +- Daha fazla işlev uygulamak [H3](https://github.com/uber/h3) API: `h3GetBaseCell`, `h3HexAreaM2`, `h3IndexesAreNeighbors`, `h3ToChildren`, `h3ToString` ve `stringToH3` [\#8938](https://github.com/ClickHouse/ClickHouse/pull/8938) ([Nico Mandery](https://github.com/nmandery)) +- Yeni ayar tanıt introducedıldı: `max_parser_depth` maksimum yığın boyutunu kontrol etmek ve büyük karmaşık sorgulara izin vermek. Bu düzeltmeler [\#6681](https://github.com/ClickHouse/ClickHouse/issues/6681) ve [\#7668](https://github.com/ClickHouse/ClickHouse/issues/7668). [\#8647](https://github.com/ClickHouse/ClickHouse/pull/8647) ([Maxim Smirnov](https://github.com/qMBQx8GH)) +- Ayar ekleme `force_optimize_skip_unused_shards` kullanılmayan parçaların atlanması mümkün değilse atma ayarı [\#8805](https://github.com/ClickHouse/ClickHouse/pull/8805) ([Azat Khuzhin](https://github.com/azat)) +- Göndermek için veri depolamak için birden fazla Disk/birim yapılandırmaya izin ver `Distributed` motor [\#8756](https://github.com/ClickHouse/ClickHouse/pull/8756) ([Azat Khuzhin](https://github.com/azat)) +- Destek depolama politikası (``) geçici veri depolamak için. [\#8750](https://github.com/ClickHouse/ClickHouse/pull/8750) ([Azat Khuzhin](https://github.com/azat)) +- Katma `X-ClickHouse-Exception-Code` Özel durum veri göndermeden önce atıldı, ayarlanmış http üstbilgisi. Bu uygular [\#4971](https://github.com/ClickHouse/ClickHouse/issues/4971). [\#8786](https://github.com/ClickHouse/ClickHouse/pull/8786) ([Mikhail Korotov](https://github.com/millb)) +- Eklendi fonksiyonu `ifNotFinite`. Bu sadece sözdizimsel bir şeker: `ifNotFinite(x, y) = isFinite(x) ? x : y`. [\#8710](https://github.com/ClickHouse/ClickHouse/pull/8710) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Katma `last_successful_update_time` sütun içinde `system.dictionaries` Tablo [\#9394](https://github.com/ClickHouse/ClickHouse/pull/9394) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Eklemek `blockSerializedSize` işlev (sıkıştırma olmadan diskte boyut) [\#8952](https://github.com/ClickHouse/ClickHouse/pull/8952) ([Azat Khuzhin](https://github.com/azat)) +- Fonksiyon Ekle `moduloOrZero` [\#9358](https://github.com/ClickHouse/ClickHouse/pull/9358) ([hcz](https://github.com/hczhcz)) +- Eklenen sistem tabloları `system.zeros` ve `system.zeros_mt` yanı sıra masal fonksiyonları `zeros()` ve `zeros_mt()`. Tablolar (ve tablo işlevleri) adı ile tek sütun içerir `zero` ve tip `UInt8`. Bu sütun sıfır içerir. Birçok satır oluşturmak için en hızlı yöntem olarak test amaçları için gereklidir. Bu düzeltmeler [\#6604](https://github.com/ClickHouse/ClickHouse/issues/6604) [\#9593](https://github.com/ClickHouse/ClickHouse/pull/9593) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +#### Deneysel Özellik {#experimental-feature} + +- Yeni kompakt parça formatı ekleyin `MergeTree`- tüm sütunların tek bir dosyada saklandığı aile tabloları. Küçük ve sık insertlerin performansını artırmaya yardımcı olur. Eski biçim (sütun başına bir dosya) şimdi geniş olarak adlandırılır. Veri depolama biçimi ayarları tarafından kontrol edilir `min_bytes_for_wide_part` ve `min_rows_for_wide_part`. [\#8290](https://github.com/ClickHouse/ClickHouse/pull/8290) ([Anton Popov](https://github.com/CurtizJ)) +- S3 depolama için destek `Log`, `TinyLog` ve `StripeLog` Tablolar. [\#8862](https://github.com/ClickHouse/ClickHouse/pull/8862) ([Pavel Kovalenko](https://github.com/Jokser)) + +#### Hata Düzeltme {#bug-fix-2} + +- Günlük mesajlarında sabit tutarsız whitespaces. [\#9322](https://github.com/ClickHouse/ClickHouse/pull/9322) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İsimsiz dizilerin dizilerinin tablo oluşturulmasında iç içe geçmiş yapılar olarak düzleştirildiği hatayı düzeltin. [\#8866](https://github.com/ClickHouse/ClickHouse/pull/8866) ([achulkov2](https://github.com/achulkov2)) +- Sorunu ne zaman düzeltildi “Too many open files” glob deseniyle eşleşen çok fazla dosya varsa hata oluşabilir `File` tablo veya `file` tablo işlevi. Şimdi dosyalar tembel açılır. Bu düzeltmeler [\#8857](https://github.com/ClickHouse/ClickHouse/issues/8857) [\#8861](https://github.com/ClickHouse/ClickHouse/pull/8861) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bırak geçici tablo şimdi sadece geçici tablo düşer. [\#8907](https://github.com/ClickHouse/ClickHouse/pull/8907) ([Vitaly Baranov](https://github.com/vitlibar)) +- Sunucuyu kapattığımızda veya bir tabloyu AYIRDIĞIMIZDA/EKLEDİĞİMİZDE eski bölümü kaldırın. [\#8602](https://github.com/ClickHouse/ClickHouse/pull/8602) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Varsayılan diskin boş alanı nasıl hesapladığı için `data` dizin. Boş alan miktarı doğru hesaplanmazsa sorunu düzeltildi `data` dizin ayrı bir cihaza monte edilir (nadir durum). Bu düzeltmeler [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) ([Mikhail Korotov](https://github.com/millb)) +- In () ınside ile virgül (çapraz) birleşmesine izin ver. [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) ([Artem Zuikov](https://github.com/4ertus2)) +- WHERE bölümünde operatör gibi \[değil\] varsa, iç BİRLEŞİME çapraz yeniden yazmaya izin verin. [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) ([Artem Zuikov](https://github.com/4ertus2)) +- Sonra olası yanlış sonucu düzeltin `GROUP BY` etkin ayarı ile `distributed_aggregation_memory_efficient`. Düzeltiyor [\#9134](https://github.com/ClickHouse/ClickHouse/issues/9134). [\#9289](https://github.com/ClickHouse/ClickHouse/pull/9289) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Bulunan anahtarlar önbellek sözlüklerinin metriklerinde cevapsız olarak sayıldı. [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Çoğaltma protokolü uyumsuzluğunu düzeltme [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) +- Sabit yarış durumu `queue_task_handle` başlangıçta `ReplicatedMergeTree` Tablolar. [\#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Belirteç `NOT` işe yar inamadı `SHOW TABLES NOT LIKE` sorgu [\#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [\#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fonksiyona Aralık kontrolü eklendi `h3EdgeLengthM`. Bu kontrol olmadan, arabellek taşması mümkündür. [\#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Birden argümanlar (10'dan fazla) üçlü mantıksal OPs toplu hesaplamalarda bir hata düzeltildi. [\#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) +- Segfaults'a yol açabilecek prewhere optimizasyonunun hatasını düzeltin veya `Inconsistent number of columns got from MergeTreeRangeReader` özel. [\#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([Anton Popov](https://github.com/CurtizJ)) +- Beklenmeyen düzeltme `Timeout exceeded while reading from socket` zaman aşımı gerçekten aşıldı ve sorgu profiler etkinleştirildiğinde önce güvenli bağlantıda rasgele olur özel durum. Ayrıca ekleyin `connect_timeout_with_failover_secure_ms` ayarlar (varsayılan 100 MS), hangi benzer `connect_timeout_with_failover_ms`, ancak güvenli bağlantılar için kullanılır (çünkü SSL el sıkışması normal TCP bağlantısından daha yavaştır) [\#9026](https://github.com/ClickHouse/ClickHouse/pull/9026) ([tavplubix](https://github.com/tavplubix)) +- Mutasyon ile devlet asmak olabilir mutasyonlar sonuçlandırılması ile Fix hata `parts_to_do=0` ve `is_done=0`. [\#9022](https://github.com/ClickHouse/ClickHouse/pull/9022) ([alesapin](https://github.com/alesapin)) +- İle Yeni herhangi bir birleştirme mantığı kullanın `partial_merge_join` ayar. Yapmak mümkün `ANY|ALL|SEMI LEFT` ve `ALL INNER` ile birleş joinsir `partial_merge_join=1` şimdi. [\#8932](https://github.com/ClickHouse/ClickHouse/pull/8932) ([Artem Zuikov](https://github.com/4ertus2)) +- Shard şimdi bir istisna atmak yerine başlatıcıdan gelen ayarları shard'ın constaintlerine sıkıştırıyor. Bu düzeltme, başka bir kısıtlamalarla bir parçaya sorgular göndermeye izin verir. [\#9447](https://github.com/ClickHouse/ClickHouse/pull/9447) ([Vitaly Baranov](https://github.com/vitlibar)) +- Sabit bellek yönetimi sorunu `MergeTreeReadPool`. [\#8791](https://github.com/ClickHouse/ClickHouse/pull/8791) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Düzeltmek `toDecimal*OrNull()` dize ile çağrıldığında işlevler ailesi `e`. Düzeltiyor [\#8312](https://github.com/ClickHouse/ClickHouse/issues/8312) [\#8764](https://github.com/ClickHouse/ClickHouse/pull/8764) ([Artem Zuikov](https://github.com/4ertus2)) +- Emin olun `FORMAT Null` istemciye hiçbir veri gönderir. [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Bu zaman damgasını düzeltin `LiveViewBlockInputStream` güncellenmeyecek. `LIVE VIEW` deneysel bir özelliktir. [\#8644](https://github.com/ClickHouse/ClickHouse/pull/8644) ([vxider](https://github.com/Vxider)) [\#8625](https://github.com/ClickHouse/ClickHouse/pull/8625) ([vxider](https://github.com/Vxider)) +- Sabit `ALTER MODIFY TTL` eski TTL ifadelerini silmeye izin vermeyen yanlış davranış. [\#8422](https://github.com/ClickHouse/ClickHouse/pull/8422) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Mergetreeındexset sabit UBSan raporu. Bu düzeltmeler [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Davranışı Düzelt theildi `match` ve `extract` haystack sıfır bayt olduğunda işlevler. Haystack sabit olduğunda davranış yanlıştı. Bu düzeltmeler [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Apache Avro 3. parti kütüphanesinde yıkıcıdan atmaktan kaçının. [\#9066](https://github.com/ClickHouse/ClickHouse/pull/9066) ([Andrew Onyshchuk](https://github.com/oandrew)) +- Bir toplu işlem yoklama `Kafka` kısmen verilerde deliklere yol açabilir. [\#8876](https://github.com/ClickHouse/ClickHouse/pull/8876) ([filimonov](https://github.com/filimonov)) +- Düzeltmek `joinGet` null dönüş türleri ile. https://github.com/ClickHouse/ClickHouse/issues/8919 [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) ([Amos Kuşu](https://github.com/amosbird)) +- Sıkıştırıldığında veri uyumsuzluğunu düzeltin `T64` codec. [\#9016](https://github.com/ClickHouse/ClickHouse/pull/9016) ([Artem Zuikov](https://github.com/4ertus2)) Veri türü kimliklerini düzeltin `T64` etkilenen sürümlerde yanlış (de)sıkıştırmaya yol açan sıkıştırma codec bileşeni. [\#9033](https://github.com/ClickHouse/ClickHouse/pull/9033) ([Artem Zuikov](https://github.com/4ertus2)) +- Ayar Ekle `enable_early_constant_folding` ve hatalara yol açan bazı durumlarda devre dışı bırakın. [\#9010](https://github.com/ClickHouse/ClickHouse/pull/9010) ([Artem Zuikov](https://github.com/4ertus2)) +- Pushdown predicate optimizer'ı görünümle düzeltin ve Testi etkinleştirin [\#9011](https://github.com/ClickHouse/ClickHouse/pull/9011) ([Kış Zhang](https://github.com/zhang2014)) +- Segfault'u düzeltin `Merge` tablo readinglardan, okuma sırasında meydana gelebilecek `File` depolamalar [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) ([tavplubix](https://github.com/tavplubix)) +- Depolama politikası için bir kontrol eklendi `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE`. Aksi takdirde, yeniden başlatıldıktan sonra parçanın verilerini erişilemez hale getirebilir ve Clickhouse'un başlatılmasını önleyebilir. [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Tablo için ayarlanmış TTL varsa, değişiklikleri düzeltin. [\#8800](https://github.com/ClickHouse/ClickHouse/pull/8800) ([Anton Popov](https://github.com/CurtizJ)) +- Ne zaman gerçekleşebilecek yarış durumunu düzeltin `SYSTEM RELOAD ALL DICTIONARIES` bazı sözlük değiştirilirken/eklenirken/kaldırılırken yürütülür. [\#8801](https://github.com/ClickHouse/ClickHouse/pull/8801) ([Vitaly Baranov](https://github.com/vitlibar)) +- Önceki sürümlerde `Memory` Veritabanı Altyapısı boş veri yolu kullanır, böylece tablolar `path` directory (e.g. `/var/lib/clickhouse/`), not in data directory of database (e.g. `/var/lib/clickhouse/db_name`). [\#8753](https://github.com/ClickHouse/ClickHouse/pull/8753) ([tavplubix](https://github.com/tavplubix)) +- Varsayılan disk veya ilke eksik hakkında sabit yanlış günlük mesajları. [\#9530](https://github.com/ClickHouse/ClickHouse/pull/9530) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Dizi türlerinin bloom\_filter dizini için değil(has()) düzeltin. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) +- Bir tablodaki ilk sütunlara izin ver `Log` motor takma isim ol [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) ([I ivanvan](https://github.com/abyss7)) +- Okurken aralıkların sırasını düzeltin `MergeTree` bir iş parçacığı tablo. İstisn fromalara yol açabilir `MergeTreeRangeReader` veya yanlış sorgu sonuçları. [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) ([Anton Popov](https://github.com/CurtizJ)) +- Yapmak `reinterpretAsFixedString` dönmek `FixedString` yerine `String`. [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) ([Andrew Onyshchuk](https://github.com/oandrew)) +- Kullanıcı yanlış hata mesajı alabildiğinde son derece nadir durumlardan kaçının (`Success` ayrıntılı hata açıklaması yerine). [\#9457](https://github.com/ClickHouse/ClickHouse/pull/9457) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kullanırken çökmesine etmeyin `Template` boş satır şablonu ile biçimlendirin. [\#8785](https://github.com/ClickHouse/ClickHouse/pull/8785) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Sistem tabloları için meta veri dosyaları yanlış yerde oluşturulabilir [\#8653](https://github.com/ClickHouse/ClickHouse/pull/8653) ([tavplubix](https://github.com/tavplubix)) Düzeltiyor [\#8581](https://github.com/ClickHouse/ClickHouse/issues/8581). +- Önbellek sözlüğünde exception\_ptr üzerindeki veri yarışını düzeltin [\#8303](https://github.com/ClickHouse/ClickHouse/issues/8303). [\#9379](https://github.com/ClickHouse/ClickHouse/pull/9379) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Sorgu için bir özel durum atmayın `ATTACH TABLE IF NOT EXISTS`. Daha önce tablo zaten varsa atıldı, buna rağmen `IF NOT EXISTS` yan. [\#8967](https://github.com/ClickHouse/ClickHouse/pull/8967) ([Anton Popov](https://github.com/CurtizJ)) +- Özel durum mesajında eksik kapanış paren düzeltildi. [\#8811](https://github.com/ClickHouse/ClickHouse/pull/8811) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Mesajdan kaçının `Possible deadlock avoided` etkileşimli modda clickhouse-client başlangıcında. [\#9455](https://github.com/ClickHouse/ClickHouse/pull/9455) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Base64 kodlanmış değerin sonunda dolgu hatalı biçimlendirilmiş olabilir sorunu düzeltildi. Base64 kütüphanesini güncelleyin. Bu düzeltmeler [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), yaklar [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Veri kaybını önlemek `Kafka` nadir durumlarda istisna sonek okuduktan sonra ancak taahhütten önce gerçekleşir. Düzeltiyor [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378) [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) ([filimonov](https://github.com/filimonov)) +- Sabit istisna `DROP TABLE IF EXISTS` [\#8663](https://github.com/ClickHouse/ClickHouse/pull/8663) ([Nikita Vasilev](https://github.com/nikvas0)) +- Bir kullanıcı denediğinde çökmeyi düzeltin `ALTER MODIFY SETTING` eski formatlı için `MergeTree` masa motorları ailesi. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) +- Json ile ilgili işlevlerde Int64'e uymayan Uİnt64 numaraları için destek. Master simdjson güncelleyin. Bu düzeltmeler [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kesinlikle monotinik olmayan fonksiyonel indeks kullanıldığında ters yüklemlerin sabit yürütülmesi. [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) ([Alexander Kazakov](https://github.com/Akazz)) +- Kat trylamaya çalış trymayın `IN` sabit in `GROUP BY` [\#8868](https://github.com/ClickHouse/ClickHouse/pull/8868) ([Amos Kuşu](https://github.com/amosbird)) +- Hatayı düzeltin `ALTER DELETE` endeks bozulmasına yol açan mutasyonlar. Bu düzeltmeler [\#9019](https://github.com/ClickHouse/ClickHouse/issues/9019) ve [\#8982](https://github.com/ClickHouse/ClickHouse/issues/8982). Ayrıca son derece nadir yarış koşullarını düzeltin `ReplicatedMergeTree` `ALTER` sorgular. [\#9048](https://github.com/ClickHouse/ClickHouse/pull/9048) ([alesapin](https://github.com/alesapin)) +- Zaman ayarı `compile_expressions` etkin mi, alabilirsiniz `unexpected column` içinde `LLVMExecutableFunction` kullan whendığımızda `Nullable` tür [\#8910](https://github.com/ClickHouse/ClickHouse/pull/8910) ([Guillaume Tassery](https://github.com/YiuRULE)) +- İçin çoklu düzeltmeler `Kafka` motor: 1) tüketici grubu rebalance sırasında görünen çiftleri düzeltin. 2) nadir düzeltmek ‘holes’ veriler bir anket ile birkaç bölümden yoklandığında ve kısmen işlendiğinde ortaya çıktı (şimdi her zaman tüm anketli mesaj bloğunu işliyoruz / işliyoruz). 3) blok boyutuna göre yıkamaları düzeltin (bundan önce sadece zaman aşımı ile kızarma düzgün çalışıyordu). 4) daha iyi abonelik prosedürü (atama geri bildirimi ile). 5) testlerin daha hızlı çalışmasını sağlayın (varsayılan aralıklarla ve zaman aşımlarıyla). Verilerin daha önce blok boyutuna göre temizlenmemesi nedeniyle (belgelere göre olması gerektiği gibi), bu PR, varsayılan ayarlarla bazı performans düşüşlerine yol açabilir(daha sık ve daha az optimal olan daha küçük yıkama nedeniyle). Bu değişiklikten sonra performans sorunuyla karşılaşırsanız-lütfen artırın `kafka_max_block_size` tabloda daha büyük değere (örneğin `CREATE TABLE ...Engine=Kafka ... SETTINGS ... kafka_max_block_size=524288`). Düzeltiyor [\#7259](https://github.com/ClickHouse/ClickHouse/issues/7259) [\#8917](https://github.com/ClickHouse/ClickHouse/pull/8917) ([filimonov](https://github.com/filimonov)) +- Düzeltmek `Parameter out of bound` prewhere optimizasyonlarından sonra bazı sorgularda istisna. [\#8914](https://github.com/ClickHouse/ClickHouse/pull/8914) ([Baudouin Giard](https://github.com/bgiard)) +- Fonksiyon argümanlarının karışık sabitliği durumu düzeltildi `arrayZip`. [\#8705](https://github.com/ClickHouse/ClickHouse/pull/8705) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yürüt whenürken `CREATE` sorgu, depolama motoru argümanlarında sabit ifadeleri katlayın. Boş veritabanı adı geçerli veritabanı ile değiştirin. Düzeltiyor [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492) [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) ([tavplubix](https://github.com/tavplubix)) +- Artık basit döngüsel takma adlarla sütunlar oluşturmak veya eklemek mümkün değil `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) +- Bozuk orijinal parçası olabilir çift hareket ile bir hata düzeltildi. Eğer kullanıyorsanız bu önemlidir `ALTER TABLE MOVE` [\#8680](https://github.com/ClickHouse/ClickHouse/pull/8680) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Vermek `interval` backticks olmadan doğru ayrıştırmak için tanımlayıcı. Bir sorgu bile yürütülemez sabit sorun `interval` tanımlayıcı backticks veya çift tırnak içine alınır. Bu düzeltmeler [\#9124](https://github.com/ClickHouse/ClickHouse/issues/9124). [\#9142](https://github.com/ClickHouse/ClickHouse/pull/9142) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit fuzz testi ve yanlış davranış `bitTestAll`/`bitTestAny` işlevler. [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Olası kilitlenme/yanlış satır sayısını düzeltin `LIMIT n WITH TIES` n'th satırına eşit çok sayıda satır olduğunda. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) +- Enabled ile yazılmış parçalarla mutasyonları düzeltin `insert_quorum`. [\#9463](https://github.com/ClickHouse/ClickHouse/pull/9463) ([alesapin](https://github.com/alesapin)) +- İmha veri yarışı Fix `Poco::HTTPServer`. Sunucu başlatıldığında ve hemen kapatıldığında gerçekleşebilir. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) +- Çalışırken yanıltıcı bir hata mesajının gösterildiği hatayı düzeltin `SHOW CREATE TABLE a_table_that_does_not_exist`. [\#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2](https://github.com/achulkov2)) +- Sabit `Parameters are out of bound` bazı nadir durumlarda istisna `SELECT` CLA anuse when we have an `ORDER BY` ve bir `LIMIT` yan. [\#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Mutasyonlar kesinleşmesini düzeltin, zaten mutasyon yapıldığında durum olabilir `is_done=0`. [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([alesapin](https://github.com/alesapin)) +- Yürütül executingmesini engellemek `ALTER ADD INDEX` eski sözdizimi ile MergeTree tabloları için, çünkü çalışmıyor. [\#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) +- Sunucu başlatma sırasında erişim tablosu, hangi `LIVE VIEW` bağlıdır, böylece sunucu başlatmak mümkün olacak. Ayrıca kaldırmak `LIVE VIEW` ayırma sırasında bağımlılıklar `LIVE VIEW`. `LIVE VIEW` deneysel bir özelliktir. [\#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) +- Olası segfault'u düzeltin `MergeTreeRangeReader`, Yürüt whileürken `PREWHERE`. [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([Anton Popov](https://github.com/CurtizJ)) +- Sütun TTLs ile olası eşleşmeyen sağlama toplamlarını düzeltin. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) +- Sadece bir birim olduğunda parçalar durumunda TTL kurallarına göre arka planda taşınmadığında bir hata düzeltildi. [\#8672](https://github.com/ClickHouse/ClickHouse/pull/8672) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Sorunu düzelt fixedildi `Method createColumn() is not implemented for data type Set`. Bu düzeltmeler [\#7799](https://github.com/ClickHouse/ClickHouse/issues/7799). [\#8674](https://github.com/ClickHouse/ClickHouse/pull/8674) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi mutasyonları daha sık sonuçlandırmaya çalışacağız. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) +- Düzeltmek `intDiv` eksi bir sabit tarafından [\#9351](https://github.com/ClickHouse/ClickHouse/pull/9351) ([hcz](https://github.com/hczhcz)) +- Olası yarış durumunu düzeltin `BlockIO`. [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Kullanmaya / bırakmaya çalışırken sunucu sonlandırmasına giden hatayı düzeltin `Kafka` tablo yanlış parametrelerle oluşturuldu. [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) ([filimonov](https://github.com/filimonov)) +- OS için yanlış sonuç döndürürse geçici çözüm eklendi `timer_create` işlev. [\#8837](https://github.com/ClickHouse/ClickHouse/pull/8837) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kullanımında Sabit hata `min_marks_for_seek` parametre. Dağıtılmış tabloda sharding anahtarı olmadığında hata mesajı düzeltildi ve kullanılmayan parçaları atlamaya çalışıyoruz. [\#8908](https://github.com/ClickHouse/ClickHouse/pull/8908) ([Azat Khuzhin](https://github.com/azat)) + +#### Geliştirme {#improvement} + +- Uygulamak `ALTER MODIFY/DROP` için Mut topasyon topların üstüne sorgular `ReplicatedMergeTree*` motorlar ailesi. Şimdi `ALTERS` yalnızca meta veri güncelleme aşamasında engeller ve bundan sonra engellemez. [\#8701](https://github.com/ClickHouse/ClickHouse/pull/8701) ([alesapin](https://github.com/alesapin)) +- İle iç Birleşimlere çapraz yeniden yazma yeteneği ekleyin `WHERE` unqialified adları içeren bölüm. [\#9512](https://github.com/ClickHouse/ClickHouse/pull/9512) ([Artem Zuikov](https://github.com/4ertus2)) +- Yapmak `SHOW TABLES` ve `SHOW DATABASES` sorgular destek `WHERE` ifadeler ve `FROM`/`IN` [\#9076](https://github.com/ClickHouse/ClickHouse/pull/9076) ([sundyli](https://github.com/sundy-li)) +- Bir ayar eklendi `deduplicate_blocks_in_dependent_materialized_views`. [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) ([urykhy](https://github.com/urykhy)) +- Son değişikliklerden sonra MySQL istemcisi onaltılık ikili dizeleri yazdırmaya başladı ve böylece onları okunabilir hale getirdi ([\#9032](https://github.com/ClickHouse/ClickHouse/issues/9032)). Clickhouse'daki geçici çözüm, dize sütunlarını her zaman değil, genellikle durum olan UTF-8 olarak işaretlemektir. [\#9079](https://github.com/ClickHouse/ClickHouse/pull/9079) ([Yuriy Baranov](https://github.com/yurriy)) +- İçin dize ve FixedString tuşları desteği ekleyin `sumMap` [\#8903](https://github.com/ClickHouse/ClickHouse/pull/8903) ([Baudouin Giard](https://github.com/bgiard)) +- SummingMergeTree haritalarında dize anahtarlarını destekleyin [\#8933](https://github.com/ClickHouse/ClickHouse/pull/8933) ([Baudouin Giard](https://github.com/bgiard)) +- İş parçacığı özel durum atılmış olsa bile iş parçacığı havuzu için iş parçacığı sonlandırma sinyali [\#8736](https://github.com/ClickHouse/ClickHouse/pull/8736) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +- Ayarlamak için izin ver `query_id` içinde `clickhouse-benchmark` [\#9416](https://github.com/ClickHouse/ClickHouse/pull/9416) ([Anton Popov](https://github.com/CurtizJ)) +- Garip ifadelere izin verme `ALTER TABLE ... PARTITION partition` sorgu. Bu adresler [\#7192](https://github.com/ClickHouse/ClickHouse/issues/7192) [\#8835](https://github.com/ClickHouse/ClickHouse/pull/8835) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tablo `system.table_engines` şimdi özellik desteği (gibi) hakkında bilgi sağlar `supports_ttl` veya `supports_sort_order`). [\#8830](https://github.com/ClickHouse/ClickHouse/pull/8830) ([Max Akhmedov](https://github.com/zlobober)) +- Etkinleştirmek `system.metric_log` varsayılan olarak. ProfileEvents, CurrentMetrics değerleri ile toplanan satırları içerecektir “collect\_interval\_milliseconds” aralığı (varsayılan olarak bir saniye). Tablo çok küçüktür (genellikle megabayt sırasına göre) ve bu verileri varsayılan olarak toplamak mantıklıdır. [\#9225](https://github.com/ClickHouse/ClickHouse/pull/9225) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries. Fixes [\#6964](https://github.com/ClickHouse/ClickHouse/issues/6964) [\#8874](https://github.com/ClickHouse/ClickHouse/pull/8874) ([I ivanvan](https://github.com/abyss7)) +- Şimdi geçici `LIVE VIEW` tarafından oluşturulur `CREATE LIVE VIEW name WITH TIMEOUT [42] ...` yerine `CREATE TEMPORARY LIVE VIEW ...`, önceki sözdizimi ile tutarlı olmadığı için `CREATE TEMPORARY TABLE ...` [\#9131](https://github.com/ClickHouse/ClickHouse/pull/9131) ([tavplubix](https://github.com/tavplubix)) +- Text\_log ekleyin.giden girişleri sınırlamak için seviye yapılandırma parametresi `system.text_log` Tablo [\#8809](https://github.com/ClickHouse/ClickHouse/pull/8809) ([Azat Khuzhin](https://github.com/azat)) +- İndir partdiğiniz parçayı TTL kurallarına göre disk /lere/birim putlere koy allowmaya izin verin [\#8598](https://github.com/ClickHouse/ClickHouse/pull/8598) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Harici MySQL sözlükleri için, MySQL bağlantı havuzunun mutualize edilmesine izin verin “share” sözlükler arasında. Bu seçenek, MySQL sunucularına bağlantı sayısını önemli ölçüde azaltır. [\#9409](https://github.com/ClickHouse/ClickHouse/pull/9409) ([Clément Rodriguez](https://github.com/clemrodriguez)) +- Quantiles için en yakın sorgu yürütme süresini göster `clickhouse-benchmark` enterpolasyonlu değerler yerine çıktı. Bazı sorguların yürütme süresine karşılık gelen değerleri göstermek daha iyidir. [\#8712](https://github.com/ClickHouse/ClickHouse/pull/8712) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kafka'ya veri eklerken mesaj için anahtar ve zaman damgası ekleme imkanı. Düzeltiyor [\#7198](https://github.com/ClickHouse/ClickHouse/issues/7198) [\#8969](https://github.com/ClickHouse/ClickHouse/pull/8969) ([filimonov](https://github.com/filimonov)) +- Sunucu terminalden çalıştırılırsa, iş parçacığı numarası, sorgu kimliği ve günlük önceliğini renklere göre vurgulayın. Bu, geliştiriciler için ilişkili günlük iletilerinin daha iyi okunabilirliği içindir. [\#8961](https://github.com/ClickHouse/ClickHouse/pull/8961) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tablolar yüklenirken daha iyi istisna mesajı `Ordinary` veritabanı. [\#9527](https://github.com/ClickHouse/ClickHouse/pull/9527) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Uygulamak `arraySlice` toplama işlev durumları olan diziler için. Bu düzeltmeler [\#9388](https://github.com/ClickHouse/ClickHouse/issues/9388) [\#9391](https://github.com/ClickHouse/ClickHouse/pull/9391) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- In operatörünün sağ tarafında sabit işlevlerin ve sabit dizilerin kullanılmasına izin verin. [\#8813](https://github.com/ClickHouse/ClickHouse/pull/8813) ([Anton Popov](https://github.com/CurtizJ)) +- Sistem için veri getirilirken zookeeper istisnası olduysa.yinelemeler, ayrı bir sütunda görüntüler. Bu uygular [\#9137](https://github.com/ClickHouse/ClickHouse/issues/9137) [\#9138](https://github.com/ClickHouse/ClickHouse/pull/9138) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Atomically kaldırmak mergetree veri parçaları üzerinde yok. [\#8402](https://github.com/ClickHouse/ClickHouse/pull/8402) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Dağıtılmış tablolar için satır düzeyinde güvenliği destekleyin. [\#8926](https://github.com/ClickHouse/ClickHouse/pull/8926) ([I ivanvan](https://github.com/abyss7)) +- Now we recognize suffix (like KB, KiB…) in settings values. [\#8072](https://github.com/ClickHouse/ClickHouse/pull/8072) ([Mikhail Korotov](https://github.com/millb)) +- Büyük bir birleştirme sonucu oluştururken bellek yetersizliğini önleyin. [\#8637](https://github.com/ClickHouse/ClickHouse/pull/8637) ([Artem Zuikov](https://github.com/4ertus2)) +- Etkileşimli modda önerilere kümelerin isimleri eklendi `clickhouse-client`. [\#8709](https://github.com/ClickHouse/ClickHouse/pull/8709) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Initialize query profiler for all threads in a group, e.g. it allows to fully profile insert-queries [\#8820](https://github.com/ClickHouse/ClickHouse/pull/8820) ([I ivanvan](https://github.com/abyss7)) +- Eklenen sütun `exception_code` içinde `system.query_log` Tablo. [\#8770](https://github.com/ClickHouse/ClickHouse/pull/8770) ([Mikhail Korotov](https://github.com/millb)) +- Bağlantı noktasında etkin MySQL uyumluluk sunucusu `9004` varsayılan sunucu yapılandırma dosyasında. Yapılandırmada örnekte sabit şifre oluşturma komutu. [\#8771](https://github.com/ClickHouse/ClickHouse/pull/8771) ([Yuriy Baranov](https://github.com/yurriy)) +- Dosya sistemi readonly ise kapatma üzerinde iptal önleyin. Bu düzeltmeler [\#9094](https://github.com/ClickHouse/ClickHouse/issues/9094) [\#9100](https://github.com/ClickHouse/ClickHouse/pull/9100) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- HTTP POST sorgusunda uzunluk gerektiğinde daha iyi özel durum iletisi. [\#9453](https://github.com/ClickHouse/ClickHouse/pull/9453) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eklemek `_path` ve `_file` sanal sütunlar için `HDFS` ve `File` motorlar ve `hdfs` ve `file` tablo fonksiyonları [\#8489](https://github.com/ClickHouse/ClickHouse/pull/8489) ([Olga Khvostikova](https://github.com/stavrolia)) +- Hatayı Düzelt `Cannot find column` içine takarken `MATERIALIZED VIEW` durumda yeni sütun view'ın iç tabloya eklendi eğer. [\#8766](https://github.com/ClickHouse/ClickHouse/pull/8766) [\#8788](https://github.com/ClickHouse/ClickHouse/pull/8788) ([vzakaznikov](https://github.com/vzakaznikov)) [\#8788](https://github.com/ClickHouse/ClickHouse/issues/8788) [\#8806](https://github.com/ClickHouse/ClickHouse/pull/8806) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8803](https://github.com/ClickHouse/ClickHouse/pull/8803) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Son güncellemeden sonra ilerleme göndererek (günlük gibi) yerel istemci-sunucu protokolü üzerinden ilerlemeyi düzeltin. Bu, yalnızca yerel protokolü kullanan bazı üçüncü taraf araçlarla ilgili olabilir. [\#9495](https://github.com/ClickHouse/ClickHouse/pull/9495) ([Azat Khuzhin](https://github.com/azat)) +- MySQL protokolünü kullanarak istemci bağlantılarının sayısını izleyen bir sistem metriği ekleyin ([\#9013](https://github.com/ClickHouse/ClickHouse/issues/9013)). [\#9015](https://github.com/ClickHouse/ClickHouse/pull/9015) ([Eugene Klimov](https://github.com/Slach)) +- Şu andan itibaren, HTTP yanıtları olacak `X-ClickHouse-Timezone` başlık, aynı zaman dilimi değerine ayarlanmış `SELECT timezone()` rapor verecek. [\#9493](https://github.com/ClickHouse/ClickHouse/pull/9493) ([Denis Glazachev](https://github.com/traceon)) + +#### Performans İyileştirme {#performance-improvement} + +- In ile analiz endeksinin performansını artırın [\#9261](https://github.com/ClickHouse/ClickHouse/pull/9261) ([Anton Popov](https://github.com/CurtizJ)) +- Mantıksal işlevlerde daha basit ve daha verimli kod + kod temizleme. Bir takip için [\#8718](https://github.com/ClickHouse/ClickHouse/issues/8718) [\#8728](https://github.com/ClickHouse/ClickHouse/pull/8728) ([Alexander Kazakov](https://github.com/Akazz)) +- Genel performans iyileştirme (%5 aralığında..Etkilenen sorgular için %200) C++20 özellikleri ile daha sıkı takma sağlayarak. [\#9304](https://github.com/ClickHouse/ClickHouse/pull/9304) ([Amos Kuşu](https://github.com/amosbird)) +- Karşılaştırma işlevlerinin iç döngüleri için daha sıkı takma ad. [\#9327](https://github.com/ClickHouse/ClickHouse/pull/9327) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Aritmetik fonksiyonların iç döngüleri için daha sıkı takma ad. [\#9325](https://github.com/ClickHouse/ClickHouse/pull/9325) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- A ~ 3 kat daha hızlı uygulama için ColumnVector::replicate(), hangi aracılığıyla ColumnConst:: convertToFullColumn () uygulanır. Sabitleri gerçekleştirirken testlerde de yararlı olacaktır. [\#9293](https://github.com/ClickHouse/ClickHouse/pull/9293) ([Alexander Kazakov](https://github.com/Akazz)) +- Bir başka küçük performans iyileştirme `ColumnVector::replicate()` (bu hızlandırır `materialize` fonksiyonu ve daha yüksek mertebeden fonksiyonları) daha da geliştirilmesi için [\#9293](https://github.com/ClickHouse/ClickHouse/issues/9293) [\#9442](https://github.com/ClickHouse/ClickHouse/pull/9442) ([Alexander Kazakov](https://github.com/Akazz)) +- Geliştirilmiş performans `stochasticLinearRegression` toplama işlevi. Bu yama Intel tarafından katkıda bulunmuştur. [\#8652](https://github.com/ClickHouse/ClickHouse/pull/8652) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Performansını artırmak `reinterpretAsFixedString` işlev. [\#9342](https://github.com/ClickHouse/ClickHouse/pull/9342) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin istemciye blok gönder notmeyin `Null` işlemciler boru hattında Biçimlendir. [\#8797](https://github.com/ClickHouse/ClickHouse/pull/8797) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) [\#8767](https://github.com/ClickHouse/ClickHouse/pull/8767) ([Alexander Kuzmenkov](https://github.com/akuzm)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement} + +- Özel durum işleme artık Linux için Windows Alt Sisteminde düzgün çalışıyor. Bkz. https://github.com/ClickHouse-Extras/libunwind/pull/3 bu düzeltmeler [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) +- Değişmek `readline` ile `replxx` etkileşimli çizgi düzenleme için `clickhouse-client` [\#8416](https://github.com/ClickHouse/ClickHouse/pull/8416) ([I ivanvan](https://github.com/abyss7)) +- FunctionsComparison daha iyi inşa süresi ve daha az şablon örnekleri. [\#9324](https://github.com/ClickHouse/ClickHouse/pull/9324) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İle entegrasyon eklendi `clang-tidy` CI. Ayrıca bakınız [\#6044](https://github.com/ClickHouse/ClickHouse/issues/6044) [\#9566](https://github.com/ClickHouse/ClickHouse/pull/9566) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi kullanarak cı içinde ClickHouse bağlantı `lld` için bile `gcc`. [\#9049](https://github.com/ClickHouse/ClickHouse/pull/9049) ([alesapin](https://github.com/alesapin)) +- İş parçacığı zamanlama rastgele izin ve glitches eklemek zaman `THREAD_FUZZER_*` ortam değişkenleri ayarlanır. Bu test yardımcı olur. [\#9459](https://github.com/ClickHouse/ClickHouse/pull/9459) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Durumsuz testlerde güvenli soketleri etkinleştirme [\#9288](https://github.com/ClickHouse/ClickHouse/pull/9288) ([tavplubix](https://github.com/tavplubix)) +- Split\_shared\_libraries = daha sağlam yap [\#9156](https://github.com/ClickHouse/ClickHouse/pull/9156) ([Azat Khuzhin](https://github.com/azat)) +- Yapmak “performance\_introspection\_and\_logging” rastgele sunucuya güvenilir test sıkışmış. Bu CI ortamında gerçekleşebilir. Ayrıca bakınız [\#9515](https://github.com/ClickHouse/ClickHouse/issues/9515) [\#9528](https://github.com/ClickHouse/ClickHouse/pull/9528) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- XML stil onayında doğrulayın. [\#9550](https://github.com/ClickHouse/ClickHouse/pull/9550) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Testte sabit yarış durumu `00738_lock_for_inner_table`. Bu test uykuya dayanıyordu. [\#9555](https://github.com/ClickHouse/ClickHouse/pull/9555) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tür performans testlerini kaldırın `once`. Bu, tüm performans testlerini istatistiksel karşılaştırma modunda çalıştırmak için gereklidir (daha güvenilir). [\#9557](https://github.com/ClickHouse/ClickHouse/pull/9557) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Aritmetik fonksiyonlar için performans testi eklendi. [\#9326](https://github.com/ClickHouse/ClickHouse/pull/9326) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin performans testi eklendi `sumMap` ve `sumMapWithOverflow` toplama fonksiyonları. İçin takip [\#8933](https://github.com/ClickHouse/ClickHouse/issues/8933) [\#8947](https://github.com/ClickHouse/ClickHouse/pull/8947) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Üslup ile ErrorCodes stil sağlamak. [\#9370](https://github.com/ClickHouse/ClickHouse/pull/9370) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Testler geçmişi için komut dosyası ekleyin. [\#8796](https://github.com/ClickHouse/ClickHouse/pull/8796) ([alesapin](https://github.com/alesapin)) +- GCC uyarısı Ekle `-Wsuggest-override` tüm yerleri bulmak ve düzeltmek için `override` anahtar kelime kullanılmalıdır. [\#8760](https://github.com/ClickHouse/ClickHouse/pull/8760) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +- Tanımlanmalıdır çünkü Mac OS X altında zayıf sembolü Yoksay [\#9538](https://github.com/ClickHouse/ClickHouse/pull/9538) ([Silinmiş kullanıcı](https://github.com/ghost)) +- Performans testlerinde bazı sorguların çalışma süresini normalleştirin. Bu, tüm performans testlerini karşılaştırma modunda çalıştırmak için hazırlık olarak yapılır. [\#9565](https://github.com/ClickHouse/ClickHouse/pull/9565) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sorgu testleriyle pytest'i desteklemek için bazı testleri düzeltin [\#9062](https://github.com/ClickHouse/ClickHouse/pull/9062) ([I ivanvan](https://github.com/abyss7)) +- Ssl'yi msan ile oluşturmada etkinleştirin, böylece durumsuz testler çalıştırırken sunucu başlangıçta başarısız olmaz [\#9531](https://github.com/ClickHouse/ClickHouse/pull/9531) ([tavplubix](https://github.com/tavplubix)) +- Test sonuçlarında veritabanı ikamesini düzeltin [\#9384](https://github.com/ClickHouse/ClickHouse/pull/9384) ([Ilya Yatsishin](https://github.com/qoega)) +- Çeşitli platformlar için düzeltmeler oluşturun [\#9381](https://github.com/ClickHouse/ClickHouse/pull/9381) ([proller](https://github.com/proller)) [\#8755](https://github.com/ClickHouse/ClickHouse/pull/8755) ([proller](https://github.com/proller)) [\#8631](https://github.com/ClickHouse/ClickHouse/pull/8631) ([proller](https://github.com/proller)) +- Stateless-with-coverage test docker görüntüsüne diskler bölümü eklendi [\#9213](https://github.com/ClickHouse/ClickHouse/pull/9213) ([Pavel Kovalenko](https://github.com/Jokser)) +- GRPC ile oluştururken kaynak ağacı dosyalarından kurtulun [\#9588](https://github.com/ClickHouse/ClickHouse/pull/9588) ([Amos Kuşu](https://github.com/amosbird)) +- Sessioncleaner'ı bağlamdan kaldırarak biraz daha hızlı oluşturma süresi. SessionCleaner kodunu daha basit hale getirin. [\#9232](https://github.com/ClickHouse/ClickHouse/pull/9232) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Clickhouse-test komut dosyasında asılı sorgular için güncelleme denetimi [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) +- Depodan bazı gereksiz dosyaları kaldırıldı. [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Değişen matematik perftests türü `once` -e doğru `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Bizim kod tabanı için interaktif kod tarayıcı HTML raporu oluşturmanıza olanak sağlar docker görüntü ekleyin. [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) Görmek [Woboq Kod Tarayıcı](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/src/index.html) +- MSan altında bazı test hatalarını bastırın. [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Hızlanma “exception while insert” test. Bu sınama genellikle hata ayıklama-with-coverage derlemede zaman aşımına uğradı. [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Güncel `libcxx` ve `libcxxabi` yenmek. Hazırlan preparationmasında [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Flacky testi düzeltin `00910_zookeeper_test_alter_compression_codecs`. [\#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yinelenen bağlayıcı bayrakları temizleyin. Bağlayıcının beklenmedik bir sembol aramayacağından emin olun. [\#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([Amos Kuşu](https://github.com/amosbird)) +- Eklemek `clickhouse-odbc` test görüntüleri içine sürücü. Bu, ClickHouse ile clickhouse etkileşimini kendi ODBC sürücüsü aracılığıyla test etmeyi sağlar. [\#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov)) +- Birim testlerinde birkaç hatayı düzeltin. [\#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([alesapin](https://github.com/alesapin)) +- Etkinleştirmek `-Wmissing-include-dirs` Mevcut olmayan tüm içeriklerin ortadan kaldırılması için GCC uyarısı-çoğunlukla cmake komut dosyası hatalarının bir sonucu olarak [\#8704](https://github.com/ClickHouse/ClickHouse/pull/8704) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +- Sorgu profiler çalışamaz, nedenleri açıklayın. Bu için tasarlanmıştır [\#9049](https://github.com/ClickHouse/ClickHouse/issues/9049) [\#9144](https://github.com/ClickHouse/ClickHouse/pull/9144) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Openssl'yi upstream master'a güncelleyin. TLS bağlantıları mesajla başarısız olabilir sorunu düzeltildi `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error` ve `SSL Exception: error:2400006E:random number generator::error retrieving entropy`. Sorun 20.1 sürümünde mevcuttu. [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sunucu için Dockerfile güncelleme [\#8893](https://github.com/ClickHouse/ClickHouse/pull/8893) ([Ilya Mazaev](https://github.com/ne-ray)) +- Build-gcc-from-sources komut dosyasında küçük düzeltmeler [\#8774](https://github.com/ClickHouse/ClickHouse/pull/8774) ([Michael Nacharov](https://github.com/mnach)) +- Değişmek `numbers` -e doğru `zeros` perftests nerede `number` sütun kullanılmaz. Bu daha temiz test sonuçlarına yol açacaktır. [\#9600](https://github.com/ClickHouse/ClickHouse/pull/9600) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Sütun yapıcılarında ınitializer\_list kullanırken yığın taşması sorununu düzeltin. [\#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([Silinmiş kullanıcı](https://github.com/ghost)) +- V1.3.0 için librdkafka yükseltin. Birlikte etkinleştir `rdkafka` ve `gsasl` Mac OS X kitaplıkları. [\#9000](https://github.com/ClickHouse/ClickHouse/pull/9000) ([Andrew Onyshchuk](https://github.com/oandrew)) +- gcc 9.2.0 üzerinde düzeltme oluşturun [\#9306](https://github.com/ClickHouse/ClickHouse/pull/9306) ([vxider](https://github.com/Vxider)) + +## ClickHouse yayın v20. 1 {#clickhouse-release-v20-1} + +### ClickHouse yayın v20. 1. 8. 41, 2020-03-20 {#clickhouse-release-v20-1-8-41-2020-03-20} + +#### Hata Düzeltme {#bug-fix-3} + +- Olası kalıcı düzeltme `Cannot schedule a task` hata (işlenmeyen özel durum nedeniyle `ParallelAggregatingBlockInputStream::Handler::onFinish/onFinishThread`). Bu düzeltmeler [\#6833](https://github.com/ClickHouse/ClickHouse/issues/6833). [\#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) +- Aşırı bellek tüketimini düzeltin `ALTER` sorgular (Mut (asyonlar). Bu düzeltmeler [\#9533](https://github.com/ClickHouse/ClickHouse/issues/9533) ve [\#9670](https://github.com/ClickHouse/ClickHouse/issues/9670). [\#9754](https://github.com/ClickHouse/ClickHouse/pull/9754) ([alesapin](https://github.com/alesapin)) +- Dış sözlükler DDL backquoting hatayı düzeltin. Bu düzeltmeler [\#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [\#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) + +### ClickHouse yayın v20.1. 7. 38, 2020-03-18 {#clickhouse-release-v20-1-7-38-2020-03-18} + +#### Hata Düzeltme {#bug-fix-4} + +- Sabit yanlış iç işlev adları için `sumKahan` ve `sumWithOverflow`. Bu işlevleri uzak sorgularda kullanırken istisnaya yol açarım. [\#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). Bu sorun tüm ClickHouse sürümlerindeydi. +- Vermek `ALTER ON CLUSTER` -den `Distributed` iç çoğaltma ile tablolar. Bu düzeltmeler [\#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [\#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)). Bu sorun tüm ClickHouse sürümlerindeydi. +- Olası istisnaları düzeltin `Size of filter doesn't match size of column` ve `Invalid number of rows in Chunk` içinde `MergeTreeRangeReader`. Yürüt whileürken görün theyebilirler `PREWHERE` bazı durumlarda. Düzeltiyor [\#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [\#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) +- Sorunu düzeltildi: Eğer gibi basit bir aritmetik ifade yazarsanız zaman dilimi korunmuş değildi `time + 1` (gibi bir ifadenin aksine `time + INTERVAL 1 SECOND`). Bu düzeltmeler [\#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [\#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)). Bu sorun tüm ClickHouse sürümlerindeydi. +- Artık basit döngüsel takma adlarla sütunlar oluşturmak veya eklemek mümkün değil `a DEFAULT b, b DEFAULT a`. [\#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) +- Base64 kodlanmış değerin sonunda dolgu hatalı biçimlendirilmiş olabilir sorunu düzeltildi. Base64 kütüphanesini güncelleyin. Bu düzeltmeler [\#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), yaklar [\#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [\#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İmha veri yarışı Fix `Poco::HTTPServer`. Sunucu başlatıldığında ve hemen kapatıldığında gerçekleşebilir. [\#9468](https://github.com/ClickHouse/ClickHouse/pull/9468) ([Anton Popov](https://github.com/CurtizJ)) +- Olası kilitlenme/yanlış satır sayısını düzeltin `LIMIT n WITH TIES` n'th satırına eşit çok sayıda satır olduğunda. [\#9464](https://github.com/ClickHouse/ClickHouse/pull/9464) ([tavplubix](https://github.com/tavplubix)) +- Sütun TTLs ile olası eşleşmeyen sağlama toplamlarını düzeltin. [\#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) +- Bir kullanıcı denediğinde çökmeyi düzeltin `ALTER MODIFY SETTING` eski formatlı için `MergeTree` masa motorları ailesi. [\#9435](https://github.com/ClickHouse/ClickHouse/pull/9435) ([alesapin](https://github.com/alesapin)) +- Şimdi mutasyonları daha sık sonuçlandırmaya çalışacağız. [\#9427](https://github.com/ClickHouse/ClickHouse/pull/9427) ([alesapin](https://github.com/alesapin)) +- Çoğaltma protokolü uyumsuzluğunu düzeltme [\#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [\#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) +- Dizi türlerinin bloom\_filter dizini için değil(has()) düzeltin. [\#9407](https://github.com/ClickHouse/ClickHouse/pull/9407) ([achimbab](https://github.com/achimbab)) +- Davranışı Düzelt theildi `match` ve `extract` haystack sıfır bayt olduğunda işlevler. Haystack sabit olduğunda davranış yanlıştı. Bu düzeltmeler [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) ([alexey-milovidov](https://github.com/alexey-milovidov)) [\#9345](https://github.com/ClickHouse/ClickHouse/pull/9345) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-1} + +- Özel durum işleme artık Linux için Windows Alt Sisteminde düzgün çalışıyor. Bkz. https://github.com/ClickHouse-Extras/libunwind/pull/3 bu düzeltmeler [\#6480](https://github.com/ClickHouse/ClickHouse/issues/6480) [\#9564](https://github.com/ClickHouse/ClickHouse/pull/9564) ([sobolevsv](https://github.com/sobolevsv)) + +### ClickHouse yayın v20. 1. 6. 30, 2020-03-05 {#clickhouse-release-v20-1-6-30-2020-03-05} + +#### Hata Düzeltme {#bug-fix-5} + +- Sıkıştırıldığında veri uyumsuzluğunu düzeltin `T64` codec. + [\#9039](https://github.com/ClickHouse/ClickHouse/pull/9039) [(abyss7)](https://github.com/abyss7) +- Mergetree tablosundan bir iş parçacığında okurken aralıkların sırasını düzeltin. Düzeltiyor [\#8964](https://github.com/ClickHouse/ClickHouse/issues/8964). + [\#9050](https://github.com/ClickHouse/ClickHouse/pull/9050) [(CurtizJ)](https://github.com/CurtizJ) +- Olası segfault'u düzeltin `MergeTreeRangeReader`, Yürüt whileürken `PREWHERE`. Düzeltiyor [\#9064](https://github.com/ClickHouse/ClickHouse/issues/9064). + [\#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) [(CurtizJ)](https://github.com/CurtizJ) +- Düzeltmek `reinterpretAsFixedString` dönmek `FixedString` yerine `String`. + [\#9052](https://github.com/ClickHouse/ClickHouse/pull/9052) [(oandrew)](https://github.com/oandrew) +- Düzeltmek `joinGet` null dönüş türleri ile. Düzeltiyor [\#8919](https://github.com/ClickHouse/ClickHouse/issues/8919) + [\#9014](https://github.com/ClickHouse/ClickHouse/pull/9014) [(amosbird)](https://github.com/amosbird) +- Fuzz testini ve bittestall/bitTestAny işlevlerinin yanlış davranışını düzeltin. + [\#9143](https://github.com/ClickHouse/ClickHouse/pull/9143) [(alexey-milovidov)](https://github.com/alexey-milovidov) +- Haystack sıfır bayt olduğunda match ve extract işlevlerinin davranışını düzeltin. Haystack sabit olduğunda davranış yanlıştı. Düzeltiyor [\#9160](https://github.com/ClickHouse/ClickHouse/issues/9160) + [\#9163](https://github.com/ClickHouse/ClickHouse/pull/9163) [(alexey-milovidov)](https://github.com/alexey-milovidov) +- Kesinlikle monotinik olmayan fonksiyonel indeks kullanıldığında ters yüklemlerin sabit yürütülmesi. Düzeltiyor [\#9034](https://github.com/ClickHouse/ClickHouse/issues/9034) + [\#9223](https://github.com/ClickHouse/ClickHouse/pull/9223) [(Akazz)](https://github.com/Akazz) +- Yeniden yazmak için izin ver `CROSS` -e doğru `INNER JOIN` varsa `[NOT] LIKE` operat inor in `WHERE` bölme. Düzeltiyor [\#9191](https://github.com/ClickHouse/ClickHouse/issues/9191) + [\#9229](https://github.com/ClickHouse/ClickHouse/pull/9229) [(4ertus2)](https://github.com/4ertus2) +- Log engine ile bir tablodaki ilk sütunların bir diğer ad olmasına izin verin. + [\#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(abyss7)](https://github.com/abyss7) +- Virgülle birleşmeye izin ver `IN()` için. Düzeltiyor [\#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). + [\#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4ertus2)](https://github.com/4ertus2) +- Geliştirmek `ALTER MODIFY/ADD` sorgu mantığı. Şimdi yapamazsın `ADD` türü olmayan sütun, `MODIFY` varsayılan ifade sütun türünü değiştirmez ve `MODIFY` tür varsayılan ifade değerini kaybetmez. Düzeltiyor [\#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). + [\#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) +- Mutasyonlar kesinleşmesini düzeltin, zaten mutasyon yapıldığında is\_done = 0 durumuna sahip olabilir. + [\#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) +- Destek “Processors” sistem için boru hattı.sayılar ve sistem.numbers\_mt. Bu aynı zamanda hatayı giderir `max_execution_time` saygın değildir. + [\#7796](https://github.com/ClickHouse/ClickHouse/pull/7796) [(KochetovNicolai)](https://github.com/KochetovNicolai) +- Yanlış saymayı düzeltin `DictCacheKeysRequestedFound` ölçü. + [\#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) [(nikitamikhaylov)](https://github.com/nikitamikhaylov) +- Depolama politikası için bir kontrol eklendi `ATTACH PARTITION FROM`, `REPLACE PARTITION`, `MOVE TO TABLE` aksi halde, yeniden başlatıldıktan sonra parçanın verilerini erişilemez hale getirebilir ve Clickhouse'un başlatılmasını önleyebilir. + [\#9383](https://github.com/ClickHouse/ClickHouse/pull/9383) [(excitoon)](https://github.com/excitoon) +- Sabit UBSan raporu `MergeTreeIndexSet`. Bu düzeltmeler [\#9250](https://github.com/ClickHouse/ClickHouse/issues/9250) + [\#9365](https://github.com/ClickHouse/ClickHouse/pull/9365) [(alexey-milovidov)](https://github.com/alexey-milovidov) +- BlockİO Olası datarace Fix. + [\#9356](https://github.com/ClickHouse/ClickHouse/pull/9356) [(KochetovNicolai)](https://github.com/KochetovNicolai) +- İçin destek `UInt64` json ile ilgili işlevlerde Int64'e uymayan sayılar. Güncelleştirme `SIMDJSON` yenmek. Bu düzeltmeler [\#9209](https://github.com/ClickHouse/ClickHouse/issues/9209) + [\#9344](https://github.com/ClickHouse/ClickHouse/pull/9344) [(alexey-milovidov)](https://github.com/alexey-milovidov) +- Veri dizini ayrı bir aygıta takılıysa, boş alan miktarı doğru hesaplanmadığında sorunu düzeltin. Varsayılan disk için veri alt dizinindeki boş alanı hesaplayın. Bu düzeltmeler [\#7441](https://github.com/ClickHouse/ClickHouse/issues/7441) + [\#9257](https://github.com/ClickHouse/ClickHouse/pull/9257) [(millb)](https://github.com/millb) +- Mesajla TLS bağlantıları başarısız olduğunda sorunu düzeltin `OpenSSL SSL_read: error:14094438:SSL routines:ssl3_read_bytes:tlsv1 alert internal error and SSL Exception: error:2400006E:random number generator::error retrieving entropy.` Openssl'yi upstream master'a güncelleyin. + [\#8956](https://github.com/ClickHouse/ClickHouse/pull/8956) [(alexey-milovidov)](https://github.com/alexey-milovidov) +- Yürüt whenürken `CREATE` sorgu, depolama motoru argümanlarında sabit ifadeleri katlayın. Boş veritabanı adı geçerli veritabanı ile değiştirin. Düzeltiyor [\#6508](https://github.com/ClickHouse/ClickHouse/issues/6508), [\#3492](https://github.com/ClickHouse/ClickHouse/issues/3492). Ayrıca ClickHouseDictionarySource yerel adresi için onay düzeltin. + [\#9262](https://github.com/ClickHouse/ClickHouse/pull/9262) [(tabplubix)](https://github.com/tavplubix) +- Segfault'u düzeltin `StorageMerge`, StorageFile okurken olabilir. + [\#9387](https://github.com/ClickHouse/ClickHouse/pull/9387) [(tabplubix)](https://github.com/tavplubix) +- Veri kaybını önlemek `Kafka` nadir durumlarda istisna sonek okuduktan sonra ancak taahhütten önce gerçekleşir. Düzeltiyor [\#9378](https://github.com/ClickHouse/ClickHouse/issues/9378). İlgili: [\#7175](https://github.com/ClickHouse/ClickHouse/issues/7175) + [\#9507](https://github.com/ClickHouse/ClickHouse/pull/9507) [(filimonov)](https://github.com/filimonov) +- Kullanmaya / bırakmaya çalışırken sunucu sonlandırmasına giden hatayı düzeltin `Kafka` tablo yanlış parametrelerle oluşturuldu. Düzeltiyor [\#9494](https://github.com/ClickHouse/ClickHouse/issues/9494). Birleşiyor [\#9507](https://github.com/ClickHouse/ClickHouse/issues/9507). + [\#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov) + +#### Yenilik {#new-feature-1} + +- Eklemek `deduplicate_blocks_in_dependent_materialized_views` hayata manzaralı tablolara idempotent ekler davranışlarını kontrol etmek için Seçenek. Bu yeni özellik, altınity'den özel bir istek ile bugfix sürümüne eklendi. + [\#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy) + +### ClickHouse yayın v20.1. 2. 4, 2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22} + +#### Geriye Dönük Uyumsuz Değişim {#backward-incompatible-change-1} + +- Ayarı yap `merge_tree_uniform_read_distribution` eski. Sunucu hala bu ayarı tanır, ancak hiçbir etkisi yoktur. [\#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fonksiyonun dönüş türünü değiştirdi `greatCircleDistance` -e doğru `Float32` çünkü şimdi hesaplamanın sonucu `Float32`. [\#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi sorgu parametrelerinin şu şekilde temsil edilmesi bekleniyor “escaped” biçimli. Örneğin, dize geçmek için `ab` yazmak zorundasın `a\tb` veya `a\b` ve sırasıyla, `a%5Ctb` veya `a%5C%09b` URL in'de. Bu, NULL olarak geçme olasılığını eklemek için gereklidir `\N`. Bu düzeltmeler [\#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [\#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Etkinleştirmek `use_minimalistic_part_header_in_zookeeper` için ayar `ReplicatedMergeTree` varsayılan olarak. Bu, Zookeeper'da depolanan veri miktarını önemli ölçüde azaltacaktır. Bu ayar 19.1 sürümünden beri desteklenmektedir ve zaten yarım yıldan fazla bir süredir herhangi bir sorun yaşamadan birden fazla hizmette üretimde kullanıyoruz. 19.1'den eski sürümlere geçiş yapma şansınız varsa bu ayarı devre dışı bırakın. [\#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Veri atlama endeksleri üretim hazır ve varsayılan olarak etkindir. Ayar `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` ve `allow_experimental_multiple_joins_emulation` artık eskimiş ve hiçbir şey yapma. [\#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yeni Ekle `ANY JOIN` için mantık `StorageJoin` ile tutarlı `JOIN` işleyiş. Davranış değişiklikleri olmadan yükseltmek için eklemeniz gerekir `SETTINGS any_join_distinct_right_table_keys = 1` motor tabloları meta katılmak veya yükseltmeden sonra bu tabloları yeniden oluşturmak için. [\#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) +- Günlük yapılandırma değişiklikleri uygulamak için yeniden başlatılması için sunucu gerektirir. Bu, sunucunun silinmiş bir günlük dosyasına oturum açtığı hatayı önlemek için geçici bir geçici çözümdür (bkz. [\#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [\#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) + +#### Yenilik {#new-feature-2} + +- Parça yolları hakkında bilgi eklendi `system.merges`. [\#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Yürütme yeteneği Ekle `SYSTEM RELOAD DICTIONARY` sorgu içinde `ON CLUSTER` modu. [\#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) +- Yürütme yeteneği Ekle `CREATE DICTIONARY` sorgular `ON CLUSTER` modu. [\#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) +- Şimdi kullanıcının profili `users.xml` birden çok profil devralır. [\#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +- Katma `system.stack_trace` tüm sunucu iş parçacıklarının yığın izlerine bakmayı sağlayan tablo. Bu, geliştiricilerin sunucu durumunu iç gözlemlemesi için kullanışlıdır. Bu düzeltmeler [\#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [\#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eklemek `DateTime64` yapılandırılabilir alt ikinci hassasiyetle veri türü. [\#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) +- Tablo fonksiyonu Ekle `clusterAllReplicas` kümedeki tüm düğümleri sorgulamaya izin verir. [\#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) +- Toplama işlevi Ekle `categoricalInformationValue` ayrık bir özelliğin bilgi değerini hesaplar. [\#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) +- Veri dosyalarının ayrıştırılmasını hızlandırın `CSV`, `TSV` ve `JSONEachRow` paralel olarak yaparak Biçimlendir .in. [\#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Fonksiyon Ekle `bankerRound` bankacı yuvarlama yapar. [\#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) +- Bölge adları için gömülü sözlükte daha fazla dil desteği: ‘ru’, ‘en’, ‘ua’, ‘uk’, ‘by’, ‘kz’, ‘tr’, ‘de’, ‘uz’, ‘lv’, ‘lt’, ‘et’, ‘pt’, ‘he’, ‘vi’. [\#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tutarlılıktaki gelişmeler `ANY JOIN` mantıksal. Şimdi `t1 ANY LEFT JOIN t2` eşitlikler `t2 ANY RIGHT JOIN t1`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +- Ayar Ekle `any_join_distinct_right_table_keys` için eski davranışları sağlayan `ANY INNER JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +- Yeni Ekle `SEMI` ve `ANTI JOIN`. Yaş `ANY INNER JOIN` davranış şu anda mevcut `SEMI LEFT JOIN`. [\#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +- Katma `Distributed` format for `File` motor ve `file` okumak için izin veren tablo işlevi `.bin` asenkron ekler tarafından oluşturulan dosyalar `Distributed` Tablo. [\#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- İsteğe bağlı Sıfırla sütun argümanı Ekle `runningAccumulate` hangi her yeni anahtar değeri için toplama sonuçlarını sıfırlamak için izin verir. [\#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) +- Prometheus bitiş noktası olarak ClickHouse kullanma yeteneği ekleyin. [\#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) +- Bölüm Ekle `` içinde `config.xml` uzak tablo motorları ve tablo işlevleri için izin verilen ana bilgisayarları kısıtlayan `URL`, `S3`, `HDFS`. [\#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) +- Eklendi fonksiyonu `greatCircleAngle` bir küre üzerindeki mesafeyi derece cinsinden hesaplar. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- H3 kütüphanesi ile tutarlı olacak şekilde dünya yarıçapı değiştirildi. [\#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Katma `JSONCompactEachRow` ve `JSONCompactEachRowWithNamesAndTypes` giriş ve çıkış biçimleri. [\#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) +- Dosya ile ilgili tablo motorları ve tablo işlevleri için özellik eklendi (`File`, `S3`, `URL`, `HDFS`) okuma ve yaz allowsmayı sağlayan `gzip` ek motor parametresi veya dosya uzantısına dayalı dosyalar. [\#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) +- Add theed the `randomASCII(length)` fonksiyon, rastgele bir dizi ile bir dize oluşturma [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) yazdırılabilir karakterler. [\#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([Süngü](https://github.com/BayoNet)) +- Eklendi fonksiyonu `JSONExtractArrayRaw` ayrıştırılmamış json dizi öğelerinde bir dizi döndürür `JSON` dize. [\#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) +- Eklemek `arrayZip` eşit uzunluktaki birden fazla diziyi bir dizi diziye birleştirmeyi sağlayan işlev. [\#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([Kış Zhang](https://github.com/zhang2014)) +- Yapılandırılmış göre diskler arasında veri taşımak için yeteneği ekleyin `TTL`- ifadeler için `*MergeTree` masa motorları ailesi. [\#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Yeni toplama fonksiyonu eklendi `avgWeighted` hangi ağırlıklı ortalama hesaplamak için izin verir. [\#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) +- Şimdi paralel ayrıştırma için varsayılan olarak etkindir `TSV`, `TSKV`, `CSV` ve `JSONEachRow` biçimliler. [\#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Birkaç coğrafi işlev Ekle `H3` kitaplık: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` ve `h3kRing`. [\#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) +- Brotli için destek eklendi (`br`) dosya ile ilgili Depolarda ve tablo işlevlerinde sıkıştırma. Bu düzeltmeler [\#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [\#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Eklemek `groupBit*` fonksiyonlar için `SimpleAggregationFunction` tür. [\#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) + +#### Hata Düzeltme {#bug-fix-6} + +- Tabloların yeniden adlandırılmasını düzeltin `Distributed` motor. Düzeltmeler sorunu [\#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [\#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) +- Şimdi sözlükler desteği `EXPRESSION` non-ClickHouse SQL lehçesinde keyfi dizedeki öznitelikler için. [\#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) +- Kırık düzeltmek `INSERT SELECT FROM mysql(...)` sorgu. Bu düzeltmeler [\#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) ve [\#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [\#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) +- Hatayı Düzelt “Mismatch column sizes” varsayılan eklerken `Tuple` itibaren `JSONEachRow`. Bu düzeltmeler [\#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [\#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) +- Şimdi kullanılması durumunda bir istisna atılacaktır `WITH TIES` birlikte `LIMIT BY`. Ayrıca kullanma yeteneği ekleyin `TOP` ile `LIMIT BY`. Bu düzeltmeler [\#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [\#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Taze glibc sürümünden istenmeyen bağımlılığı düzeltin `clickhouse-odbc-bridge` ikilik. [\#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Kuşu](https://github.com/amosbird)) +- Kontrol fonksiyonunda hatayı düzeltin `*MergeTree` motorlar ailesi. Şimdi, son granül ve son işarette (nihai olmayan) eşit miktarda satıra sahip olduğumuzda başarısız olmaz. [\#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) +- Insert Into Fix `Enum*` sonra sütunlar `ALTER` sorgu, altta yatan sayısal tür tablo belirtilen türe eşit olduğunda. Bu düzeltmeler [\#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [\#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) +- İzin verilen sabit olmayan negatif “size” fonksiyon için argüman `substring`. Yanlışlıkla izin verilmedi. Bu düzeltmeler [\#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [\#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yanlış sayıda argüman geçtiğinde ayrıştırma hatasını düzeltin `(O|J)DBC` masa motoru. [\#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) +- Syslog için günlükleri gönderirken çalışan clickhouse işleminin komut adını kullanma. Önceki sürümlerde, komut adı yerine boş dize kullanıldı. [\#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) +- İzin verilen ana bilgisayarların kontrolünü düzeltin `localhost`. Bu PR, sağlanan çözümü düzeltir [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [\#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) +- Nadir kazayı düzeltin `argMin` ve `argMax` sonuç kullanıldığında, uzun dize bağımsız değişkenleri için işlevler `runningAccumulate` işlev. Bu düzeltmeler [\#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [\#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinozor](https://github.com/769344359)) +- İle tablolar için bellek overcommit Fix `Buffer` motor. [\#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) +- Alabilir fonksiyonları sabit potansiyel hata `NULL` bağımsız değişkenlerden biri olarak ve NULL olmayan döndürür. [\#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Arka plan işlemleri için iş parçacığı havuzunda daha iyi metrik hesaplamaları `MergeTree` masa motorları. [\#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Düzeltme fonksiyonu `IN` için `WHERE` satır düzeyinde tablo filtresi varsa deyimi. Düzeltiyor [\#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [\#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([I ivanvan](https://github.com/abyss7)) +- Şimdi, integral değeri ayarlar değerleri için tamamen ayrıştırılmazsa bir istisna atılır. [\#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) +- Toplama işlevi, sorguda ikiden fazla yerel parça içeren dağıtılmış tabloya kullanıldığında özel durumu düzeltin. [\#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) +- Şimdi bloom filtresi sıfır uzunluk dizilerini işleyebilir ve gereksiz hesaplamalar yapmaz. [\#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) +- Bir istemci ana bilgisayar için istemci ana bilgisayar eşleştirerek izin verilip verilmediğini kontrol sabit `host_regexp` belirtilen `users.xml`. [\#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) +- Çoklu olarak yanlış pozitiflere yol açan belirsiz sütun kontrolünü rahatlatın `JOIN ON` bölme. [\#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) +- Sabit Olası sunucu çökmesi (`std::terminate`) sunucu veri gönder cannotem whenediğinde veya yaz theama thedığında `JSON` veya `XML` değerleri ile biçim `String` veri türü (gerektiren `UTF-8` doğrulama) veya sonuç verilerini Brotli algoritması ile sıkıştırırken veya diğer bazı nadir durumlarda. Bu düzeltmeler [\#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [\#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yarış durumunu düzeltin `StorageDistributedDirectoryMonitor` CI tarafından bulundu. Bu düzeltmeler [\#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [\#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Şimdi arka plan birleşir `*MergeTree` tablo motorları ailesi depolama ilkesi hacim sırasını daha doğru bir şekilde korur. [\#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Şimdi masa motoru `Kafka` ile düzgün çalışır `Native` biçimli. Bu düzeltmeler [\#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [\#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [\#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +- Başlıklarla sabit formatlar (gibi `CSVWithNames`) tablo motoru için EOF hakkında istisna atıyorlardı `Kafka`. [\#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +- Sağ kısmında alt sorgudan set yapma ile bir hata düzeltildi `IN` bölme. Bu düzeltmeler [\#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) ve [\#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [\#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Depolama alanından okurken Olası kazayı düzeltin `File`. [\#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Dosyaların sabit okuma `Parquet` tür sütunları içeren biçim `list`. [\#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) +- Hatayı Düzelt `Not found column` ile dağıtılmış sorgular için `PREWHERE` örnekleme anahtarına bağımlı durum `max_parallel_replicas > 1`. [\#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Hatayı Düzelt `Not found column` sorgu kullanılırsa `PREWHERE` tablonun diğer adı ve sonuç kümesi bağımlı birincil anahtar koşulu nedeniyle boştu. [\#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fonksiyonlar için sabit dönüş tipi `rand` ve `randConstant` durumunda `Nullable` değişken. Şimdi fonksiyonlar her zaman geri döner `UInt32` ve asla `Nullable(UInt32)`. [\#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Engelli yüklem push-down için `WITH FILL` ifade. Bu düzeltmeler [\#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [\#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Kış Zhang](https://github.com/zhang2014)) +- Sabit yanlış `count()` res forult for `SummingMergeTree` ne zaman `FINAL` bölüm kullanılır. [\#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [\#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Uzak sunuculardan sabit işlevler için olası yanlış sonucu düzeltin. Gibi işlevlerle sorgular için oldu `version()`, `uptime()` vb. farklı sunucular için farklı sabit değerler döndürür. Bu düzeltmeler [\#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [\#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Yanlış sonuçlara yol açan push-down yüklemi optimizasyonunda karmaşık hatayı düzeltin. Bu, push-down yüklemi optimizasyonu ile ilgili birçok sorunu giderir. [\#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Kış Zhang](https://github.com/zhang2014)) +- Çökmeyi düzeltin `CREATE TABLE .. AS dictionary` sorgu. [\#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) +- Çeşitli iyileştirmeler clickhouse dilbilgisi `.g4` Dosya. [\#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) +- Çökmelere yol açan hatayı düzeltin `JOIN`motorlu tablolar ile s `Join`. Bu düzeltmeler [\#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [\#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [\#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [\#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [\#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) +- Gereksiz sözlükleri yeniden yükleyin `CREATE DATABASE`. [\#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) +- Okuma için maksimum akış sayısını sınırlayın `StorageFile` ve `StorageHDFS`. Düzeltmeler https://github.com/ClickHouse/ClickHouse/issues/7650. [\#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) +- Hatayı düzeltin `ALTER ... MODIFY ... CODEC` sorgu, kullanıcı hem varsayılan ifade hem de codec belirttiğinde. Düzeltiyor [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [\#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) +- Sütunların arka plan birleşimindeki hatayı düzeltin `SimpleAggregateFunction(LowCardinality)` tür. [\#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Sabit tip kontrol fonksiyonu `toDateTime64`. [\#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) +- Şimdi sunucu üzerinde çökme yok `LEFT` veya `FULL JOIN` ve Birleştirme motoru ve desteklenmeyen `join_use_nulls` ayarlar. [\#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) +- Şimdi `DROP DICTIONARY IF EXISTS db.dict` sorgu, istisna atmıyor `db` yok. [\#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +- Tablo işlevlerinde Olası çökmeleri düzeltin (`file`, `mysql`, `remote`) referans kullanımı nedeniyle Kaldır toılan `IStorage` nesne. Tablo işlevine ekleme sırasında belirtilen sütunların yanlış ayrıştırılmasını düzeltin. [\#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) +- Başlamadan önce ağ olun `clickhouse-server`. Bu düzeltmeler [\#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [\#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) +- Güvenli bağlantılar için zaman aşımlarını düzeltin, böylece sorgular belirsiz bir şekilde askıda kalmaz. Bu düzeltmeler [\#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [\#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltmek `clickhouse-copier`'In eşzamanlı işçiler arasındaki gereksiz çekişme. [\#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +- Şimdi mutasyonlar, mutasyon versiyonları mevcut mutasyon versiyonundan daha büyük olsa bile, ekli parçaları atlamaz. [\#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [\#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) +- Gereksiz kopyaları Yoksay `*MergeTree` başka bir diske ve sunucuya taşındıktan sonra veri parçaları yeniden başlatılır. [\#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Çökmeyi düzeltin `FULL JOIN` ile `LowCardinality` içinde `JOIN` anahtar. [\#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) +- Sütun adını birden çok kez ekleme sorgusu gibi kullanmak yasaktır `INSERT INTO tbl (x, y, x)`. Bu düzeltmeler [\#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [\#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [\#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) +- Bilinmeyen CPU'lar için fiziksel CPU çekirdeği sayısını tespit etmek için geri dönüş eklendi (mantıksal CPU çekirdeği sayısını kullanarak). Bu düzeltmeler [\#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [\#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltmek `There's no column` materialized ve alias sütunlar için hata. [\#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) +- Sabit sever kazasında zaman `EXISTS` sorgu olmadan kullanıldı `TABLE` veya `DICTIONARY` niteleyici. Gibi `EXISTS t`. Bu düzeltmeler [\#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). Bu hata 19.17 sürümünde tanıtıldı. [\#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Nadir hatayı hata ile düzeltin `"Sizes of columns doesn't match"` kullanırken bu görünebilir `SimpleAggregateFunction` sütun. [\#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +- Boş olan kullanıcının bulunduğu hatayı düzeltin `allow_databases` tüm veritabanlarına erişim var (ve aynı `allow_dictionaries`). [\#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) +- Sunucu zaten istemciden bağlantısı kesildiğinde istemci çökmesini düzeltin. [\#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) +- Düzeltmek `ORDER BY` birincil anahtar öneki ve birincil anahtar soneki ile sıralama durumunda davranış. [\#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) +- Tabloda nitelikli sütun mevcut olup olmadığını kontrol edin. Bu düzeltmeler [\#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [\#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) +- İle sabit davranış `ALTER MOVE` belirtilen birleştirme bitiş hamle superpart hemen sonra koştu. Düzeltiyor [\#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [\#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Kullanırken Olası sunucu çökmesini düzeltin `UNION` farklı sayıda sütun ile. Düzeltiyor [\#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [\#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- İşlev için sonuç alt dizesinin boyutunu düzeltin `substr` negatif boyutu ile. [\#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Şimdi server part Mut mutasyonu Yürüt inmüyor `MergeTree` arka plan havuzunda yeterli boş iş parçacığı yoksa. [\#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) +- Biçimlendirmede küçük bir yazım hatası düzeltildi `UNION ALL` AST. [\#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) +- Negatif sayılar için sabit yanlış bloom filtre sonuçları. Bu düzeltmeler [\#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [\#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([Kış Zhang](https://github.com/zhang2014)) +- Sıkıştırmada sabit potansiyel tampon taşması. Kötü niyetli kullanıcı, arabellekten sonra okunmasına neden olacak sıkıştırılmış verileri iletebilir. Bu sorun Yandex bilgi güvenliği ekibinden Eldar Zaitov tarafından bulundu. [\#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Tamsayılar taşması nedeniyle yanlış sonucu düzeltin `arrayIntersect`. [\#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Şimdi `OPTIMIZE TABLE` sorgu işlemi gerçekleştirmek çevrimdışı yinelemeler için beklemez. [\#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) +- Sabit `ALTER TTL` par forser for `Replicated*MergeTree` Tablolar. [\#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Sunucu ve istemci arasındaki iletişimi düzeltin, böylece sunucu sorgu hatasından sonra geçici tablolar bilgilerini okur. [\#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) +- Düzeltmek `bitmapAnd` birleştirilmiş bir bit eşlem ve bir skaler bit eşlem kesişen işlev hatası. [\#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) +- Tanımı rafine `ZXid` hata düzeltmeleri ZooKeeper programcı Kılavuzu göre `clickhouse-cluster-copier`. [\#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +- `odbc` tablo fonksiyonu şimdi saygı `external_table_functions_use_nulls` ayar. [\#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) +- Nadir bir veri yarışına yol açan Sabit hata. [\#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) +- Şimdi `SYSTEM RELOAD DICTIONARY` göz ardı ederek bir sözlüğü tamamen yeniden yükler `update_field`. Bu düzeltmeler [\#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [\#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) +- Sözlüğün sorgu oluştur'da olup olmadığını kontrol etme yeteneği ekleyin. [\#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alesapin](https://github.com/alesapin)) +- Düzeltmek `Float*` ayrıştırma `Values` biçimli. Bu düzeltmeler [\#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [\#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) +- Bazı arka plan işlemlerinde yer ayıramadığımızda çökmeyi düzeltin `*MergeTree` masa motorları ailesi. [\#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Tablo içerdiğinde birleştirme işleminin çökmesini düzeltin `SimpleAggregateFunction(LowCardinality)` sütun. Bu düzeltmeler [\#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [\#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) +- Tüm YBÜ yerel destek geri yükleme ve sabit ifadeler için harmanlama uygulamak için yeteneği ekleyin. Ayrıca dil adı ekleyin `system.collations` Tablo. [\#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) +- Sıfır minimum ömrü olan harici sözlükler olduğunda hatayı düzeltin (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) arka planda güncelleme yapmayın. [\#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alesapin](https://github.com/alesapin)) +- ClickHouse kaynağı ile harici sözlük sorguda alt sorgu olduğunda çökmesini düzeltin. [\#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Motor ile tablodaki dosya uzantısının yanlış ayrıştırılmasını düzeltin `URL`. Bu düzeltmeler [\#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [\#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) +- Düzeltmek `CHECK TABLE` sorgu için `*MergeTree` anahtarsız tablolar. Düzeltiyor [\#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [\#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) +- Sabit dönüşüm `Float64` MySQL türü için. [\#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) +- Şimdi Eğer tablo tamamen sunucu çökmesi nedeniyle düştü değildi, sunucu geri yüklemek ve yüklemek için çalışacağız. [\#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) +- Tablo işlevinde sabit çökme `file` mevcut olmayan dosyaya eklerken. Şimdi bu durumda dosya oluşturulur ve daha sonra insert işlenir. [\#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +- Ne zaman olabilir nadir kilitlenme Fix `trace_log` etkin olduğunu. [\#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) +- Yanında farklı türleri ile çalışmak için yeteneği ekleyin `Date` içinde `RangeHashed` DDL sorgusundan oluşturulan dış sözlük. Düzeltiyor [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [\#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) +- Düzeltmeler çökme zaman `now64()` başka bir işlevin sonucu ile çağrılır. [\#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) +- Mysql Tel protokolü üzerinden bağlantılar için istemci IP tespit ile Sabit hata. [\#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) +- Boş dizi işlemeyi düzeltin `arraySplit` işlev. Bu düzeltmeler [\#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [\#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) +- Sorunu ne zaman düzeltildi `pid-file` başka bir koşu `clickhouse-server` silinebilir. [\#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) +- Varsa sözlük yeniden yükle `invalidate_query`, güncellemeleri durdurdu ve önceki güncelleme denemelerinde bazı istisnalar. [\#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) +- Fonksiyonda Sabit hata `arrayReduce` bu yol açabilir “double free” ve toplu işlev birleştiricisinde hata `Resample` bu bellek sızıntısına neden olabilir. Toplama fonksiyonu eklendi `aggThrow`. Bu fonksiyon test amaçlı kullanılabilir. [\#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Geliştirme {#improvement-1} + +- İle çalışırken geliştirilmiş günlük kaydı `S3` masa motoru. [\#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +- Çağrılırken hiçbir bağımsız değişken geçirildiğinde yazdırılan yardım iletisi `clickhouse-local`. Bu düzeltmeler [\#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [\#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) +- Ayar Ekle `mutations_sync` hangi beklemek sağlar `ALTER UPDATE/DELETE` eşzamanlı sorgular. [\#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) +- Göreli ayarlamak için izin ver `user_files_path` içinde `config.xml` (bir şekilde benzer `format_schema_path`). [\#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) +- Dönüştürme işlevleri için yasadışı türler için istisna Ekle `-OrZero` postfix. [\#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) +- Bir dağıtılmış sorgu bir parça için veri gönderme başlığındaki kolay bir biçim. [\#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) +- `Live View` tablo motoru refactoring. [\#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) +- DDL sorgularından oluşturulan dış sözlükler için ek kontroller ekleyin. [\#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) +- Hatayı Düzelt `Column ... already exists` kullanırken `FINAL` ve `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Düzeltiyor [\#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [\#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Şimdi tablo ilk argüman `joinGet` fonksiyon tablo indentifier olabilir. [\#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Kuşu](https://github.com/amosbird)) +- Kullanmaya izin ver `MaterializedView` yukarıdaki alt sorgularla `Kafka` Tablolar. [\#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) +- Şimdi arka plan diskler arasında hareket eder, seprate iş parçacığı havuzunu çalıştırır. [\#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) +- `SYSTEM RELOAD DICTIONARY` şimdi eşzamanlı olarak yürütür. [\#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) +- Yığın izleri artık sanal bellek adresleri yerine (nesne dosyasının yüklendiği) fiziksel adresleri (nesne dosyasındaki uzaklıklar) görüntüler. Bu kullanımına izin verir `addr2line` ikili konum bağımsız ve ASLR etkin olduğunda. Bu düzeltmeler [\#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [\#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Satır düzeyinde güvenlik filtreleri için yeni sözdizimi desteği: `
…
`. Düzeltiyor [\#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [\#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([I ivanvan](https://github.com/abyss7)) +- Şimdi `cityHash` fonksiyonu ile çalışabilir `Decimal` ve `UUID` türler. Düzeltiyor [\#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [\#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) +- Uyarlanabilir tanecikliliğin uygulanmasından sonra eskimiş olduğu için sistem günlüklerinden sabit dizin tanecikliliği (1024 idi) kaldırıldı. [\#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- ClickHouse SSL olmadan derlendiğinde etkin MySQL uyumluluk sunucusu. [\#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([Yuriy Baranov](https://github.com/yurriy)) +- Şimdi sunucu sağlama toplamları toplu halde bozuk veri durumunda daha ayrıntılı hatalar veren partiler dağıttı. [\#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) +- Destek `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` ve `ATTACH TABLE` için `MySQL` veritabanı motoru. [\#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([Kış Zhang](https://github.com/zhang2014)) +- S3 tablo fonksiyonu ve tablo motoru kimlik doğrulama ekleyin. [\#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) +- İlave parçalar için kontrol eklendi `MergeTree` farklı disklerde, tanımlanmamış disklerdeki veri parçalarını kaçırmamak için. [\#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Mac istemci ve sunucu için SSL desteğini etkinleştirin. [\#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([I ivanvan](https://github.com/abyss7)) +- Şimdi ClickHouse MySQL Federe sunucu olarak çalışabilir (bkz https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [\#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) +- `clickhouse-client` şimdi sadece etkinleştir `bracketed-paste` multiquery açık ve çok satırlı kapalı olduğunda. Bu düzeltmeler (\#7757) \[https://github.com/ClickHouse/ClickHouse/issues/7757\]. [\#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Kuşu](https://github.com/amosbird)) +- Destek `Array(Decimal)` içinde `if` işlev. [\#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) +- Destek ondalık `arrayDifference`, `arrayCumSum` ve `arrayCumSumNegative` işlevler. [\#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) +- Katma `lifetime` sütun için `system.dictionaries` Tablo. [\#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [\#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekule](https://github.com/kekekekule)) +- İçin farklı disklerde mevcut parçalar için geliştirilmiş kontrol `*MergeTree` masa motorları. Kur [\#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [\#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) +- İle entegrasyon `AWS SDK` için `S3` tüm S3 özelliklerini kutudan çıkarmaya izin veren etkileşimler. [\#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) +- Alt sorgular için destek eklendi `Live View` Tablolar. [\#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) +- Kullanmak için kontrol edin `Date` veya `DateTime` sütun fromundan `TTL` ifadeler kaldırıldı. [\#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Disk hakkında bilgi eklendi `system.detached_parts` Tablo. [\#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Şimdi ayarlar `max_(table|partition)_size_to_drop` yeniden başlatma olmadan değiştirilebilir. [\#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +- Hata mesajlarının biraz daha iyi kullanılabilirliği. Kullanıcıdan aşağıdaki satırları kaldırmamasını isteyin `Stack trace:`. [\#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Daha iyi okuma mesajları `Kafka` sonra çeşitli biçim enginelerde motor [\#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [\#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([I ivanvan](https://github.com/abyss7)) +- Desteklemeyen MySQL istemcileri ile daha iyi uyumluluk `sha2_password` auth eklentisi. [\#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) +- MySQL uyumluluk sunucusunda daha fazla sütun türünü destekleyin. [\#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) +- Uygulamak `ORDER BY` için optimizasyon `Merge`, `Buffer` ve `Materilized View` altta yatan depolar `MergeTree` Tablolar. [\#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) +- Şimdi her zaman POSIX uygulamasını kullanıyoruz `getrandom` eski çekirdeklerle daha iyi uyumluluğa sahip olmak (\<3.17). [\#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Kuşu](https://github.com/amosbird)) +- Bir hareket TTL kuralında geçerli hedef için daha iyi kontrol edin. [\#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) +- Kırık ekleme partileri için daha iyi kontroller `Distributed` masa motoru. [\#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) +- Gelecekte işlem mustesi gereken parça adı diz ofisiyle sütun ekleme `system.mutations` Tablo. [\#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) +- İşlemciler için paralel birleştirme sıralama optimizasyonu. [\#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Ayar `mark_cache_min_lifetime` artık eskimiş ve hiçbir şey yapmıyor. Önceki sürümlerde, Mark cache bellek daha büyük büyüyebilir `mark_cache_size` içinde veri karşılamak için `mark_cache_min_lifetime` ikincilikler. Bu, karışıklığa ve beklenenden daha yüksek bellek kullanımına yol açıyordu, bu özellikle bellek kısıtlı sistemlerde kötüydü. Bu sürümü yükledikten sonra performans düşüşü görecekseniz, `mark_cache_size`. [\#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kullanıma hazırlık `tid` her yerde. Bu için gereklidir [\#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [\#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +#### Performans İyileştirme {#performance-improvement-1} + +- İşlemciler boru hattında performans iyileştirmeleri. [\#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Önbellek sözlüklerinde süresi dolmuş anahtarların engellenmeyen güncellemeleri (eskileri okuma izni ile). [\#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Olmadan ClickHouse derlemek `-fno-omit-frame-pointer` küresel bir daha kayıt yedek. [\#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Kuşu](https://github.com/amosbird)) +- Hızlanma `greatCircleDistance` fonksiyon ve bunun için performans testleri ekleyin. [\#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) +- Fonksiyonun geliştirilmiş performansı `roundDown`. [\#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Geliştirilmiş performans `max`, `min`, `argMin`, `argMax` için `DateTime64` veri türü. [\#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) +- Bir sınırı olmadan veya büyük sınırı ve dış sıralama ile sıralama geliştirilmiş performans. [\#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kayan nokta sayılarını 6 kata kadar biçimlendirmenin geliştirilmiş performansı. [\#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Geliştirilmiş performans `modulo` işlev. [\#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Kuşu](https://github.com/amosbird)) +- Optimum `ORDER BY` ve tek sütun tuşu ile birleştirme. [\#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin daha iyi uygulama `arrayReduce`, `-Array` ve `-State` birleştiriciler. [\#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Kuşu](https://github.com/amosbird)) +- Şimdi `PREWHERE` olarak en az verimli olacak şekilde optimize edilmelidir `WHERE`. [\#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Kuşu](https://github.com/amosbird)) +- Yolu geliştirmek `round` ve `roundBankers` negatif sayılar işleme. [\#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) +- Geliştirilmiş çözme performansı `DoubleDelta` ve `Gorilla` codec bileşenleri yaklaşık %30-40 oranında. Bu düzeltmeler [\#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [\#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) +- Geliştirilmiş performans `base64` ilgili fonksiyonlar. [\#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Bir işlev eklendi `geoDistance`. Bu benzer `greatCircleDistance` ancak wgs-84 elipsoid modeline yaklaşım kullanır. Her iki fonksiyonun performansı da aynıdır. [\#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Hızla `min` ve `max` için toplama fonksiyonları `Decimal` veri türü. [\#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) +- Vectorize işleme `arrayReduce`. [\#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Kuşu](https://github.com/amosbird)) +- `if` zincirler şimdi olarak optimize edilmiştir `multiIf`. [\#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) +- Performans gerilemesini düzeltin `Kafka` masa motoru 19.15'te tanıtıldı. Bu düzeltmeler [\#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [\#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) +- Kaldırıyordu “pie” kod üretimi `gcc` Debian paketlerinden bazen varsayılan olarak getirir. [\#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Paralel ayrıştırma veri formatları [\#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +- Optimize edilmiş ayrıştırıcıyı etkinleştir `Values` varsayılan olarak ifadelerle (`input_format_values_deduce_templates_of_expressions=1`). [\#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) + +#### Yapı / Test / Ambalaj Geliştirme {#buildtestingpackaging-improvement-2} + +- İçin düzeltmeler oluşturun `ARM` ve minimal modda. [\#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) +- İçin kapsama dosyası Ekle `clickhouse-server` std:: atexit çağrılmadığında. Ayrıca kapsama ile vatansız testlerde biraz geliştirilmiş günlüğü. [\#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) +- Contrib'de llvm kütüphanesini güncelleyin. OS paketlerinden LLVM kullanmaktan kaçının. [\#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Paket yapmak `curl` tamamen sessiz inşa. [\#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [\#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) +- Bazı düzeltmek `MemorySanitizer` uyarmalar. [\#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Kullanmak `add_warning` ve `no_warning` makro inlar içinde `CMakeLists.txt`. [\#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([I ivanvan](https://github.com/abyss7)) +- Minio S3 uyumlu nesne desteği ekleyin (https://min.io/) daha iyi entegrasyon testleri için. [\#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [\#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) +- İthal `libc` başlıklar contrib için. Oluşturur daha tutarlı (sadece için çeşitli sistemler üzerinde yapmak için izin verir `x86_64-linux-gnu`). [\#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kaldırmak `-fPIC` bazı kütüphanelerden. [\#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Temizlemek `CMakeLists.txt` kıvırmak için. Bkz. https://github.com/ClickHouse/ClickHouse/pull/8011\#issuecomment-569478910 [\#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sessiz uyarılar `CapNProto` kitaplık. [\#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kısa dize optimize karma tablolar için performans testleri ekleyin. [\#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Kuşu](https://github.com/amosbird)) +- Şimdi ClickHouse üzerine inşa edecek `AArch64` bile `MADV_FREE` kullanılamaz. Bu düzeltmeler [\#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [\#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Kuşu](https://github.com/amosbird)) +- Güncelleştirme `zlib-ng` bellek dezenfektanı sorunları düzeltmek için. [\#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [\#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Linux olmayan sistemde dahili MySQL kütüphanesini etkinleştirin, çünkü OS paketlerinin kullanımı çok kırılgandır ve genellikle hiç çalışmaz. Bu düzeltmeler [\#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [\#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Etkinleştirildikten sonra bazı sistemlerde sabit yapı `libc++`. Bu yerini alır [\#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [\#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yapmak `Field` yöntemleri daha fazla tür-daha fazla hata bulmak için güvenli. [\#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [\#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Eksik dosyalar eklendi `libc-headers` alt modül. [\#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yanlış Düzelt `JSON` performans testi çıktısında alıntı. [\#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Şimdi yığın izleme için görüntülenir `std::exception` ve `Poco::Exception`. Önceki sürümlerde sadece mevcuttu `DB::Exception`. Bu teşhis geliştirir. [\#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Taşıma `clock_gettime` ve `clock_nanosleep` taze glibc sürümleri için. [\#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Kuşu](https://github.com/amosbird)) +- Etkinleştirmek `part_log` geliştiriciler için örnek yapılandırmada. [\#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yeniden yüklemenin async doğasını düzeltin `01036_no_superfluous_dict_reload_on_create_database*`. [\#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) +- Sabit codec performans testleri. [\#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) +- İçin yükleme s scriptscript addleri Ekle `.tgz` onlar için yapı ve belgeler. [\#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [\#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) +- Eski kaldırıldı `ZSTD` test (2016 yılında zstd'nin 1.0 sürümünün olduğu hatayı yeniden üretmek için oluşturuldu). Bu düzeltmeler [\#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [\#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Mac OS Catalina üzerinde sabit yapı. [\#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) +- Sonuç fark yapmak için codec performans testlerinde satır sayısı arttı. [\#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) +- Hata ayıklama yapılarında, tedavi `LOGICAL_ERROR` onaylama işlemi hatası gibi özel durumlar, dikkat kolaylaştırmak. [\#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Biçimlerle ilgili performans testini daha deterministik hale getirin. [\#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Güncelleştirme `lz4` bir hafızayı düzeltmek İçinvatandaş arızası. [\#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Bilinen bir MemorySanitizer yanlış pozitif özel durum işleme bastırın. [\#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Güncelleştirme `gcc` ve `g++` sürüm 9 için `build/docker/build.sh` [\#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) +- Bunu test etmek için performans testi örneği ekleyin `PREWHERE` daha kötü `WHERE`. [\#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Kuşu](https://github.com/amosbird)) +- Bir gevşek testi düzeltmek için ilerleme. [\#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Veri için MemorySanitizer raporundan kaçının `libunwind`. [\#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Güncel `libc++` en son sürüme. [\#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Kaynaklardan YBÜ Kütüphanesi oluşturun. Bu düzeltmeler [\#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [\#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Anahtar fromlı `libressl` -e doğru `openssl`. ClickHouse, bu değişiklikten sonra TLS 1.3 ve SNI'Yİ desteklemelidir. Bu düzeltmeler [\#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [\#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Sabit UBSan raporu kullanırken `chacha20_poly1305` SS SSLL ('den (Bağlan (ış happensta olur https://yandex.ru/). [\#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin varsayılan şifre dosyasının düzeltme modu `.deb` linux dağıtımları. [\#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) +- Almak için geliştirilmiş ifade `clickhouse-server` Pidıd ın `clickhouse-test`. [\#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) +- V1.10.0 için contrib/googletest güncellendi. [\#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) +- Sabit ThreadSaninitizer raporu `base64` kitaplık. Ayrıca bu kütüphaneyi en son sürüme güncelledi, ancak önemli değil. Bu düzeltmeler [\#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [\#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Düzeltmek `00600_replace_running_query` işlemciler için. [\#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- İçin destek Kaldır `tcmalloc` yapmak `CMakeLists.txt` basit. [\#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yayın gcc şimdi kullanmak oluşturur `libc++` yerine `libstdc++`. Son zamanlarda `libc++` sadece clang ile kullanıldı. Bu yapı yapılandırmaları ve taşınabilirlik tutarlılığını artıracaktır. [\#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- MemorySanitizer ile inşa etmek için YBÜ kütüphanesini etkinleştirin. [\#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Gelen uyarıları bastır `CapNProto` kitaplık. [\#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- İçin özel kod durumları kaldırıldı `tcmalloc`, çünkü artık desteklenmiyor. [\#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- CI kapsama görevinde, kapsama raporunu kaydetmesine izin vermek için sunucuyu incelikle öldürün. Bu, son zamanlarda gördüğümüz eksik kapsama raporlarını düzeltir. [\#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alesapin](https://github.com/alesapin)) +- Karşı tüm codec bileşenleri için performans testleri `Float64` ve `UInt64` değerler. [\#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) +- `termcap` (f çeşitli sorunlar için çok kaldırılmış ve kurşundur.g. eksik “up” kap ve yankılanan `^J` çok satır yerine). İyilik `terminfo` veya paketlenmiş `ncurses`. [\#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Kuşu](https://github.com/amosbird)) +- Düzeltmek `test_storage_s3` entegrasyon testi. [\#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Destek `StorageFile(, null)` aslında Diske yazmadan verilen format dosyasına blok eklemek için. Bu performans testleri için gereklidir. [\#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Kuşu](https://github.com/amosbird)) +- Eklenen argüman `--print-time` test başına yürütme süresini basan işlevsel testlere. [\#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Eklenen iddialar `KeyCondition` rpn değerlendirirken. Bu, gcc-9'dan gelen uyarıyı düzeltir. [\#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Cı yapılarında cmake seçeneklerini dökümü. [\#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Bazı fat kitaplıkları için hata ayıklama bilgisi oluşturmayın. [\#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Yapmak `log_to_console.xml` her zaman ne olursa olsun interaktif ya da değil, stderr oturum açın. [\#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) +- Kullanılmayan bazı özellikler kaldırıldı `clickhouse-performance-test` aracı. [\#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Şimdi de arayacağız `lld-X` karşılık gelen ile `clang-X` sürüm. [\#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alesapin](https://github.com/alesapin)) +- Parke inşa iyileştirme. [\#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) +- Daha fazla GCC uyarısı [\#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +- Arch Linux için paket şimdi ClickHouse sunucusu çalıştırmak için izin verir, ve sadece istemci. [\#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) +- İşlemcilerle testi düzeltin. Küçük performans düzeltmeleri. [\#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Contrib/protobuf güncelleyin. [\#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) +- Yeni yıl kutlaması olarak C++20'ye geçmenin hazırlanmasında. “May the C++ force be with ClickHouse.” [\#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Kuşu](https://github.com/amosbird)) + +#### Deneysel Özellik {#experimental-feature-1} + +- Deneysel ayar eklendi `min_bytes_to_use_mmap_io`. Bu userspace çekirdekten veri kopyalamadan büyük dosyaları okumak için izin verir. Bu ayar varsayılan olarak devre dışıdır. Önerilen eşik yaklaşık 64 MB'dir, çünkü mmap / munmap yavaştır. [\#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Erişim kontrol sisteminin bir parçası olarak yeniden işlenmiş kotalar. Yeni tablo eklendi `system.quotas` yeni fonksiyonlar `currentQuota`, `currentQuotaKey`, yeni SQL sözdizimi `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [\#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) +- İstisnalar atmak yerine bilinmeyen ayarları uyarılarla atlamaya izin verin. [\#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) +- Erişim kontrol sisteminin bir parçası olarak reworked satır politikaları. Yeni tablo eklendi `system.row_policies` yeni işlev `currentRowPolicies()`, yeni SQL sözdizimi `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [\#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) + +#### Güvenlik Düzeltme {#security-fix} + +- İle tablolarda dizin yapısını okuma imkanı Düzelt theildi `File` masa motoru. Bu düzeltmeler [\#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [\#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +## [2019 için Changelog](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md) {#changelog-for-2019} diff --git a/docs/tr/whats_new/index.md b/docs/tr/whats_new/index.md new file mode 100644 index 00000000000..17464a36cb0 --- /dev/null +++ b/docs/tr/whats_new/index.md @@ -0,0 +1,8 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_folder_title: Ne yeni +toc_priority: 72 +--- + + diff --git a/docs/tr/whats_new/roadmap.md b/docs/tr/whats_new/roadmap.md new file mode 100644 index 00000000000..fc43396d834 --- /dev/null +++ b/docs/tr/whats_new/roadmap.md @@ -0,0 +1,19 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 74 +toc_title: "Yol haritas\u0131" +--- + +# Yol haritası {#roadmap} + +## Q1 2020 {#q1-2020} + +- Rol tabanlı erişim denetimi + +## Q2 2020 {#q2-2020} + +- Dış kimlik doğrulama hizmetleri ile entegrasyon +- Kullanıcılar arasında küme kapasitesinin daha hassas dağılımı için kaynak havuzları + +{## [Orijinal makale](https://clickhouse.tech/docs/en/roadmap/) ##} diff --git a/docs/tr/whats_new/security_changelog.md b/docs/tr/whats_new/security_changelog.md new file mode 100644 index 00000000000..117d1766cdb --- /dev/null +++ b/docs/tr/whats_new/security_changelog.md @@ -0,0 +1,76 @@ +--- +machine_translated: true +machine_translated_rev: e8cd92bba3269f47787db090899f7c242adf7818 +toc_priority: 76 +toc_title: "G\xFCvenlik Changelog" +--- + +## ClickHouse sürümünde düzeltildi 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} + +### CVE-2019-15024 {#cve-2019-15024} + +Аn attacker that has write access to ZooKeeper and who ican run a custom server available from the network where ClickHouse runs, can create a custom-built malicious server that will act as a ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from the malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. + +Kredi: Yandex Bilgi Güvenliği ekibinden Eldar Zaitov + +### CVE-2019-16535 {#cve-2019-16535} + +Аn OOB read, OOB write and integer underflow in decompression algorithms can be used to achieve RCE or DoS via native protocol. + +Kredi: Yandex Bilgi Güvenliği ekibinden Eldar Zaitov + +### CVE-2019-16536 {#cve-2019-16536} + +DOS'A giden yığın taşması, kötü amaçlı kimliği doğrulanmış bir istemci tarafından tetiklenebilir. + +Kredi: Yandex Bilgi Güvenliği ekibinden Eldar Zaitov + +## ClickHouse sürümü 19.13.6.1, 2019-09-20'de düzeltildi {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20} + +### CVE-2019-18657 {#cve-2019-18657} + +Tablo fonksiyonu `url` güvenlik açığı saldırganın istekte rasgele HTTP üstbilgileri enjekte etmesine izin vermişti. + +Krediler: [Nikita Tikhomirov](https://github.com/NSTikhomirov) + +## ClickHouse sürümünde sabit 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10} + +### CVE-2018-14672 {#cve-2018-14672} + +Catboost modellerini yüklemek için işlevler, yol geçişine izin verdi ve hata mesajları aracılığıyla keyfi dosyaları okudu. + +Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov + +## ClickHouse sürüm 18.10.3, 2018-08-13 sabit {#fixed-in-clickhouse-release-18-10-3-2018-08-13} + +### CVE-2018-14671 {#cve-2018-14671} + +unixODBC, dosya sisteminden rasgele paylaşılan nesnelerin yüklenmesine izin verdi ve bu da uzaktan kod yürütme güvenlik açığına yol açtı. + +Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov ve Evgeny Sidorov + +## ClickHouse sürüm 1.1.54388, 2018-06-28 sabit {#fixed-in-clickhouse-release-1-1-54388-2018-06-28} + +### CVE-2018-14668 {#cve-2018-14668} + +“remote” tablo fonksiyonu izin keyfi semboller “user”, “password” ve “default\_database” çapraz Protokol isteği sahtecilik saldırılarına yol açan alanlar. + +Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov + +## ClickHouse sürüm 1.1.54390, 2018-07-06 sabit {#fixed-in-clickhouse-release-1-1-54390-2018-07-06} + +### CVE-2018-14669 {#cve-2018-14669} + +ClickHouse MySQL istemcisi vardı “LOAD DATA LOCAL INFILE” işlevsellik, kötü niyetli bir MySQL veritabanının bağlı ClickHouse sunucusundan rasgele dosyaları okumasına izin verdi. + +Kredi: Yandex Bilgi Güvenliği ekibinden Andrey Krasichkov ve Evgeny Sidorov + +## ClickHouse sürüm 1.1.54131, 2017-01-10 sabit {#fixed-in-clickhouse-release-1-1-54131-2017-01-10} + +### CVE-2018-14670 {#cve-2018-14670} + +Deb paketindeki yanlış yapılandırma, veritabanının yetkisiz kullanımına neden olabilir. + +Kredi: İngiltere'nin Ulusal siber güvenlik merkezi (NCSC) + +{## [Orijinal makale](https://clickhouse.tech/docs/en/security_changelog/) ##} diff --git a/docs/zh/changelog/index.md b/docs/zh/changelog/index.md index 90bb7abe0b0..c79e32ceaf3 100644 --- a/docs/zh/changelog/index.md +++ b/docs/zh/changelog/index.md @@ -1,6 +1,7 @@ --- machine_translated: true machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 +toc_title: '变更日志' --- ## 碌莽禄release拢.0755-88888888 {#clickhouse-release-v20-3} @@ -246,7 +247,7 @@ machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 - 更新了clickhouse-test脚本中挂起查询的检查 [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) - 从存储库中删除了一些无用的文件。 [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 更改类型的数学perftests从 `once` 到 `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 添加码头镜像，它允许为我们的代码库构建交互式代码浏览器HTML报告。 [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([阿利沙平](https://github.com/alesapin)）见 [Woboq代码浏览器](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html) +- 添加码头镜像，它允许为我们的代码库构建交互式代码浏览器HTML报告。 [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([阿利沙平](https://github.com/alesapin)）见 [Woboq代码浏览器](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html) - 抑制MSan下的一些测试失败。 [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) - 加速 “exception while insert” 测试 此测试通常在具有复盖率的调试版本中超时。 [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 更新 `libcxx` 和 `libcxxabi` 为了主人 在准备 [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) diff --git a/docs/zh/development/browse_code.md b/docs/zh/development/browse_code.md index 10d3ffecd15..d098675b6a0 100644 --- a/docs/zh/development/browse_code.md +++ b/docs/zh/development/browse_code.md @@ -7,7 +7,7 @@ toc_title: "\u6D4F\u89C8ClickHouse\u6E90\u4EE3\u7801" # 浏览ClickHouse源代码 {#browse-clickhouse-source-code} -您可以使用 **Woboq** 在线代码浏览器可用 [这里](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/src/index.html). 它提供了代码导航和语义突出显示，搜索和索引。 代码快照每天更新。 +您可以使用 **Woboq** 在线代码浏览器可用 [这里](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). 它提供了代码导航和语义突出显示，搜索和索引。 代码快照每天更新。 此外，您还可以浏览源 [GitHub](https://github.com/ClickHouse/ClickHouse) 像往常一样 diff --git a/docs/zh/engines/database_engines/mysql.md b/docs/zh/engines/database_engines/mysql.md index 78844154bce..80ff82ec2d3 100644 --- a/docs/zh/engines/database_engines/mysql.md +++ b/docs/zh/engines/database_engines/mysql.md @@ -7,8 +7,6 @@ MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并 但您无法对其执行以下操作： -- `ATTACH`/`DETACH` -- `DROP` - `RENAME` - `CREATE TABLE` - `ALTER` @@ -17,7 +15,7 @@ MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并 ``` sql CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', 'database', 'user', 'password') +ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') ``` **MySQL数据库引擎参数** diff --git a/docs/zh/engines/table_engines/index.md b/docs/zh/engines/table_engines/index.md index 9603ebe78c8..bb0443e4868 100644 --- a/docs/zh/engines/table_engines/index.md +++ b/docs/zh/engines/table_engines/index.md @@ -1,4 +1,3 @@ - # 表引擎 {#biao-yin-qing} 表引擎（即表的类型）决定了： @@ -14,16 +13,16 @@ ## MergeTree {#mergetree} -适用于高负载任务的最通用和功能最强大的表引擎。这些引擎的共同特点是可以快速插入数据并进行后续的后台数据处理。 MergeTree系列引擎支持数据复制（使用[复制\*](mergetree_family/replication.md) 的引擎版本），分区和一些其他引擎不支持的其他功能。 +适用于高负载任务的最通用和功能最强大的表引擎。这些引擎的共同特点是可以快速插入数据并进行后续的后台数据处理。 MergeTree系列引擎支持数据复制（使用[Replicated\*](mergetree_family/replication.md) 的引擎版本），分区和一些其他引擎不支持的其他功能。 该类型的引擎： -\* [MergeTree](mergetree_family/mergetree.md) -\* [更换麦树](mergetree_family/replacingmergetree.md) -\* [SummingMergeTree](mergetree_family/summingmergetree.md) -\* [AggregatingMergeTree](mergetree_family/aggregatingmergetree.md) -\* [折叠树](mergetree_family/collapsingmergetree.md) -\* [版本集合在新树](mergetree_family/versionedcollapsingmergetree.md) -\* [GraphiteMergeTree](mergetree_family/graphitemergetree.md) +- [MergeTree](mergetree_family/mergetree.md) +- [ReplacingMergeTree](mergetree_family/replacingmergetree.md) +- [SummingMergeTree](mergetree_family/summingmergetree.md) +- [AggregatingMergeTree](mergetree_family/aggregatingmergetree.md) +- [CollapsingMergeTree](mergetree_family/collapsingmergetree.md) +- [VersionedCollapsingMergeTree](mergetree_family/versionedcollapsingmergetree.md) +- [GraphiteMergeTree](mergetree_family/graphitemergetree.md) ## 日志 {#log} @@ -33,14 +32,14 @@ - [TinyLog](log_family/tinylog.md) - [StripeLog](log_family/stripelog.md) -- [日志](log_family/log.md) +- [Log](log_family/log.md) ## 集成引擎 {#integration-engines} 用于与其他的数据存储与处理系统集成的引擎。 该类型的引擎： -- [卡夫卡](integrations/kafka.md) +- [Kafka](integrations/kafka.md) - [MySQL](integrations/mysql.md) - [ODBC](integrations/odbc.md) - [JDBC](integrations/jdbc.md) @@ -50,18 +49,18 @@ 该类型的引擎： -- [分布](special/distributed.md) +- [Distributed](special/distributed.md) - [MaterializedView](special/materializedview.md) -- [字典](special/dictionary.md) -- [合并](special/merge.md) -- [文件](special/file.md) +- [Dictionary](special/dictionary.md) +- [Merge](special/merge.md) +- [File](special/file.md) - [Null](special/null.md) -- [设置](special/set.md) -- [加入我们](special/join.md) +- [Set](special/set.md) +- [Join](special/join.md) - [URL](special/url.md) -- [查看](special/view.md) -- [记忆](special/memory.md) -- [缓冲区](special/buffer.md) +- [View](special/view.md) +- [Memory](special/memory.md) +- [Buffer](special/buffer.md) # 虚拟列 {#xu-ni-lie} @@ -69,6 +68,6 @@ 您不能在 `CREATE TABLE` 中指定虚拟列，并且虚拟列不会包含在 `SHOW CREATE TABLE` 和 `DESCRIBE TABLE` 的查询结果中。虚拟列是只读的，所以您不能向虚拟列中写入数据。 -如果想要查询虚拟列中的数据，您必须在SELECT查询中包含虚拟列的名字。SELECT \* 不会返回虚拟列的内容。 +如果想要查询虚拟列中的数据，您必须在SELECT查询中包含虚拟列的名字。`SELECT *` 不会返回虚拟列的内容。 若您创建的表中有一列与虚拟列的名字相同，那么虚拟列将不能再被访问。我们不建议您这样做。为了避免这种列名的冲突，虚拟列的名字一般都以下划线开头。 diff --git a/docs/zh/getting_started/example_datasets/amplab_benchmark.md b/docs/zh/getting_started/example_datasets/amplab_benchmark.md index 4c3b26819b1..30d55f8b28d 100644 --- a/docs/zh/getting_started/example_datasets/amplab_benchmark.md +++ b/docs/zh/getting_started/example_datasets/amplab_benchmark.md @@ -1,5 +1,5 @@ -# AMPLab 大数据基准测试 {#amplab-da-shu-ju-ji-zhun-ce-shi} +# AMPLab大数据基准测试 {#amplab-da-shu-ju-ji-zhun-ce-shi} 参考 https://amplab.cs.berkeley.edu/benchmark/ diff --git a/docs/zh/getting_started/example_datasets/index.md b/docs/zh/getting_started/example_datasets/index.md index 4faf3b0ecfc..c610af8a269 100644 --- a/docs/zh/getting_started/example_datasets/index.md +++ b/docs/zh/getting_started/example_datasets/index.md @@ -1,6 +1,4 @@ --- -machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 toc_folder_title: "\u793A\u4F8B\u6570\u636E\u96C6" toc_priority: 12 toc_title: "\u5BFC\u8A00" @@ -11,12 +9,12 @@ toc_title: "\u5BFC\u8A00" 本节介绍如何获取示例数据集并将其导入ClickHouse。 对于某些数据集示例查询也可用。 -- [匿名Yandex的。梅里卡数据集](metrica.md) -- [星型架构基准测试](star_schema.md) -- [WikiStat](wikistat.md) -- [来自Criteo的万兆字节点击日志](criteo.md) -- [AMPLab大数据基准](amplab_benchmark.md) +- [脱敏的Yandex.Metrica数据集](metrica.md) +- [星型基准测试](star_schema.md) +- [维基访问数据](wikistat.md) +- [Criteo TB级别点击日志](criteo.md) +- [AMPLab大数据基准测试](amplab_benchmark.md) - [纽约出租车数据](nyc_taxi.md) -- [时间](ontime.md) +- [航班飞行数据](ontime.md) [原始文章](https://clickhouse.tech/docs/en/getting_started/example_datasets) diff --git a/docs/zh/getting_started/example_datasets/metrica.md b/docs/zh/getting_started/example_datasets/metrica.md index f7e0c86d324..6e349a1135d 100644 --- a/docs/zh/getting_started/example_datasets/metrica.md +++ b/docs/zh/getting_started/example_datasets/metrica.md @@ -1,11 +1,9 @@ --- -machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 toc_priority: 21 toc_title: "Yandex\u6885\u7279\u91CC\u5361\u6570\u636E" --- -# 匿名Yandex的。梅特里卡数据 {#anonymized-yandex-metrica-data} +# 脱敏的Yandex.Metrica数据集 {#anonymized-yandex-metrica-data} Dataset由两个表组成，其中包含有关命中的匿名数据 (`hits_v1`）和访问 (`visits_v1`）的Yandex的。梅特里卡 你可以阅读更多关于Yandex的。梅特里卡 [ClickHouse历史](../../introduction/history.md) 科。 diff --git a/docs/zh/getting_started/example_datasets/star_schema.md b/docs/zh/getting_started/example_datasets/star_schema.md index 4fed13923ff..b575abe63fa 100644 --- a/docs/zh/getting_started/example_datasets/star_schema.md +++ b/docs/zh/getting_started/example_datasets/star_schema.md @@ -1,5 +1,5 @@ -# 星型架构基准测试 {#star-schema-benchmark} +# 星型基准测试 {#star-schema-benchmark} 编译 dbgen: diff --git a/docs/zh/getting_started/install.md b/docs/zh/getting_started/install.md index 6a9aae286ad..111c362caf7 100644 --- a/docs/zh/getting_started/install.md +++ b/docs/zh/getting_started/install.md @@ -1,3 +1,4 @@ +# 安装 {#clickhouse-an-zhuang} ## 系统要求 {#xi-tong-yao-qiu} diff --git a/docs/zh/getting_started/playground.md b/docs/zh/getting_started/playground.md index a09d615ba21..252b2eae611 100644 --- a/docs/zh/getting_started/playground.md +++ b/docs/zh/getting_started/playground.md @@ -1,14 +1,12 @@ --- -machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 toc_priority: 14 toc_title: "\u266A\u64CD\u573A\u266A" --- -# ﾂ环板Playgroundｮﾂ嘉ｯ {#clickhouse-playground} +# ClickHouse体验平台 {#clickhouse-playground} -[ﾂ环板Playgroundｮﾂ嘉ｯ](https://play.clickhouse.tech?file=welcome) 允许人们通过即时运行查询来尝试ClickHouse，而无需设置他们的服务器或集群。 -Playground中提供了几个示例数据集以及显示ClickHouse要素的示例查询。 +[ClickHouse体验平台](https://play.clickhouse.tech?file=welcome) 允许人们通过即时运行查询来尝试ClickHouse，而无需设置他们的服务器或集群。 +体验平台中提供了几个示例数据集以及显示ClickHouse特性的示例查询。 查询以只读用户身份执行。 这意味着一些局限性: @@ -21,25 +19,26 @@ Playground中提供了几个示例数据集以及显示ClickHouse要素的示例 - [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode) - [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) -ClickHouse游乐场给m2的经验。小 -[管理服务ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) -实例托管在 [Yandex云](https://cloud.yandex.com/). +ClickHouse体验还有如下： +[ClickHouse管理服务](https://cloud.yandex.com/services/managed-clickhouse) +实例托管 [Yandex云](https://cloud.yandex.com/). 更多信息 [云提供商](../commercial/cloud.md). -ClickHouse游乐场网的界面使请求通过ClickHouse [HTTP API](../interfaces/http.md). -Playground后端只是一个ClickHouse集群，没有任何额外的服务器端应用程序。 -隆隆隆隆路虏脢..陇.貌.垄拢卢虏禄.陇.貌路.隆拢脳枚脢虏 +ClickHouse体验平台界面实际上是通过ClickHouse [HTTP API](../interfaces/http.md)接口实现的. +体验平台后端只是一个ClickHouse集群，没有任何额外的服务器端应用程序。 +体验平台也同样提供了ClickHouse HTTPS服务端口。 -您可以使用任何HTTP客户端向playground进行查询，例如 [卷曲的](https://curl.haxx.se) 或 [wget](https://www.gnu.org/software/wget/)，或使用以下方式建立连接 [JDBC](../interfaces/jdbc.md) 或 [ODBC](../interfaces/odbc.md) 司机 + +您可以使用任何HTTP客户端向体验平台进行查询，例如 [curl](https://curl.haxx.se) 或 [wget](https://www.gnu.org/software/wget/)，或使用以下方式建立连接 [JDBC](../interfaces/jdbc.md) 或 [ODBC](../interfaces/odbc.md) 司机 有关支持ClickHouse的软件产品的更多信息，请访问 [这里](../interfaces/index.md). -| 参数 | 价值 | +| 参数 | 值 | |:-----|:--------------------------------------| -| 端点 | https://play-api.克莱克豪斯技术：8443 | +| 服务端口 | https://play-api.clickhouse.tech:8443 | | 用户 | `playground` | | 密码 | `clickhouse` | -请注意，此端点需要安全连接。 +请注意，此服务端口需要安全连接。 示例: diff --git a/docs/zh/getting_started/tutorial.md b/docs/zh/getting_started/tutorial.md index 4fc2fbdf290..bd86e19a037 100644 --- a/docs/zh/getting_started/tutorial.md +++ b/docs/zh/getting_started/tutorial.md @@ -5,7 +5,7 @@ toc_priority: 12 toc_title: "\u6559\u7A0B" --- -# 点击教程 {#clickhouse-tutorial} +# 教程 {#clickhouse-tutorial} ## 从本教程中可以期待什么？ {#what-to-expect-from-this-tutorial} diff --git a/docs/zh/index.md b/docs/zh/index.md index cb9ccf0420a..c6cc5069b14 100644 --- a/docs/zh/index.md +++ b/docs/zh/index.md @@ -1,8 +1,3 @@ ---- -machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 ---- - # 什么是ClickHouse？ {#shi-yao-shi-clickhouse} ClickHouse是一个用于联机分析(OLAP)的列式数据库管理系统(DBMS)。 @@ -81,54 +76,6 @@ ClickHouse是一个用于联机分析(OLAP)的列式数据库管理系统(DBMS) 例如，查询«统计每个广告平台的记录数量»需要读取«广告平台ID»这一列，它在未压缩的情况下需要1个字节进行存储。如果大部分流量不是来自广告平台，那么这一列至少可以以十倍的压缩率被压缩。当采用快速压缩算法，它的解压速度最少在十亿字节(未压缩数据)每秒。换句话说，这个查询可以在单个服务器上以每秒大约几十亿行的速度进行处理。这实际上是当前实现的速度。 -
- -示例 - - $ clickhouse-client - ClickHouse client version 0.0.52053. - Connecting to localhost:9000. - Connected to ClickHouse server version 0.0.52053. - - :) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 - - SELECT - CounterID, - count() - FROM hits - GROUP BY CounterID - ORDER BY count() DESC - LIMIT 20 - - ┌─CounterID─┬──count()─┐ - │ 114208 │ 56057344 │ - │ 115080 │ 51619590 │ - │ 3228 │ 44658301 │ - │ 38230 │ 42045932 │ - │ 145263 │ 42042158 │ - │ 91244 │ 38297270 │ - │ 154139 │ 26647572 │ - │ 150748 │ 24112755 │ - │ 242232 │ 21302571 │ - │ 338158 │ 13507087 │ - │ 62180 │ 12229491 │ - │ 82264 │ 12187441 │ - │ 232261 │ 12148031 │ - │ 146272 │ 11438516 │ - │ 168777 │ 11403636 │ - │ 4120072 │ 11227824 │ - │ 10938808 │ 10519739 │ - │ 74088 │ 9047015 │ - │ 115079 │ 8837972 │ - │ 337234 │ 8205961 │ - └───────────┴──────────┘ - - 20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) - - :) - -
- ### CPU {#cpu} 由于执行一个查询需要处理大量的行，因此在整个向量上执行所有操作将比在每一行上执行所有操作更加高效。同时这将有助于实现一个几乎没有调用成本的查询引擎。如果你不这样做，使用任何一个机械硬盘，查询引擎都不可避免的停止CPU进行等待。所以，在数据按列存储并且按列执行是很有意义的。 diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md index aac3d7a1b11..128a4060c2d 100644 --- a/docs/zh/interfaces/third-party/integrations.md +++ b/docs/zh/interfaces/third-party/integrations.md @@ -1,4 +1,3 @@ - # 第三方集成库 {#di-san-fang-ji-cheng-ku} !!! warning "声明" @@ -8,18 +7,21 @@ - 关系数据库管理系统 - [MySQL](https://www.mysql.com) - - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - - [horgh-复制器](https://github.com/larsnovikov/horgh-replicator) + - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) + - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) + - [horgh-复制器](https://github.com/larsnovikov/horgh-replicator) - [PostgreSQL](https://www.postgresql.org) - - [clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) - - [infi.clickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (使用 [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - - [pg2ch](https://github.com/mkabilov/pg2ch) + - [clickhousedb\_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) + - [infi.clickhouse\_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (使用 [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [pg2ch](https://github.com/mkabilov/pg2ch) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - - [ClickHouseMightrator](https://github.com/zlzforever/ClickHouseMigrator) + - [ClickHouseMightrator](https://github.com/zlzforever/ClickHouseMigrator) - 消息队列 - [卡夫卡](https://kafka.apache.org) - - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (使用 [去客户](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (使用 [去客户](https://github.com/ClickHouse/clickhouse-go/)) +- 流处理 + - [Flink](https://flink.apache.org) + - [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink) - 对象存储 - [S3](https://en.wikipedia.org/wiki/Amazon_S3) - [ﾂ环板backupｮﾂ嘉ｯﾂ偲](https://github.com/AlexAkulov/clickhouse-backup) @@ -32,41 +34,44 @@ - [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse) - 监控 - [石墨](https://graphiteapp.org) - - [graphouse](https://github.com/yandex/graphouse) - - [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/lomik/carbon-clickhouse) + - - [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/lomik/graphite-clickhouse) - - [石墨-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) -优化静态分区 [\*GraphiteMergeTree](../../engines/table_engines/mergetree_family/graphitemergetree.md#graphitemergetree) 如果从规则 [汇总配置](../../engines/table_engines/mergetree_family/graphitemergetree.md#rollup-configuration) 可以应用 + - [graphouse](https://github.com/yandex/graphouse) + - [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/lomik/carbon-clickhouse) + + - [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/lomik/graphite-clickhouse) + - [石墨-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) -优化静态分区 [\*GraphiteMergeTree](../../engines/table_engines/mergetree_family/graphitemergetree.md#graphitemergetree) 如果从规则 [汇总配置](../../engines/table_engines/mergetree_family/graphitemergetree.md#rollup-configuration) 可以应用 - [Grafana](https://grafana.com/) - - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) + - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) - [普罗米修斯号](https://prometheus.io/) - - [clickhouse\_exporter](https://github.com/f1yegor/clickhouse_exporter) - - [PromHouse](https://github.com/Percona-Lab/PromHouse) - - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) （用途 [去客户](https://github.com/kshvakov/clickhouse/)) + - [clickhouse\_exporter](https://github.com/f1yegor/clickhouse_exporter) + - [PromHouse](https://github.com/Percona-Lab/PromHouse) + - [clickhouse\_exporter](https://github.com/hot-wifi/clickhouse_exporter) （用途 [去客户](https://github.com/kshvakov/clickhouse/)) - [Nagios](https://www.nagios.org/) - - [check\_clickhouse](https://github.com/exogroup/check_clickhouse/) - - [check\_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) + - [check\_clickhouse](https://github.com/exogroup/check_clickhouse/) + - [check\_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py) - [Zabbix](https://www.zabbix.com) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/Altinity/clickhouse-zabbix-template) + - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/Altinity/clickhouse-zabbix-template) - [Sematext](https://sematext.com/) - - [clickhouse积分](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) + - [clickhouse积分](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse) - 记录 - [rsyslog](https://www.rsyslog.com/) - - [鹿茫house om omhousehousehousehouse酶](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) + - [鹿茫house omhousee酶](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - - [loghouse](https://github.com/flant/loghouse) (对于 [Kubernetes](https://kubernetes.io)) - - [logagent](https://www.sematext.com/logagent) - - [logagent输出-插件-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) + - [loghouse](https://github.com/flant/loghouse) (对于 [Kubernetes](https://kubernetes.io)) + - [Sematext](https://www.sematext.com/logagent) + - [logagent输出-插件-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/) - 地理 - [MaxMind](https://dev.maxmind.com/geoip/) - - [ﾂ环板-ｮﾂ嘉ｯﾂ偲青clickｼﾂ氾ｶﾂ鉄ﾂ工ﾂ渉](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) + - [ﾂ环板-ｮﾂ嘉ｯﾂ偲青clickｼﾂ氾ｶﾂ鉄ﾂ工ﾂ渉](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip) ## 编程语言生态系统 {#bian-cheng-yu-yan-sheng-tai-xi-tong} - Python - [SQLAlchemy](https://www.sqlalchemy.org) - - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/cloudflare/sqlalchemy-clickhouse) (使用 [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) + - [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/cloudflare/sqlalchemy-clickhouse) (使用 [infi.clickhouse\_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [熊猫](https://pandas.pydata.org) - - [pandahouse](https://github.com/kszucs/pandahouse) + - [pandahouse](https://github.com/kszucs/pandahouse) +- PHP + - [Doctrine](https://www.doctrine-project.org/) + - [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse) - R - [dplyr](https://db.rstudio.com/dplyr/) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) (使用 [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/artpaul/clickhouse-cpp)) diff --git a/docs/zh/introduction/adopters.md b/docs/zh/introduction/adopters.md index 8a69e67264e..7e18cd9cdef 100644 --- a/docs/zh/introduction/adopters.md +++ b/docs/zh/introduction/adopters.md @@ -1,82 +1,84 @@ --- -machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 toc_priority: 8 toc_title: "\u91C7\u7528\u8005" --- -# ﾂ环板tersｮﾂ嘉ｯﾂ偲 {#clickhouse-adopters} +# ClickHouse用户 {#clickhouse-adopters} !!! warning "免责声明" - 使用ClickHouse的公司和他们的成功故事下面的名单是从公共来源组装,因此可能不同于当前的现实. 如果您分享您公司采用ClickHouse的故事，我们将不胜感激 [将其添加到列表](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md)，但请确保你不会有任何保密协议的问题，这样做。 提供来自其他公司的出版物的更新也很有用。 + 如下使用ClickHouse的公司和他们的成功案例来源于公开资源，因此和实际情况可能有所出入。如果您分享您公司使用ClickHouse的故事，我们将不胜感激 [将其添加到列表](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md)，但请确保你这样做不会有任何保密协议的问题。也欢迎提供来自其他公司的出版物的更新。 | 公司简介 | 行业 | 用例 | 群集大小 | (Un)压缩数据大小\* | 参考资料 | |-----------------------------------------------------------------|-------------------|----------------|---------------------------------------------------|----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [2gis](https://2gis.ru) | 地图 | 监测 | — | — | [讲俄语，2019年7月](https://youtu.be/58sPkXfq6nw) | -| [阿罗哈浏览器](https://alohabrowser.com/) | 移动应用程序 | 浏览器后端 | — | — | [俄罗斯幻灯片，2019年5月](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | -| [阿玛迪斯](https://amadeus.com/) | 旅费 | 分析 | — | — | [新闻稿,四月2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | -| [Appsflyer](https://www.appsflyer.com) | 移动分析 | 主要产品 | — | — | [讲俄语，2019年7月](https://www.youtube.com/watch?v=M3wbRlcpBbY) | -| [ArenaData](https://arenadata.tech/) | 数据平台 | 主要产品 | — | — | [幻灯片在俄罗斯，十二月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | -| [Badoo](https://badoo.com) | 约会 | 时间序列 | — | — | [幻灯片在俄罗斯，十二月2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | -| [Benocs](https://www.benocs.com/) | 网络遥测和分析 | 主要产品 | — | — | [幻灯片英文，2017年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | -| [彭博](https://www.bloomberg.com/) | 金融、媒体 | 监测 | 102个服务器 | — | [幻灯片，2018年5月](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | -| [Bloxy](https://bloxy.info) | 区块链 | 分析 | — | — | [幻灯片在俄罗斯，八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| [2gis](https://2gis.ru) | 地图 | 监测 | — | — | [俄文，2019年7月](https://youtu.be/58sPkXfq6nw) | +| [Aloha 浏览器](https://alohabrowser.com/) | 移动应用程序 | 浏览器后端 | — | — | [俄文幻灯片，2019年5月](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | +| [阿玛迪斯](https://amadeus.com/) | 旅行 | 分析 | — | — | [新闻稿,四月2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | +| [Appsflyer](https://www.appsflyer.com) | 移动分析 | 主要产品 | — | — | [俄文，2019年7月](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| [ArenaData](https://arenadata.tech/) | 数据平台 | 主要产品 | — | — | [俄文幻灯片，十二月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | +| [Badoo](https://badoo.com) | 约会 | 时间序列 | — | — | [俄文幻灯片，十二月2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| [Benocs](https://www.benocs.com/) | 网络遥测和分析 | 主要产品 | — | — | [英文幻灯片，2017年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | +| [彭博社](https://www.bloomberg.com/) | 金融、媒体 | 监测 | 102个服务器 | — | [幻灯片，2018年5月](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| [Bloxy](https://bloxy.info) | 区块链 | 分析 | — | — | [俄文幻灯片，八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | | `Dataliance/UltraPower` | 电信 | 分析 | — | — | [中文幻灯片，2018年1月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | | [CARTO](https://carto.com/) | 商业智能 | 地理分析 | — | — | [地理空间处理与ClickHouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | | [CERN](http://public.web.cern.ch/public/) | 研究 | 实验 | — | — | [新闻稿,四月2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://cisco.com/) | 碌莽禄Networking: | 流量分析 | — | — | [闪电对话，十月2019](https://youtu.be/-hI1vDR2oPY?t=5057) | -| [城堡证券](https://www.citadelsecurities.com/) | 财政 | — | — | — | [贡献，2019年3月](https://github.com/ClickHouse/ClickHouse/pull/4774) | -| [Citymobil](https://city-mobil.ru) | 出租车 | 分析 | — | — | [博客文章在俄罗斯，三月2020](https://habr.com/en/company/citymobil/blog/490660/) | -| [内容广场](https://contentsquare.com) | 网站分析 | 主要产品 | — | — | [博客文章在法国，十一月2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| [思科](http://cisco.com/) | 网络 | 流量分析 | — | — | [闪电对话，十月2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| [城堡证券](https://www.citadelsecurities.com/) | 金融 | — | — | — | [贡献，2019年3月](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| [Citymobil](https://city-mobil.ru) | 出租车 | 分析 | — | — | [俄文博客文章，三月2020](https://habr.com/en/company/citymobil/blog/490660/) | +| [内容广场](https://contentsquare.com) | 网站分析 | 主要产品 | — | — | [法文博客文章，十一月2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | [Cloudflare](https://cloudflare.com) | CDN | 流量分析 | 36服务器 | — | [博客文章,五月2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [博客文章,三月2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | | [Corunet](https://coru.net/) | 分析 | 主要产品 | — | — | [英文幻灯片，2019年4月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [CraiditX 氪信](https://creditx.com) | 金融AI | 分析 | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | -| [ﾂ环板/ｮﾂ嘉ｯﾂ偲](https://www.criteo.com/) | 零售 | 主要产品 | — | — | [幻灯片中的英文，十月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| [德意志银行](https://db.com) | 财政 | 商业智能分析 | — | — | [幻灯片中的英文，十月2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| [CraiditX 氪信](https://www.creditx.com) | 金融AI | 分析 | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| [Criteo/Storetail](https://www.criteo.com/) | 零售 | 主要产品 | — | — | [英文幻灯片，十月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| [德意志银行](https://db.com) | 金融 | 商业智能分析 | — | — | [英文幻灯片，十月2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | | [Diva-e](https://www.diva-e.com) | 数字咨询 | 主要产品 | — | — | [英文幻灯片，2019年9月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | | [Exness](https://www.exness.com) | 交易 | 指标，日志记录 | — | — | [俄语交谈，2019年5月](https://youtu.be/_rpU-TvSfZ8?t=3215) | | [精灵](https://geniee.co.jp) | 广告网络 | 主要产品 | — | — | [日文博客，2017年7月](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| [HUYA](https://www.huya.com/) | 视频流 | 分析 | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com) | 房地产 | 分析 | — | — | [博客文章英文,四月2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| [Infovista](https://www.infovista.com/) | 网络 | 分析 | — | — | [幻灯片中的英文，十月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | -| [InnoGames](https://www.innogames.com) | 游戏 | 指标，日志记录 | — | — | [俄罗斯幻灯片，2019年9月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | -| [Integros](https://integros.com) | 视频服务平台 | 分析 | — | — | [俄罗斯幻灯片，2019年5月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| [虎牙](https://www.huya.com/) | 视频流 | 分析 | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | +| [Idealista](https://www.idealista.com) | 房地产 | 分析 | — | — | [英文博客文章,四月2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Infovista](https://www.infovista.com/) | 网络 | 分析 | — | — | [英文幻灯片，十月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| [InnoGames](https://www.innogames.com) | 游戏 | 指标，日志记录 | — | — | [俄文幻灯片，2019年9月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| [Integros](https://integros.com) | 视频服务平台 | 分析 | — | — | [俄文幻灯片，2019年5月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | | [科迪亚克数据](https://www.kodiakdata.com/) | 云 | 主要产品 | — | — | [虏茅驴麓卤戮碌禄路戮鲁拢](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | | [Kontur](https://kontur.ru) | 软件开发 | 指标 | — | — | [俄语交谈，2018年11月](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [LifeStreet](https://lifestreet.com/) | 广告网络 | 主要产品 | 75台服务器（3个副本) | 5.27PiB | [博客文章在俄罗斯，2017年2月](https://habr.com/en/post/322620/) | +| [LifeStreet](https://lifestreet.com/) | 广告网络 | 主要产品 | 75台服务器（3个副本) | 5.27PiB | [俄文博客文章，2017年2月](https://habr.com/en/post/322620/) | | [Mail.ru 云解决方案](https://mcs.mail.ru/) | 云服务 | 主要产品 | — | — | [运行ClickHouse实例，俄语](https://mcs.mail.ru/help/db-create/clickhouse#) | | [MessageBird](https://www.messagebird.com) | 电信 | 统计 | — | — | [英文幻灯片，2018年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | -| [MGID](https://www.mgid.com/) | 广告网络 | 网络分析 | — | — | [我们在实施分析DBMS ClickHouse的经验，在俄罗斯](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| [MGID](https://www.mgid.com/) | 广告网络 | 网络分析 | — | — | [我们在实施分析DBMS ClickHouse的经验，俄文](http://gs-studio.com/news-about-it/32777----clickhouse---c) | | [OneAPM](https://www.oneapm.com/) | 监测和数据分析 | 主要产品 | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | -| [ﾂ环板Innovationｮﾂ嘉ｯ](http://www.pragma-innovation.fr/) | 遥测和大数据分析 | 主要产品 | — | — | [幻灯片中的英文，十月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | -| [QINGCLOUD](https://www.qingcloud.com/) | 云服务 | 主要产品 | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | +| [Pragma Innovation](http://www.pragma-innovation.fr/) | 遥测和大数据分析 | 主要产品 | — | — | [英文幻灯片，十月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| [青云](https://www.qingcloud.com/) | 云服务 | 主要产品 | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | | [Qrator](https://qrator.net) | DDoS保护 | 主要产品 | — | — | [博客文章,三月2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [北京百分之信息技术有限公司,Ltd.](https://www.percent.cn/) | 分析 | 主要产品 | — | — | [中文幻灯片，2019年6月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | -| [漫步者](https://rambler.ru) | 互联网服务 | 分析 | — | — | [俄语交谈，2018年4月](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | -| [腾讯](https://www.tencent.com) | 消息传递 | 日志记录 | — | — | [中文讲座，2019年11月](https://youtu.be/T-iVQRuw-QY?t=5050) | -| [交通明星](https://trafficstars.com/) | 广告网络 | — | — | — | [幻灯片在俄罗斯，2018年5月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | -| [S7航空公司](https://www.s7.ru) | 航空公司 | 指标，日志记录 | — | — | [讲俄语，2019年3月](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| [SEMrush](https://www.semrush.com/) | 碌莽禄Marketing: | 主要产品 | — | — | [幻灯片在俄罗斯，八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| [百分点](https://www.percent.cn/) | 分析 | 主要产品 | — | — | [中文幻灯片，2019年6月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| [漫步者](https://rambler.ru) | 互联网服务 | 分析 | — | — | [俄语讲座，2018年4月](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | +| [腾讯](https://www.tencent.com) | 通讯软件 | 日志记录 | — | — | [中文讲座，2019年11月](https://youtu.be/T-iVQRuw-QY?t=5050) | +| [流量之星](https://trafficstars.com/) | 广告网络 | — | — | — | [俄文幻灯片，2018年5月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | +| [S7航空公司](https://www.s7.ru) | 航空公司 | 指标，日志记录 | — | — | [俄文，2019年3月](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| [SEMrush](https://www.semrush.com/) | 营销 | 主要产品 | — | — | [俄文幻灯片，八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | | [scireum GmbH](https://www.scireum.de/) | 电子商务 | 主要产品 | — | — | [德语讲座，2020年2月](https://www.youtube.com/watch?v=7QWAn5RbyR4) | -| [哨兵](https://sentry.io/) | 软件开发人员 | 产品后端 | — | — | [博客文章英文,五月2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| [Sentry](https://sentry.io/) | 软件开发 | 产品后端 | — | — | [英文博客文章,五月2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | | [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | 政府社会保障 | 分析 | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | | [seo.do](https://seo.do/) | 分析 | 主要产品 | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | -| [新浪](http://english.sina.com/index.html) | 新闻 | — | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | -| [SMI2](https://smi2.ru/) | 新闻 | 分析 | — | — | [博客文章在俄罗斯，2017年11月](https://habr.com/ru/company/smi2/blog/314558/) | +| [新浪](http://sina.com/) | 新闻 | — | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| [SMI2](https://smi2.ru/) | 新闻 | 分析 | — | — | [俄文博客文章，2017年11月](https://habr.com/ru/company/smi2/blog/314558/) | | [Splunk](https://www.splunk.com/) | 业务分析 | 主要产品 | — | — | [英文幻灯片，2018年1月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | -| [Spotify的](https://www.spotify.com) | 音乐 | 实验 | — | — | [幻灯片，七月2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| [Spotify](https://www.spotify.com) | 音乐 | 实验 | — | — | [幻灯片，七月2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | | [腾讯](https://www.tencent.com) | 大数据 | 数据处理 | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | | [优步](https://www.uber.com) | 出租车 | 日志记录 | — | — | [幻灯片，二月2020](https://presentations.clickhouse.tech/meetup40/uber.pdf) | -| [VKontakte](https://vk.com) | 社交网络 | 统计，日志记录 | — | — | [幻灯片在俄罗斯，八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | -| [Wisebits](https://wisebits.com/) | IT解决方案 | 分析 | — | — | [俄罗斯幻灯片，2019年5月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | -| [小新科技](https://www.xiaoheiban.cn/) | 教育 | 共同目的 | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | -| [西马拉亚](https://www.ximalaya.com/) | 音频共享 | OLAP | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | -| [Yandex云](https://cloud.yandex.ru/services/managed-clickhouse) | 公有云 | 主要产品 | — | — | [讲俄语，2019年12月](https://www.youtube.com/watch?v=pgnak9e_E0o) | -| [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | 商业智能 | 主要产品 | — | — | [幻灯片在俄罗斯，十二月2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | -| [Yandex市场](https://market.yandex.ru/) | 电子商务 | 指标，日志记录 | — | — | [讲俄语，2019年1月](https://youtu.be/_l1qP0DyBcA?t=478) | +| [VKontakte](https://vk.com) | 社交网络 | 统计，日志记录 | — | — | [俄文幻灯片，八月2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| [Wisebits](https://wisebits.com/) | IT解决方案 | 分析 | — | — | [俄文幻灯片，2019年5月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| [晓信科技](https://www.xiaoheiban.cn/) | 教育 | 共同目的 | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| [喜马拉雅](https://www.ximalaya.com/) | 音频共享 | OLAP | — | — | [英文幻灯片，2019年11月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | +| [Yandex云](https://cloud.yandex.ru/services/managed-clickhouse) | 公有云 | 主要产品 | — | — | [俄文，2019年12月](https://www.youtube.com/watch?v=pgnak9e_E0o) | +| [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | 商业智能 | 主要产品 | — | — | [俄文幻灯片，十二月2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| [Yandex市场](https://market.yandex.ru/) | 电子商务 | 指标，日志记录 | — | — | [俄文，2019年1月](https://youtu.be/_l1qP0DyBcA?t=478) | | [Yandex Metrica](https://metrica.yandex.com) | 网站分析 | 主要产品 | 一个集群中的360台服务器，一个部门中的1862台服务器 | 66.41PiB/5.68PiB | [幻灯片，二月2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | -| [ЦВТ](https://htc-cs.ru/) | 软件开发 | 指标，日志记录 | — | — | [博客文章,三月2019,在俄罗斯](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| [МКБ](https://mkb.ru/) | 银行 | 网络系统监控 | — | — | [俄罗斯幻灯片，2019年9月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| [ЦВТ](https://htc-cs.ru/) | 软件开发 | 指标，日志记录 | — | — | [博客文章,三月2019，俄文](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| [МКБ](https://mkb.ru/) | 银行 | 网络系统监控 | — | — | [俄文幻灯片，2019年9月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | | [金数据](https://jinshuju.net) | 商业智能分析 | 主要产品 | — | — | [中文幻灯片，2019年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | +| [Instana](https://www.instana.com) | APM 平台 | 主要产品 | — | — | [推特消息](https://twitter.com/mieldonkers/status/1248884119158882304) | +| [Wargaming](https://wargaming.com/en/) | 游戏 | | — | — | [采访](https://habr.com/en/post/496954/) | +| [Crazypanda](https://crazypanda.ru/en/) | 游戏 | | — | — | ClickHouse 社区会议 | +| [FunCorp](https://fun.co/rp) | 游戏 | | — | — | [文章](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) | [原始文章](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/zh/introduction/distinctive_features.md b/docs/zh/introduction/distinctive_features.md index 3b1e7a8c716..e27f3317bf0 100644 --- a/docs/zh/introduction/distinctive_features.md +++ b/docs/zh/introduction/distinctive_features.md @@ -1,5 +1,5 @@ -# ClickHouse的独特功能 {#clickhousede-du-te-gong-neng} +# ClickHouse的特性 {#clickhouse-de-te-xing} ## 真正的列式数据库管理系统 {#zhen-zheng-de-lie-shi-shu-ju-ku-guan-li-xi-tong} @@ -62,4 +62,10 @@ ClickHouse使用异步的多主复制技术。当数据被写入任何一个可 更多信息，参见 [数据复制](../engines/table_engines/mergetree_family/replication.md)。 +# 限制 {#clickhouseke-xian-zhi} + +1. 没有完整的事务支持。 +2. 缺少高频率，低延迟的修改或删除已存在数据的能力。仅能用于批量删除或修改数据，但这符合 [GDPR](https://gdpr-info.eu)。 +3. 稀疏索引使得ClickHouse不适合通过其键检索单行的点查询。 + [来源文章](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/zh/introduction/features_considered_disadvantages.md b/docs/zh/introduction/features_considered_disadvantages.md deleted file mode 100644 index efc967e90ac..00000000000 --- a/docs/zh/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1,8 +0,0 @@ - -# ClickHouse的限制 {#clickhouseke-yi-ren-wei-shi-que-dian-de-gong-neng} - -1. 没有完整的事务支持。 -2. 缺少高频率，低延迟的修改或删除已存在数据的能力。仅能用于批量删除或修改数据，但这符合 [GDPR](https://gdpr-info.eu)。 -3. 稀疏索引使得ClickHouse不适合通过其键检索单行的点查询。 - -[来源文章](https://clickhouse.tech/docs/zh/introduction/features_considered_disadvantages/) diff --git a/docs/zh/introduction/performance.md b/docs/zh/introduction/performance.md index 9c5ce29df6f..ef48661d304 100644 --- a/docs/zh/introduction/performance.md +++ b/docs/zh/introduction/performance.md @@ -1,7 +1,7 @@ # 性能 {#performance} -根据Yandex的内部测试结果，ClickHouse表现出了比同类可比较产品更优的性能。你可以在 [这里](https://clickhouse.tech/benchmark.html) 查看具体的测试结果。 +根据Yandex的内部测试结果，ClickHouse表现出了比同类可比较产品更优的性能。你可以在 [这里](https://clickhouse.tech/benchmark/dbms/) 查看具体的测试结果。 许多其他的测试也证实这一点。你可以使用互联网搜索到它们，或者你也可以从 [我们收集的部分相关连接](https://clickhouse.tech/#independent-benchmarks) 中查看。 diff --git a/docs/zh/operations/server_configuration_parameters/settings.md b/docs/zh/operations/server_configuration_parameters/settings.md index b78f8173741..f10b6311b27 100644 --- a/docs/zh/operations/server_configuration_parameters/settings.md +++ b/docs/zh/operations/server_configuration_parameters/settings.md @@ -1,17 +1,15 @@ --- -machine_translated: true -machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 toc_priority: 57 toc_title: "\u670D\u52A1\u5668\u8BBE\u7F6E" --- -# 服务器设置 {#server-settings} +# 服务器配置 {#server-settings} ## builtin\_dictionaries\_reload\_interval {#builtin-dictionaries-reload-interval} -重新加载内置字典之前的时间间隔（以秒为单位）。 +重新加载内置字典的间隔时间（以秒为单位）。 -ClickHouse每x秒重新加载内置字典。 这使得编辑字典成为可能 “on the fly” 无需重新启动服务器。 +ClickHouse每x秒重新加载内置字典。 这使得编辑字典 “on the fly”，而无需重新启动服务器。 默认值:3600. @@ -23,7 +21,7 @@ ClickHouse每x秒重新加载内置字典。 这使得编辑字典成为可能 ## 压缩 {#server-settings-compression} -数据压缩设置 [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)-发动机表。 +数据压缩配置 [MergeTree](../../engines/table_engines/mergetree_family/mergetree.md)-引擎表。 !!! warning "警告" 如果您刚开始使用ClickHouse，请不要使用它。 @@ -41,7 +39,7 @@ ClickHouse每x秒重新加载内置字典。 这使得编辑字典成为可能 ``` -`` 字段: +`` 参数: - `min_part_size` – The minimum size of a data part. - `min_part_size_ratio` – The ratio of the data part size to the table size. @@ -82,9 +80,9 @@ ClickHouse每x秒重新加载内置字典。 这使得编辑字典成为可能 ## default\_profile {#default-profile} -默认设置配置文件。 +默认配置文件。 -设置配置文件位于参数中指定的文件中 `user_config`. +配置文件位于`user_config`参数指定的文件中 . **示例** @@ -774,11 +772,11 @@ TCP端口，用于与客户端进行安全通信。 使用它与 [OpenSSL](#serv users.xml ``` -## 动物园管理员 {#server-settings_zookeeper} +## zookeeper {#server-settings_zookeeper} -包含允许ClickHouse与 [动物园管理员](http://zookeeper.apache.org/) 集群。 +包含允许ClickHouse与 [zookpeer](http://zookeeper.apache.org/) 集群。 -ClickHouse使用ZooKeeper在使用复制表时存储副本的元数据。 如果未使用复制的表，则可以省略此部分参数。 +ClickHouse使用ZooKeeper存储复制表副本的元数据。 如果未使用复制的表，则可以省略此部分参数。 本节包含以下参数: diff --git a/docs/zh/whats_new/changelog/index.md b/docs/zh/whats_new/changelog/index.md index 90bb7abe0b0..33bb7bfd5f1 100644 --- a/docs/zh/whats_new/changelog/index.md +++ b/docs/zh/whats_new/changelog/index.md @@ -246,7 +246,7 @@ machine_translated_rev: b111334d6614a02564cf32f379679e9ff970d9b1 - 更新了clickhouse-test脚本中挂起查询的检查 [\#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) - 从存储库中删除了一些无用的文件。 [\#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 更改类型的数学perftests从 `once` 到 `loop`. [\#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 添加码头镜像，它允许为我们的代码库构建交互式代码浏览器HTML报告。 [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([阿利沙平](https://github.com/alesapin)）见 [Woboq代码浏览器](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html) +- 添加码头镜像，它允许为我们的代码库构建交互式代码浏览器HTML报告。 [\#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([阿利沙平](https://github.com/alesapin)）见 [Woboq代码浏览器](https://clickhouse.tech/codebrowser/html_report///ClickHouse/dbms/index.html) - 抑制MSan下的一些测试失败。 [\#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) - 加速 “exception while insert” 测试 此测试通常在具有复盖率的调试版本中超时。 [\#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 更新 `libcxx` 和 `libcxxabi` 为了主人 在准备 [\#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [\#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 624712504b7..ce59f5cac7f 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -64,7 +64,8 @@ public: concurrency(concurrency_), delay(delay_), queue(concurrency), randomize(randomize_), cumulative(cumulative_), max_iterations(max_iterations_), max_time(max_time_), json_path(json_path_), confidence(confidence_), query_id(query_id_), settings(settings_), - global_context(Context::createGlobal()), pool(concurrency) + shared_context(Context::createShared()), global_context(Context::createGlobal(shared_context.get())), + pool(concurrency) { const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; size_t connections_cnt = std::max(ports_.size(), hosts_.size()); @@ -149,6 +150,7 @@ private: size_t confidence; std::string query_id; Settings settings; + SharedContextHolder shared_context; Context global_context; QueryProcessingStage::Enum query_processing_stage; diff --git a/programs/benchmark/CMakeLists.txt b/programs/benchmark/CMakeLists.txt index 58096985037..be999aafe80 100644 --- a/programs/benchmark/CMakeLists.txt +++ b/programs/benchmark/CMakeLists.txt @@ -1,6 +1,5 @@ set(CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp) set(CLICKHOUSE_BENCHMARK_LINK PRIVATE dbms clickhouse_aggregate_functions clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY}) -set(CLICKHOUSE_BENCHMARK_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) clickhouse_program_add(benchmark) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 9cd1332b513..64d80eeaee9 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -82,16 +82,8 @@ #endif /// http://en.wikipedia.org/wiki/ANSI_escape_code - -/// Similar codes \e[s, \e[u don't work in VT100 and Mosh. -#define SAVE_CURSOR_POSITION "\033""7" -#define RESTORE_CURSOR_POSITION "\033""8" - #define CLEAR_TO_END_OF_LINE "\033[K" -/// This codes are possibly not supported everywhere. -#define DISABLE_LINE_WRAPPING "\033[?7l" -#define ENABLE_LINE_WRAPPING "\033[?7h" namespace DB { @@ -133,8 +125,6 @@ private: bool stdin_is_a_tty = false; /// stdin is a terminal. bool stdout_is_a_tty = false; /// stdout is a terminal. - uint16_t terminal_width = 0; /// Terminal width is needed to render progress bar. - std::unique_ptr connection; /// Connection to DB. String query_id; /// Current query_id. String query; /// Current query. @@ -148,7 +138,8 @@ private: bool has_vertical_output_suffix = false; /// Is \G present at the end of the query string? - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); /// Buffer that reads from stdin in batch mode. ReadBufferFromFileDescriptor std_in {STDIN_FILENO}; @@ -694,7 +685,7 @@ private: if (ignore_error) { Tokens tokens(begin, end); - IParser::Pos token_iterator(tokens); + IParser::Pos token_iterator(tokens, context.getSettingsRef().max_parser_depth); while (token_iterator->type != TokenType::Semicolon && token_iterator.isValid()) ++token_iterator; begin = token_iterator->end; @@ -968,10 +959,15 @@ private: ParserQuery parser(end, true); ASTPtr res; + const auto & settings = context.getSettingsRef(); + size_t max_length = 0; + if (!allow_multi_statements) + max_length = settings.max_query_size; + if (is_interactive || ignore_error) { String message; - res = tryParseQuery(parser, pos, end, message, true, "", allow_multi_statements, 0); + res = tryParseQuery(parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth); if (!res) { @@ -980,7 +976,7 @@ private: } } else - res = parseQueryAndMovePosition(parser, pos, end, "", allow_multi_statements, 0); + res = parseQueryAndMovePosition(parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth); if (is_interactive) { @@ -1122,11 +1118,16 @@ private: /// to avoid losing sync. if (!cancelled) { - auto cancel_query = [&] { + auto cancel_query = [&] + { connection->sendCancel(); cancelled = true; if (is_interactive) + { + if (written_progress_chars) + clearProgress(); std::cout << "Cancelling query." << std::endl; + } /// Pressing Ctrl+C twice results in shut down. interrupt_listener.unblock(); @@ -1436,7 +1437,7 @@ private: { written_progress_chars = 0; if (!send_logs) - std::cerr << RESTORE_CURSOR_POSITION CLEAR_TO_END_OF_LINE; + std::cerr << "\r" CLEAR_TO_END_OF_LINE; } @@ -1461,20 +1462,14 @@ private: "\033[1m↗\033[0m", }; - if (!send_logs) - { - if (written_progress_chars) - message << RESTORE_CURSOR_POSITION CLEAR_TO_END_OF_LINE; - else - message << SAVE_CURSOR_POSITION; - } + auto indicator = indicators[increment % 8]; - message << DISABLE_LINE_WRAPPING; + if (!send_logs && written_progress_chars) + message << '\r'; size_t prefix_size = message.count(); - message << indicators[increment % 8] - << " Progress: "; + message << indicator << " Progress: "; message << formatReadableQuantity(progress.read_rows) << " rows, " @@ -1488,7 +1483,7 @@ private: else message << ". "; - written_progress_chars = message.count() - prefix_size - (increment % 8 == 7 ? 10 : 13); /// Don't count invisible output (escape sequences). + written_progress_chars = message.count() - prefix_size - (strlen(indicator) - 2); /// Don't count invisible output (escape sequences). /// If the approximate number of rows to process is known, we can display a progress bar and percentage. if (progress.total_rows_to_read > 0) @@ -1506,7 +1501,7 @@ private: if (show_progress_bar) { - ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); + ssize_t width_of_progress_bar = static_cast(getTerminalWidth()) - written_progress_chars - strlen(" 99%"); if (width_of_progress_bar > 0) { std::string bar = UnicodeBar::render(UnicodeBar::getWidth(progress.read_rows, 0, total_rows_corrected, width_of_progress_bar)); @@ -1521,7 +1516,8 @@ private: message << ' ' << (99 * progress.read_rows / total_rows_corrected) << '%'; } - message << ENABLE_LINE_WRAPPING; + message << CLEAR_TO_END_OF_LINE; + if (send_logs) message << '\n'; @@ -1589,7 +1585,11 @@ private: resetOutput(); if (is_interactive && !written_first_block) + { + if (written_progress_chars) + clearProgress(); std::cout << "Ok." << std::endl; + } } static void showClientVersion() @@ -1687,6 +1687,7 @@ public: stdin_is_a_tty = isatty(STDIN_FILENO); stdout_is_a_tty = isatty(STDOUT_FILENO); + uint64_t terminal_width = 0; if (stdin_is_a_tty) terminal_width = getTerminalWidth(); @@ -1715,7 +1716,6 @@ public: ("database,d", po::value(), "database") ("pager", po::value(), "pager") ("disable_suggestion,A", "Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client.") - ("always_load_suggestion_data", "Load suggestion data even if clickhouse-client is run in non-interactive mode. Used for testing.") ("suggestion_limit", po::value()->default_value(10000), "Suggestion limit for how many databases, tables and columns to fetch.") ("multiline,m", "multiline") diff --git a/programs/client/Suggest.cpp b/programs/client/Suggest.cpp index f7141449f54..8fffbec4fab 100644 --- a/programs/client/Suggest.cpp +++ b/programs/client/Suggest.cpp @@ -67,16 +67,19 @@ void Suggest::load(const ConnectionParameters & connection_parameters, size_t su Suggest::Suggest() { /// Keywords may be not up to date with ClickHouse parser. - words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", - "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", - "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", - "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", - "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", - "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", - "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", - "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", - "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", - "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"}; + words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", + "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", + "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", + "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", + "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", + "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", + "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", + "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", + "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", + "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", + "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", + "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "FOR", "RANDOMIZED", + "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP"}; } void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 98a3055da28..fecdad9bcea 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB @@ -123,7 +124,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) DB::ParserCodec codec_parser; std::string codecs_line = boost::algorithm::join(codecs, ","); - auto ast = DB::parseQuery(codec_parser, "(" + codecs_line + ")", 0); + auto ast = DB::parseQuery(codec_parser, "(" + codecs_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); codec = DB::CompressionCodecFactory::instance().get(ast, nullptr); } else diff --git a/programs/copier/CMakeLists.txt b/programs/copier/CMakeLists.txt index 5573fbc5e5d..76db6ce1ffc 100644 --- a/programs/copier/CMakeLists.txt +++ b/programs/copier/CMakeLists.txt @@ -12,6 +12,6 @@ set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_dictionaries string_utils ${Poco_XML_LIBRARY} PUBLIC daemon) -set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) clickhouse_program_add(copier) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index c4714ff201f..e8b999534e2 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -1185,19 +1185,26 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( { String query; query += "SELECT " + fields + " FROM " + getQuotedTable(from_table); + + if (enable_splitting && experimental_use_sample_offset) + query += " SAMPLE 1/" + toString(number_of_splits) + " OFFSET " + toString(current_piece_number) + "/" + toString(number_of_splits); + /// TODO: Bad, it is better to rewrite with ASTLiteral(partition_key_field) query += " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + task_partition.name + " AS partition_key))"; - if (enable_splitting) + if (enable_splitting && !experimental_use_sample_offset) query += " AND ( cityHash64(" + primary_key_comma_separated + ") %" + toString(number_of_splits) + " = " + toString(current_piece_number) + " )"; if (!task_table.where_condition_str.empty()) query += " AND (" + task_table.where_condition_str + ")"; + if (!limit.empty()) query += " LIMIT " + limit; ParserQuery p_query(query.data() + query.size()); - return parseQuery(p_query, query, 0); + + const auto & settings = context.getSettingsRef(); + return parseQuery(p_query, query, settings.max_query_size, settings.max_parser_depth); }; /// Load balancing @@ -1409,7 +1416,8 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( query += "INSERT INTO " + getQuotedTable(split_table_for_current_piece) + " VALUES "; ParserQuery p_query(query.data() + query.size()); - query_insert_ast = parseQuery(p_query, query, 0); + const auto & settings = context.getSettingsRef(); + query_insert_ast = parseQuery(p_query, query, settings.max_query_size, settings.max_parser_depth); LOG_DEBUG(log, "Executing INSERT query: " << query); } @@ -1634,7 +1642,8 @@ ASTPtr ClusterCopier::getCreateTableForPullShard(const ConnectionTimeouts & time &task_cluster->settings_pull); ParserCreateQuery parser_create_query; - return parseQuery(parser_create_query, create_query_pull_str, 0); + const auto & settings = context.getSettingsRef(); + return parseQuery(parser_create_query, create_query_pull_str, settings.max_query_size, settings.max_parser_depth); } /// If it is implicitly asked to create split Distributed table for certain piece on current shard, we will do it. @@ -1712,7 +1721,8 @@ std::set ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti } ParserQuery parser_query(query.data() + query.size()); - ASTPtr query_ast = parseQuery(parser_query, query, 0); + const auto & settings = context.getSettingsRef(); + ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth); LOG_DEBUG(log, "Computing destination partition set, executing query: " << query); @@ -1759,7 +1769,8 @@ bool ClusterCopier::checkShardHasPartition(const ConnectionTimeouts & timeouts, << partition_quoted_name << " existence, executing query: " << query); ParserQuery parser_query(query.data() + query.size()); - ASTPtr query_ast = parseQuery(parser_query, query, 0); +const auto & settings = context.getSettingsRef(); + ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth); Context local_context = context; local_context.setSettings(task_cluster->settings_pull); @@ -1772,16 +1783,22 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi createShardInternalTables(timeouts, task_shard, false); TaskTable & task_table = task_shard.task_table; - - std::string query = "SELECT 1 FROM " + getQuotedTable(task_shard.table_read_shard) - + " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) - + " = (" + partition_quoted_name + " AS partition_key))"; - const size_t number_of_splits = task_table.number_of_splits; const String & primary_key_comma_separated = task_table.primary_key_comma_separated; - query += " AND (cityHash64(" + primary_key_comma_separated + ") % " - + std::to_string(number_of_splits) + " = " + std::to_string(current_piece_number) + " )"; + UNUSED(primary_key_comma_separated); + + std::string query = "SELECT 1 FROM " + getQuotedTable(task_shard.table_read_shard); + + if (experimental_use_sample_offset) + query += " SAMPLE 1/" + toString(number_of_splits) + " OFFSET " + toString(current_piece_number) + "/" + toString(number_of_splits); + + query += " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + + " = (" + partition_quoted_name + " AS partition_key))"; + + if (!experimental_use_sample_offset) + query += " AND (cityHash64(" + primary_key_comma_separated + ") % " + + std::to_string(number_of_splits) + " = " + std::to_string(current_piece_number) + " )"; if (!task_table.where_condition_str.empty()) query += " AND (" + task_table.where_condition_str + ")"; @@ -1793,7 +1810,8 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi << "existence, executing query: " << query); ParserQuery parser_query(query.data() + query.size()); - ASTPtr query_ast = parseQuery(parser_query, query, 0); + const auto & settings = context.getSettingsRef(); + ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth); Context local_context = context; local_context.setSettings(task_cluster->settings_pull); @@ -1826,7 +1844,8 @@ UInt64 ClusterCopier::executeQueryOnCluster( if (query_ast_ == nullptr) { ParserQuery p_query(query.data() + query.size()); - query_ast = parseQuery(p_query, query, 0); + const auto & settings = context.getSettingsRef(); + query_ast = parseQuery(p_query, query, settings.max_query_size, settings.max_parser_depth); } else query_ast = query_ast_; diff --git a/programs/copier/ClusterCopier.h b/programs/copier/ClusterCopier.h index 4ad6265bb3c..3d6400f51d4 100644 --- a/programs/copier/ClusterCopier.h +++ b/programs/copier/ClusterCopier.h @@ -61,6 +61,11 @@ public: move_fault_probability = move_fault_probability_; } + void setExperimentalUseSampleOffset(bool value) + { + experimental_use_sample_offset = value; + } + protected: String getWorkersPath() const @@ -211,6 +216,8 @@ private: double copy_fault_probability = 0.0; double move_fault_probability = 0.0; + bool experimental_use_sample_offset{false}; + Context & context; Poco::Logger * log; diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index 14bfd3a3339..52a37c75c72 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -20,6 +20,11 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) if (config().has("move-fault-probability")) move_fault_probability = std::max(std::min(config().getDouble("move-fault-probability"), 1.0), 0.0); base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current(); + + + if (config().has("experimental-use-sample-offset")) + experimental_use_sample_offset = config().getBool("experimental-use-sample-offset"); + // process_id is '#_' time_t timestamp = Poco::Timestamp().epochTime(); auto curr_pid = Poco::Process::id(); @@ -75,6 +80,8 @@ void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options) .argument("log-level").binding("log-level")); options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consecutive copier launches will populate /base-dir/launch_id/* directories") .argument("base-dir").binding("base-dir")); + options.addOption(Poco::Util::Option("experimental-use-sample-offset", "", "Use SAMPLE OFFSET query instead of cityHash64(PRIMARY KEY) % n == k") + .argument("experimental-use-sample-offset").binding("experimental-use-sample-offset")); using Me = std::decay_t; options.addOption(Poco::Util::Option("help", "", "produce this help message").binding("help") @@ -94,7 +101,8 @@ void ClusterCopierApp::mainImpl() << "path " << process_path << ", " << "revision " << ClickHouseRevision::get() << ")"); - auto context = std::make_unique(Context::createGlobal()); + SharedContextHolder shared_context = Context::createShared(); + auto context = std::make_unique(Context::createGlobal(shared_context.get())); context->makeGlobalContext(); SCOPE_EXIT(context->shutdown()); @@ -121,6 +129,8 @@ void ClusterCopierApp::mainImpl() copier->setCopyFaultProbability(copy_fault_probability); copier->setMoveFaultProbability(move_fault_probability); + copier->setExperimentalUseSampleOffset(experimental_use_sample_offset); + auto task_file = config().getString("task-file", ""); if (!task_file.empty()) copier->uploadTaskDescription(task_path, task_file, config().getBool("task-upload-force", false)); diff --git a/programs/copier/ClusterCopierApp.h b/programs/copier/ClusterCopierApp.h index 8cadd9d5dff..173aacc4361 100644 --- a/programs/copier/ClusterCopierApp.h +++ b/programs/copier/ClusterCopierApp.h @@ -82,6 +82,8 @@ private: double move_fault_probability = 0.0; bool is_help = false; + bool experimental_use_sample_offset{false}; + std::string base_dir; std::string process_path; std::string process_id; diff --git a/programs/copier/TaskTableAndShard.h b/programs/copier/TaskTableAndShard.h index 615ad297b79..32841e93a14 100644 --- a/programs/copier/TaskTableAndShard.h +++ b/programs/copier/TaskTableAndShard.h @@ -4,6 +4,9 @@ #include "Internals.h" #include "ClusterPartition.h" +#include + + namespace DB { namespace ErrorCodes @@ -260,9 +263,10 @@ inline TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConf + "." + escapeForFileName(table_push.second); engine_push_str = config.getString(table_prefix + "engine"); + { ParserStorage parser_storage; - engine_push_ast = parseQuery(parser_storage, engine_push_str, 0); + engine_push_ast = parseQuery(parser_storage, engine_push_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); engine_push_partition_key_ast = extractPartitionKey(engine_push_ast); primary_key_comma_separated = createCommaSeparatedStringFrom(extractPrimaryKeyColumnNames(engine_push_ast)); engine_push_zk_path = extractReplicatedTableZookeeperPath(engine_push_ast); @@ -273,7 +277,7 @@ inline TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConf auxiliary_engine_split_asts.reserve(number_of_splits); { ParserExpressionWithOptionalAlias parser_expression(false); - sharding_key_ast = parseQuery(parser_expression, sharding_key_str, 0); + sharding_key_ast = parseQuery(parser_expression, sharding_key_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); main_engine_split_ast = createASTStorageDistributed(cluster_push_name, table_push.first, table_push.second, sharding_key_ast); @@ -291,7 +295,7 @@ inline TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConf if (!where_condition_str.empty()) { ParserExpressionWithOptionalAlias parser_expression(false); - where_condition_ast = parseQuery(parser_expression, where_condition_str, 0); + where_condition_ast = parseQuery(parser_expression, where_condition_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); // Will use canonical expression form where_condition_str = queryToString(where_condition_ast); diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index f826d6394bc..b5a4e2d1603 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -53,7 +53,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) const char * end = pos + query.size(); ParserQuery parser(end); - ASTPtr res = parseQuery(parser, pos, end, "query", 0); + ASTPtr res = parseQuery(parser, pos, end, "query", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (!quiet) { diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 26752da5d87..d2d19a395bb 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -148,7 +148,8 @@ try return Application::EXIT_OK; } - context = std::make_unique(Context::createGlobal()); + shared_context = Context::createShared(); + context = std::make_unique(Context::createGlobal(shared_context.get())); context->makeGlobalContext(); context->setApplicationType(Context::ApplicationType::LOCAL); tryInitPath(); @@ -267,8 +268,10 @@ void LocalServer::processQueries() String initial_create_query = getInitialCreateTableQuery(); String queries_str = initial_create_query + config().getRawString("query"); + const auto & settings = context->getSettingsRef(); + std::vector queries; - auto parse_res = splitMultipartQuery(queries_str, queries); + auto parse_res = splitMultipartQuery(queries_str, queries, settings.max_query_size, settings.max_parser_depth); if (!parse_res.second) throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index 874319c1ea5..77d0b9ff5dc 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -41,6 +41,7 @@ private: void setupUsers(); protected: + SharedContextHolder shared_context; std::unique_ptr context; /// Settings specified via command line args diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 0352eba0a0a..9a80dc8d035 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -1080,7 +1080,8 @@ try header.insert(std::move(column)); } - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); ReadBufferFromFileDescriptor file_in(STDIN_FILENO); diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index b89d50569f6..c5fb149284b 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -120,12 +120,14 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques SCOPE_EXIT(SQLFreeStmt(hstmt, SQL_DROP)); + const auto & context_settings = context.getSettingsRef(); + /// TODO Why not do SQLColumns instead? std::string name = schema_name.empty() ? table_name : schema_name + "." + table_name; std::stringstream ss; std::string input = "SELECT * FROM " + name + " WHERE 1 = 0"; ParserQueryWithOutput parser; - ASTPtr select = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr select = parseQuery(parser, input.data(), input.data() + input.size(), "", context_settings.max_query_size, context_settings.max_parser_depth); IAST::FormatSettings settings(ss, true); settings.always_quote_identifiers = true; diff --git a/programs/odbc-bridge/ColumnInfoHandler.h b/programs/odbc-bridge/ColumnInfoHandler.h index 426cea15b34..e1a5ad33b32 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.h +++ b/programs/odbc-bridge/ColumnInfoHandler.h @@ -14,7 +14,7 @@ namespace DB class ODBCColumnsInfoHandler : public Poco::Net::HTTPRequestHandler { public: - ODBCColumnsInfoHandler(size_t keep_alive_timeout_, std::shared_ptr context_) + ODBCColumnsInfoHandler(size_t keep_alive_timeout_, Context & context_) : log(&Poco::Logger::get("ODBCColumnsInfoHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_) { } @@ -24,7 +24,7 @@ public: private: Poco::Logger * log; size_t keep_alive_timeout; - std::shared_ptr context; + Context & context; }; } #endif diff --git a/programs/odbc-bridge/HandlerFactory.h b/programs/odbc-bridge/HandlerFactory.h index f1a6803f65b..35835de5dad 100644 --- a/programs/odbc-bridge/HandlerFactory.h +++ b/programs/odbc-bridge/HandlerFactory.h @@ -21,7 +21,7 @@ namespace DB class HandlerFactory : public Poco::Net::HTTPRequestHandlerFactory { public: - HandlerFactory(const std::string & name_, size_t keep_alive_timeout_, std::shared_ptr context_) + HandlerFactory(const std::string & name_, size_t keep_alive_timeout_, Context & context_) : log(&Poco::Logger::get(name_)), name(name_), keep_alive_timeout(keep_alive_timeout_), context(context_) { pool_map = std::make_shared(); @@ -33,7 +33,7 @@ private: Poco::Logger * log; std::string name; size_t keep_alive_timeout; - std::shared_ptr context; + Context & context; std::shared_ptr pool_map; }; } diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.h b/programs/odbc-bridge/IdentifierQuoteHandler.h index 2d4cf0277be..86230d8ad17 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.h +++ b/programs/odbc-bridge/IdentifierQuoteHandler.h @@ -12,8 +12,8 @@ namespace DB class IdentifierQuoteHandler : public Poco::Net::HTTPRequestHandler { public: - IdentifierQuoteHandler(size_t keep_alive_timeout_, std::shared_ptr context_) - : log(&Poco::Logger::get("IdentifierQuoteHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_) + IdentifierQuoteHandler(size_t keep_alive_timeout_, Context &) + : log(&Poco::Logger::get("IdentifierQuoteHandler")), keep_alive_timeout(keep_alive_timeout_) { } @@ -22,7 +22,6 @@ public: private: Poco::Logger * log; size_t keep_alive_timeout; - std::shared_ptr context; }; } #endif diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 3ae5f49f24b..15954c63e47 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -129,7 +129,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); try { - BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, *context); + BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context); auto pool = getPool(connection_string); ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); copyData(inp, *writer); diff --git a/programs/odbc-bridge/MainHandler.h b/programs/odbc-bridge/MainHandler.h index ae139f393f8..806313fc9b2 100644 --- a/programs/odbc-bridge/MainHandler.h +++ b/programs/odbc-bridge/MainHandler.h @@ -24,7 +24,7 @@ public: ODBCHandler(std::shared_ptr pool_map_, size_t keep_alive_timeout_, - std::shared_ptr context_) + Context & context_) : log(&Poco::Logger::get("ODBCHandler")) , pool_map(pool_map_) , keep_alive_timeout(keep_alive_timeout_) @@ -39,7 +39,7 @@ private: std::shared_ptr pool_map; size_t keep_alive_timeout; - std::shared_ptr context; + Context & context; static inline std::mutex mutex; diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index e7e8aca7147..1cfba4b3aa8 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -176,8 +176,9 @@ int ODBCBridge::main(const std::vector & /*args*/) http_params->setTimeout(http_timeout); http_params->setKeepAliveTimeout(keep_alive_timeout); - context = std::make_shared(Context::createGlobal()); - context->makeGlobalContext(); + auto shared_context = Context::createShared(); + Context context(Context::createGlobal(shared_context.get())); + context.makeGlobalContext(); if (config().has("query_masking_rules")) { diff --git a/programs/odbc-bridge/ODBCBridge.h b/programs/odbc-bridge/ODBCBridge.h index 4ae11ad7301..9a0d37fa0f9 100644 --- a/programs/odbc-bridge/ODBCBridge.h +++ b/programs/odbc-bridge/ODBCBridge.h @@ -35,7 +35,5 @@ private: size_t keep_alive_timeout; Poco::Logger * log; - - std::shared_ptr context; /// need for settings only }; } diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index ec890c0a96d..77236210741 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -36,6 +35,11 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + + namespace DB { diff --git a/programs/server/MySQLHandler.cpp b/programs/server/MySQLHandler.cpp index 3e1432dbfce..97d9f1919ac 100644 --- a/programs/server/MySQLHandler.cpp +++ b/programs/server/MySQLHandler.cpp @@ -1,10 +1,8 @@ -#include - #include "MySQLHandler.h" + #include #include #include -#include #include #include #include @@ -16,12 +14,17 @@ #include #include #include +#include + +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_POCO_NETSSL -#include -#include -#include -#include +# include +# include +# include +# include #endif namespace DB @@ -268,7 +271,8 @@ void MySQLHandler::comPing() packet_sender->sendPacket(OK_Packet(0x0, client_capability_flags, 0, 0, 0), true); } -static bool isFederatedServerSetupCommand(const String & query); +static bool isFederatedServerSetupSetCommand(const String & query); +static bool isFederatedServerSetupSelectVarCommand(const String & query); void MySQLHandler::comQuery(ReadBuffer & payload) { @@ -276,22 +280,25 @@ void MySQLHandler::comQuery(ReadBuffer & payload) // This is a workaround in order to support adding ClickHouse to MySQL using federated server. // As Clickhouse doesn't support these statements, we just send OK packet in response. - if (isFederatedServerSetupCommand(query)) + if (isFederatedServerSetupSetCommand(query)) { packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); } else { - String replacement_query = "select ''"; + String replacement_query = "SELECT ''"; bool should_replace = false; bool with_output = false; // Translate query from MySQL to ClickHouse. - // This is a temporary workaround until ClickHouse supports the syntax "@@var_name". - if (query == "select @@version_comment limit 1") // MariaDB client starts session with that query + // Required parameters when setup: + // * max_allowed_packet, default 64MB, https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet + if (isFederatedServerSetupSelectVarCommand(query)) { should_replace = true; + replacement_query = "SELECT 67108864 AS max_allowed_packet"; } + // This is a workaround in order to support adding ClickHouse to MySQL using federated server. if (0 == strncasecmp("SHOW TABLE STATUS LIKE", query.c_str(), 22)) { @@ -358,11 +365,27 @@ void MySQLHandlerSSL::finishHandshakeSSL(size_t packet_size, char * buf, size_t #endif -static bool isFederatedServerSetupCommand(const String & query) +static bool isFederatedServerSetupSetCommand(const String & query) { - return 0 == strncasecmp("SET NAMES", query.c_str(), 9) || 0 == strncasecmp("SET character_set_results", query.c_str(), 25) - || 0 == strncasecmp("SET FOREIGN_KEY_CHECKS", query.c_str(), 22) || 0 == strncasecmp("SET AUTOCOMMIT", query.c_str(), 14) - || 0 == strncasecmp("SET SESSION TRANSACTION ISOLATION LEVEL", query.c_str(), 39); + static const std::regex expr{ + "(^(SET NAMES(.*)))" + "|(^(SET character_set_results(.*)))" + "|(^(SET FOREIGN_KEY_CHECKS(.*)))" + "|(^(SET AUTOCOMMIT(.*)))" + "|(^(SET sql_mode(.*)))" + "|(^(SET SESSION TRANSACTION ISOLATION LEVEL(.*)))" + , std::regex::icase}; + return 1 == std::regex_match(query, expr); +} + +static bool isFederatedServerSetupSelectVarCommand(const String & query) +{ + static const std::regex expr{ + "|(^(SELECT @@(.*)))" + "|(^((/\\*(.*)\\*/)([ \t]*)(SELECT([ \t]*)@@(.*))))" + "|(^((/\\*(.*)\\*/)([ \t]*)(SHOW VARIABLES(.*))))" + , std::regex::icase}; + return 1 == std::regex_match(query, expr); } const String MySQLHandler::show_table_status_replacement_query("SELECT" diff --git a/programs/server/MySQLHandler.h b/programs/server/MySQLHandler.h index ca5d045beb0..01634dd68dc 100644 --- a/programs/server/MySQLHandler.h +++ b/programs/server/MySQLHandler.h @@ -1,13 +1,17 @@ #pragma once -#include + #include #include #include #include #include "IServer.h" +#if !defined(ARCADIA_BUILD) +# include +#endif + #if USE_POCO_NETSSL -#include +# include #endif namespace CurrentMetrics diff --git a/programs/server/MySQLHandlerFactory.h b/programs/server/MySQLHandlerFactory.h index b7df9fa60e0..74f0bb35a40 100644 --- a/programs/server/MySQLHandlerFactory.h +++ b/programs/server/MySQLHandlerFactory.h @@ -1,11 +1,15 @@ #pragma once -#include #include #include #include "IServer.h" + +#if !defined(ARCADIA_BUILD) +# include +#endif + #if USE_SSL -#include +# include #endif namespace DB diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5322514e7c2..843acfa5e22 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -26,7 +25,6 @@ #include #include #include -#include "config_core.h" #include #include #include @@ -59,25 +57,31 @@ #include "MetricsTransmitter.h" #include #include "TCPHandlerFactory.h" -#include "Common/config_version.h" #include #include #include "MySQLHandlerFactory.h" +#if !defined(ARCADIA_BUILD) +# include +# include "config_core.h" +# include "Common/config_version.h" +#endif + #if defined(OS_LINUX) -#include -#include +# include +# include #endif #if USE_POCO_NETSSL -#include -#include +# include +# include #endif namespace CurrentMetrics { extern const Metric Revision; extern const Metric VersionInteger; + extern const Metric MemoryTracking; } namespace @@ -227,7 +231,10 @@ int Server::main(const std::vector & /*args*/) /** Context contains all that query execution is dependent: * settings, available functions, data types, aggregate functions, databases... */ - global_context = std::make_unique(Context::createGlobal()); + auto shared_context = Context::createShared(); + auto global_context = std::make_unique(Context::createGlobal(shared_context.get())); + global_context_ptr = global_context.get(); + global_context->makeGlobalContext(); global_context->setApplicationType(Context::ApplicationType::SERVER); @@ -248,7 +255,7 @@ int Server::main(const std::vector & /*args*/) const auto memory_amount = getMemoryAmount(); -#if defined(__linux__) +#if defined(OS_LINUX) std::string executable_path = getExecutablePath(); if (executable_path.empty()) executable_path = "/usr/bin/clickhouse"; /// It is used for information messages. @@ -324,7 +331,9 @@ int Server::main(const std::vector & /*args*/) /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. * At this moment, no one could own shared part of Context. */ + global_context_ptr = nullptr; global_context.reset(); + shared_context.reset(); LOG_DEBUG(log, "Destroyed global context."); }); @@ -552,6 +561,28 @@ int Server::main(const std::vector & /*args*/) global_context->setFormatSchemaPath(format_schema_path.path()); format_schema_path.createDirectories(); + /// Limit on total memory usage + size_t max_server_memory_usage = config().getUInt64("max_server_memory_usage", 0); + + double max_server_memory_usage_to_ram_ratio = config().getDouble("max_server_memory_usage_to_ram_ratio", 0.9); + size_t default_max_server_memory_usage = memory_amount * max_server_memory_usage_to_ram_ratio; + + if (max_server_memory_usage == 0) + { + max_server_memory_usage = default_max_server_memory_usage; + LOG_INFO(log, "Setting max_server_memory_usage was set to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage)); + } + else if (max_server_memory_usage > default_max_server_memory_usage) + { + max_server_memory_usage = default_max_server_memory_usage; + LOG_INFO(log, "Setting max_server_memory_usage was lowered to " << formatReadableSizeWithBinarySuffix(max_server_memory_usage) + << " because the system has low amount of memory"); + } + + total_memory_tracker.setOrRaiseHardLimit(max_server_memory_usage); + total_memory_tracker.setDescription("(total)"); + total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); + LOG_INFO(log, "Loading metadata from " + path); try @@ -586,9 +617,19 @@ int Server::main(const std::vector & /*args*/) /// Look at compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h /// #if USE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) - /// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache. + /// Profilers cannot work reliably with any other libunwind or without PHDR cache. if (hasPHDRCache()) + { global_context->initializeTraceCollector(); + + /// Set up server-wide memory profiler (for total memory tracker). + UInt64 total_memory_profiler_step = config().getUInt64("total_memory_profiler_step", 0); + if (total_memory_profiler_step) + { + total_memory_tracker.setOrRaiseProfilerLimit(total_memory_profiler_step); + total_memory_tracker.setProfilerStep(total_memory_profiler_step); + } + } #endif /// Describe multiple reasons when query profiler cannot work. @@ -631,7 +672,7 @@ int Server::main(const std::vector & /*args*/) dns_cache_updater = std::make_unique(*global_context, config().getInt("dns_cache_update_period", 15)); } -#if defined(__linux__) +#if defined(OS_LINUX) if (!TaskStatsInfoGetter::checkPermissions()) { LOG_INFO(log, "It looks like the process has no CAP_NET_ADMIN capability, 'taskstats' performance statistics will be disabled." diff --git a/programs/server/Server.h b/programs/server/Server.h index 337d1551b70..ffd89df6af4 100644 --- a/programs/server/Server.h +++ b/programs/server/Server.h @@ -35,7 +35,7 @@ public: Context & context() const override { - return *global_context; + return *global_context_ptr; } bool isCancelled() const override @@ -56,7 +56,7 @@ protected: std::string getDefaultCorePath() const override; private: - std::unique_ptr global_context; + Context * global_context_ptr = nullptr; }; } diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index 725aa8453b3..70825cb4f7e 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -33,6 +32,10 @@ #include "TCPHandler.h" +#if !defined(ARCADIA_BUILD) +# include +#endif + namespace DB { @@ -147,10 +150,6 @@ void TCPHandler::runImpl() if (server.isCancelled() || in->eof()) break; - /// receiveHello() has set the default settings for the current user, - /// but this default itself could change while we were waiting for a packet from the client. - connection_context.resetSettingsToDefault(); - /// Set context of request. query_context = connection_context; diff --git a/programs/server/TCPHandlerFactory.h b/programs/server/TCPHandlerFactory.h index 0eb8be13a2d..3b764af96ec 100644 --- a/programs/server/TCPHandlerFactory.h +++ b/programs/server/TCPHandlerFactory.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include "IServer.h" #include "TCPHandler.h" @@ -16,6 +17,13 @@ private: IServer & server; Poco::Logger * log; + class DummyTCPHandler : public Poco::Net::TCPServerConnection + { + public: + using Poco::Net::TCPServerConnection::TCPServerConnection; + void run() override {} + }; + public: explicit TCPHandlerFactory(IServer & server_, bool secure_ = false) : server(server_) @@ -25,12 +33,16 @@ public: Poco::Net::TCPServerConnection * createConnection(const Poco::Net::StreamSocket & socket) override { - LOG_TRACE(log, - "TCP Request. " - << "Address: " - << socket.peerAddress().toString()); - - return new TCPHandler(server, socket); + try + { + LOG_TRACE(log, "TCP Request. Address: " << socket.peerAddress().toString()); + return new TCPHandler(server, socket); + } + catch (const Poco::Net::NetException & e) + { + LOG_TRACE(log, "TCP Request. Client is not connected (most likely RST packet was sent)."); + return new DummyTCPHandler(socket); + } } }; diff --git a/programs/server/config.d/path.xml b/programs/server/config.d/path.xml index 14b7deb9de0..8db1d18e8c7 100644 --- a/programs/server/config.d/path.xml +++ b/programs/server/config.d/path.xml @@ -3,4 +3,5 @@ ./tmp/ ./user_files/ ./format_schemas/ + ./access/ diff --git a/programs/server/config.xml b/programs/server/config.xml index f55ab02d903..94a2a04e8db 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -37,7 +37,7 @@ true - + true true sslv2,sslv3 @@ -90,6 +90,29 @@ 100 + + 0 + + + 0.9 + + + 4194304 + diff --git a/programs/server/ya.make b/programs/server/ya.make new file mode 100644 index 00000000000..7f0a3969fd2 --- /dev/null +++ b/programs/server/ya.make @@ -0,0 +1,30 @@ +PROGRAM(clickhouse-server) + +PEERDIR( + clickhouse/base/common + clickhouse/base/daemon + clickhouse/base/loggers + clickhouse/src + contrib/libs/poco/NetSSL_OpenSSL +) + +SRCS( + clickhouse-server.cpp + + HTTPHandler.cpp + HTTPHandlerFactory.cpp + InterserverIOHTTPHandler.cpp + MetricsTransmitter.cpp + MySQLHandler.cpp + MySQLHandlerFactory.cpp + NotFoundHandler.cpp + PingRequestHandler.cpp + PrometheusMetricsWriter.cpp + PrometheusRequestHandler.cpp + ReplicasStatusHandler.cpp + RootRequestHandler.cpp + Server.cpp + TCPHandler.cpp +) + +END() diff --git a/programs/ya.make b/programs/ya.make new file mode 100644 index 00000000000..6c773c312b8 --- /dev/null +++ b/programs/ya.make @@ -0,0 +1,3 @@ +RECURSE( + server +) diff --git a/src/Access/AccessFlags.h b/src/Access/AccessFlags.h index c8f57fcd419..cbba295be1a 100644 --- a/src/Access/AccessFlags.h +++ b/src/Access/AccessFlags.h @@ -253,7 +253,7 @@ private: } else { - if (nodes.contains(keyword)) + if (nodes.count(keyword)) throw Exception(keyword + " declared twice", ErrorCodes::LOGICAL_ERROR); node = std::make_unique(keyword, node_type); nodes[node->keyword] = node.get(); @@ -279,7 +279,7 @@ private: { auto parent_node = std::make_unique(parent_keyword); it_parent = nodes.emplace(parent_node->keyword, parent_node.get()).first; - assert(!owned_nodes.contains(parent_node->keyword)); + assert(!owned_nodes.count(parent_node->keyword)); std::string_view parent_keyword_as_string_view = parent_node->keyword; owned_nodes[parent_keyword_as_string_view] = std::move(parent_node); } @@ -299,9 +299,9 @@ private: #undef MAKE_ACCESS_FLAGS_TO_KEYWORD_TREE_NODE - if (!owned_nodes.contains("NONE")) + if (!owned_nodes.count("NONE")) throw Exception("'NONE' not declared", ErrorCodes::LOGICAL_ERROR); - if (!owned_nodes.contains("ALL")) + if (!owned_nodes.count("ALL")) throw Exception("'ALL' not declared", ErrorCodes::LOGICAL_ERROR); flags_to_keyword_tree = std::move(owned_nodes["ALL"]); diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 9c3b5e36ec8..79d77e3f352 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -265,17 +265,7 @@ public: } private: - Node * tryGetChild(const std::string_view & name) - { - if (!children) - return nullptr; - auto it = children->find(name); - if (it == children->end()) - return nullptr; - return &it->second; - } - - const Node * tryGetChild(const std::string_view & name) const + Node * tryGetChild(const std::string_view & name) const { if (!children) return nullptr; diff --git a/src/Access/AccessRightsElement.cpp b/src/Access/AccessRightsElement.cpp index b99cffc191c..81237a334e2 100644 --- a/src/Access/AccessRightsElement.cpp +++ b/src/Access/AccessRightsElement.cpp @@ -72,7 +72,7 @@ String AccessRightsElements::toString() const { String res; bool need_comma = false; - for (auto & element : *this) + for (const auto & element : *this) { if (std::exchange(need_comma, true)) res += ", "; diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 14775f7a4de..ab504e32579 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -147,9 +147,9 @@ void ContextAccess::setUser(const UserPtr & user_) const current_roles.reserve(params.current_roles.size()); for (const auto & id : params.current_roles) { - if (user->granted_roles.contains(id)) + if (user->granted_roles.count(id)) current_roles.push_back(id); - if (user->granted_roles_with_admin_option.contains(id)) + if (user->granted_roles_with_admin_option.count(id)) current_roles_with_admin_option.push_back(id); } } @@ -196,7 +196,7 @@ bool ContextAccess::isClientHostAllowed() const template -bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const +bool ContextAccess::calculateResultAccessAndCheck(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const { auto access = calculateResultAccess(grant_option); bool is_granted = access->isGranted(flags, args...); @@ -267,6 +267,22 @@ bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & fla } +template +bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags) const +{ + return calculateResultAccessAndCheck(log_, flags); +} + +template +bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const Args &... args) const +{ + if (database.empty()) + return calculateResultAccessAndCheck(log_, flags, params.current_database, args...); + else + return calculateResultAccessAndCheck(log_, flags, database, args...); +} + + template bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & element) const { @@ -276,24 +292,15 @@ bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessRightsEleme } else if (element.any_table) { - if (element.database.empty()) - return checkAccessImpl(log_, element.access_flags, params.current_database); - else - return checkAccessImpl(log_, element.access_flags, element.database); + return checkAccessImpl(log_, element.access_flags, element.database); } else if (element.any_column) { - if (element.database.empty()) - return checkAccessImpl(log_, element.access_flags, params.current_database, element.table); - else - return checkAccessImpl(log_, element.access_flags, element.database, element.table); + return checkAccessImpl(log_, element.access_flags, element.database, element.table); } else { - if (element.database.empty()) - return checkAccessImpl(log_, element.access_flags, params.current_database, element.table, element.columns); - else - return checkAccessImpl(log_, element.access_flags, element.database, element.table, element.columns); + return checkAccessImpl(log_, element.access_flags, element.database, element.table, element.columns); } } @@ -351,7 +358,7 @@ void ContextAccess::checkAdminOption(const UUID & role_id) const return; auto roles_with_admin_option_loaded = roles_with_admin_option.load(); - if (roles_with_admin_option_loaded && roles_with_admin_option_loaded->contains(role_id)) + if (roles_with_admin_option_loaded && roles_with_admin_option_loaded->count(role_id)) return; std::optional role_name = manager->readName(role_id); @@ -408,9 +415,10 @@ boost::shared_ptr ContextAccess::calculateResultAccess(bool static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY; static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl; static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE; + static const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS; if (readonly_) - merged_access->revoke(write_table_access | table_and_dictionary_ddl | AccessType::SYSTEM | AccessType::KILL_QUERY | AccessType::ACCESS_MANAGEMENT); + merged_access->revoke(write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY); if (readonly_ == 1) { diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index bee63103793..e0fbf58dbe8 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -130,8 +130,11 @@ private: void setRolesInfo(const std::shared_ptr & roles_info_) const; void setSettingsAndConstraints() const; + template + bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags) const; + template - bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const; + bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const Args &... args) const; template bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & element) const; @@ -139,6 +142,9 @@ private: template bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElements & elements) const; + template + bool calculateResultAccessAndCheck(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const; + boost::shared_ptr calculateResultAccess(bool grant_option) const; boost::shared_ptr calculateResultAccess(bool grant_option, UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const; diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 12c65e7df1e..bf4316cc195 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -93,7 +94,7 @@ namespace const char * end = begin + file_contents.size(); while (pos < end) { - queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0)); + queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH)); while (isWhitespaceASCII(*pos) || *pos == ';') ++pos; } @@ -108,42 +109,42 @@ namespace for (const auto & query : queries) { - if (auto create_user_query = query->as()) + if (auto * create_user_query = query->as()) { if (res) throw Exception("Two access entities in one file " + file_path.string(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); res = user = std::make_unique(); InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query); } - else if (auto create_role_query = query->as()) + else if (auto * create_role_query = query->as()) { if (res) throw Exception("Two access entities in one file " + file_path.string(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); res = role = std::make_unique(); InterpreterCreateRoleQuery::updateRoleFromQuery(*role, *create_role_query); } - else if (auto create_policy_query = query->as()) + else if (auto * create_policy_query = query->as()) { if (res) throw Exception("Two access entities in one file " + file_path.string(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); res = policy = std::make_unique(); InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(*policy, *create_policy_query); } - else if (auto create_quota_query = query->as()) + else if (auto * create_quota_query = query->as()) { if (res) throw Exception("Two access entities are attached in the same file " + file_path.string(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); res = quota = std::make_unique(); InterpreterCreateQuotaQuery::updateQuotaFromQuery(*quota, *create_quota_query); } - else if (auto create_profile_query = query->as()) + else if (auto * create_profile_query = query->as()) { if (res) throw Exception("Two access entities are attached in the same file " + file_path.string(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); res = profile = std::make_unique(); InterpreterCreateSettingsProfileQuery::updateSettingsProfileFromQuery(*profile, *create_profile_query); } - else if (auto grant_query = query->as()) + else if (auto * grant_query = query->as()) { if (!user && !role) throw Exception("A user or role should be attached before grant in file " + file_path.string(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); @@ -560,7 +561,7 @@ std::vector DiskAccessStorage::findAllImpl(std::type_index type) const bool DiskAccessStorage::existsImpl(const UUID & id) const { std::lock_guard lock{mutex}; - return id_to_entry_map.contains(id); + return id_to_entry_map.count(id); } @@ -571,7 +572,7 @@ AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id) const if (it == id_to_entry_map.end()) throwNotFound(id); - auto & entry = it->second; + const auto & entry = it->second; if (!entry.entity) entry.entity = readAccessEntityFromDisk(id); return entry.entity; @@ -709,7 +710,7 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ if (name_changed) { const auto & name_to_id_map = name_to_id_maps.at(type); - if (name_to_id_map.contains(new_name)) + if (name_to_id_map.count(new_name)) throwNameCollisionCannotRename(type, String{old_name}, new_name); scheduleWriteLists(type); } diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp index a525fb65606..56c73aaf40d 100644 --- a/src/Access/EnabledRowPolicies.cpp +++ b/src/Access/EnabledRowPolicies.cpp @@ -1,7 +1,5 @@ #include -#include -#include -#include +#include #include #include @@ -35,19 +33,17 @@ ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & table_name, ConditionType type, const ASTPtr & extra_condition) const { - ASTPtr main_condition = getCondition(database, table_name, type); - if (!main_condition) - return extra_condition; - if (!extra_condition) - return main_condition; - auto function = std::make_shared(); - auto exp_list = std::make_shared(); - function->name = "and"; - function->arguments = exp_list; - function->children.push_back(exp_list); - exp_list->children.push_back(main_condition); - exp_list->children.push_back(extra_condition); - return function; + ASTPtr condition = getCondition(database, table_name, type); + if (condition && extra_condition) + condition = makeASTForLogicalAnd({condition, extra_condition}); + else if (!condition) + condition = extra_condition; + + bool value; + if (tryGetLiteralBool(condition.get(), value) && value) + condition = nullptr; /// The condition is always true, no need to check it. + + return condition; } diff --git a/src/Access/ExtendedRoleSet.cpp b/src/Access/ExtendedRoleSet.cpp index eed475bc3cc..145bd0fe7e5 100644 --- a/src/Access/ExtendedRoleSet.cpp +++ b/src/Access/ExtendedRoleSet.cpp @@ -253,44 +253,44 @@ void ExtendedRoleSet::add(const boost::container::flat_set & ids_) bool ExtendedRoleSet::match(const UUID & id) const { - return (all || ids.contains(id)) && !except_ids.contains(id); + return (all || ids.count(id)) && !except_ids.count(id); } bool ExtendedRoleSet::match(const UUID & user_id, const std::vector & enabled_roles) const { - if (!all && !ids.contains(user_id)) + if (!all && !ids.count(user_id)) { bool found_enabled_role = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.count(enabled_role); }); if (!found_enabled_role) return false; } - if (except_ids.contains(user_id)) + if (except_ids.count(user_id)) return false; bool in_except_list = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.count(enabled_role); }); return !in_except_list; } bool ExtendedRoleSet::match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const { - if (!all && !ids.contains(user_id)) + if (!all && !ids.count(user_id)) { bool found_enabled_role = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.count(enabled_role); }); if (!found_enabled_role) return false; } - if (except_ids.contains(user_id)) + if (except_ids.count(user_id)) return false; bool in_except_list = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.count(enabled_role); }); return !in_except_list; } diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 3474ab9dbd5..740fe1dac04 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -66,7 +66,7 @@ std::optional MultipleAccessStorage::findImpl(std::type_index type, const std::vector storages_with_duplicates; for (const auto & id : ids) { - auto * storage = findStorage(id); + const auto * storage = findStorage(id); if (storage) storages_with_duplicates.push_back(storage); } @@ -207,7 +207,7 @@ void MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & updat ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const { - auto storage = findStorage(id); + const auto * storage = findStorage(id); if (!storage) return {}; return storage->subscribeForChanges(id, handler); diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index 6db3eb66c5d..dca844bf746 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -145,7 +145,7 @@ boost::shared_ptr QuotaCache::QuotaInfo::rebuildI continue; /// Found an interval with the same duration, we need to copy its usage information to `result`. - auto & current_interval = *lower_bound; + const auto & current_interval = *lower_bound; for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { new_interval.used[resource_type].store(current_interval.used[resource_type].load()); @@ -167,7 +167,12 @@ QuotaCache::QuotaCache(const AccessControlManager & access_control_manager_) QuotaCache::~QuotaCache() = default; -std::shared_ptr QuotaCache::getEnabledQuota(const UUID & user_id, const String & user_name, const std::vector & enabled_roles, const Poco::Net::IPAddress & client_address, const String & client_key) +std::shared_ptr QuotaCache::getEnabledQuota( + const UUID & user_id, + const String & user_name, + const std::vector & enabled_roles, + const Poco::Net::IPAddress & client_address, + const String & client_key) { std::lock_guard lock{mutex}; ensureAllQuotasRead(); @@ -250,16 +255,18 @@ void QuotaCache::quotaRemoved(const UUID & quota_id) void QuotaCache::chooseQuotaToConsume() { /// `mutex` is already locked. - std::erase_if( - enabled_quotas, - [&](const std::pair> & pr) + + for (auto i = enabled_quotas.begin(), e = enabled_quotas.end(); i != e;) + { + auto elem = i->second.lock(); + if (!elem) + i = enabled_quotas.erase(i); + else { - auto elem = pr.second.lock(); - if (!elem) - return true; // remove from the `enabled_quotas` list. chooseQuotaToConsumeFor(*elem); - return false; // keep in the `enabled_quotas` list. - }); + ++i; + } + } } void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled) diff --git a/src/Access/QuotaCache.h b/src/Access/QuotaCache.h index 81734f385c1..8399c5f73eb 100644 --- a/src/Access/QuotaCache.h +++ b/src/Access/QuotaCache.h @@ -20,7 +20,13 @@ public: QuotaCache(const AccessControlManager & access_control_manager_); ~QuotaCache(); - std::shared_ptr getEnabledQuota(const UUID & user_id, const String & user_name, const std::vector & enabled_roles, const Poco::Net::IPAddress & address, const String & client_key); + std::shared_ptr getEnabledQuota( + const UUID & user_id, + const String & user_name, + const std::vector & enabled_roles, + const Poco::Net::IPAddress & address, + const String & client_key); + std::vector getUsageInfo() const; private: diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 63e19a3cb40..0263b793017 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -103,16 +103,17 @@ void RoleCache::collectRolesInfo() { /// `mutex` is already locked. - std::erase_if( - enabled_roles, - [&](const std::pair> & pr) + for (auto i = enabled_roles.begin(), e = enabled_roles.end(); i != e;) + { + auto elem = i->second.lock(); + if (!elem) + i = enabled_roles.erase(i); + else { - auto elem = pr.second.lock(); - if (!elem) - return true; // remove from the `enabled_roles` map. collectRolesInfoFor(*elem); - return false; // keep in the `enabled_roles` map. - }); + ++i; + } + } } diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp index 9509923adbf..0c9fea69dc4 100644 --- a/src/Access/RowPolicyCache.cpp +++ b/src/Access/RowPolicyCache.cpp @@ -1,97 +1,20 @@ #include #include #include -#include -#include #include #include +#include #include #include #include #include -#include -#include +#include namespace DB { namespace { - bool tryGetLiteralBool(const IAST & ast, bool & value) - { - try - { - if (const ASTLiteral * literal = ast.as()) - { - value = !literal->value.isNull() && applyVisitor(FieldVisitorConvertToNumber(), literal->value); - return true; - } - return false; - } - catch (...) - { - return false; - } - } - - ASTPtr applyFunctionAND(ASTs arguments) - { - bool const_arguments = true; - boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool - { - bool b; - if (!tryGetLiteralBool(*argument, b)) - return false; - const_arguments &= b; - return true; - }); - - if (!const_arguments) - return std::make_shared(Field{UInt8(0)}); - if (arguments.empty()) - return std::make_shared(Field{UInt8(1)}); - if (arguments.size() == 1) - return arguments[0]; - - auto function = std::make_shared(); - auto exp_list = std::make_shared(); - function->name = "and"; - function->arguments = exp_list; - function->children.push_back(exp_list); - exp_list->children = std::move(arguments); - return function; - } - - - ASTPtr applyFunctionOR(ASTs arguments) - { - bool const_arguments = false; - boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool - { - bool b; - if (!tryGetLiteralBool(*argument, b)) - return false; - const_arguments |= b; - return true; - }); - - if (const_arguments) - return std::make_shared(Field{UInt8(1)}); - if (arguments.empty()) - return std::make_shared(Field{UInt8(0)}); - if (arguments.size() == 1) - return arguments[0]; - - auto function = std::make_shared(); - auto exp_list = std::make_shared(); - function->name = "or"; - function->arguments = exp_list; - function->children.push_back(exp_list); - exp_list->children = std::move(arguments); - return function; - } - - using ConditionType = RowPolicy::ConditionType; constexpr size_t MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; @@ -111,10 +34,16 @@ namespace ASTPtr getResult() && { /// Process permissive conditions. - restrictions.push_back(applyFunctionOR(std::move(permissions))); + restrictions.push_back(makeASTForLogicalOr(std::move(permissions))); /// Process restrictive conditions. - return applyFunctionAND(std::move(restrictions)); + auto condition = makeASTForLogicalAnd(std::move(restrictions)); + + bool value; + if (tryGetLiteralBool(condition.get(), value) && value) + condition = nullptr; /// The condition is always true, no need to check it. + + return condition; } private: @@ -137,7 +66,7 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) continue; auto previous_range = std::pair(std::begin(policy->conditions), std::begin(policy->conditions) + type); - auto previous_it = std::find(previous_range.first, previous_range.second, condition); + const auto * previous_it = std::find(previous_range.first, previous_range.second, condition); if (previous_it != previous_range.second) { /// The condition is already parsed before. @@ -149,7 +78,7 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) try { ParserExpression parser; - parsed_conditions[type] = parseQuery(parser, condition, 0); + parsed_conditions[type] = parseQuery(parser, condition, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); } catch (...) { @@ -250,16 +179,17 @@ void RowPolicyCache::rowPolicyRemoved(const UUID & policy_id) void RowPolicyCache::mixConditions() { /// `mutex` is already locked. - std::erase_if( - enabled_row_policies, - [&](const std::pair> & pr) + for (auto i = enabled_row_policies.begin(), e = enabled_row_policies.end(); i != e;) + { + auto elem = i->second.lock(); + if (!elem) + i = enabled_row_policies.erase(i); + else { - auto elem = pr.second.lock(); - if (!elem) - return true; // remove from the `enabled_row_policies` map. mixConditionsFor(*elem); - return false; // keep in the `enabled_row_policies` map. - }); + ++i; + } + } } diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index 552ed324635..f283715e129 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -104,16 +104,17 @@ void SettingsProfilesCache::setDefaultProfileName(const String & default_profile void SettingsProfilesCache::mergeSettingsAndConstraints() { /// `mutex` is already locked. - std::erase_if( - enabled_settings, - [&](const std::pair> & pr) + for (auto i = enabled_settings.begin(), e = enabled_settings.end(); i != e;) + { + auto enabled = i->second.lock(); + if (!enabled) + i = enabled_settings.erase(i); + else { - auto enabled = pr.second.lock(); - if (!enabled) - return true; // remove from the `enabled_settings` list. mergeSettingsAndConstraintsFor(*enabled); - return false; // keep in the `enabled_settings` list. - }); + ++i; + } + } } @@ -161,7 +162,7 @@ void SettingsProfilesCache::substituteProfiles(SettingsProfileElements & element auto parent_profile_id = *element.parent_profile; element.parent_profile.reset(); - if (already_substituted.contains(parent_profile_id)) + if (already_substituted.count(parent_profile_id)) continue; already_substituted.insert(parent_profile_id); diff --git a/src/Access/ya.make b/src/Access/ya.make new file mode 100644 index 00000000000..fb2e23e0684 --- /dev/null +++ b/src/Access/ya.make @@ -0,0 +1,40 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + AccessControlManager.cpp + AccessRights.cpp + AccessRightsElement.cpp + AllowedClientHosts.cpp + Authentication.cpp + ContextAccess.cpp + DiskAccessStorage.cpp + EnabledQuota.cpp + EnabledRoles.cpp + EnabledRolesInfo.cpp + EnabledRowPolicies.cpp + EnabledSettings.cpp + ExtendedRoleSet.cpp + IAccessEntity.cpp + IAccessStorage.cpp + MemoryAccessStorage.cpp + MultipleAccessStorage.cpp + Quota.cpp + QuotaCache.cpp + QuotaUsageInfo.cpp + Role.cpp + RoleCache.cpp + RowPolicy.cpp + RowPolicyCache.cpp + SettingsConstraints.cpp + SettingsProfile.cpp + SettingsProfileElement.cpp + SettingsProfilesCache.cpp + User.cpp + UsersConfigAccessStorage.cpp +) + +END() diff --git a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp index d1022fd859f..ba0de7e1b46 100644 --- a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp +++ b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp @@ -31,7 +31,7 @@ AggregateFunctionPtr createAggregateFunctionCategoricalIV( "Aggregate function " + name + " requires two or more arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - for (auto & argument : arguments) + for (const auto & argument : arguments) { if (!WhichDataType(argument).isUInt8()) throw Exception( diff --git a/src/AggregateFunctions/AggregateFunctionEntropy.cpp b/src/AggregateFunctions/AggregateFunctionEntropy.cpp index e9db84949f7..a30b27f5311 100644 --- a/src/AggregateFunctions/AggregateFunctionEntropy.cpp +++ b/src/AggregateFunctions/AggregateFunctionEntropy.cpp @@ -27,7 +27,7 @@ AggregateFunctionPtr createAggregateFunctionEntropy(const std::string & name, co if (num_args == 1) { /// Specialized implementation for single argument of numeric type. - if (auto res = createWithNumericBasedType(*argument_types[0], argument_types)) + if (auto * res = createWithNumericBasedType(*argument_types[0], argument_types)) return AggregateFunctionPtr(res); } diff --git a/src/AggregateFunctions/AggregateFunctionHistogram.h b/src/AggregateFunctions/AggregateFunctionHistogram.h index 673d78d807a..96ee01652de 100644 --- a/src/AggregateFunctions/AggregateFunctionHistogram.h +++ b/src/AggregateFunctions/AggregateFunctionHistogram.h @@ -34,7 +34,7 @@ namespace ErrorCodes } /** - * distance compression algorigthm implementation + * distance compression algorithm implementation * http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf */ class AggregateFunctionHistogramData diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index e028e585d5c..d6dce6fe080 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -245,7 +245,7 @@ void Adam::read(ReadBuffer & buf) void Adam::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) { - auto & adam_rhs = static_cast(rhs); + const auto & adam_rhs = static_cast(rhs); if (adam_rhs.average_gradient.empty()) return; @@ -330,7 +330,7 @@ void Nesterov::write(WriteBuffer & buf) const void Nesterov::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) { - auto & nesterov_rhs = static_cast(rhs); + const auto & nesterov_rhs = static_cast(rhs); if (accumulated_gradient.empty()) accumulated_gradient.resize(nesterov_rhs.accumulated_gradient.size(), Float64{0.0}); @@ -395,7 +395,7 @@ void Momentum::write(WriteBuffer & buf) const void Momentum::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) { - auto & momentum_rhs = static_cast(rhs); + const auto & momentum_rhs = static_cast(rhs); for (size_t i = 0; i < accumulated_gradient.size(); ++i) { accumulated_gradient[i] = accumulated_gradient[i] * frac + momentum_rhs.accumulated_gradient[i] * rhs_frac; @@ -473,7 +473,7 @@ void LogisticRegression::predict( if (!isNativeNumber(cur_col.type)) throw Exception("Prediction arguments must have numeric type", ErrorCodes::BAD_ARGUMENTS); - auto & features_column = cur_col.column; + const auto & features_column = cur_col.column; for (size_t row_num = 0; row_num < limit; ++row_num) results[row_num] += weights[i - 1] * features_column->getFloat64(offset + row_num); diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp index e329cedd1df..bbdb7622edf 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp @@ -40,11 +40,11 @@ AggregateFunctionPtr createAggregateFunctionSequenceBase(const std::string & nam + toString(max_events) + " event arguments.", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION}; - const auto time_arg = argument_types.front().get(); + const auto * time_arg = argument_types.front().get(); for (const auto i : ext::range(1, arg_count)) { - const auto cond_arg = argument_types[i].get(); + const auto * cond_arg = argument_types[i].get(); if (!isUInt8(cond_arg)) throw Exception{"Illegal type " + cond_arg->getName() + " of argument " + toString(i + 1) + " of aggregate function " + name + ", must be UInt8", diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index e627160e04d..872c70c2b98 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -34,7 +34,7 @@ AggregateFunctionPtr createAggregateFunctionWindowFunnel(const std::string & nam for (const auto i : ext::range(1, arguments.size())) { - auto cond_arg = arguments[i].get(); + const auto * cond_arg = arguments[i].get(); if (!isUInt8(cond_arg)) throw Exception{"Illegal type " + cond_arg->getName() + " of argument " + toString(i + 1) + " of aggregate function " + name + ", must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; diff --git a/src/AggregateFunctions/parseAggregateFunctionParameters.cpp b/src/AggregateFunctions/parseAggregateFunctionParameters.cpp index bcb73f1e9d9..2a6b9e3b499 100644 --- a/src/AggregateFunctions/parseAggregateFunctionParameters.cpp +++ b/src/AggregateFunctions/parseAggregateFunctionParameters.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB @@ -65,7 +66,7 @@ void getAggregateFunctionNameAndParametersArray( ParserExpressionList params_parser(false); ASTPtr args_ast = parseQuery(params_parser, parameters_str.data(), parameters_str.data() + parameters_str.size(), - "parameters of aggregate function in " + error_context, 0); + "parameters of aggregate function in " + error_context, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (args_ast->children.empty()) throw Exception("Incorrect list of parameters to aggregate function " diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index a4fc41e9c06..a9ab1d4f8ea 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -32,7 +32,9 @@ void registerAggregateFunctions() registerAggregateFunctionUniqUpTo(factory); registerAggregateFunctionTopK(factory); registerAggregateFunctionsBitwise(factory); +#if !defined(ARCADIA_BUILD) registerAggregateFunctionsBitmap(factory); +#endif registerAggregateFunctionsMaxIntersections(factory); registerAggregateFunctionHistogram(factory); registerAggregateFunctionRetention(factory); diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make new file mode 100644 index 00000000000..bfa32b6dd78 --- /dev/null +++ b/src/AggregateFunctions/ya.make @@ -0,0 +1,54 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + AggregateFunctionAggThrow.cpp + AggregateFunctionArray.cpp + AggregateFunctionAvg.cpp + AggregateFunctionAvgWeighted.cpp + AggregateFunctionBitwise.cpp + AggregateFunctionBoundingRatio.cpp + AggregateFunctionCategoricalInformationValue.cpp + AggregateFunctionCombinatorFactory.cpp + AggregateFunctionCount.cpp + AggregateFunctionEntropy.cpp + AggregateFunctionFactory.cpp + AggregateFunctionForEach.cpp + AggregateFunctionGroupArray.cpp + AggregateFunctionGroupArrayInsertAt.cpp + AggregateFunctionGroupArrayMoving.cpp + AggregateFunctionGroupUniqArray.cpp + AggregateFunctionHistogram.cpp + AggregateFunctionIf.cpp + AggregateFunctionMaxIntersections.cpp + AggregateFunctionMerge.cpp + AggregateFunctionMinMaxAny.cpp + AggregateFunctionMLMethod.cpp + AggregateFunctionNull.cpp + AggregateFunctionOrFill.cpp + AggregateFunctionQuantile.cpp + AggregateFunctionResample.cpp + AggregateFunctionRetention.cpp + AggregateFunctionSequenceMatch.cpp + AggregateFunctionSimpleLinearRegression.cpp + AggregateFunctionState.cpp + AggregateFunctionStatistics.cpp + AggregateFunctionStatisticsSimple.cpp + AggregateFunctionSum.cpp + AggregateFunctionSumMap.cpp + AggregateFunctionTimeSeriesGroupSum.cpp + AggregateFunctionTopK.cpp + AggregateFunctionUniq.cpp + AggregateFunctionUniqCombined.cpp + AggregateFunctionUniqUpTo.cpp + AggregateFunctionWindowFunnel.cpp + parseAggregateFunctionParameters.cpp + registerAggregateFunctions.cpp + UniqCombinedBiasData.cpp + UniqVariadicHash.cpp +) + +END() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 35f90529fd9..68f1beadb2b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -311,6 +311,8 @@ add_object_library(clickhouse_processors_formats Processors/Formats) add_object_library(clickhouse_processors_formats_impl Processors/Formats/Impl) add_object_library(clickhouse_processors_transforms Processors/Transforms) add_object_library(clickhouse_processors_sources Processors/Sources) +add_object_library(clickhouse_processors_merges Processors/Merges) +add_object_library(clickhouse_processors_merges_algorithms Processors/Merges/Algorithms) if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES) @@ -398,6 +400,7 @@ endif() target_link_libraries(clickhouse_common_io PUBLIC ${CITYHASH_LIBRARIES} + pcg_random PRIVATE ${Poco_XML_LIBRARY} ${ZLIB_LIBRARIES} @@ -453,9 +456,6 @@ dbms_target_link_libraries ( target_include_directories(clickhouse_common_io PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/Core/include) # uses some includes from core dbms_target_include_directories(PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/Core/include) -target_include_directories(clickhouse_common_io SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) -dbms_target_include_directories(SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) - dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) if (NOT USE_INTERNAL_LZ4_LIBRARY AND LZ4_INCLUDE_DIR) @@ -574,6 +574,8 @@ target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) +target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MSGPACK_INCLUDE_DIR}) + if (ENABLE_TESTS AND USE_GTEST) macro (grep_gtest_sources BASE_DIR DST_VAR) # Cold match files that are not in tests/ directories diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index f530652caae..cc79e3ec2c8 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -19,16 +19,19 @@ #include #include #include -#include #include #include #include #include #include -#include +#if !defined(ARCADIA_BUILD) +# include +# include +#endif + #if USE_POCO_NETSSL -#include +# include #endif namespace CurrentMetrics diff --git a/src/Client/ya.make b/src/Client/ya.make new file mode 100644 index 00000000000..685e0540ad7 --- /dev/null +++ b/src/Client/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/poco/NetSSL_OpenSSL +) + +SRCS( + Connection.cpp + ConnectionPoolWithFailover.cpp + MultiplexedConnections.cpp + TimeoutSetter.cpp +) + +END() diff --git a/src/Columns/Collator.cpp b/src/Columns/Collator.cpp index c3915125fa1..77735564077 100644 --- a/src/Columns/Collator.cpp +++ b/src/Columns/Collator.cpp @@ -1,17 +1,19 @@ #include -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif #if USE_ICU - #include - #include - #include - #include +# include +# include +# include +# include #else - #ifdef __clang__ - #pragma clang diagnostic ignored "-Wunused-private-field" - #pragma clang diagnostic ignored "-Wmissing-noreturn" - #endif +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wunused-private-field" +# pragma clang diagnostic ignored "-Wmissing-noreturn" +# endif #endif #include diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 474dd418186..e52d3e5303f 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -30,7 +30,7 @@ namespace ErrorCodes ColumnAggregateFunction::~ColumnAggregateFunction() { if (!func->hasTrivialDestructor() && !src) - for (auto val : data) + for (auto * val : data) func->destroy(val); } @@ -82,7 +82,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues() const MutableColumnPtr res = func->getReturnType()->createColumn(); res->reserve(data.size()); - for (auto val : data) + for (auto * val : data) func->insertResultInto(val, *res); return res; @@ -93,7 +93,7 @@ MutableColumnPtr ColumnAggregateFunction::predictValues(Block & block, const Col MutableColumnPtr res = func->getReturnTypeToPredict()->createColumn(); res->reserve(data.size()); - auto machine_learning_function = func.get(); + auto * machine_learning_function = func.get(); if (machine_learning_function) { if (data.size() == 1) @@ -105,7 +105,7 @@ MutableColumnPtr ColumnAggregateFunction::predictValues(Block & block, const Col { /// Case for non-constant column. Use different aggregate function for each row. size_t row_num = 0; - for (auto val : data) + for (auto * val : data) { machine_learning_function->predictValues(val, *res, block, row_num, 1, arguments, context); ++row_num; @@ -425,7 +425,7 @@ void ColumnAggregateFunction::insert(const Field & x) throw Exception(String("Inserting field of type ") + x.getTypeName() + " into ColumnAggregateFunction. " "Expected " + Field::Types::toString(Field::Types::AggregateFunctionState), ErrorCodes::LOGICAL_ERROR); - auto & field_name = x.get().name; + const auto & field_name = x.get().name; if (type_string != field_name) throw Exception("Cannot insert filed with type " + field_name + " into column with type " + type_string, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 943539fb551..19ef1f7dd0b 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -226,7 +226,7 @@ void ColumnArray::updateWeakHash32(WeakHash32 & hash) const data->updateWeakHash32(internal_hash); Offset prev_offset = 0; - auto & offsets_data = getOffsets(); + const auto & offsets_data = getOffsets(); auto & hash_data = hash.getData(); auto & internal_hash_data = internal_hash.getData(); @@ -589,8 +589,8 @@ ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_h auto array_of_nested = ColumnArray::create(nullable_elems.getNestedColumnPtr(), offsets); auto filtered_array_of_nested_owner = array_of_nested->filter(filt, result_size_hint); - auto & filtered_array_of_nested = assert_cast(*filtered_array_of_nested_owner); - auto & filtered_offsets = filtered_array_of_nested.getOffsetsPtr(); + const auto & filtered_array_of_nested = assert_cast(*filtered_array_of_nested_owner); + const auto & filtered_offsets = filtered_array_of_nested.getOffsetsPtr(); auto res_null_map = ColumnUInt8::create(); diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 511ed7c1a8b..5396389294a 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -42,7 +42,7 @@ int ColumnDecimal::compareAt(size_t n, size_t m, const IColumn & rhs_, int) c template StringRef ColumnDecimal::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const { - auto pos = arena.allocContinue(sizeof(T), begin); + auto * pos = arena.allocContinue(sizeof(T), begin); memcpy(pos, &data[n], sizeof(T)); return StringRef(pos, sizeof(T)); } diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index b66ff5370b0..57ae4cbdedf 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -86,7 +86,7 @@ void ColumnFixedString::insertData(const char * pos, size_t length) StringRef ColumnFixedString::serializeValueIntoArena(size_t index, Arena & arena, char const *& begin) const { - auto pos = arena.allocContinue(n, begin); + auto * pos = arena.allocContinue(n, begin); memcpy(pos, &chars[n * index], n); return StringRef(pos, n); } diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp index aec9067efaa..24c074187a4 100644 --- a/src/Columns/ColumnFunction.cpp +++ b/src/Columns/ColumnFunction.cpp @@ -134,7 +134,7 @@ std::vector ColumnFunction::scatter(IColumn::ColumnIndex num_c size_t ColumnFunction::byteSize() const { size_t total_size = 0; - for (auto & column : captured_columns) + for (const auto & column : captured_columns) total_size += column.column->byteSize(); return total_size; @@ -143,7 +143,7 @@ size_t ColumnFunction::byteSize() const size_t ColumnFunction::allocatedBytes() const { size_t total_size = 0; - for (auto & column : captured_columns) + for (const auto & column : captured_columns) total_size += column.column->allocatedBytes(); return total_size; diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index aecc175fe0d..e87b3b4cbf6 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -143,7 +143,7 @@ void ColumnLowCardinality::insertDefault() void ColumnLowCardinality::insertFrom(const IColumn & src, size_t n) { - auto * low_cardinality_src = typeid_cast(&src); + const auto * low_cardinality_src = typeid_cast(&src); if (!low_cardinality_src) throw Exception("Expected ColumnLowCardinality, got" + src.getName(), ErrorCodes::ILLEGAL_COLUMN); @@ -174,7 +174,7 @@ void ColumnLowCardinality::insertFromFullColumn(const IColumn & src, size_t n) void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, size_t length) { - auto * low_cardinality_src = typeid_cast(&src); + const auto * low_cardinality_src = typeid_cast(&src); if (!low_cardinality_src) throw Exception("Expected ColumnLowCardinality, got " + src.getName(), ErrorCodes::ILLEGAL_COLUMN); @@ -250,7 +250,7 @@ void ColumnLowCardinality::updateWeakHash32(WeakHash32 & hash) const throw Exception("Size of WeakHash32 does not match size of column: column size is " + std::to_string(s) + ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR); - auto & dict = getDictionary().getNestedColumn(); + const auto & dict = getDictionary().getNestedColumn(); WeakHash32 dict_hash(dict->size()); dict->updateWeakHash32(dict_hash); diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 79b3bff1ccc..55ce1401073 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -54,7 +54,7 @@ void ColumnNullable::updateWeakHash32(WeakHash32 & hash) const WeakHash32 old_hash = hash; nested_column->updateWeakHash32(hash); - auto & null_map_data = getNullMapData(); + const auto & null_map_data = getNullMapData(); auto & hash_data = hash.getData(); auto & old_hash_data = old_hash.getData(); @@ -123,7 +123,7 @@ StringRef ColumnNullable::serializeValueIntoArena(size_t n, Arena & arena, char const auto & arr = getNullMapData(); static constexpr auto s = sizeof(arr[0]); - auto pos = arena.allocContinue(s, begin); + auto * pos = arena.allocContinue(s, begin); memcpy(pos, &arr[n], s); if (arr[n]) @@ -424,25 +424,25 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const const auto & null_map_data = getNullMapData(); - if (const auto col_i8 = typeid_cast(nested_column.get())) + if (const auto * col_i8 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_i8, null_map_data, min, max); - else if (const auto col_i16 = typeid_cast(nested_column.get())) + else if (const auto * col_i16 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_i16, null_map_data, min, max); - else if (const auto col_i32 = typeid_cast(nested_column.get())) + else if (const auto * col_i32 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_i32, null_map_data, min, max); - else if (const auto col_i64 = typeid_cast(nested_column.get())) + else if (const auto * col_i64 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_i64, null_map_data, min, max); - else if (const auto col_u8 = typeid_cast(nested_column.get())) + else if (const auto * col_u8 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_u8, null_map_data, min, max); - else if (const auto col_u16 = typeid_cast(nested_column.get())) + else if (const auto * col_u16 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_u16, null_map_data, min, max); - else if (const auto col_u32 = typeid_cast(nested_column.get())) + else if (const auto * col_u32 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_u32, null_map_data, min, max); - else if (const auto col_u64 = typeid_cast(nested_column.get())) + else if (const auto * col_u64 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_u64, null_map_data, min, max); - else if (const auto col_f32 = typeid_cast(nested_column.get())) + else if (const auto * col_f32 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_f32, null_map_data, min, max); - else if (const auto col_f64 = typeid_cast(nested_column.get())) + else if (const auto * col_f64 = typeid_cast(nested_column.get())) getExtremesFromNullableContent(*col_f64, null_map_data, min, max); } diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 84757c776ba..12f7a5632db 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -78,7 +78,7 @@ void ColumnString::updateWeakHash32(WeakHash32 & hash) const UInt32 * hash_data = hash.getData().data(); Offset prev_offset = 0; - for (auto & offset : offsets) + for (const auto & offset : offsets) { auto str_size = offset - prev_offset; /// Skip last zero byte. diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 1d331377e2b..59552c67f14 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -121,7 +121,7 @@ void ColumnTuple::insertData(const char *, size_t) void ColumnTuple::insert(const Field & x) { - auto & tuple = DB::get(x); + const auto & tuple = DB::get(x); const size_t tuple_size = columns.size(); if (tuple.size() != tuple_size) @@ -158,7 +158,7 @@ void ColumnTuple::popBack(size_t n) StringRef ColumnTuple::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const { StringRef res(begin, 0); - for (auto & column : columns) + for (const auto & column : columns) { auto value_ref = column->serializeValueIntoArena(n, arena, begin); res.data = value_ref.data - res.size; @@ -178,7 +178,7 @@ const char * ColumnTuple::deserializeAndInsertFromArena(const char * pos) void ColumnTuple::updateHashWithValue(size_t n, SipHash & hash) const { - for (auto & column : columns) + for (const auto & column : columns) column->updateHashWithValue(n, hash); } @@ -190,7 +190,7 @@ void ColumnTuple::updateWeakHash32(WeakHash32 & hash) const throw Exception("Size of WeakHash32 does not match size of column: column size is " + std::to_string(s) + ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR); - for (auto & column : columns) + for (const auto & column : columns) column->updateWeakHash32(hash); } @@ -385,7 +385,7 @@ void ColumnTuple::forEachSubcolumn(ColumnCallback callback) bool ColumnTuple::structureEquals(const IColumn & rhs) const { - if (auto rhs_tuple = typeid_cast(&rhs)) + if (const auto * rhs_tuple = typeid_cast(&rhs)) { const size_t tuple_size = columns.size(); if (tuple_size != rhs_tuple->columns.size()) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index b5634a5814c..814de64a0b6 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -36,7 +36,7 @@ namespace ErrorCodes template StringRef ColumnVector::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const { - auto pos = arena.allocContinue(sizeof(T), begin); + auto * pos = arena.allocContinue(sizeof(T), begin); unalignedStore(pos, data[n]); return StringRef(pos, sizeof(T)); } @@ -372,10 +372,10 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); - auto it = res->getData().begin(); + auto * it = res->getData().begin(); for (size_t i = 0; i < size; ++i) { - const auto span_end = res->getData().begin() + offsets[i]; + const auto * span_end = res->getData().begin() + offsets[i]; for (; it != span_end; ++it) *it = data[i]; } diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index a363e26a690..14ace8f8a4b 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -65,7 +65,7 @@ bool memoryIsByte(const void * data, size_t size, uint8_t byte) { if (size == 0) return true; - auto ptr = reinterpret_cast(data); + const auto * ptr = reinterpret_cast(data); return *ptr == byte && memcmp(ptr, ptr + 1, size - 1) == 0; } @@ -121,7 +121,7 @@ namespace if (diff_offset > 0) { - const auto res_offsets_pos = &res_offsets[offsets_size_old]; + auto * res_offsets_pos = &res_offsets[offsets_size_old]; /// adjust offsets for (size_t i = 0; i < SIMD_BYTES; ++i) @@ -172,10 +172,10 @@ namespace } const UInt8 * filt_pos = filt.data(); - const auto filt_end = filt_pos + size; + const auto * filt_end = filt_pos + size; - auto offsets_pos = src_offsets.data(); - const auto offsets_begin = offsets_pos; + const auto * offsets_pos = src_offsets.data(); + const auto * offsets_begin = offsets_pos; /// copy array ending at *end_offset_ptr const auto copy_array = [&] (const IColumn::Offset * offset_ptr) @@ -193,7 +193,7 @@ namespace #ifdef __SSE2__ const __m128i zero_vec = _mm_setzero_si128(); static constexpr size_t SIMD_BYTES = 16; - const auto filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; + const auto * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; while (filt_pos < filt_end_aligned) { diff --git a/src/Columns/FilterDescription.cpp b/src/Columns/FilterDescription.cpp index 272cc4d92de..27336d3db58 100644 --- a/src/Columns/FilterDescription.cpp +++ b/src/Columns/FilterDescription.cpp @@ -61,7 +61,7 @@ FilterDescription::FilterDescription(const IColumn & column_) return; } - if (auto * nullable_column = checkAndGetColumn(column)) + if (const auto * nullable_column = checkAndGetColumn(column)) { ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate(); diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 090537d6770..4af593bb658 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -44,7 +44,7 @@ public: /// Name of a Column kind, without parameters (example: FixedString, Array). virtual const char * getFamilyName() const = 0; - /** If column isn't constant, returns nullptr (or itself). + /** If column isn't constant, returns itself. * If column is constant, transforms constant to full column (if column type allows such transform) and return it. */ virtual Ptr convertToFullColumnIfConst() const { return getPtr(); } diff --git a/src/Columns/tests/gtest_column_unique.cpp b/src/Columns/tests/gtest_column_unique.cpp index 601d34e6302..23a356afff2 100644 --- a/src/Columns/tests/gtest_column_unique.cpp +++ b/src/Columns/tests/gtest_column_unique.cpp @@ -43,7 +43,7 @@ TEST(ColumnUnique, InsertRange) ASSERT_EQ(indexes[i] + 1, idx->getUInt(i)) << "Different indexes at position " << i; } - auto & nested = column_unique->getNestedColumn(); + const auto & nested = column_unique->getNestedColumn(); ASSERT_EQ(nested->size(), mod_to + 1); for (size_t i = 0; i < mod_to; ++i) @@ -87,7 +87,7 @@ TEST(ColumnUnique, InsertRangeWithOverflow) ASSERT_EQ(indexes[i] + 1, idx->getUInt(i)) << "Different indexes at position " << i; } - auto & nested = column_unique->getNestedColumn(); + const auto & nested = column_unique->getNestedColumn(); ASSERT_EQ(nested->size(), max_dict_size); ASSERT_EQ(add_keys->size(), mod_to - max_val); @@ -196,15 +196,15 @@ TEST(ColumnVector, CorrectnessOfReplicate) column->insertValue(1); const auto empty_column = column->replicate({0, 0, 0}); - const auto empty_column_ptr = typeid_cast(empty_column.get()); + const auto * empty_column_ptr = typeid_cast(empty_column.get()); EXPECT_NE(empty_column_ptr, nullptr); EXPECT_EQ(empty_column_ptr->size(), 0); const auto new_column = column->replicate({1, 1, 5}); - const auto new_column_ptr = typeid_cast(new_column.get()); + const auto * new_column_ptr = typeid_cast(new_column.get()); EXPECT_NE(new_column_ptr, nullptr); EXPECT_EQ(new_column_ptr->size(), 5); - auto it = new_column_ptr->getData().cbegin(); + const auto * it = new_column_ptr->getData().cbegin(); for (const auto num : {3, 1, 1, 1, 1}) { EXPECT_EQ(*it, num); diff --git a/src/Columns/tests/gtest_weak_hash_32.cpp b/src/Columns/tests/gtest_weak_hash_32.cpp index c79188e9e88..5d87d2aa2e1 100644 --- a/src/Columns/tests/gtest_weak_hash_32.cpp +++ b/src/Columns/tests/gtest_weak_hash_32.cpp @@ -108,7 +108,7 @@ TEST(WeakHash32, ColumnVectorU8) auto col = ColumnUInt8::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (size_t i = 0; i < 265; ++i) data.push_back(i); @@ -125,7 +125,7 @@ TEST(WeakHash32, ColumnVectorI8) auto col = ColumnInt8::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int i = -128; i < 128; ++i) data.push_back(i); @@ -142,7 +142,7 @@ TEST(WeakHash32, ColumnVectorU16) auto col = ColumnUInt16::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (size_t i = 0; i < 65536; ++i) data.push_back(i); @@ -159,7 +159,7 @@ TEST(WeakHash32, ColumnVectorI16) auto col = ColumnInt16::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int i = -32768; i < 32768; ++i) data.push_back(i); @@ -176,7 +176,7 @@ TEST(WeakHash32, ColumnVectorU32) auto col = ColumnUInt32::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (uint64_t i = 0; i < 65536; ++i) data.push_back(i << 16u); @@ -193,7 +193,7 @@ TEST(WeakHash32, ColumnVectorI32) auto col = ColumnInt32::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) data.push_back(i << 16); //-V610 @@ -210,7 +210,7 @@ TEST(WeakHash32, ColumnVectorU64) auto col = ColumnUInt64::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (uint64_t i = 0; i < 65536; ++i) data.push_back(i << 32u); @@ -227,7 +227,7 @@ TEST(WeakHash32, ColumnVectorI64) auto col = ColumnInt64::create(); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) data.push_back(i << 32); //-V610 @@ -246,7 +246,7 @@ TEST(WeakHash32, ColumnVectorU128) auto eq = ColumnUInt32::create(); auto & eq_data = eq->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (uint64_t i = 0; i < 65536; ++i) { @@ -267,7 +267,7 @@ TEST(WeakHash32, ColumnDecimal32) auto col = ColumnDecimal::create(0, 0); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) data.push_back(i << 16); //-V610 @@ -284,7 +284,7 @@ TEST(WeakHash32, ColumnDecimal64) auto col = ColumnDecimal::create(0, 0); auto & data = col->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) data.push_back(i << 32); //-V610 @@ -303,7 +303,7 @@ TEST(WeakHash32, ColumnDecimal128) auto eq = ColumnUInt32::create(); auto & eq_data = eq->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (uint64_t i = 0; i < 65536; ++i) { @@ -319,13 +319,13 @@ TEST(WeakHash32, ColumnDecimal128) checkColumn(hash.getData(), eq_data, [&](size_t row) { return getHexUIntLowercase(col->getElement(row)); }); } -TEST(WeakHash32, ColumnString_1) +TEST(WeakHash32, ColumnString1) { auto col = ColumnString::create(); auto eq = ColumnUInt32::create(); auto & data = eq->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = 0; i < 65536; ++i) { @@ -341,7 +341,7 @@ TEST(WeakHash32, ColumnString_1) checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); }); } -TEST(WeakHash32, ColumnString_2) +TEST(WeakHash32, ColumnString2) { auto col = ColumnString::create(); auto eq = ColumnUInt32::create(); @@ -356,7 +356,7 @@ TEST(WeakHash32, ColumnString_2) * bb * bbb */ - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { size_t max_size = 3000; char letter = 'a'; @@ -381,7 +381,7 @@ TEST(WeakHash32, ColumnString_2) checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); }, allowed_collisions); } -TEST(WeakHash32, ColumnString_3) +TEST(WeakHash32, ColumnString3) { auto col = ColumnString::create(); auto eq = ColumnUInt32::create(); @@ -396,7 +396,7 @@ TEST(WeakHash32, ColumnString_3) * b\0 * b\0\0 */ - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { size_t max_size = 3000; char letter = 'a'; @@ -426,7 +426,7 @@ TEST(WeakHash32, ColumnFixedString) auto eq = ColumnUInt32::create(); auto & data = eq->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { char letter = 'a'; for (int64_t i = 0; i < 65536; ++i) @@ -467,7 +467,7 @@ TEST(WeakHash32, ColumnArray) * ... */ UInt64 cur_off = 0; - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { UInt32 cur = 0; for (int64_t i = 0; i < 65536; ++i) @@ -502,7 +502,7 @@ TEST(WeakHash32, ColumnArray) checkColumn(hash.getData(), eq_data, print_function); } -TEST(WeakHash32, ColumnArray_2) +TEST(WeakHash32, ColumnArray2) { auto val = ColumnUInt32::create(); auto off = ColumnUInt64::create(); @@ -512,7 +512,7 @@ TEST(WeakHash32, ColumnArray_2) auto & off_data = off->getData(); UInt64 cur_off = 0; - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = 0; i < 1000; ++i) { @@ -568,7 +568,7 @@ TEST(WeakHash32, ColumnArrayArray) */ UInt64 cur_off = 0; UInt64 cur_off2 = 0; - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { UInt32 cur = 1; for (int64_t i = 0; i < 3000; ++i) @@ -603,8 +603,8 @@ TEST(WeakHash32, ColumnArrayArray) { auto & offsets2 = col_arr_arr->getOffsets(); size_t s2 = offsets2[row2] - offsets2[row2 - 1]; - auto & arr2 = typeid_cast(col_arr_arr->getData()); - auto & offsets = arr2.getOffsets(); + const auto & arr2 = typeid_cast(col_arr_arr->getData()); + const auto & offsets = arr2.getOffsets(); size_t row = offsets2[row2]; size_t s = offsets[row] - offsets[row - 1]; auto value = arr2.getData().getUInt(offsets[row]); @@ -640,7 +640,7 @@ TEST(WeakHash32, ColumnLowcardinality) auto eq = ColumnUInt8::create(); auto & data = eq->getData(); - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (size_t i = 0; i < 65536; ++i) { @@ -663,7 +663,7 @@ TEST(WeakHash32, ColumnNullable) auto & mask_data = mask->getData(); PaddedPODArray eq; - for (int _i [[maybe_unused]] : {1, 2}) + for (int idx [[maybe_unused]] : {1, 2}) { for (uint64_t i = 0; i < 65536; ++i) { @@ -681,7 +681,7 @@ TEST(WeakHash32, ColumnNullable) checkColumn(hash.getData(), eq, [&](size_t row) { return eq[row] == -1 ? "Null" : std::to_string(eq[row]); }); } -TEST(WeakHash32, ColumnTuple_UInt64_UInt64) +TEST(WeakHash32, ColumnTupleUInt64UInt64) { auto col1 = ColumnUInt64::create(); auto col2 = ColumnUInt64::create(); @@ -689,9 +689,9 @@ TEST(WeakHash32, ColumnTuple_UInt64_UInt64) auto & data2 = col2->getData(); PaddedPODArray eq; - for (int _i [[maybe_unused]] : {0, 1, 2, 3}) + for (int idx : {0, 1, 2, 3}) { - auto l = _i % 2; + auto l = idx % 2; for (uint64_t i = 0; i < 65536; ++i) { @@ -719,16 +719,16 @@ TEST(WeakHash32, ColumnTuple_UInt64_UInt64) checkColumn(hash.getData(), eq, print_func); } -TEST(WeakHash32, ColumnTuple_UInt64_String) +TEST(WeakHash32, ColumnTupleUInt64String) { auto col1 = ColumnUInt64::create(); auto col2 = ColumnString::create(); auto & data1 = col1->getData(); PaddedPODArray eq; - for (int _i [[maybe_unused]] : {0, 1, 2, 3}) + for (int idx : {0, 1, 2, 3}) { - auto l = _i % 2; + auto l = idx % 2; size_t max_size = 3000; char letter = 'a'; @@ -765,7 +765,7 @@ TEST(WeakHash32, ColumnTuple_UInt64_String) checkColumn(hash.getData(), eq, print_func, allowed_collisions); } -TEST(WeakHash32, ColumnTuple_UInt64_FixedString) +TEST(WeakHash32, ColumnTupleUInt64FixedString) { size_t max_size = 3000; auto col1 = ColumnUInt64::create(); @@ -773,9 +773,9 @@ TEST(WeakHash32, ColumnTuple_UInt64_FixedString) auto & data1 = col1->getData(); PaddedPODArray eq; - for (int _i [[maybe_unused]] : {0, 1, 2, 3}) + for (int idx : {0, 1, 2, 3}) { - auto l = _i % 2; + auto l = idx % 2; char letter = 'a'; for (int64_t i = 0; i < 65536; ++i) @@ -810,7 +810,7 @@ TEST(WeakHash32, ColumnTuple_UInt64_FixedString) checkColumn(hash.getData(), eq, print_func); } -TEST(WeakHash32, ColumnTuple_UInt64_Array) +TEST(WeakHash32, ColumnTupleUInt64Array) { size_t max_size = 3000; auto val = ColumnUInt32::create(); @@ -824,9 +824,9 @@ TEST(WeakHash32, ColumnTuple_UInt64_Array) auto & data1 = col1->getData(); UInt64 cur_off = 0; - for (int _i [[maybe_unused]] : {0, 1, 2, 3}) + for (int idx : {0, 1, 2, 3}) { - auto l = _i % 2; + auto l = idx % 2; UInt32 cur = 0; for (int64_t i = 0; i < 65536; ++i) @@ -858,8 +858,8 @@ TEST(WeakHash32, ColumnTuple_UInt64_Array) { std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row)); - auto * col_arr = typeid_cast(col_tuple->getColumnPtr(1).get()); - auto & offsets = col_arr->getOffsets(); + const auto * col_arr = typeid_cast(col_tuple->getColumnPtr(1).get()); + const auto & offsets = col_arr->getOffsets(); size_t s = offsets[row] - offsets[row - 1]; auto value = col_arr->getData().getUInt(offsets[row]); auto r = std::string("[array of size ") + std::to_string(s) + " with values " + std::to_string(value) + "]"; diff --git a/src/Columns/ya.make b/src/Columns/ya.make new file mode 100644 index 00000000000..b36a524e775 --- /dev/null +++ b/src/Columns/ya.make @@ -0,0 +1,34 @@ +LIBRARY() + +ADDINCL( + contrib/libs/icu/common + contrib/libs/icu/i18n + contrib/libs/pdqsort +) + +PEERDIR( + clickhouse/src/Common + contrib/libs/icu + contrib/libs/pdqsort +) + +SRCS( + Collator.cpp + ColumnAggregateFunction.cpp + ColumnArray.cpp + ColumnConst.cpp + ColumnDecimal.cpp + ColumnFixedString.cpp + ColumnFunction.cpp + ColumnLowCardinality.cpp + ColumnNullable.cpp + ColumnsCommon.cpp + ColumnString.cpp + ColumnTuple.cpp + ColumnVector.cpp + FilterDescription.cpp + getLeastSuperColumn.cpp + IColumn.cpp +) + +END() diff --git a/src/Common/Arena.h b/src/Common/Arena.h index 32c0f4c12d1..5febed8ddf6 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -220,7 +220,8 @@ public: // This method only works for extending the last allocation. For lack of // original size, check a weaker condition: that 'begin' is at least in // the current Chunk. - assert(range_start >= head->begin && range_start < head->end); + assert(range_start >= head->begin); + assert(range_start < head->end); if (head->pos + additional_bytes <= head->end) { diff --git a/src/Common/BitHelpers.h b/src/Common/BitHelpers.h index ba6a4c60a49..bc6d7413def 100644 --- a/src/Common/BitHelpers.h +++ b/src/Common/BitHelpers.h @@ -53,12 +53,10 @@ inline size_t getLeadingZeroBits(T x) } } +// Unsafe since __builtin_ctz()-family explicitly state that result is undefined on x == 0 template -inline size_t getTrailingZeroBits(T x) +inline size_t getTrailingZeroBitsUnsafe(T x) { - if (!x) - return sizeof(x) * 8; - if constexpr (sizeof(T) <= sizeof(unsigned int)) { return __builtin_ctz(x); @@ -73,6 +71,15 @@ inline size_t getTrailingZeroBits(T x) } } +template +inline size_t getTrailingZeroBits(T x) +{ + if (!x) + return sizeof(x) * 8; + + return getTrailingZeroBitsUnsafe(x); +} + /** Returns a mask that has '1' for `bits` LSB set: * maskLowBits(3) => 00000111 */ diff --git a/src/Common/ClickHouseRevision.cpp b/src/Common/ClickHouseRevision.cpp index 5513922a655..0b81026adca 100644 --- a/src/Common/ClickHouseRevision.cpp +++ b/src/Common/ClickHouseRevision.cpp @@ -1,5 +1,8 @@ #include -#include + +#if !defined(ARCADIA_BUILD) +# include +#endif namespace ClickHouseRevision { diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 7c39518d30b..1e765a0b8a0 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -101,7 +101,7 @@ static ElementIdentifier getElementIdentifier(Node * element) { const Node * node = attrs->item(i); std::string name = node->nodeName(); - auto subst_name_pos = std::find(ConfigProcessor::SUBSTITUTION_ATTRS.begin(), ConfigProcessor::SUBSTITUTION_ATTRS.end(), name); + const auto * subst_name_pos = std::find(ConfigProcessor::SUBSTITUTION_ATTRS.begin(), ConfigProcessor::SUBSTITUTION_ATTRS.end(), name); if (name == "replace" || name == "remove" || subst_name_pos != ConfigProcessor::SUBSTITUTION_ATTRS.end()) continue; @@ -279,7 +279,7 @@ void ConfigProcessor::doIncludesRecursive( size_t substs_count = 0; for (const auto & attr_name : SUBSTITUTION_ATTRS) { - auto subst = attributes->getNamedItem(attr_name); + const auto * subst = attributes->getNamedItem(attr_name); attr_nodes[attr_name] = subst; substs_count += static_cast(subst == nullptr); } diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 2a566e3efff..01fb484b449 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -30,7 +30,7 @@ M(QueryThread, "Number of query processing threads") \ M(ReadonlyReplica, "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.") \ M(LeaderReplica, "Number of Replicated tables that are leaders. Leader replica is responsible for assigning merges, cleaning old blocks for deduplications and a few more bookkeeping tasks. There may be no more than one leader across all replicas at one moment of time. If there is no leader it will be elected soon or it indicate an issue.") \ - M(MemoryTracking, "Total amount of memory (bytes) allocated in currently executing queries. Note that some memory allocations may not be accounted.") \ + M(MemoryTracking, "Total amount of memory (bytes) allocated by the server.") \ M(MemoryTrackingInBackgroundProcessingPool, "Total amount of memory (bytes) allocated in background processing pool (that is dedicated for backround merges, mutations and fetches). Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks.") \ M(MemoryTrackingInBackgroundMoveProcessingPool, "Total amount of memory (bytes) allocated in background processing pool (that is dedicated for backround moves). Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks.") \ M(MemoryTrackingInBackgroundSchedulePool, "Total amount of memory (bytes) allocated in background schedule pool (that is dedicated for bookkeeping tasks of Replicated tables).") \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 27381811a8d..1f10893b08e 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -491,6 +491,7 @@ namespace ErrorCodes extern const int CANNOT_ASSIGN_ALTER = 517; extern const int CANNOT_COMMIT_OFFSET = 518; extern const int NO_REMOTE_SHARD_AVAILABLE = 519; + extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE = 520; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index bc3d4a78969..354895b1361 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -8,11 +8,14 @@ #include #include #include -#include #include #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + namespace DB { @@ -155,12 +158,12 @@ static std::string getExtraExceptionInfo(const std::exception & e) String msg; try { - if (auto file_exception = dynamic_cast(&e)) + if (const auto * file_exception = dynamic_cast(&e)) { if (file_exception->code() == ENOSPC) getNoSpaceLeftInfoMessage(file_exception->message(), msg); } - else if (auto errno_exception = dynamic_cast(&e)) + else if (const auto * errno_exception = dynamic_cast(&e)) { if (errno_exception->getErrno() == ENOSPC && errno_exception->getPath()) getNoSpaceLeftInfoMessage(errno_exception->getPath().value(), msg); @@ -263,7 +266,7 @@ int getCurrentExceptionCode() void rethrowFirstException(const Exceptions & exceptions) { - for (auto & exception : exceptions) + for (const auto & exception : exceptions) if (exception) std::rethrow_exception(exception); } diff --git a/src/Common/HashTable/HashSet.h b/src/Common/HashTable/HashSet.h index 2589329e5ef..c1970e898a1 100644 --- a/src/Common/HashTable/HashSet.h +++ b/src/Common/HashTable/HashSet.h @@ -43,7 +43,7 @@ public: for (size_t i = 0; i < rhs.grower.bufSize(); ++i) if (!rhs.buf[i].isZero(*this)) - this->insert(Cell::getKey(rhs.buf[i].getValue())); + this->insert(rhs.buf[i].getValue()); } @@ -60,7 +60,7 @@ public: { Cell x; x.read(rb); - this->insert(Cell::getKey(x.getValue())); + this->insert(x.getValue()); } } }; diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index ea85076fa5f..f9dae0b540f 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -981,7 +981,7 @@ public: { Cell x; x.read(rb); - insert(Cell::getKey(x.getValue())); + insert(x.getValue()); } } @@ -1006,7 +1006,7 @@ public: Cell x; DB::assertChar(',', rb); x.readText(rb); - insert(Cell::getKey(x.getValue())); + insert(x.getValue()); } } diff --git a/src/Common/MemoryStatisticsOS.cpp b/src/Common/MemoryStatisticsOS.cpp new file mode 100644 index 00000000000..adc5bf5d904 --- /dev/null +++ b/src/Common/MemoryStatisticsOS.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include + +#include "MemoryStatisticsOS.h" + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; +} + +static constexpr auto filename = "/proc/self/statm"; +static constexpr size_t PAGE_SIZE = 4096; + +MemoryStatisticsOS::MemoryStatisticsOS() +{ + fd = ::open(filename, O_RDONLY | O_CLOEXEC); + + if (-1 == fd) + throwFromErrno("Cannot open file " + std::string(filename), errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); +} + +MemoryStatisticsOS::~MemoryStatisticsOS() +{ + if (0 != ::close(fd)) + tryLogCurrentException(__PRETTY_FUNCTION__); +} + +MemoryStatisticsOS::Data MemoryStatisticsOS::get() const +{ + Data data; + + constexpr size_t buf_size = 1024; + char buf[buf_size]; + + ssize_t res = 0; + + do + { + res = ::pread(fd, buf, buf_size, 0); + + if (-1 == res) + { + if (errno == EINTR) + continue; + + throwFromErrno("Cannot read from file " + std::string(filename), ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR); + } + + assert(res >= 0); + break; + } while (true); + + ReadBufferFromMemory in(buf, res); + + uint64_t unused; + readIntText(data.virt, in); + skipWhitespaceIfAny(in); + readIntText(data.resident, in); + skipWhitespaceIfAny(in); + readIntText(data.shared, in); + skipWhitespaceIfAny(in); + readIntText(data.code, in); + skipWhitespaceIfAny(in); + readIntText(unused, in); + skipWhitespaceIfAny(in); + readIntText(data.data_and_stack, in); + + data.virt *= PAGE_SIZE; + data.resident *= PAGE_SIZE; + data.shared *= PAGE_SIZE; + data.code *= PAGE_SIZE; + data.data_and_stack *= PAGE_SIZE; + + return data; +} + +} diff --git a/src/Common/MemoryStatisticsOS.h b/src/Common/MemoryStatisticsOS.h new file mode 100644 index 00000000000..97caf4e8fbe --- /dev/null +++ b/src/Common/MemoryStatisticsOS.h @@ -0,0 +1,40 @@ +#pragma once +#include + + +namespace DB +{ + +/** Opens a file /proc/self/mstat. Keeps it open and reads memory statistics via 'pread'. + * This is Linux specific. + * See: man procfs + * + * Note: a class is used instead of a single function to avoid excessive file open/close on every use. + * pread is used to avoid lseek. + * + * Actual performance is from 1 to 5 million iterations per second. + */ +class MemoryStatisticsOS +{ +public: + /// In number of bytes. + struct Data + { + uint64_t virt; + uint64_t resident; + uint64_t shared; + uint64_t code; + uint64_t data_and_stack; + }; + + MemoryStatisticsOS(); + ~MemoryStatisticsOS(); + + /// Thread-safe. + Data get() const; + +private: + int fd; +}; + +} diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index efc1ff7785e..10d3b375cd1 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -25,10 +25,16 @@ static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; /// Each thread could new/delete memory in range of (-untracked_memory_limit, untracked_memory_limit) without access to common counters. static constexpr Int64 untracked_memory_limit = 4 * 1024 * 1024; +MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); + + +MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} +MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} + MemoryTracker::~MemoryTracker() { - if (static_cast(level) < static_cast(VariableContext::Process) && peak) + if ((level == VariableContext::Process || level == VariableContext::User) && peak) { try { @@ -39,19 +45,6 @@ MemoryTracker::~MemoryTracker() /// Exception in Logger, intentionally swallow. } } - - /** This is needed for next memory tracker to be consistent with sum of all referring memory trackers. - * - * Sometimes, memory tracker could be destroyed before memory was freed, and on destruction, amount > 0. - * For example, a query could allocate some data and leave it in cache. - * - * If memory will be freed outside of context of this memory tracker, - * but in context of one of the 'next' memory trackers, - * then memory usage of 'next' memory trackers will be underestimated, - * because amount will be decreased twice (first - here, second - when real 'free' happens). - */ - if (auto value = amount.load(std::memory_order_relaxed)) - free(value); } @@ -62,10 +55,11 @@ void MemoryTracker::logPeakMemoryUsage() const << ": " << formatReadableSizeWithBinarySuffix(peak) << "."); } -static void logMemoryUsage(Int64 amount) +void MemoryTracker::logMemoryUsage(Int64 current) const { LOG_DEBUG(&Logger::get("MemoryTracker"), - "Current memory usage: " << formatReadableSizeWithBinarySuffix(amount) << "."); + "Current memory usage" << (description ? " " + std::string(description) : "") + << ": " << formatReadableSizeWithBinarySuffix(current) << "."); } @@ -131,17 +125,24 @@ void MemoryTracker::alloc(Int64 size) throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); } + updatePeak(will_be); + + if (auto * loaded_next = parent.load(std::memory_order_relaxed)) + loaded_next->alloc(size); +} + + +void MemoryTracker::updatePeak(Int64 will_be) +{ auto peak_old = peak.load(std::memory_order_relaxed); if (will_be > peak_old) /// Races doesn't matter. Could rewrite with CAS, but not worth. { peak.store(will_be, std::memory_order_relaxed); - if (level == VariableContext::Process && will_be / log_peak_memory_usage_every > peak_old / log_peak_memory_usage_every) + if ((level == VariableContext::Process || level == VariableContext::Global) + && will_be / log_peak_memory_usage_every > peak_old / log_peak_memory_usage_every) logMemoryUsage(will_be); } - - if (auto loaded_next = parent.load(std::memory_order_relaxed)) - loaded_next->alloc(size); } @@ -172,7 +173,7 @@ void MemoryTracker::free(Int64 size) } } - if (auto loaded_next = parent.load(std::memory_order_relaxed)) + if (auto * loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->free(size); if (metric != CurrentMetrics::end()) @@ -198,6 +199,13 @@ void MemoryTracker::reset() } +void MemoryTracker::set(Int64 to) +{ + amount.store(to, std::memory_order_relaxed); + updatePeak(to); +} + + void MemoryTracker::setOrRaiseHardLimit(Int64 value) { /// This is just atomic set to maximum. @@ -219,7 +227,7 @@ namespace CurrentMemoryTracker { void alloc(Int64 size) { - if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) + if (auto * memory_tracker = DB::CurrentThread::getMemoryTracker()) { Int64 & untracked = DB::CurrentThread::getUntrackedMemory(); untracked += size; @@ -242,7 +250,7 @@ namespace CurrentMemoryTracker void free(Int64 size) { - if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) + if (auto * memory_tracker = DB::CurrentThread::getMemoryTracker()) { Int64 & untracked = DB::CurrentThread::getUntrackedMemory(); untracked -= size; @@ -257,7 +265,7 @@ namespace CurrentMemoryTracker DB::SimpleActionLock getCurrentMemoryTrackerActionLock() { - auto memory_tracker = DB::CurrentThread::getMemoryTracker(); + auto * memory_tracker = DB::CurrentThread::getMemoryTracker(); if (!memory_tracker) return {}; return memory_tracker->blocker.cancel(); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 98f416cec40..1cd85d0a8b2 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -13,6 +13,7 @@ */ class MemoryTracker { +private: std::atomic amount {0}; std::atomic peak {0}; std::atomic hard_limit {0}; @@ -33,9 +34,12 @@ class MemoryTracker /// This description will be used as prefix into log messages (if isn't nullptr) const char * description = nullptr; + void updatePeak(Int64 will_be); + void logMemoryUsage(Int64 current) const; + public: - MemoryTracker(VariableContext level_ = VariableContext::Thread) : level(level_) {} - MemoryTracker(MemoryTracker * parent_, VariableContext level_ = VariableContext::Thread) : parent(parent_), level(level_) {} + MemoryTracker(VariableContext level_ = VariableContext::Thread); + MemoryTracker(MemoryTracker * parent_, VariableContext level_ = VariableContext::Thread); ~MemoryTracker(); @@ -113,6 +117,9 @@ public: /// Reset the accumulated data and the parent. void reset(); + /// Reset current counter to a new value. + void set(Int64 to); + /// Prints info about peak memory consumption into log. void logPeakMemoryUsage() const; @@ -120,6 +127,8 @@ public: DB::SimpleActionBlocker blocker; }; +extern MemoryTracker total_memory_tracker; + /// Convenience methods, that use current thread's memory_tracker if it is available. namespace CurrentMemoryTracker diff --git a/src/Common/OpenSSLHelpers.h b/src/Common/OpenSSLHelpers.h index 8155ffd30ef..e77fc3037c1 100644 --- a/src/Common/OpenSSLHelpers.h +++ b/src/Common/OpenSSLHelpers.h @@ -1,8 +1,11 @@ #pragma once -#include -#if USE_SSL -#include +#if !defined(ARCADIA_BUILD) +# include +#endif + +#if USE_SSL +# include namespace DB diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index 0c31558cecb..fddefe596c4 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -5,12 +5,16 @@ #include #include #include -#include #include + +#if !defined(ARCADIA_BUILD) +# include +#endif + #if USE_RE2_ST - #include +# include #else - #define re2_st re2 +# define re2_st re2 #endif diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index a923088db3d..e0f251e9ed5 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -274,12 +274,7 @@ PoolWithFailoverBase::getMany( < std::forward_as_tuple(!right.is_up_to_date, right.staleness); }); - if (up_to_date_count >= min_entries) - { - /// There is enough up-to-date entries. - try_results.resize(up_to_date_count); - } - else if (fallback_to_stale_replicas) + if (fallback_to_stale_replicas) { /// There is not enough up-to-date entries but we are allowed to return stale entries. /// Gather all up-to-date ones and least-bad stale ones. @@ -287,6 +282,11 @@ PoolWithFailoverBase::getMany( size_t size = std::min(try_results.size(), max_entries); try_results.resize(size); } + else if (up_to_date_count >= min_entries) + { + /// There is enough up-to-date entries. + try_results.resize(up_to_date_count); + } else throw DB::Exception( "Could not find enough connections to up-to-date replicas. Got: " + std::to_string(up_to_date_count) diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index dd9f36fb3ae..a78c2b2ea6c 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include diff --git a/src/Common/QueryProfiler.h b/src/Common/QueryProfiler.h index 17280372237..f00a081972e 100644 --- a/src/Common/QueryProfiler.h +++ b/src/Common/QueryProfiler.h @@ -1,11 +1,14 @@ #pragma once #include -#include -#include #include #include +#if !defined(ARCADIA_BUILD) +# include +# include +#endif + namespace Poco { diff --git a/src/Common/RWLock.cpp b/src/Common/RWLock.cpp index 5dfc1b55c63..a8dba490fac 100644 --- a/src/Common/RWLock.cpp +++ b/src/Common/RWLock.cpp @@ -29,19 +29,17 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int DEADLOCK_AVOIDED; } -/** A single-use object that represents lock's ownership +/** A one-time-use-object that represents lock ownership * For the purpose of exception safety guarantees LockHolder is to be used in two steps: - * 1. Create an instance (allocating all the memory needed) + * 1. Create an instance (allocating all the needed memory) * 2. Associate the instance with the lock (attach to the lock and locking request group) */ class RWLockImpl::LockHolderImpl { bool bound{false}; - Type lock_type; String query_id; CurrentMetrics::Increment active_client_increment; RWLock parent; @@ -53,24 +51,30 @@ public: /// Implicit memory allocation for query_id is done here LockHolderImpl(const String & query_id_, Type type) - : lock_type{type}, query_id{query_id_}, - active_client_increment{ + : query_id{query_id_} + , active_client_increment{ type == Type::Read ? CurrentMetrics::RWLockActiveReaders : CurrentMetrics::RWLockActiveWriters} { } - ~LockHolderImpl(); + ~LockHolderImpl() + { + if (bound && parent != nullptr) + parent->unlock(it_group, query_id); + else + active_client_increment.destroy(); + } private: /// A separate method which binds the lock holder to the owned lock /// N.B. It is very important that this method produces no allocations bool bindWith(RWLock && parent_, GroupsContainer::iterator it_group_) noexcept { - if (bound) + if (bound || parent_ == nullptr) return false; it_group = it_group_; parent = std::move(parent_); - ++it_group->refererrs; + ++it_group->requests; bound = true; return true; } @@ -79,56 +83,27 @@ private: }; -namespace -{ - /// Global information about all read locks that query has. It is needed to avoid some type of deadlocks. - - class QueryLockInfo - { - private: - mutable std::mutex mutex; - std::map queries; - - public: - void add(const String & query_id) - { - std::lock_guard lock(mutex); - - const auto res = queries.emplace(query_id, 1); // may throw - if (!res.second) - ++res.first->second; - } - - void remove(const String & query_id) noexcept - { - std::lock_guard lock(mutex); - - const auto query_it = queries.find(query_id); - if (query_it != queries.cend() && --query_it->second == 0) - queries.erase(query_it); - } - - void check(const String & query_id) const - { - std::lock_guard lock(mutex); - - if (queries.find(query_id) != queries.cend()) - throw Exception("Possible deadlock avoided. Client should retry.", ErrorCodes::DEADLOCK_AVOIDED); - } - }; - - QueryLockInfo all_read_locks; -} - - -/** To guarantee that we do not get any piece of our data corrupted: +/** General algorithm: + * Step 1. Try the FastPath (for both Reads/Writes) + * Step 2. Find ourselves request group: attach to existing or create a new one + * Step 3. Wait/timed wait for ownership signal + * Step 3a. Check if we must handle timeout and exit + * Step 4. Persist lock ownership + * + * To guarantee that we do not get any piece of our data corrupted: * 1. Perform all actions that include allocations before changing lock's internal state * 2. Roll back any changes that make the state inconsistent * * Note: "SM" in the commentaries below stands for STATE MODIFICATION */ -RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id) +RWLockImpl::LockHolder +RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id, const std::chrono::milliseconds & lock_timeout_ms) { + const auto lock_deadline_tp = + (lock_timeout_ms == std::chrono::milliseconds(0)) + ? std::chrono::time_point::max() + : std::chrono::steady_clock::now() + lock_timeout_ms; + const bool request_has_query_id = query_id != NO_QUERY; Stopwatch watch(CLOCK_MONOTONIC_COARSE); @@ -145,100 +120,98 @@ RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String & /// This object is placed above unique_lock, because it may lock in destructor. auto lock_holder = std::make_shared(query_id, type); - std::unique_lock lock(mutex); + std::unique_lock state_lock(internal_state_mtx); /// The FastPath: /// Check if the same query_id already holds the required lock in which case we can proceed without waiting if (request_has_query_id) { - const auto it_query = owner_queries.find(query_id); - if (it_query != owner_queries.end()) + const auto owner_query_it = owner_queries.find(query_id); + if (owner_query_it != owner_queries.end()) { - const auto current_owner_group = queue.begin(); + if (wrlock_owner != writers_queue.end()) + throw Exception( + "RWLockImpl::getLock(): RWLock is already locked in exclusive mode", + ErrorCodes::LOGICAL_ERROR); - /// XXX: it means we can't upgrade lock from read to write! + /// Lock upgrading is not supported if (type == Write) throw Exception( "RWLockImpl::getLock(): Cannot acquire exclusive lock while RWLock is already locked", ErrorCodes::LOGICAL_ERROR); - if (current_owner_group->type == Write) - throw Exception( - "RWLockImpl::getLock(): RWLock is already locked in exclusive mode", - ErrorCodes::LOGICAL_ERROR); - /// N.B. Type is Read here, query_id is not empty and it_query is a valid iterator - all_read_locks.add(query_id); /// SM1: may throw on insertion (nothing to roll back) - ++it_query->second; /// SM2: nothrow - lock_holder->bindWith(shared_from_this(), current_owner_group); /// SM3: nothrow + ++owner_query_it->second; /// SM1: nothrow + lock_holder->bindWith(shared_from_this(), rdlock_owner); /// SM2: nothrow finalize_metrics(); return lock_holder; } } - /** If the query already has any active read lock and tries to acquire another read lock - * but it is not in front of the queue and has to wait, deadlock is possible: - * - * Example (four queries, two RWLocks - 'a' and 'b'): - * - * --> time --> - * - * q1: ra rb - * q2: wa - * q3: rb ra - * q4: wb - * - * We will throw an exception instead. - */ - - if (type == Type::Write || queue.empty() || queue.back().type == Type::Write) + if (type == Type::Write) { - if (type == Type::Read && request_has_query_id && !queue.empty()) - all_read_locks.check(query_id); - - /// Create a new group of locking requests - queue.emplace_back(type); /// SM1: may throw (nothing to roll back) + writers_queue.emplace_back(type); /// SM1: may throw (nothing to roll back) } - else if (request_has_query_id && queue.size() > 1) - all_read_locks.check(query_id); + else if (readers_queue.empty() || + (rdlock_owner == readers_queue.begin() && readers_queue.size() == 1 && !writers_queue.empty())) + { + readers_queue.emplace_back(type); /// SM1: may throw (nothing to roll back) + } + GroupsContainer::iterator it_group = + (type == Type::Write) ? std::prev(writers_queue.end()) : std::prev(readers_queue.end()); - GroupsContainer::iterator it_group = std::prev(queue.end()); + /// Lock is free to acquire + if (rdlock_owner == readers_queue.end() && wrlock_owner == writers_queue.end()) + { + (type == Read ? rdlock_owner : wrlock_owner) = it_group; /// SM2: nothrow + } + else + { + /// Wait until our group becomes the lock owner + const auto predicate = [&] () { return it_group == (type == Read ? rdlock_owner : wrlock_owner); }; - /// We need to reference the associated group before waiting to guarantee - /// that this group does not get deleted prematurely - ++it_group->refererrs; + if (lock_deadline_tp == std::chrono::time_point::max()) + { + ++it_group->requests; + it_group->cv.wait(state_lock, predicate); + --it_group->requests; + } + else + { + ++it_group->requests; + const auto wait_result = it_group->cv.wait_until(state_lock, lock_deadline_tp, predicate); + --it_group->requests; - /// Wait a notification until we will be the only in the group. - it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); }); + /// Step 3a. Check if we must handle timeout and exit + if (!wait_result) /// Wait timed out! + { + /// Rollback(SM1): nothrow + if (it_group->requests == 0) + { + (type == Read ? readers_queue : writers_queue).erase(it_group); + } - --it_group->refererrs; + return nullptr; + } + } + } if (request_has_query_id) { try { - if (type == Type::Read) - all_read_locks.add(query_id); /// SM2: may throw on insertion - /// and is safe to roll back unconditionally const auto emplace_res = - owner_queries.emplace(query_id, 1); /// SM3: may throw on insertion + owner_queries.emplace(query_id, 1); /// SM2: may throw on insertion if (!emplace_res.second) - ++emplace_res.first->second; /// SM4: nothrow + ++emplace_res.first->second; /// SM3: nothrow } catch (...) { /// Methods std::list<>::emplace_back() and std::unordered_map<>::emplace() provide strong exception safety - /// We only need to roll back the changes to these objects: all_read_locks and the locking queue - if (type == Type::Read) - all_read_locks.remove(query_id); /// Rollback(SM2): nothrow - - if (it_group->refererrs == 0) - { - const auto next = queue.erase(it_group); /// Rollback(SM1): nothrow - if (next != queue.end()) - next->cv.notify_all(); - } + /// We only need to roll back the changes to these objects: owner_queries and the readers/writers queue + if (it_group->requests == 0) + dropOwnerGroupAndPassOwnership(it_group); /// Rollback(SM1): nothrow throw; } @@ -251,10 +224,9 @@ RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String & } -/** The sequence points of acquiring lock's ownership by an instance of LockHolderImpl: - * 1. all_read_locks is updated - * 2. owner_queries is updated - * 3. request group is updated by LockHolderImpl which in turn becomes "bound" +/** The sequence points of acquiring lock ownership by an instance of LockHolderImpl: + * 1. owner_queries is updated + * 2. request group is updated by LockHolderImpl which in turn becomes "bound" * * If by the time when destructor of LockHolderImpl is called the instance has been "bound", * it is guaranteed that all three steps have been executed successfully and the resulting state is consistent. @@ -262,38 +234,74 @@ RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String & * * We do not employ try-catch: if something bad happens, there is nothing we can do =( */ -RWLockImpl::LockHolderImpl::~LockHolderImpl() +void RWLockImpl::unlock(GroupsContainer::iterator group_it, const String & query_id) noexcept { - if (!bound || parent == nullptr) + std::lock_guard state_lock(internal_state_mtx); + + /// All of theses are Undefined behavior and nothing we can do! + if (rdlock_owner == readers_queue.end() && wrlock_owner == writers_queue.end()) return; - - std::lock_guard lock(parent->mutex); - - /// The associated group must exist (and be the beginning of the queue?) - if (parent->queue.empty() || it_group != parent->queue.begin()) + if (rdlock_owner != readers_queue.end() && group_it != rdlock_owner) + return; + if (wrlock_owner != writers_queue.end() && group_it != wrlock_owner) return; /// If query_id is not empty it must be listed in parent->owner_queries - if (query_id != RWLockImpl::NO_QUERY) + if (query_id != NO_QUERY) { - const auto owner_it = parent->owner_queries.find(query_id); - if (owner_it != parent->owner_queries.end()) + const auto owner_query_it = owner_queries.find(query_id); + if (owner_query_it != owner_queries.end()) { - if (--owner_it->second == 0) /// SM: nothrow - parent->owner_queries.erase(owner_it); /// SM: nothrow - - if (lock_type == RWLockImpl::Read) - all_read_locks.remove(query_id); /// SM: nothrow + if (--owner_query_it->second == 0) /// SM: nothrow + owner_queries.erase(owner_query_it); /// SM: nothrow } } - /// If we are the last remaining referrer, remove the group and notify the next group - if (--it_group->refererrs == 0) /// SM: nothrow - { - const auto next = parent->queue.erase(it_group); /// SM: nothrow - if (next != parent->queue.end()) - next->cv.notify_all(); - } + /// If we are the last remaining referrer, remove this QNode and notify the next one + if (--group_it->requests == 0) /// SM: nothrow + dropOwnerGroupAndPassOwnership(group_it); } + +void RWLockImpl::dropOwnerGroupAndPassOwnership(GroupsContainer::iterator group_it) noexcept +{ + rdlock_owner = readers_queue.end(); + wrlock_owner = writers_queue.end(); + + if (group_it->type == Read) + { + readers_queue.erase(group_it); + /// Prepare next phase + if (!writers_queue.empty()) + { + wrlock_owner = writers_queue.begin(); + } + else + { + rdlock_owner = readers_queue.begin(); + } + } + else + { + writers_queue.erase(group_it); + /// Prepare next phase + if (!readers_queue.empty()) + { + rdlock_owner = readers_queue.begin(); + } + else + { + wrlock_owner = writers_queue.begin(); + } + } + + if (rdlock_owner != readers_queue.end()) + { + rdlock_owner->cv.notify_all(); + } + else if (wrlock_owner != writers_queue.end()) + { + wrlock_owner->cv.notify_one(); + } +} } diff --git a/src/Common/RWLock.h b/src/Common/RWLock.h index a7084720d6c..ad0a3f139fc 100644 --- a/src/Common/RWLock.h +++ b/src/Common/RWLock.h @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -18,8 +19,15 @@ class RWLockImpl; using RWLock = std::shared_ptr; -/// Implements shared lock with FIFO service -/// Can be acquired recursively (several calls for the same query) in Read mode +/// Implements Readers-Writers locking algorithm that serves requests in "Phase Fair" order. +/// (Phase Fair RWLock as suggested in https://www.cs.unc.edu/~anderson/papers/rtsj10-for-web.pdf) +/// It is used for synchronizing access to various objects on query level (i.e. Storages). +/// +/// In general, ClickHouse processes queries by multiple threads of execution in parallel. +/// As opposed to the standard OS synchronization primitives (mutexes), this implementation allows +/// unlock() to be called by a thread other than the one, that called lock(). +/// It is also possible to acquire RWLock in Read mode without waiting (FastPath) by multiple threads, +/// that execute the same query (share the same query_id). /// /// NOTE: it is important to allow acquiring the same lock in Read mode without waiting if it is already /// acquired by another thread of the same query. Otherwise the following deadlock is possible: @@ -42,37 +50,44 @@ public: friend class LockHolderImpl; using LockHolder = std::shared_ptr; - /// Waits in the queue and returns appropriate lock - /// Empty query_id means the lock is acquired out of the query context (e.g. in a background thread). - LockHolder getLock(Type type, const String & query_id); + /// Empty query_id means the lock is acquired from outside of query context (e.g. in a background thread). + LockHolder getLock(Type type, const String & query_id, + const std::chrono::milliseconds & lock_timeout_ms = std::chrono::milliseconds(0)); /// Use as query_id to acquire a lock outside the query context. inline static const String NO_QUERY = String(); + inline static const auto default_locking_timeout_ms = std::chrono::milliseconds(120000); private: - RWLockImpl() = default; - - struct Group; - using GroupsContainer = std::list; - using OwnerQueryIds = std::unordered_map; - - /// Group of locking requests that should be granted concurrently - /// i.e. a group can contain several readers, but only one writer + /// Group of locking requests that should be granted simultaneously + /// i.e. one or several readers or a single writer struct Group { const Type type; - size_t refererrs; + size_t requests; std::condition_variable cv; /// all locking requests of the group wait on this condvar - explicit Group(Type type_) : type{type_}, refererrs{0} {} + explicit Group(Type type_) : type{type_}, requests{0} {} }; - GroupsContainer queue; + using GroupsContainer = std::list; + using OwnerQueryIds = std::unordered_map; + +private: + mutable std::mutex internal_state_mtx; + + GroupsContainer readers_queue; + GroupsContainer writers_queue; + GroupsContainer::iterator rdlock_owner{readers_queue.end()}; /// equals to readers_queue.begin() in read phase + /// or readers_queue.end() otherwise + GroupsContainer::iterator wrlock_owner{writers_queue.end()}; /// equals to writers_queue.begin() in write phase + /// or writers_queue.end() otherwise OwnerQueryIds owner_queries; - mutable std::mutex mutex; +private: + RWLockImpl() = default; + void unlock(GroupsContainer::iterator group_it, const String & query_id) noexcept; + void dropOwnerGroupAndPassOwnership(GroupsContainer::iterator group_it) noexcept; }; - - } diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index 8a0f9361207..9bd692a9c64 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -165,7 +165,7 @@ void SensitiveDataMasker::addMaskingRule( size_t SensitiveDataMasker::wipeSensitiveData(std::string & data) const { size_t matches = 0; - for (auto & rule : all_masking_rules) + for (const auto & rule : all_masking_rules) matches += rule->apply(data); return matches; } diff --git a/src/Common/SettingsChanges.h b/src/Common/SettingsChanges.h index 2e037a50a17..004a08c3b4b 100644 --- a/src/Common/SettingsChanges.h +++ b/src/Common/SettingsChanges.h @@ -10,6 +10,11 @@ struct SettingChange { String name; Field value; + SettingChange() {} + + SettingChange(const String & name_, const Field value_) + : name(name_) + , value(value_) {} friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); } friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); } diff --git a/src/Common/SharedLibrary.cpp b/src/Common/SharedLibrary.cpp index 1cc512f925d..689179be7d8 100644 --- a/src/Common/SharedLibrary.cpp +++ b/src/Common/SharedLibrary.cpp @@ -35,7 +35,7 @@ void * SharedLibrary::getImpl(const std::string & name, bool no_throw) { dlerror(); - auto res = dlsym(handle, name.c_str()); + auto * res = dlsym(handle, name.c_str()); if (char * error = dlerror()) { diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 785a97e4d74..6d0b6a0f7d2 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -14,8 +13,12 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + #if USE_UNWIND -# include +# include #endif std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext_t & context) @@ -261,7 +264,7 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs for (size_t i = offset; i < size; ++i) { const void * virtual_addr = frames[i]; - auto object = symbol_index.findObject(virtual_addr); + const auto * object = symbol_index.findObject(virtual_addr); uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; const void * physical_addr = reinterpret_cast(uintptr_t(virtual_addr) - virtual_offset); @@ -279,7 +282,7 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs } } - auto symbol = symbol_index.findSymbol(virtual_addr); + const auto * symbol = symbol_index.findSymbol(virtual_addr); if (symbol) { int status = 0; diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index 14c4d627d61..482c6fd0bad 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -105,7 +105,7 @@ void collectSymbolsFromProgramHeaders(dl_phdr_info * info, */ size_t sym_cnt = 0; - for (auto it = dyn_begin; it->d_tag != DT_NULL; ++it) + for (const auto * it = dyn_begin; it->d_tag != DT_NULL; ++it) { // TODO: this branch leads to invalid address of the hash table. Need further investigation. // if (it->d_tag == DT_HASH) @@ -148,7 +148,7 @@ void collectSymbolsFromProgramHeaders(dl_phdr_info * info, continue; const char * strtab = nullptr; - for (auto it = dyn_begin; it->d_tag != DT_NULL; ++it) + for (const auto * it = dyn_begin; it->d_tag != DT_NULL; ++it) { if (it->d_tag == DT_STRTAB) { @@ -160,7 +160,7 @@ void collectSymbolsFromProgramHeaders(dl_phdr_info * info, if (!strtab) continue; - for (auto it = dyn_begin; it->d_tag != DT_NULL; ++it) + for (const auto * it = dyn_begin; it->d_tag != DT_NULL; ++it) { if (it->d_tag == DT_SYMTAB) { diff --git a/src/Common/TaskStatsInfoGetter.cpp b/src/Common/TaskStatsInfoGetter.cpp index 9c4025624fb..689cd3ea7c1 100644 --- a/src/Common/TaskStatsInfoGetter.cpp +++ b/src/Common/TaskStatsInfoGetter.cpp @@ -258,7 +258,7 @@ TaskStatsInfoGetter::TaskStatsInfoGetter() } -void TaskStatsInfoGetter::getStat(::taskstats & out_stats, pid_t tid) +void TaskStatsInfoGetter::getStat(::taskstats & out_stats, pid_t tid) const { NetlinkMessage answer = query(netlink_socket_fd, taskstats_family_id, tid, TASKSTATS_CMD_GET, TASKSTATS_CMD_ATTR_PID, &tid, sizeof(tid)); @@ -307,7 +307,7 @@ bool TaskStatsInfoGetter::checkPermissions() TaskStatsInfoGetter::TaskStatsInfoGetter() = default; TaskStatsInfoGetter::~TaskStatsInfoGetter() = default; -void TaskStatsInfoGetter::getStat(::taskstats &, pid_t) +void TaskStatsInfoGetter::getStat(::taskstats &, pid_t) const { } diff --git a/src/Common/TaskStatsInfoGetter.h b/src/Common/TaskStatsInfoGetter.h index abf946c216f..989843a471c 100644 --- a/src/Common/TaskStatsInfoGetter.h +++ b/src/Common/TaskStatsInfoGetter.h @@ -16,7 +16,7 @@ public: TaskStatsInfoGetter(); ~TaskStatsInfoGetter(); - void getStat(::taskstats & out_stats, pid_t tid); + void getStat(::taskstats & out_stats, pid_t tid) const; /// Whether the current process has permissions (sudo or cap_net_admin capabilties) to get taskstats info static bool checkPermissions(); diff --git a/src/Common/ThreadFuzzer.cpp b/src/Common/ThreadFuzzer.cpp index 42e133b4561..0e7c5429ade 100644 --- a/src/Common/ThreadFuzzer.cpp +++ b/src/Common/ThreadFuzzer.cpp @@ -8,6 +8,7 @@ #include +#include #include #include @@ -18,8 +19,15 @@ #include + /// We will also wrap some thread synchronization functions to inject sleep/migration before or after. -#if defined(OS_LINUX) +#if defined(OS_LINUX) && !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) + #define THREAD_FUZZER_WRAP_PTHREAD 1 +#else + #define THREAD_FUZZER_WRAP_PTHREAD 0 +#endif + +#if THREAD_FUZZER_WRAP_PTHREAD # define FOR_EACH_WRAPPED_FUNCTION(M) \ M(int, pthread_mutex_lock, pthread_mutex_t * arg) \ M(int, pthread_mutex_unlock, pthread_mutex_t * arg) @@ -66,7 +74,7 @@ static void initFromEnv(std::atomic & what, const char * name) static std::atomic num_cpus = 0; -#if defined(OS_LINUX) +#if THREAD_FUZZER_WRAP_PTHREAD # define DEFINE_WRAPPER_PARAMS(RET, NAME, ...) \ static std::atomic NAME##_before_yield_probability = 0; \ static std::atomic NAME##_before_migrate_probability = 0; \ @@ -97,7 +105,7 @@ void ThreadFuzzer::initConfiguration() initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY"); initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US"); -#if defined(OS_LINUX) +#if THREAD_FUZZER_WRAP_PTHREAD # define INIT_WRAPPER_PARAMS(RET, NAME, ...) \ initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \ initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \ @@ -118,7 +126,7 @@ void ThreadFuzzer::initConfiguration() bool ThreadFuzzer::isEffective() const { -#if defined(OS_LINUX) +#if THREAD_FUZZER_WRAP_PTHREAD # define CHECK_WRAPPER_PARAMS(RET, NAME, ...) \ if (NAME##_before_yield_probability.load(std::memory_order_relaxed)) \ return true; \ @@ -199,7 +207,7 @@ void ThreadFuzzer::signalHandler(int) errno = saved_errno; } -void ThreadFuzzer::setup() +void ThreadFuzzer::setup() const { struct sigaction sa{}; sa.sa_handler = signalHandler; @@ -236,7 +244,7 @@ void ThreadFuzzer::setup() /// We expect that for every function like pthread_mutex_lock there is the same function with two underscores prefix. /// NOTE We cannot use dlsym(... RTLD_NEXT), because it will call pthread_mutex_lock and it will lead to infinite recursion. -#if defined(OS_LINUX) +#if THREAD_FUZZER_WRAP_PTHREAD # define MAKE_WRAPPER(RET, NAME, ...) \ extern "C" RET __##NAME(__VA_ARGS__); /* NOLINT */ \ extern "C" RET NAME(__VA_ARGS__) /* NOLINT */ \ diff --git a/src/Common/ThreadFuzzer.h b/src/Common/ThreadFuzzer.h index c9041d1f7b6..d0693945cb0 100644 --- a/src/Common/ThreadFuzzer.h +++ b/src/Common/ThreadFuzzer.h @@ -65,7 +65,7 @@ private: ThreadFuzzer(); void initConfiguration(); - void setup(); + void setup() const; static void signalHandler(int); }; diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 9bed96552ea..13f27f7e3ef 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -114,7 +114,7 @@ void ThreadStatus::updatePerformanceCounters() } } -void ThreadStatus::assertState(const std::initializer_list & permitted_states, const char * description) +void ThreadStatus::assertState(const std::initializer_list & permitted_states, const char * description) const { for (auto permitted_state : permitted_states) { diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 58af6d4efff..31c1a1f55f2 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -163,7 +163,7 @@ protected: void logToQueryThreadLog(QueryThreadLog & thread_log); - void assertState(const std::initializer_list & permitted_states, const char * description = nullptr); + void assertState(const std::initializer_list & permitted_states, const char * description = nullptr) const; ThreadGroupStatusPtr thread_group; diff --git a/src/Common/TraceCollector.cpp b/src/Common/TraceCollector.cpp index 399a2404b21..be2020f4351 100644 --- a/src/Common/TraceCollector.cpp +++ b/src/Common/TraceCollector.cpp @@ -65,7 +65,7 @@ void TraceCollector::collect(TraceType trace_type, const StackTrace & stack_trac sizeof(StackTrace::Frames) + // collected stack trace, maximum capacity sizeof(TraceType) + // trace type sizeof(UInt64) + // thread_id - sizeof(UInt64); // size + sizeof(UInt64); // size char buffer[buf_size]; WriteBufferFromFileDescriptorDiscardOnFailure out(pipe.fds_rw[1], buf_size, buffer); diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index b9d1f2cdcdf..41aeb421394 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -13,10 +13,13 @@ #include #include #include -#include #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + namespace DB { namespace ErrorCodes diff --git a/src/Common/ZooKeeper/CMakeLists.txt b/src/Common/ZooKeeper/CMakeLists.txt index aa6efcd3ca1..4dbf999419e 100644 --- a/src/Common/ZooKeeper/CMakeLists.txt +++ b/src/Common/ZooKeeper/CMakeLists.txt @@ -7,6 +7,10 @@ add_library(clickhouse_common_zookeeper ${clickhouse_common_zookeeper_headers} $ target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY}) target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR}) +if (USE_POCO_NETSSL) + target_link_libraries (clickhouse_common_zookeeper PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) +endif() + if (ENABLE_TESTS) add_subdirectory (tests) endif () diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 0bcef6cd75f..4f736e66aab 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -140,20 +140,20 @@ struct TestKeeperMultiRequest final : MultiRequest, TestKeeperRequest for (const auto & generic_request : generic_requests) { - if (auto * concrete_request_create = dynamic_cast(generic_request.get())) + if (const auto * concrete_request_create = dynamic_cast(generic_request.get())) { auto create = std::make_shared(*concrete_request_create); requests.push_back(create); } - else if (auto * concrete_request_remove = dynamic_cast(generic_request.get())) + else if (const auto * concrete_request_remove = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_remove)); } - else if (auto * concrete_request_set = dynamic_cast(generic_request.get())) + else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_set)); } - else if (auto * concrete_request_check = dynamic_cast(generic_request.get())) + else if (const auto * concrete_request_check = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_check)); } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 99c3f115021..032d1e90ff5 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -59,30 +59,35 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho if (implementation == "zookeeper") { if (hosts.empty()) - throw KeeperException("No addresses passed to ZooKeeper constructor.", Coordination::ZBADARGUMENTS); + throw KeeperException("No hosts passed to ZooKeeper constructor.", Coordination::ZBADARGUMENTS); - std::vector addresses_strings; - splitInto<','>(addresses_strings, hosts); - Coordination::ZooKeeper::Addresses addresses; - addresses.reserve(addresses_strings.size()); + std::vector hosts_strings; + splitInto<','>(hosts_strings, hosts); + Coordination::ZooKeeper::Nodes nodes; + nodes.reserve(hosts_strings.size()); - for (const auto & address_string : addresses_strings) + for (auto & host_string : hosts_strings) { try { - addresses.emplace_back(address_string); + bool secure = bool(startsWith(host_string, "secure://")); + + if (secure) + host_string.erase(0, strlen("secure://")); + + nodes.emplace_back(Coordination::ZooKeeper::Node{Poco::Net::SocketAddress{host_string}, secure}); } catch (const Poco::Net::DNSException & e) { - LOG_ERROR(log, "Cannot use ZooKeeper address " << address_string << ", reason: " << e.displayText()); + LOG_ERROR(log, "Cannot use ZooKeeper host " << host_string << ", reason: " << e.displayText()); } } - if (addresses.empty()) - throw KeeperException("Cannot use any of provided ZooKeeper addresses", Coordination::ZBADARGUMENTS); + if (nodes.empty()) + throw KeeperException("Cannot use any of provided ZooKeeper nodes", Coordination::ZBADARGUMENTS); impl = std::make_unique( - addresses, + nodes, chroot, identity_.empty() ? "" : "digest", identity_, @@ -130,6 +135,7 @@ struct ZooKeeperArgs if (startsWith(key, "node")) { hosts_strings.push_back( + (config.getBool(config_name + "." + key + ".secure", false) ? "secure://" : "") + config.getString(config_name + "." + key + ".host") + ":" + config.getString(config_name + "." + key + ".port", "2181") ); diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 2d4d449b1a6..db166314a07 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -63,10 +63,14 @@ public: example1 2181 + + 1 example2 2181 + + 1 30000 10000 diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index b8700a93e35..fd873658d7a 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -11,6 +11,14 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + +#if USE_POCO_NETSSL +#include +#endif + #include @@ -44,6 +52,13 @@ namespace CurrentMetrics extern const Metric ZooKeeperWatch; } +namespace DB +{ + namespace ErrorCodes + { + extern const int SUPPORT_IS_DISABLED; + } +} /** ZooKeeper wire protocol. @@ -656,22 +671,22 @@ struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest for (const auto & generic_request : generic_requests) { - if (auto * concrete_request_create = dynamic_cast(generic_request.get())) + if (const auto * concrete_request_create = dynamic_cast(generic_request.get())) { auto create = std::make_shared(*concrete_request_create); if (create->acls.empty()) create->acls = default_acls; requests.push_back(create); } - else if (auto * concrete_request_remove = dynamic_cast(generic_request.get())) + else if (const auto * concrete_request_remove = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_remove)); } - else if (auto * concrete_request_set = dynamic_cast(generic_request.get())) + else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_set)); } - else if (auto * concrete_request_check = dynamic_cast(generic_request.get())) + else if (const auto * concrete_request_check = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_check)); } @@ -817,7 +832,7 @@ ZooKeeper::~ZooKeeper() ZooKeeper::ZooKeeper( - const Addresses & addresses, + const Nodes & nodes, const String & root_path_, const String & auth_scheme, const String & auth_data, @@ -851,7 +866,7 @@ ZooKeeper::ZooKeeper( default_acls.emplace_back(std::move(acl)); } - connect(addresses, connection_timeout); + connect(nodes, connection_timeout); if (!auth_scheme.empty()) sendAuth(auth_scheme, auth_data); @@ -864,11 +879,11 @@ ZooKeeper::ZooKeeper( void ZooKeeper::connect( - const Addresses & addresses, + const Nodes & nodes, Poco::Timespan connection_timeout) { - if (addresses.empty()) - throw Exception("No addresses passed to ZooKeeper constructor", ZBADARGUMENTS); + if (nodes.empty()) + throw Exception("No nodes passed to ZooKeeper constructor", ZBADARGUMENTS); static constexpr size_t num_tries = 3; bool connected = false; @@ -876,12 +891,25 @@ void ZooKeeper::connect( WriteBufferFromOwnString fail_reasons; for (size_t try_no = 0; try_no < num_tries; ++try_no) { - for (const auto & address : addresses) + for (const auto & node : nodes) { try { - socket = Poco::Net::StreamSocket(); /// Reset the state of previous attempt. - socket.connect(address, connection_timeout); + /// Reset the state of previous attempt. + if (node.secure) + { +#if USE_POCO_NETSSL + socket = Poco::Net::SecureStreamSocket(); +#else + throw Exception{"Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED}; +#endif + } + else + { + socket = Poco::Net::StreamSocket(); + } + + socket.connect(node.address, connection_timeout); socket.setReceiveTimeout(operation_timeout); socket.setSendTimeout(operation_timeout); @@ -915,7 +943,7 @@ void ZooKeeper::connect( } catch (...) { - fail_reasons << "\n" << getCurrentExceptionMessage(false) << ", " << address.toString(); + fail_reasons << "\n" << getCurrentExceptionMessage(false) << ", " << node.address.toString(); } } @@ -926,15 +954,19 @@ void ZooKeeper::connect( if (!connected) { WriteBufferFromOwnString message; - message << "All connection tries failed while connecting to ZooKeeper. Addresses: "; + message << "All connection tries failed while connecting to ZooKeeper. nodes: "; bool first = true; - for (const auto & address : addresses) + for (const auto & node : nodes) { if (first) first = false; else message << ", "; - message << address.toString(); + + if (node.secure) + message << "secure://"; + + message << node.address.toString(); } message << fail_reasons.str() << "\n"; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 88e949dbd45..840cbdbde3f 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -93,17 +93,23 @@ struct ZooKeeperRequest; class ZooKeeper : public IKeeper { public: - using Addresses = std::vector; + struct Node + { + Poco::Net::SocketAddress address; + bool secure; + }; + + using Nodes = std::vector; using XID = int32_t; using OpNum = int32_t; - /** Connection to addresses is performed in order. If you want, shuffle them manually. + /** Connection to nodes is performed in order. If you want, shuffle them manually. * Operation timeout couldn't be greater than session timeout. * Operation timeout applies independently for network read, network write, waiting for events and synchronization. */ ZooKeeper( - const Addresses & addresses, + const Nodes & nodes, const String & root_path, const String & auth_scheme, const String & auth_data, @@ -213,7 +219,7 @@ private: ThreadFromGlobalPool receive_thread; void connect( - const Addresses & addresses, + const Nodes & node, Poco::Timespan connection_timeout); void sendHandshake(); diff --git a/src/Common/ZooKeeper/tests/CMakeLists.txt b/src/Common/ZooKeeper/tests/CMakeLists.txt index 06716e49918..45a48ddc7a9 100644 --- a/src/Common/ZooKeeper/tests/CMakeLists.txt +++ b/src/Common/ZooKeeper/tests/CMakeLists.txt @@ -2,7 +2,7 @@ add_executable(zkutil_test_commands zkutil_test_commands.cpp) target_link_libraries(zkutil_test_commands PRIVATE clickhouse_common_zookeeper) add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp) -target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zookeeper) +target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zookeeper string_utils) add_executable(zkutil_test_lock zkutil_test_lock.cpp) target_link_libraries(zkutil_test_lock PRIVATE clickhouse_common_zookeeper) diff --git a/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp b/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp index aa348163adf..d9d3402fa32 100644 --- a/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp +++ b/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -23,15 +24,23 @@ try Poco::Logger::root().setChannel(channel); Poco::Logger::root().setLevel("trace"); - std::string addresses_arg = argv[1]; - std::vector addresses_strings; - splitInto<','>(addresses_strings, addresses_arg); - ZooKeeper::Addresses addresses; - addresses.reserve(addresses_strings.size()); - for (const auto & address_string : addresses_strings) - addresses.emplace_back(address_string); + std::string hosts_arg = argv[1]; + std::vector hosts_strings; + splitInto<','>(hosts_strings, hosts_arg); + ZooKeeper::Nodes nodes; + nodes.reserve(hosts_strings.size()); + for (auto & host_string : hosts_strings) + { + bool secure = bool(startsWith(host_string, "secure://")); - ZooKeeper zk(addresses, {}, {}, {}, {5, 0}, {0, 50000}, {0, 50000}); + if (secure) + host_string.erase(0, strlen("secure://")); + + nodes.emplace_back(ZooKeeper::Node{Poco::Net::SocketAddress{host_string},secure}); + } + + + ZooKeeper zk(nodes, {}, {}, {}, {5, 0}, {0, 50000}, {0, 50000}); Poco::Event event(true); diff --git a/src/Common/ZooKeeper/tests/zookeeper_impl.cpp b/src/Common/ZooKeeper/tests/zookeeper_impl.cpp index da609a7bc72..74ba63514f2 100644 --- a/src/Common/ZooKeeper/tests/zookeeper_impl.cpp +++ b/src/Common/ZooKeeper/tests/zookeeper_impl.cpp @@ -5,7 +5,7 @@ int main() try { - Coordination::ZooKeeper zookeeper({Poco::Net::SocketAddress{"localhost:2181"}}, "", "", "", {30, 0}, {0, 50000}, {0, 50000}); + Coordination::ZooKeeper zookeeper({Coordination::ZooKeeper::Node{Poco::Net::SocketAddress{"localhost:2181"}, false}}, "", "", "", {30, 0}, {0, 50000}, {0, 50000}); zookeeper.create("/test", "hello", false, false, {}, [](const Coordination::CreateResponse & response) { diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index bbd0b592ee5..209a798b022 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -74,7 +74,7 @@ std::filesystem::path getMountPoint(std::filesystem::path absolute_path) String getFilesystemName([[maybe_unused]] const String & mount_point) { #if defined(__linux__) - auto mounted_filesystems = setmntent("/etc/mtab", "r"); + FILE * mounted_filesystems = setmntent("/etc/mtab", "r"); if (!mounted_filesystems) throw DB::Exception("Cannot open /etc/mtab to get name of filesystem", ErrorCodes::SYSTEM_ERROR); mntent fs_info; diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 32b70b76fbd..1eaa64ae2a4 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -1,11 +1,16 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) +# include +#else +# include +#endif + #if USE_CPUID -# include +# include #elif USE_CPUINFO -# include +# include #endif diff --git a/src/Common/memcmpSmall.h b/src/Common/memcmpSmall.h index 5dc5e04a707..bafc08a9cbe 100644 --- a/src/Common/memcmpSmall.h +++ b/src/Common/memcmpSmall.h @@ -64,6 +64,80 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char } +/** Variant when memory regions may have different sizes. + * But compare the regions as the smaller one is padded with zero bytes up to the size of the larger. + * It's needed to hold that: toFixedString('abc', 5) = 'abc' + * for compatibility with SQL standard. + */ +template +inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size) +{ + size_t min_size = std::min(a_size, b_size); + + for (size_t offset = 0; offset < min_size; offset += 16) + { + uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128(reinterpret_cast(a + offset)), + _mm_loadu_si128(reinterpret_cast(b + offset)))); + mask = ~mask; + + if (mask) + { + offset += __builtin_ctz(mask); + + if (offset >= min_size) + break; + + return detail::cmp(a[offset], b[offset]); + } + } + + /// The strings are equal up to min_size. + /// If the rest of the larger string is zero bytes then the strings are considered equal. + + size_t max_size; + const Char * longest; + int cmp; + + if (a_size == b_size) + { + return 0; + } + else if (a_size > b_size) + { + max_size = a_size; + longest = a; + cmp = 1; + } + else + { + max_size = b_size; + longest = b; + cmp = -1; + } + + const __m128i zero16 = _mm_setzero_si128(); + + for (size_t offset = min_size; offset < max_size; offset += 16) + { + uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(longest + offset)), + zero16)); + + if (mask) + { + offset += __builtin_ctz(mask); + + if (offset >= max_size) + return 0; + return cmp; + } + } + + return 0; +} + + /** Variant when memory regions have same size. * TODO Check if the compiler can optimize previous function when the caller pass identical sizes. */ @@ -206,6 +280,46 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char return detail::cmp(a_size, b_size); } +template +inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size) +{ + size_t min_size; + size_t max_size; + const Char * longest; + int size_cmp; + + if (a_size == b_size) + { + min_size = a_size; + max_size = a_size; + longest = a; + size_cmp = 0; + } + else if (a_size > b_size) + { + min_size = b_size; + max_size = a_size; + longest = a; + size_cmp = 1; + } + else + { + min_size = a_size; + max_size = b_size; + longest = b; + size_cmp = -1; + } + + if (auto res = memcmp(a, b, min_size)) + return res; + + for (size_t i = min_size; i < max_size; ++i) + if (longest[i] != 0) + return size_cmp; + + return 0; +} + template inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t size) { @@ -248,3 +362,13 @@ inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size) } #endif + + +/** Compare memory regions for equality. + * But if the sizes are different, compare the regions as the smaller one is padded with zero bytes up to the size of the larger. + */ +template +inline bool memequalSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size) +{ + return 0 == memcmpSmallLikeZeroPaddedAllowOverflow15(a, a_size, b, b_size); +} diff --git a/src/Common/new_delete.cpp b/src/Common/new_delete.cpp index 0aa5f8aacce..ed0d99679e4 100644 --- a/src/Common/new_delete.cpp +++ b/src/Common/new_delete.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -14,14 +13,13 @@ /// Replace default new/delete with memory tracking versions. /// @sa https://en.cppreference.com/w/cpp/memory/new/operator_new /// https://en.cppreference.com/w/cpp/memory/new/operator_delete -#if !UNBUNDLED namespace Memory { inline ALWAYS_INLINE void trackMemory(std::size_t size) { -#if USE_JEMALLOC +#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 5 /// The nallocx() function allocates no memory, but it performs the same size computation as the mallocx() function /// @note je_mallocx() != je_malloc(). It's expected they don't differ much in allocation logic. if (likely(size != 0)) @@ -49,18 +47,18 @@ inline ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t { try { -#if USE_JEMALLOC +#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 5 /// @note It's also possible to use je_malloc_usable_size() here. if (likely(ptr != nullptr)) CurrentMemoryTracker::free(sallocx(ptr, 0)); #else if (size) CurrentMemoryTracker::free(size); -# ifdef _GNU_SOURCE +# if defined(_GNU_SOURCE) /// It's innaccurate resource free for sanitizers. malloc_usable_size() result is greater or equal to allocated size. else CurrentMemoryTracker::free(malloc_usable_size(ptr)); -# endif +# endif #endif } catch (...) @@ -130,26 +128,3 @@ void operator delete[](void * ptr, std::size_t size) noexcept Memory::untrackMemory(ptr, size); Memory::deleteSized(ptr, size); } - -#else - -/// new - -void * operator new(std::size_t size) { return Memory::newImpl(size); } -void * operator new[](std::size_t size) { return Memory::newImpl(size); } - -void * operator new(std::size_t size, const std::nothrow_t &) noexcept { return Memory::newNoExept(size); } -void * operator new[](std::size_t size, const std::nothrow_t &) noexcept { return Memory::newNoExept(size); } - -/// delete - -void operator delete(void * ptr) noexcept { Memory::deleteImpl(ptr); } -void operator delete[](void * ptr) noexcept { Memory::deleteImpl(ptr); } - -void operator delete(void * ptr, const std::nothrow_t &) noexcept { Memory::deleteImpl(ptr); } -void operator delete[](void * ptr, const std::nothrow_t &) noexcept { Memory::deleteImpl(ptr); } - -void operator delete(void * ptr, std::size_t size) noexcept { Memory::deleteSized(ptr, size); } -void operator delete[](void * ptr, std::size_t size) noexcept { Memory::deleteSized(ptr, size); } - -#endif diff --git a/src/Common/parseGlobs.cpp b/src/Common/parseGlobs.cpp index 31e7c2eb612..71ddbbd92ea 100644 --- a/src/Common/parseGlobs.cpp +++ b/src/Common/parseGlobs.cpp @@ -46,7 +46,7 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob char point; std::istringstream iss_range(buffer); iss_range >> range_begin >> point >> point >> range_end; - assert(iss_range.good()); + assert(!iss_range.fail()); bool leading_zeros = buffer[0] == '0'; size_t num_len = std::to_string(range_end).size(); if (leading_zeros) diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 4ed21ba4ef2..2ddbee43f5f 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -68,3 +68,6 @@ target_link_libraries (symbol_index PRIVATE clickhouse_common_io) add_executable (chaos_sanitizer chaos_sanitizer.cpp) target_link_libraries (chaos_sanitizer PRIVATE clickhouse_common_io) + +add_executable (memory_statistics_os_perf memory_statistics_os_perf.cpp) +target_link_libraries (memory_statistics_os_perf PRIVATE clickhouse_common_io) diff --git a/src/Common/tests/arena_with_free_lists.cpp b/src/Common/tests/arena_with_free_lists.cpp index e8cc583c0fb..d02fadca794 100644 --- a/src/Common/tests/arena_with_free_lists.cpp +++ b/src/Common/tests/arena_with_free_lists.cpp @@ -195,7 +195,7 @@ struct Dictionary const auto size = string.size(); if (size != 0) { - auto string_ptr = string_arena->alloc(size + 1); + auto * string_ptr = string_arena->alloc(size + 1); std::copy(string.data(), string.data() + size + 1, string_ptr); string_ref = StringRef{string_ptr, size}; } @@ -260,7 +260,7 @@ int main(int argc, char ** argv) for (const auto & s : data) { - auto ptr = arena.alloc(s.size() + 1); + auto * ptr = arena.alloc(s.size() + 1); memcpy(ptr, s.data(), s.size() + 1); refs.emplace_back(ptr, s.size() + 1); } @@ -286,7 +286,7 @@ int main(int argc, char ** argv) arena.free(const_cast(refs[index_to].data), refs[index_to].size); const auto & s = data[index_from]; - auto ptr = arena.alloc(s.size() + 1); + auto * ptr = arena.alloc(s.size() + 1); memcpy(ptr, s.data(), s.size() + 1); bytes += s.size() + 1; diff --git a/src/Common/tests/chaos_sanitizer.cpp b/src/Common/tests/chaos_sanitizer.cpp index 130d660f7ab..98f28f95b78 100644 --- a/src/Common/tests/chaos_sanitizer.cpp +++ b/src/Common/tests/chaos_sanitizer.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -20,31 +21,31 @@ int main(int argc, char ** argv) std::cerr << (DB::ThreadFuzzer::instance().isEffective() ? "ThreadFuzzer is enabled.\n" : "ThreadFuzzer is not enabled.\n"); - volatile size_t counter1 = 0; - volatile size_t counter2 = 0; + std::atomic counter1 = 0; + std::atomic counter2 = 0; /// These threads are synchronized by sleep (that's intentionally incorrect). std::thread t1([&] { for (size_t i = 0; i < num_iterations; ++i) - ++counter1; + counter1.store(counter1.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); sleepForNanoseconds(100000000); for (size_t i = 0; i < num_iterations; ++i) - ++counter2; + counter2.store(counter2.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); }); std::thread t2([&] { for (size_t i = 0; i < num_iterations; ++i) - ++counter2; + counter2.store(counter2.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); sleepForNanoseconds(100000000); for (size_t i = 0; i < num_iterations; ++i) - ++counter1; + counter1.store(counter1.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); }); t1.join(); diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h index 3711ab67932..05f60e01774 100644 --- a/src/Common/tests/gtest_global_context.h +++ b/src/Common/tests/gtest_global_context.h @@ -2,16 +2,24 @@ #include -inline DB::Context createContext() +struct ContextHolder { - auto context = DB::Context::createGlobal(); - context.makeGlobalContext(); - context.setPath("./"); - return context; -} + DB::SharedContextHolder shared_context; + DB::Context context; -inline const DB::Context & getContext() + ContextHolder() + : shared_context(DB::Context::createShared()) + , context(DB::Context::createGlobal(shared_context.get())) + { + } + + ContextHolder(ContextHolder &&) = default; +}; + +inline ContextHolder getContext() { - static DB::Context global_context = createContext(); - return global_context; + ContextHolder holder; + holder.context.makeGlobalContext(); + holder.context.setPath("./"); + return holder; } diff --git a/src/Common/tests/gtest_rw_lock.cpp b/src/Common/tests/gtest_rw_lock.cpp index dec4c732fd5..73987a25508 100644 --- a/src/Common/tests/gtest_rw_lock.cpp +++ b/src/Common/tests/gtest_rw_lock.cpp @@ -150,9 +150,16 @@ TEST(Common, RWLockDeadlock) usleep(100000); usleep(100000); usleep(100000); + usleep(100000); try { - auto holder2 = lock2->getLock(RWLockImpl::Read, "q1"); + auto holder2 = lock2->getLock(RWLockImpl::Read, "q1", std::chrono::milliseconds(100)); + if (!holder2) + { + throw Exception( + "Locking attempt timed out! Possible deadlock avoided. Client should retry.", + ErrorCodes::DEADLOCK_AVOIDED); + } } catch (const Exception & e) { @@ -174,9 +181,16 @@ TEST(Common, RWLockDeadlock) auto holder2 = lock2->getLock(RWLockImpl::Read, "q3"); usleep(100000); usleep(100000); + usleep(100000); try { - auto holder1 = lock1->getLock(RWLockImpl::Read, "q3"); + auto holder1 = lock1->getLock(RWLockImpl::Read, "q3", std::chrono::milliseconds(100)); + if (!holder1) + { + throw Exception( + "Locking attempt timed out! Possible deadlock avoided. Client should retry.", + ErrorCodes::DEADLOCK_AVOIDED); + } } catch (const Exception & e) { diff --git a/src/Common/tests/integer_hash_tables_and_hashes.cpp b/src/Common/tests/integer_hash_tables_and_hashes.cpp index b5ba0be6420..5b090fa6e4e 100644 --- a/src/Common/tests/integer_hash_tables_and_hashes.cpp +++ b/src/Common/tests/integer_hash_tables_and_hashes.cpp @@ -293,7 +293,7 @@ void NO_INLINE test(const Key * data, size_t size, std::function +#include + + +int main(int argc, char ** argv) +{ + using namespace DB; + + size_t num_iterations = argc >= 2 ? std::stoull(argv[1]) : 1000000; + MemoryStatisticsOS stats; + + uint64_t counter = 0; + for (size_t i = 0; i < num_iterations; ++i) + { + MemoryStatisticsOS::Data data = stats.get(); + counter += data.resident; + } + + if (num_iterations) + std::cerr << (counter / num_iterations) << '\n'; + return 0; +} + + diff --git a/src/Common/tests/parallel_aggregation.cpp b/src/Common/tests/parallel_aggregation.cpp index a4254f565e4..e39be163619 100644 --- a/src/Common/tests/parallel_aggregation.cpp +++ b/src/Common/tests/parallel_aggregation.cpp @@ -139,7 +139,7 @@ static void aggregate3(Map & local_map, Map & global_map, Mutex & mutex, Source: for (auto it = begin; it != end; ++it) { - auto found = local_map.find(*it); + auto * found = local_map.find(*it); if (found) ++found->getMapped(); @@ -199,7 +199,7 @@ static void aggregate4(Map & local_map, MapTwoLevel & global_map, Mutex * mutexe { for (; it != block_end; ++it) { - auto found = local_map.find(*it); + auto * found = local_map.find(*it); if (found) ++found->getMapped(); diff --git a/src/Common/tests/symbol_index.cpp b/src/Common/tests/symbol_index.cpp index 1c7e0ffc27d..67defa77242 100644 --- a/src/Common/tests/symbol_index.cpp +++ b/src/Common/tests/symbol_index.cpp @@ -33,7 +33,7 @@ int main(int argc, char ** argv) const void * address = reinterpret_cast(std::stoull(argv[1], nullptr, 16)); - auto symbol = symbol_index.findSymbol(address); + const auto * symbol = symbol_index.findSymbol(address); if (symbol) std::cerr << symbol->name << ": " << symbol->address_begin << " ... " << symbol->address_end << "\n"; else @@ -45,7 +45,7 @@ int main(int argc, char ** argv) else std::cerr << "dladdr: Not found\n"; - auto object = symbol_index.findObject(getAddress()); + const auto * object = symbol_index.findObject(getAddress()); Dwarf dwarf(*object->elf); Dwarf::LocationInfo location; diff --git a/src/Common/ya.make b/src/Common/ya.make new file mode 100644 index 00000000000..f4dac1dcd3e --- /dev/null +++ b/src/Common/ya.make @@ -0,0 +1,111 @@ +LIBRARY() + +ADDINCL ( + GLOBAL clickhouse/src + contrib/libs/libcpuid + contrib/libs/libunwind/include + GLOBAL contrib/restricted/ryu +) + +PEERDIR( + clickhouse/base/common + clickhouse/base/pcg-random + clickhouse/base/widechar_width + contrib/libs/libcpuid/libcpuid + contrib/libs/openssl + contrib/libs/poco/NetSSL_OpenSSL + contrib/libs/re2 + contrib/restricted/ryu +) + +# TODO: stub for config_version.h +CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\") +CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=0) +CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0) +CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0) +CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\") +CFLAGS (GLOBAL -DVERSION_INTEGER=0) +CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\") +CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\") +CFLAGS (GLOBAL -DVERSION_REVISION=0) +CFLAGS (GLOBAL -DVERSION_STRING=\"Unknown\") + +SRCS( + ActionLock.cpp + AlignedBuffer.cpp + checkStackSize.cpp + ClickHouseRevision.cpp + Config/AbstractConfigurationComparison.cpp + Config/ConfigProcessor.cpp + Config/configReadClient.cpp + Config/ConfigReloader.cpp + createHardLink.cpp + CurrentMetrics.cpp + CurrentThread.cpp + DNSResolver.cpp + Dwarf.cpp + Elf.cpp + ErrorCodes.cpp + escapeForFileName.cpp + Exception.cpp + ExternalLoaderStatus.cpp + FieldVisitors.cpp + FileChecker.cpp + filesystemHelpers.cpp + formatIPv6.cpp + formatReadable.cpp + getExecutablePath.cpp + getMultipleKeysFromConfig.cpp + getNumberOfPhysicalCPUCores.cpp + hasLinuxCapability.cpp + hex.cpp + IntervalKind.cpp + IPv6ToBinary.cpp + isLocalAddress.cpp + Macros.cpp + malloc.cpp + MemoryStatisticsOS.cpp + MemoryTracker.cpp + new_delete.cpp + OptimizedRegularExpression.cpp + parseAddress.cpp + parseGlobs.cpp + parseRemoteDescription.cpp + PipeFDs.cpp + PODArray.cpp + ProfileEvents.cpp + QueryProfiler.cpp + quoteString.cpp + randomSeed.cpp + RemoteHostFilter.cpp + RWLock.cpp + SensitiveDataMasker.cpp + setThreadName.cpp + SharedLibrary.cpp + ShellCommand.cpp + StackTrace.cpp + StatusFile.cpp + StatusInfo.cpp + Stopwatch.cpp + StringUtils/StringUtils.cpp + StudentTTest.cpp + SymbolIndex.cpp + TaskStatsInfoGetter.cpp + TerminalSize.cpp + thread_local_rng.cpp + ThreadFuzzer.cpp + ThreadPool.cpp + ThreadStatus.cpp + TraceCollector.cpp + UTF8Helpers.cpp + WeakHash.cpp + ZooKeeper/IKeeper.cpp + ZooKeeper/Lock.cpp + ZooKeeper/TestKeeper.cpp + ZooKeeper/ZooKeeper.cpp + ZooKeeper/ZooKeeperHolder.cpp + ZooKeeper/ZooKeeperImpl.cpp + ZooKeeper/ZooKeeperNodeCache.cpp +) + +END() diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 25efa6cdfb5..e76aa5d3308 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -53,7 +53,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c + ". Actual: " + getHexUIntLowercase(calculated_checksum.first) + getHexUIntLowercase(calculated_checksum.second) + ". Size of compressed block: " + toString(size); - auto message_hardware_failure = "This is most likely due to hardware failure. If you receive broken data over network and the error does not repeat every time, this can be caused by bad RAM on network interface controller or bad controller itself or bad RAM on network switches or bad CPU on network switches (look at the logs on related network switches; note that TCP checksums don't help) or bad RAM on host (look at dmesg or kern.log for enormous amount of EDAC errors, ECC-related reports, Machine Check Exceptions, mcelog; note that ECC memory can fail if the number of errors is huge) or bad CPU on host. If you read data from disk, this can be caused by disk bit rott. This exception protects ClickHouse from data corruption due to hardware failures."; + const char * message_hardware_failure = "This is most likely due to hardware failure. If you receive broken data over network and the error does not repeat every time, this can be caused by bad RAM on network interface controller or bad controller itself or bad RAM on network switches or bad CPU on network switches (look at the logs on related network switches; note that TCP checksums don't help) or bad RAM on host (look at dmesg or kern.log for enormous amount of EDAC errors, ECC-related reports, Machine Check Exceptions, mcelog; note that ECC memory can fail if the number of errors is huge) or bad CPU on host. If you read data from disk, this can be caused by disk bit rott. This exception protects ClickHouse from data corruption due to hardware failures."; auto flip_bit = [](char * buf, size_t pos) { diff --git a/src/Compression/CompressionCodecMultiple.cpp b/src/Compression/CompressionCodecMultiple.cpp index 4a43650a8a7..294a8c7629b 100644 --- a/src/Compression/CompressionCodecMultiple.cpp +++ b/src/Compression/CompressionCodecMultiple.cpp @@ -46,7 +46,7 @@ String CompressionCodecMultiple::getCodecDesc() const UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_size) const { UInt32 compressed_size = uncompressed_size; - for (auto & codec : codecs) + for (const auto & codec : codecs) compressed_size = codec->getCompressedReserveSize(compressed_size); /// TotalCodecs ByteForEachCodec data diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp index 4ae3ee95ebc..53f9fb13d58 100644 --- a/src/Compression/CompressionCodecT64.cpp +++ b/src/Compression/CompressionCodecT64.cpp @@ -167,7 +167,7 @@ TypeIndex typeIdx(const DataTypePtr & data_type) void transpose64x8(UInt64 * src_dst) { - auto * src8 = reinterpret_cast(src_dst); + const auto * src8 = reinterpret_cast(src_dst); UInt64 dst[8] = {}; for (UInt32 i = 0; i < 64; ++i) @@ -234,7 +234,7 @@ void transposeBytes(T value, UInt64 * matrix, UInt32 col) template void reverseTransposeBytes(const UInt64 * matrix, UInt32 col, T & value) { - auto * matrix8 = reinterpret_cast(matrix); + const auto * matrix8 = reinterpret_cast(matrix); if constexpr (sizeof(T) > 4) { @@ -423,8 +423,8 @@ UInt32 compressData(const char * src, UInt32 bytes_size, char * dst) T min, max; findMinMax(src, bytes_size, min, max); - MinMaxType min64 = min; - MinMaxType max64 = max; + MinMaxType min64 = min; // NOLINT + MinMaxType max64 = max; // NOLINT /// Write header { diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index a6bfdaebb14..f3652366a24 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -462,7 +462,7 @@ CompressionCodecPtr makeCodec(const std::string & codec_string, const DataTypePt { const std::string codec_statement = "(" + codec_string + ")"; Tokens tokens(codec_statement.begin().base(), codec_statement.end().base()); - IParser::Pos token_iterator(tokens); + IParser::Pos token_iterator(tokens, 0); Expected expected; ASTPtr codec_ast; diff --git a/src/Compression/ya.make b/src/Compression/ya.make new file mode 100644 index 00000000000..55b5287e3f3 --- /dev/null +++ b/src/Compression/ya.make @@ -0,0 +1,33 @@ +LIBRARY() + +ADDINCL( + contrib/libs/lz4 + contrib/libs/zstd +) + +PEERDIR( + clickhouse/src/Common + contrib/libs/lz4 + contrib/libs/zstd +) + +SRCS( + CachedCompressedReadBuffer.cpp + CompressedReadBuffer.cpp + CompressedReadBufferBase.cpp + CompressedReadBufferFromFile.cpp + CompressedWriteBuffer.cpp + CompressionCodecDelta.cpp + CompressionCodecDoubleDelta.cpp + CompressionCodecGorilla.cpp + CompressionCodecLZ4.cpp + CompressionCodecMultiple.cpp + CompressionCodecNone.cpp + CompressionCodecT64.cpp + CompressionCodecZSTD.cpp + CompressionFactory.cpp + ICompressionCodec.cpp + LZ4_decompress_faster.cpp +) + +END() diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 732be068569..521326f7e88 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -251,7 +251,7 @@ void BackgroundSchedulePool::threadFunction() attachToThreadGroup(); SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); - if (auto memory_tracker = CurrentThread::getMemoryTracker()) + if (auto * memory_tracker = CurrentThread::getMemoryTracker()) memory_tracker->setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool); while (!shutdown) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index a18d34af994..d70ae0ccdb3 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -181,25 +181,25 @@ const ColumnWithTypeAndName & Block::safeGetByPosition(size_t position) const } -ColumnWithTypeAndName & Block::getByName(const std::string & name) +const ColumnWithTypeAndName * Block::findByName(const std::string & name) const { auto it = index_by_name.find(name); if (index_by_name.end() == it) - throw Exception("Not found column " + name + " in block. There are only columns: " + dumpNames() - , ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); - - return data[it->second]; + { + return nullptr; + } + return &data[it->second]; } const ColumnWithTypeAndName & Block::getByName(const std::string & name) const { - auto it = index_by_name.find(name); - if (index_by_name.end() == it) + const auto * result = findByName(name); + if (!result) throw Exception("Not found column " + name + " in block. There are only columns: " + dumpNames() , ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); - return data[it->second]; + return *result; } diff --git a/src/Core/Block.h b/src/Core/Block.h index 82b60c83efb..ce804ddc0b5 100644 --- a/src/Core/Block.h +++ b/src/Core/Block.h @@ -28,7 +28,7 @@ class Block { private: using Container = ColumnsWithTypeAndName; - using IndexByName = std::map; + using IndexByName = std::unordered_map; Container data; IndexByName index_by_name; @@ -64,7 +64,20 @@ public: ColumnWithTypeAndName & safeGetByPosition(size_t position); const ColumnWithTypeAndName & safeGetByPosition(size_t position) const; - ColumnWithTypeAndName & getByName(const std::string & name); + ColumnWithTypeAndName* findByName(const std::string & name) + { + return const_cast( + const_cast(this)->findByName(name)); + } + + const ColumnWithTypeAndName* findByName(const std::string & name) const; + + ColumnWithTypeAndName & getByName(const std::string & name) + { + return const_cast( + const_cast(this)->getByName(name)); + } + const ColumnWithTypeAndName & getByName(const std::string & name) const; Container::iterator begin() { return data.begin(); } diff --git a/src/Core/Defines.h b/src/Core/Defines.h index f2d4a517712..5552de3b045 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -91,3 +91,13 @@ # define ASAN_UNPOISON_MEMORY_REGION(a, b) # define ASAN_POISON_MEMORY_REGION(a, b) #endif + +/// Actually, there may be multiple acquisitions of different locks for a given table within one query. +/// Check with IStorage class for the list of possible locks +#define DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC 120 + +/// Default limit on recursion depth of recursive descend parser. +#define DBMS_DEFAULT_MAX_PARSER_DEPTH 1000 + +/// Max depth of hierarchical dictionary +#define DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH 1000 diff --git a/src/Core/Field.h b/src/Core/Field.h index 5e62e4ef3e4..152ae29bd1e 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -157,7 +157,9 @@ private: template <> struct NearestFieldTypeImpl { using Type = std::conditional_t, Int64, UInt64>; }; template <> struct NearestFieldTypeImpl { using Type = Int64; }; template <> struct NearestFieldTypeImpl { using Type = UInt64; }; +#if __cplusplus > 201703L template <> struct NearestFieldTypeImpl { using Type = UInt64; }; +#endif template <> struct NearestFieldTypeImpl { using Type = UInt64; }; template <> struct NearestFieldTypeImpl { using Type = UInt64; }; diff --git a/src/Core/MySQLProtocol.h b/src/Core/MySQLProtocol.h index 1fae57517c1..20dd25b6b2d 100644 --- a/src/Core/MySQLProtocol.h +++ b/src/Core/MySQLProtocol.h @@ -22,10 +22,14 @@ #include #include #include -#include "config_core.h" + +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_SSL -#include -#include +# include +# include #endif /// Implementation of MySQL wire protocol. @@ -914,8 +918,17 @@ public: scramble.resize(SCRAMBLE_LENGTH + 1, 0); Poco::RandomInputStream generator; - for (size_t i = 0; i < SCRAMBLE_LENGTH; i++) + /** Generate a random string using ASCII characters but avoid separator character, + * produce pseudo random numbers between with about 7 bit worth of entropty between 1-127. + * https://github.com/mysql/mysql-server/blob/8.0/mysys/crypt_genhash_impl.cc#L427 + */ + for (size_t i = 0; i < SCRAMBLE_LENGTH; ++i) + { generator >> scramble[i]; + scramble[i] &= 0x7f; + if (scramble[i] == '\0' || scramble[i] == '$') + scramble[i] = scramble[i] + 1; + } } String getName() override @@ -993,8 +1006,13 @@ public: scramble.resize(SCRAMBLE_LENGTH + 1, 0); Poco::RandomInputStream generator; - for (size_t i = 0; i < SCRAMBLE_LENGTH; i++) + for (size_t i = 0; i < SCRAMBLE_LENGTH; ++i) + { generator >> scramble[i]; + scramble[i] &= 0x7f; + if (scramble[i] == '\0' || scramble[i] == '$') + scramble[i] = scramble[i] + 1; + } } String getName() override diff --git a/src/Core/NamesAndTypes.cpp b/src/Core/NamesAndTypes.cpp index b45e7d771a9..3a55a4328a7 100644 --- a/src/Core/NamesAndTypes.cpp +++ b/src/Core/NamesAndTypes.cpp @@ -137,8 +137,12 @@ NamesAndTypesList NamesAndTypesList::filter(const Names & names) const NamesAndTypesList NamesAndTypesList::addTypes(const Names & names) const { - /// NOTE It's better to make a map in `IStorage` than to create it here every time again. - ::google::dense_hash_map types; + /// NOTE: It's better to make a map in `IStorage` than to create it here every time again. +#if !defined(ARCADIA_BUILD) + google::dense_hash_map types; +#else + google::sparsehash::dense_hash_map types; +#endif types.set_empty_key(StringRef()); for (const NameAndTypePair & column : *this) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 7ab17fb12bb..7de1b474397 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -61,8 +61,8 @@ void Settings::loadSettingsFromConfig(const String & path, const Poco::Util::Abs void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_values_, bool changed_only) { /// Convert ptr and make simple check - auto column_names = (column_names_) ? &typeid_cast(*column_names_) : nullptr; - auto column_values = (column_values_) ? &typeid_cast(*column_values_) : nullptr; + auto * column_names = (column_names_) ? &typeid_cast(*column_names_) : nullptr; + auto * column_values = (column_values_) ? &typeid_cast(*column_values_) : nullptr; size_t size = 0; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 753231603b2..7b5b130ee72 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -78,9 +78,11 @@ struct Settings : public SettingsCollection M(SettingBool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ M(SettingBool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \ M(SettingBool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \ + M(SettingUInt64, background_buffer_flush_schedule_pool_size, 16, "Number of threads performing background flush for tables with Buffer engine. Only has meaning at server startup.", 0) \ M(SettingUInt64, background_pool_size, 16, "Number of threads performing background work for tables (for example, merging in merge tree). Only has meaning at server startup.", 0) \ M(SettingUInt64, background_move_pool_size, 8, "Number of threads performing background moves for tables. Only has meaning at server startup.", 0) \ - M(SettingUInt64, background_schedule_pool_size, 16, "Number of threads performing background tasks for replicated tables. Only has meaning at server startup.", 0) \ + M(SettingUInt64, background_schedule_pool_size, 16, "Number of threads performing background tasks for replicated tables, kafka streaming, dns cache updates. Only has meaning at server startup.", 0) \ + M(SettingUInt64, background_distributed_schedule_pool_size, 16, "Number of threads performing background tasks for distributed sends. Only has meaning at server startup.", 0) \ \ M(SettingMilliseconds, distributed_directory_monitor_sleep_time_ms, 100, "Sleep time for StorageDistributed DirectoryMonitors, in case of any errors delay grows exponentially.", 0) \ M(SettingMilliseconds, distributed_directory_monitor_max_sleep_time_ms, 30000, "Maximum sleep time for StorageDistributed DirectoryMonitors, it limits exponential growth too.", 0) \ @@ -149,7 +151,7 @@ struct Settings : public SettingsCollection M(SettingInt64, os_thread_priority, 0, "If non zero - set corresponding 'nice' value for query processing threads. Can be used to adjust query priority for OS scheduler.", 0) \ \ M(SettingBool, log_queries, 0, "Log requests and write the log to the system table.", 0) \ - \ + M(SettingLogQueriesType, log_queries_min_type, QueryLogElementType::QUERY_START, "query_log minimal type to log, possible values (from low to high): QUERY_START, QUERY_FINISH, EXCEPTION_BEFORE_START, EXCEPTION_WHILE_PROCESSING.", 0) \ M(SettingUInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \ \ M(SettingDistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \ @@ -337,7 +339,6 @@ struct Settings : public SettingsCollection \ M(SettingUInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ - M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Maximum memory usage for processing all concurrently running queries on the server. Zero means unlimited.", 0) \ M(SettingUInt64, memory_profiler_step, 0, "Every number of bytes the memory profiler will collect the allocating stack trace. The minimal effective step is 4 MiB (less values will work as clamped to 4 MiB). Zero means disabled memory profiler.", 0) \ \ M(SettingUInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ @@ -404,8 +405,11 @@ struct Settings : public SettingsCollection M(SettingBool, use_compact_format_in_distributed_parts_names, false, "Changes format of directories names for distributed table insert parts.", 0) \ M(SettingUInt64, multiple_joins_rewriter_version, 1, "1 or 2. Second rewriter version knows about table columns and keep not clashed names as is.", 0) \ M(SettingBool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \ - M(SettingUInt64, max_parser_depth, 1000, "Maximum parser depth.", 0) \ + M(SettingUInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \ M(SettingSeconds, temporary_live_view_timeout, DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC, "Timeout after which temporary live view is deleted.", 0) \ + M(SettingBool, transform_null_in, false, "If enabled, NULL values will be matched with 'IN' operator as if they are considered equal.", 0) \ + M(SettingBool, allow_nondeterministic_mutations, false, "Allow non-deterministic functions in ALTER UPDATE/ALTER DELETE statements", 0) \ + M(SettingSeconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "How long locking request should wait before failing", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ @@ -418,6 +422,7 @@ struct Settings : public SettingsCollection M(SettingBool, merge_tree_uniform_read_distribution, true, "Obsolete setting, does nothing. Will be removed after 2020-05-20", 0) \ M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \ + M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index 6d879b27181..238ac1c3c62 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -542,6 +542,13 @@ IMPLEMENT_SETTING_ENUM(FormatSettings::DateTimeInputFormat, DATE_TIME_INPUT_FORM M(trace, "trace") IMPLEMENT_SETTING_ENUM(LogsLevel, LOGS_LEVEL_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS) +#define LOG_QUERIES_TYPE_LIST_OF_NAMES(M) \ + M(QUERY_START, "QUERY_START") \ + M(QUERY_FINISH, "QUERY_FINISH") \ + M(EXCEPTION_BEFORE_START, "EXCEPTION_BEFORE_START") \ + M(EXCEPTION_WHILE_PROCESSING, "EXCEPTION_WHILE_PROCESSING") +IMPLEMENT_SETTING_ENUM(QueryLogElementType, LOG_QUERIES_TYPE_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS) + namespace details { diff --git a/src/Core/SettingsCollection.h b/src/Core/SettingsCollection.h index da21412b7c1..9b1813cdb86 100644 --- a/src/Core/SettingsCollection.h +++ b/src/Core/SettingsCollection.h @@ -298,6 +298,16 @@ enum class LogsLevel }; using SettingLogsLevel = SettingEnum; +// Make it signed for compatibility with DataTypeEnum8 +enum QueryLogElementType : int8_t +{ + QUERY_START = 1, + QUERY_FINISH = 2, + EXCEPTION_BEFORE_START = 3, + EXCEPTION_WHILE_PROCESSING = 4, +}; +using SettingLogQueriesType = SettingEnum; + enum class SettingsBinaryFormat { @@ -514,8 +524,8 @@ public: bool tryGet(const StringRef & name, String & value) const; /// Compares two collections of settings. - bool operator ==(const Derived & rhs) const; - bool operator!=(const Derived & rhs) const { return !(*this == rhs); } + bool operator ==(const SettingsCollection & rhs) const; + bool operator!=(const SettingsCollection & rhs) const { return !(*this == rhs); } /// Gathers all changed values (e.g. for applying them later to another collection of settings). SettingsChanges changes() const; diff --git a/src/Core/SettingsCollectionImpl.h b/src/Core/SettingsCollectionImpl.h index d5716c2a80d..8210b04e2da 100644 --- a/src/Core/SettingsCollectionImpl.h +++ b/src/Core/SettingsCollectionImpl.h @@ -173,19 +173,19 @@ bool SettingsCollection::tryGet(const StringRef & name, String & value) template -bool SettingsCollection::operator ==(const Derived & rhs) const +bool SettingsCollection::operator ==(const SettingsCollection & rhs) const { const auto & the_members = members(); for (size_t i = 0; i != the_members.size(); ++i) { const auto & member = the_members[i]; bool left_changed = member.is_changed(castToDerived()); - bool right_changed = member.is_changed(rhs); + bool right_changed = member.is_changed(rhs.castToDerived()); if (left_changed || right_changed) { if (left_changed != right_changed) return false; - if (member.get_field(castToDerived()) != member.get_field(rhs)) + if (member.get_field(castToDerived()) != member.get_field(rhs.castToDerived())) return false; } } diff --git a/src/Core/ya.make b/src/Core/ya.make new file mode 100644 index 00000000000..4999fe334bc --- /dev/null +++ b/src/Core/ya.make @@ -0,0 +1,24 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/sparsehash + contrib/restricted/boost/libs +) + +SRCS( + BackgroundSchedulePool.cpp + Block.cpp + BlockInfo.cpp + ColumnWithTypeAndName.cpp + ExternalResultDescription.cpp + ExternalTable.cpp + Field.cpp + iostream_debug_helpers.cpp + MySQLProtocol.cpp + NamesAndTypes.cpp + Settings.cpp + SettingsCollection.cpp +) + +END() diff --git a/src/DataStreams/AggregatingSortedBlockInputStream.cpp b/src/DataStreams/AggregatingSortedBlockInputStream.cpp deleted file mode 100644 index c36ba4968d3..00000000000 --- a/src/DataStreams/AggregatingSortedBlockInputStream.cpp +++ /dev/null @@ -1,249 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - - -class RemovingLowCardinalityBlockInputStream : public IBlockInputStream -{ -public: - RemovingLowCardinalityBlockInputStream(BlockInputStreamPtr input_, ColumnNumbers positions_) - : input(std::move(input_)), positions(std::move(positions_)) - { - header = transform(input->getHeader()); - } - - Block transform(Block block) - { - if (block) - { - for (auto & pos : positions) - { - auto & col = block.safeGetByPosition(pos); - col.column = recursiveRemoveLowCardinality(col.column); - col.type = recursiveRemoveLowCardinality(col.type); - } - } - - return block; - } - - String getName() const override { return "RemovingLowCardinality"; } - Block getHeader() const override { return header; } - const BlockMissingValues & getMissingValues() const override { return input->getMissingValues(); } - bool isSortedOutput() const override { return input->isSortedOutput(); } - const SortDescription & getSortDescription() const override { return input->getSortDescription(); } - -protected: - Block readImpl() override { return transform(input->read()); } - -private: - Block header; - BlockInputStreamPtr input; - ColumnNumbers positions; -}; - - -AggregatingSortedBlockInputStream::AggregatingSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_) - : MergingSortedBlockInputStream(inputs_, description_, max_block_size_) -{ - /// Fill in the column numbers that need to be aggregated. - for (size_t i = 0; i < num_columns; ++i) - { - ColumnWithTypeAndName & column = header.safeGetByPosition(i); - - /// We leave only states of aggregate functions. - if (!dynamic_cast(column.type.get()) && !dynamic_cast(column.type->getCustomName())) - { - column_numbers_not_to_aggregate.push_back(i); - continue; - } - - /// Included into PK? - SortDescription::const_iterator it = description.begin(); - for (; it != description.end(); ++it) - if (it->column_name == column.name || (it->column_name.empty() && it->column_number == i)) - break; - - if (it != description.end()) - { - column_numbers_not_to_aggregate.push_back(i); - continue; - } - - if (auto simple_aggr = dynamic_cast(column.type->getCustomName())) - { - // simple aggregate function - SimpleAggregateDescription desc{simple_aggr->getFunction(), i}; - if (desc.function->allocatesMemoryInArena()) - allocatesMemoryInArena = true; - - columns_to_simple_aggregate.emplace_back(std::move(desc)); - - if (recursiveRemoveLowCardinality(column.type).get() != column.type.get()) - converted_lc_columns.emplace_back(i); - } - else - { - // standard aggregate function - column_numbers_to_aggregate.push_back(i); - } - } - - result_header = header; - - if (!converted_lc_columns.empty()) - { - for (auto & input : children) - input = std::make_shared(input, converted_lc_columns); - - header = children.at(0)->getHeader(); - } -} - - -Block AggregatingSortedBlockInputStream::readImpl() -{ - if (finished) - return Block(); - - MutableColumns merged_columns; - init(merged_columns); - - if (has_collation) - throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR); - - if (merged_columns.empty()) - return Block(); - - columns_to_aggregate.resize(column_numbers_to_aggregate.size()); - for (size_t i = 0, size = columns_to_aggregate.size(); i < size; ++i) - columns_to_aggregate[i] = typeid_cast(merged_columns[column_numbers_to_aggregate[i]].get()); - - merge(merged_columns, queue_without_collation); - - for (auto & pos : converted_lc_columns) - { - auto & from_type = header.getByPosition(pos).type; - auto & to_type = result_header.getByPosition(pos).type; - merged_columns[pos] = (*recursiveTypeConversion(std::move(merged_columns[pos]), from_type, to_type)).mutate(); - } - - return result_header.cloneWithColumns(std::move(merged_columns)); -} - - -void AggregatingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap & queue) -{ - size_t merged_rows = 0; - - /// We take the rows in the correct order and put them in `merged_block`, while the rows are no more than `max_block_size` - while (queue.isValid()) - { - SortCursor current = queue.current(); - - setPrimaryKeyRef(next_key, current); - - bool key_differs; - - if (current_key.empty()) /// The first key encountered. - { - setPrimaryKeyRef(current_key, current); - key_differs = true; - } - else - key_differs = next_key != current_key; - - /// if there are enough rows accumulated and the last one is calculated completely - if (key_differs && merged_rows >= max_block_size) - { - /// Write the simple aggregation result for the previous group. - insertSimpleAggregationResult(merged_columns); - return; - } - - if (key_differs) - { - current_key.swap(next_key); - - /// We will write the data for the group. We copy the values of ordinary columns. - for (size_t j : column_numbers_not_to_aggregate) - merged_columns[j]->insertFrom(*current->all_columns[j], current->pos); - - /// Add the empty aggregation state to the aggregate columns. The state will be updated in the `addRow` function. - for (auto & column_to_aggregate : columns_to_aggregate) - column_to_aggregate->insertDefault(); - - /// Write the simple aggregation result for the previous group. - if (merged_rows > 0) - insertSimpleAggregationResult(merged_columns); - - /// Reset simple aggregation states for next row - for (auto & desc : columns_to_simple_aggregate) - desc.createState(); - - if (allocatesMemoryInArena) - arena = std::make_unique(); - - ++merged_rows; - } - - addRow(current); - - if (!current->isLast()) - { - queue.next(); - } - else - { - /// We fetch the next block from the appropriate source, if there is one. - fetchNextBlock(current, queue); - } - } - - /// Write the simple aggregation result for the previous group. - if (merged_rows > 0) - insertSimpleAggregationResult(merged_columns); - - finished = true; -} - - -void AggregatingSortedBlockInputStream::addRow(SortCursor & cursor) -{ - for (size_t i = 0, size = column_numbers_to_aggregate.size(); i < size; ++i) - { - size_t j = column_numbers_to_aggregate[i]; - columns_to_aggregate[i]->insertMergeFrom(*cursor->all_columns[j], cursor->pos); - } - - for (auto & desc : columns_to_simple_aggregate) - { - auto & col = cursor->all_columns[desc.column_number]; - desc.add_function(desc.function.get(), desc.state.data(), &col, cursor->pos, arena.get()); - } -} - -void AggregatingSortedBlockInputStream::insertSimpleAggregationResult(MutableColumns & merged_columns) -{ - for (auto & desc : columns_to_simple_aggregate) - { - desc.function->insertResultInto(desc.state.data(), *merged_columns[desc.column_number]); - desc.destroyState(); - } -} - -} diff --git a/src/DataStreams/AggregatingSortedBlockInputStream.h b/src/DataStreams/AggregatingSortedBlockInputStream.h deleted file mode 100644 index b0387dbcf2b..00000000000 --- a/src/DataStreams/AggregatingSortedBlockInputStream.h +++ /dev/null @@ -1,123 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - - -namespace DB -{ - -class Arena; - -/** Merges several sorted streams to one. - * During this for each group of consecutive identical values of the primary key (the columns by which the data is sorted), - * merges them into one row. When merging, the data is pre-aggregated - merge of states of aggregate functions, - * corresponding to a one value of the primary key. For columns that are not part of the primary key and which do not have the AggregateFunction type, - * when merged, the first value is selected. - */ -class AggregatingSortedBlockInputStream : public MergingSortedBlockInputStream -{ -public: - AggregatingSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_); - - String getName() const override { return "AggregatingSorted"; } - - bool isSortedOutput() const override { return true; } - - Block getHeader() const override { return result_header; } - -protected: - /// Can return 1 more records than max_block_size. - Block readImpl() override; - -private: - Logger * log = &Logger::get("AggregatingSortedBlockInputStream"); - - /// Read finished. - bool finished = false; - - struct SimpleAggregateDescription; - - /// Columns with which numbers should be aggregated. - ColumnNumbers column_numbers_to_aggregate; - ColumnNumbers column_numbers_not_to_aggregate; - std::vector columns_to_aggregate; - std::vector columns_to_simple_aggregate; - - SharedBlockRowRef current_key; /// The current primary key. - SharedBlockRowRef next_key; /// The primary key of the next row. - - Block result_header; - ColumnNumbers converted_lc_columns; - - /** We support two different cursors - with Collation and without. - * Templates are used instead of polymorphic SortCursor and calls to virtual functions. - */ - void merge(MutableColumns & merged_columns, SortingHeap & queue); - - /** Extract all states of aggregate functions and merge them with the current group. - */ - void addRow(SortCursor & cursor); - - /** Insert all values of current row for simple aggregate functions - */ - void insertSimpleAggregationResult(MutableColumns & merged_columns); - - /// Does SimpleAggregateFunction allocates memory in arena? - bool allocatesMemoryInArena = false; - /// Memory pool for SimpleAggregateFunction - /// (only when allocatesMemoryInArena == true). - std::unique_ptr arena; - - /// Stores information for aggregation of SimpleAggregateFunction columns - struct SimpleAggregateDescription - { - /// An aggregate function 'anyLast', 'sum'... - AggregateFunctionPtr function; - IAggregateFunction::AddFunc add_function; - size_t column_number; - AlignedBuffer state; - bool created = false; - - SimpleAggregateDescription(const AggregateFunctionPtr & function_, const size_t column_number_) : function(function_), column_number(column_number_) - { - add_function = function->getAddressOfAddFunction(); - state.reset(function->sizeOfData(), function->alignOfData()); - } - - void createState() - { - if (created) - return; - function->create(state.data()); - created = true; - } - - void destroyState() - { - if (!created) - return; - function->destroy(state.data()); - created = false; - } - - /// Explicitly destroy aggregation state if the stream is terminated - ~SimpleAggregateDescription() - { - destroyState(); - } - - SimpleAggregateDescription() = default; - SimpleAggregateDescription(SimpleAggregateDescription &&) = default; - SimpleAggregateDescription(const SimpleAggregateDescription &) = delete; - }; -}; - -} diff --git a/src/DataStreams/CollapsingSortedBlockInputStream.cpp b/src/DataStreams/CollapsingSortedBlockInputStream.cpp deleted file mode 100644 index ef82a6d8c5e..00000000000 --- a/src/DataStreams/CollapsingSortedBlockInputStream.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include -#include -#include -#include - -/// Maximum number of messages about incorrect data in the log. -#define MAX_ERROR_MESSAGES 10 - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int INCORRECT_DATA; - extern const int LOGICAL_ERROR; -} - - -void CollapsingSortedBlockInputStream::reportIncorrectData() -{ - std::stringstream s; - s << "Incorrect data: number of rows with sign = 1 (" << count_positive - << ") differs with number of rows with sign = -1 (" << count_negative - << ") by more than one (for key: "; - - for (size_t i = 0, size = current_key.size(); i < size; ++i) - { - if (i != 0) - s << ", "; - s << applyVisitor(FieldVisitorToString(), (*(*current_key.columns)[i])[current_key.row_num]); - } - - s << ")."; - - /** Fow now we limit ourselves to just logging such situations, - * since the data is generated by external programs. - * With inconsistent data, this is an unavoidable error that can not be easily corrected by admins. Therefore Warning. - */ - LOG_WARNING(log, s.rdbuf()); -} - - -void CollapsingSortedBlockInputStream::insertRows(MutableColumns & merged_columns, size_t block_size, MergeStopCondition & condition) -{ - if (count_positive == 0 && count_negative == 0) - { - /// No input rows have been read. - return; - } - - if (last_is_positive || count_positive != count_negative) - { - if (count_positive <= count_negative) - { - condition.addRowWithGranularity(block_size); - for (size_t i = 0; i < num_columns; ++i) - merged_columns[i]->insertFrom(*(*first_negative.columns)[i], first_negative.row_num); - - if (out_row_sources_buf) - current_row_sources[first_negative_pos].setSkipFlag(false); - } - - if (count_positive >= count_negative) - { - condition.addRowWithGranularity(block_size); - for (size_t i = 0; i < num_columns; ++i) - merged_columns[i]->insertFrom(*(*last_positive.columns)[i], last_positive.row_num); - - if (out_row_sources_buf) - current_row_sources[last_positive_pos].setSkipFlag(false); - } - - if (!(count_positive == count_negative || count_positive + 1 == count_negative || count_positive == count_negative + 1)) - { - if (count_incorrect_data < MAX_ERROR_MESSAGES) - reportIncorrectData(); - ++count_incorrect_data; - } - } - - if (out_row_sources_buf) - out_row_sources_buf->write( - reinterpret_cast(current_row_sources.data()), - current_row_sources.size() * sizeof(RowSourcePart)); -} - - -Block CollapsingSortedBlockInputStream::readImpl() -{ - if (finished) - return {}; - - MutableColumns merged_columns; - init(merged_columns); - - if (has_collation) - throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR); - - if (merged_columns.empty()) - return {}; - - merge(merged_columns, queue_without_collation); - return header.cloneWithColumns(std::move(merged_columns)); -} - - -void CollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap & queue) -{ - - MergeStopCondition stop_condition(average_block_sizes, max_block_size); - size_t current_block_granularity; - /// Take rows in correct order and put them into `merged_columns` until the rows no more than `max_block_size` - for (; queue.isValid(); ++current_pos) - { - SortCursor current = queue.current(); - current_block_granularity = current->rows; - - if (current_key.empty()) - setPrimaryKeyRef(current_key, current); - - Int8 sign = assert_cast(*current->all_columns[sign_column_number]).getData()[current->pos]; - setPrimaryKeyRef(next_key, current); - - bool key_differs = next_key != current_key; - - /// if there are enough rows and the last one is calculated completely - if (key_differs && stop_condition.checkStop()) - { - ++blocks_written; - return; - } - - if (key_differs) - { - /// We write data for the previous primary key. - insertRows(merged_columns, current_block_granularity, stop_condition); - - current_key.swap(next_key); - - count_negative = 0; - count_positive = 0; - - current_pos = 0; - first_negative_pos = 0; - last_positive_pos = 0; - last_negative_pos = 0; - current_row_sources.resize(0); - } - - /// Initially, skip all rows. On insert, unskip "corner" rows. - if (out_row_sources_buf) - current_row_sources.emplace_back(current.impl->order, true); - - if (sign == 1) - { - ++count_positive; - last_is_positive = true; - - setRowRef(last_positive, current); - last_positive_pos = current_pos; - } - else if (sign == -1) - { - if (!count_negative) - { - setRowRef(first_negative, current); - first_negative_pos = current_pos; - } - - if (!blocks_written && stop_condition.empty()) - { - setRowRef(last_negative, current); - last_negative_pos = current_pos; - } - - ++count_negative; - last_is_positive = false; - } - else - throw Exception("Incorrect data: Sign = " + toString(sign) + " (must be 1 or -1).", - ErrorCodes::INCORRECT_DATA); - - if (!current->isLast()) - { - queue.next(); - } - else - { - /// We take next block from the corresponding source, if there is one. - fetchNextBlock(current, queue); - } - } - - /// Write data for last primary key. - insertRows(merged_columns, /*some_granularity*/ 0, stop_condition); - - finished = true; -} - -} diff --git a/src/DataStreams/CollapsingSortedBlockInputStream.h b/src/DataStreams/CollapsingSortedBlockInputStream.h deleted file mode 100644 index 2b528d27339..00000000000 --- a/src/DataStreams/CollapsingSortedBlockInputStream.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include - -#include -#include - - -namespace DB -{ - -/** Merges several sorted streams to one. - * For each group of consecutive identical values of the primary key (the columns by which the data is sorted), - * keeps no more than one row with the value of the column `sign_column = -1` ("negative row") - * and no more than a row with the value of the column `sign_column = 1` ("positive row"). - * That is, it collapses the records from the change log. - * - * If the number of positive and negative rows is the same, and the last row is positive, then the first negative and last positive rows are written. - * If the number of positive and negative rows is the same, and the last line is negative, it writes nothing. - * If the positive by 1 is greater than the negative rows, then only the last positive row is written. - * If negative by 1 is greater than positive rows, then only the first negative row is written. - * Otherwise, a logical error. - */ -class CollapsingSortedBlockInputStream : public MergingSortedBlockInputStream -{ -public: - CollapsingSortedBlockInputStream( - BlockInputStreams inputs_, const SortDescription & description_, - const String & sign_column, size_t max_block_size_, - WriteBuffer * out_row_sources_buf_ = nullptr, bool average_block_sizes_ = false) - : MergingSortedBlockInputStream(inputs_, description_, max_block_size_, 0, out_row_sources_buf_, false, average_block_sizes_) - { - sign_column_number = header.getPositionByName(sign_column); - } - - String getName() const override { return "CollapsingSorted"; } - -protected: - /// Can return 1 more records than max_block_size. - Block readImpl() override; - -private: - size_t sign_column_number; - - Logger * log = &Logger::get("CollapsingSortedBlockInputStream"); - - /// Read is finished. - bool finished = false; - - SharedBlockRowRef current_key; /// The current primary key. - SharedBlockRowRef next_key; /// The primary key of the next row. - - SharedBlockRowRef first_negative; /// The first negative row for the current primary key. - SharedBlockRowRef last_positive; /// The last positive row for the current primary key. - SharedBlockRowRef last_negative; /// Last negative row. It is only stored if there is not one row is written to output. - - size_t count_positive = 0; /// The number of positive rows for the current primary key. - size_t count_negative = 0; /// The number of negative rows for the current primary key. - bool last_is_positive = false; /// true if the last row for the current primary key is positive. - - size_t count_incorrect_data = 0; /// To prevent too many error messages from writing to the log. - - size_t blocks_written = 0; - - /// Fields specific for VERTICAL merge algorithm. - /// Row numbers are relative to the start of current primary key. - size_t current_pos = 0; /// Current row number - size_t first_negative_pos = 0; /// Row number of first_negative - size_t last_positive_pos = 0; /// Row number of last_positive - size_t last_negative_pos = 0; /// Row number of last_negative - PODArray current_row_sources; /// Sources of rows with the current primary key - - /** We support two different cursors - with Collation and without. - * Templates are used instead of polymorphic SortCursors and calls to virtual functions. - */ - void merge(MutableColumns & merged_columns, SortingHeap & queue); - - /// Output to result rows for the current primary key. - void insertRows(MutableColumns & merged_columns, size_t block_size, MergeStopCondition & condition); - - void reportIncorrectData(); -}; - -} diff --git a/src/DataStreams/ConvertingBlockInputStream.cpp b/src/DataStreams/ConvertingBlockInputStream.cpp index 89864847a92..368ee7083b1 100644 --- a/src/DataStreams/ConvertingBlockInputStream.cpp +++ b/src/DataStreams/ConvertingBlockInputStream.cpp @@ -17,11 +17,11 @@ namespace ErrorCodes } -static ColumnPtr castColumnWithDiagnostic(const ColumnWithTypeAndName & src_elem, const ColumnWithTypeAndName & res_elem, const Context & context) +static ColumnPtr castColumnWithDiagnostic(const ColumnWithTypeAndName & src_elem, const ColumnWithTypeAndName & res_elem) { try { - return castColumn(src_elem, res_elem.type, context); + return castColumn(src_elem, res_elem.type); } catch (Exception & e) { @@ -32,11 +32,10 @@ static ColumnPtr castColumnWithDiagnostic(const ColumnWithTypeAndName & src_elem ConvertingBlockInputStream::ConvertingBlockInputStream( - const Context & context_, const BlockInputStreamPtr & input, const Block & result_header, MatchColumnsMode mode) - : context(context_), header(result_header), conversion(header.columns()) + : header(result_header), conversion(header.columns()) { children.emplace_back(input); @@ -85,7 +84,7 @@ ConvertingBlockInputStream::ConvertingBlockInputStream( /// Check conversion by dry run CAST function. - castColumnWithDiagnostic(src_elem, res_elem, context); + castColumnWithDiagnostic(src_elem, res_elem); } } @@ -107,7 +106,7 @@ Block ConvertingBlockInputStream::readImpl() const auto & src_elem = src.getByPosition(conversion[res_pos]); auto & res_elem = res.getByPosition(res_pos); - ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context); + ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem); if (isColumnConst(*src_elem.column) && !isColumnConst(*res_elem.column)) converted = converted->convertToFullColumnIfConst(); diff --git a/src/DataStreams/ConvertingBlockInputStream.h b/src/DataStreams/ConvertingBlockInputStream.h index 553d9221dd6..b0324618408 100644 --- a/src/DataStreams/ConvertingBlockInputStream.h +++ b/src/DataStreams/ConvertingBlockInputStream.h @@ -32,7 +32,6 @@ public: }; ConvertingBlockInputStream( - const Context & context, const BlockInputStreamPtr & input, const Block & result_header, MatchColumnsMode mode); @@ -43,7 +42,6 @@ public: private: Block readImpl() override; - const Context & context; Block header; /// How to construct result block. Position in source block, where to get each column. diff --git a/src/DataStreams/DistinctBlockInputStream.cpp b/src/DataStreams/DistinctBlockInputStream.cpp index eaebebd4a46..b1f9756d55e 100644 --- a/src/DataStreams/DistinctBlockInputStream.cpp +++ b/src/DataStreams/DistinctBlockInputStream.cpp @@ -107,7 +107,7 @@ ColumnRawPtrs DistinctBlockInputStream::getKeyColumns(const Block & block) const for (size_t i = 0; i < columns; ++i) { - auto & column = columns_names.empty() + const auto & column = columns_names.empty() ? block.safeGetByPosition(i).column : block.getByName(columns_names[i]).column; diff --git a/src/DataStreams/DistinctSortedBlockInputStream.cpp b/src/DataStreams/DistinctSortedBlockInputStream.cpp index e8e9f7278aa..aac3c374252 100644 --- a/src/DataStreams/DistinctSortedBlockInputStream.cpp +++ b/src/DataStreams/DistinctSortedBlockInputStream.cpp @@ -126,7 +126,7 @@ ColumnRawPtrs DistinctSortedBlockInputStream::getKeyColumns(const Block & block) for (size_t i = 0; i < columns; ++i) { - auto & column = columns_names.empty() + const auto & column = columns_names.empty() ? block.safeGetByPosition(i).column : block.getByName(columns_names[i]).column; @@ -144,7 +144,7 @@ ColumnRawPtrs DistinctSortedBlockInputStream::getClearingColumns(const Block & b clearing_hint_columns.reserve(description.size()); for (const auto & sort_column_description : description) { - const auto sort_column_ptr = block.safeGetByPosition(sort_column_description.column_number).column.get(); + const auto * sort_column_ptr = block.safeGetByPosition(sort_column_description.column_number).column.get(); const auto it = std::find(key_columns.cbegin(), key_columns.cend(), sort_column_ptr); if (it != key_columns.cend()) /// if found in key_columns clearing_hint_columns.emplace_back(sort_column_ptr); diff --git a/src/DataStreams/ExecutionSpeedLimits.cpp b/src/DataStreams/ExecutionSpeedLimits.cpp index ca79138655a..28a8cd94994 100644 --- a/src/DataStreams/ExecutionSpeedLimits.cpp +++ b/src/DataStreams/ExecutionSpeedLimits.cpp @@ -41,7 +41,7 @@ static void limitProgressingSpeed(size_t total_progress_size, size_t max_speed_i void ExecutionSpeedLimits::throttle( size_t read_rows, size_t read_bytes, - size_t total_rows_to_read, UInt64 total_elapsed_microseconds) + size_t total_rows_to_read, UInt64 total_elapsed_microseconds) const { if ((min_execution_rps != 0 || max_execution_rps != 0 || min_execution_bps != 0 || max_execution_bps != 0 @@ -103,7 +103,7 @@ static bool handleOverflowMode(OverflowMode mode, const String & message, int co } } -bool ExecutionSpeedLimits::checkTimeLimit(UInt64 elapsed_ns, OverflowMode overflow_mode) +bool ExecutionSpeedLimits::checkTimeLimit(UInt64 elapsed_ns, OverflowMode overflow_mode) const { if (max_execution_time != 0 && elapsed_ns > static_cast(max_execution_time.totalMicroseconds()) * 1000) diff --git a/src/DataStreams/ExecutionSpeedLimits.h b/src/DataStreams/ExecutionSpeedLimits.h index 539a5b0108b..8f098bfd6b4 100644 --- a/src/DataStreams/ExecutionSpeedLimits.h +++ b/src/DataStreams/ExecutionSpeedLimits.h @@ -23,9 +23,9 @@ public: Poco::Timespan timeout_before_checking_execution_speed = 0; /// Pause execution in case if speed limits were exceeded. - void throttle(size_t read_rows, size_t read_bytes, size_t total_rows_to_read, UInt64 total_elapsed_microseconds); + void throttle(size_t read_rows, size_t read_bytes, size_t total_rows_to_read, UInt64 total_elapsed_microseconds) const; - bool checkTimeLimit(UInt64 elapsed_ns, OverflowMode overflow_mode); + bool checkTimeLimit(UInt64 elapsed_ns, OverflowMode overflow_mode) const; }; } diff --git a/src/DataStreams/FilterBlockInputStream.cpp b/src/DataStreams/FilterBlockInputStream.cpp index 3b3cea11412..b4b00083d7f 100644 --- a/src/DataStreams/FilterBlockInputStream.cpp +++ b/src/DataStreams/FilterBlockInputStream.cpp @@ -180,7 +180,7 @@ Block FilterBlockInputStream::readImpl() } -Block FilterBlockInputStream::removeFilterIfNeed(Block && block) +Block FilterBlockInputStream::removeFilterIfNeed(Block && block) const { if (block && remove_filter) block.erase(static_cast(filter_column)); diff --git a/src/DataStreams/FilterBlockInputStream.h b/src/DataStreams/FilterBlockInputStream.h index 7e49f5bb6cd..a60f8e8d699 100644 --- a/src/DataStreams/FilterBlockInputStream.h +++ b/src/DataStreams/FilterBlockInputStream.h @@ -40,7 +40,7 @@ private: ConstantFilterDescription constant_filter_description; - Block removeFilterIfNeed(Block && block); + Block removeFilterIfNeed(Block && block) const; }; } diff --git a/src/DataStreams/FinishSortingBlockInputStream.cpp b/src/DataStreams/FinishSortingBlockInputStream.cpp index b0bbf98b28e..2288e676abe 100644 --- a/src/DataStreams/FinishSortingBlockInputStream.cpp +++ b/src/DataStreams/FinishSortingBlockInputStream.cpp @@ -119,7 +119,7 @@ Block FinishSortingBlockInputStream::readImpl() for (size_t i = 0; i < size; ++i) perm[i] = i; - auto it = std::upper_bound(perm.begin(), perm.end(), last_block.rows() - 1, less); + auto * it = std::upper_bound(perm.begin(), perm.end(), last_block.rows() - 1, less); /// We need to save tail of block, because next block may starts with the same key as in tail /// and we should sort these rows in one chunk. diff --git a/src/DataStreams/GraphiteRollupSortedBlockInputStream.h b/src/DataStreams/GraphiteRollupSortedBlockInputStream.h deleted file mode 100644 index bfaeff7733b..00000000000 --- a/src/DataStreams/GraphiteRollupSortedBlockInputStream.h +++ /dev/null @@ -1,241 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -/** Intended for implementation of "rollup" - aggregation (rounding) of older data - * for a table with Graphite data (Graphite is the system for time series monitoring). - * - * Table with graphite data has at least the following columns (accurate to the name): - * Path, Time, Value, Version - * - * Path - name of metric (sensor); - * Time - time of measurement; - * Value - value of measurement; - * Version - a number, that for equal pairs of Path and Time, need to leave only record with maximum version. - * - * Each row in a table correspond to one value of one sensor. - * - * Pattern should contain function, retention scheme, or both of them. The order of patterns does mean as well: - * * Aggregation OR retention patterns should be first - * * Then aggregation AND retention full patterns have to be placed - * * default pattern without regexp must be the last - * - * Rollup rules are specified in the following way: - * - * pattern - * regexp - * function - * pattern - * regexp - * age -> precision - * age -> precision - * ... - * pattern - * regexp - * function - * age -> precision - * age -> precision - * ... - * pattern - * ... - * default - * function - * age -> precision - * ... - * - * regexp - pattern for sensor name - * default - if no pattern has matched - * - * age - minimal data age (in seconds), to start rounding with specified precision. - * precision - rounding precision (in seconds) - * - * function - name of aggregate function to be applied for values, that time was rounded to same. - * - * Example: - * - * - * - * \.max$ - * max - * - * - * click_cost - * any - * - * 0 - * 5 - * - * - * 86400 - * 60 - * - * - * - * max - * - * 0 - * 60 - * - * - * 3600 - * 300 - * - * - * 86400 - * 3600 - * - * - * - */ - -namespace Graphite -{ - struct Retention - { - UInt32 age; - UInt32 precision; - }; - - using Retentions = std::vector; - - struct Pattern - { - std::shared_ptr regexp; - std::string regexp_str; - AggregateFunctionPtr function; - Retentions retentions; /// Must be ordered by 'age' descending. - enum { TypeUndef, TypeRetention, TypeAggregation, TypeAll } type = TypeAll; /// The type of defined pattern, filled automatically - }; - - using Patterns = std::vector; - using RetentionPattern = Pattern; - using AggregationPattern = Pattern; - - struct Params - { - String config_name; - String path_column_name; - String time_column_name; - String value_column_name; - String version_column_name; - Graphite::Patterns patterns; - }; - - using RollupRule = std::pair; -} - -/** Merges several sorted streams into one. - * - * For each group of consecutive identical values of the `path` column, - * and the same `time` values, rounded to some precision - * (where rounding accuracy depends on the template set for `path` - * and the amount of time elapsed from `time` to the specified time), - * keeps one line, - * performing the rounding of time, - * merge `value` values using the specified aggregate functions, - * as well as keeping the maximum value of the `version` column. - */ -class GraphiteRollupSortedBlockInputStream : public MergingSortedBlockInputStream -{ -public: - GraphiteRollupSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_, - const Graphite::Params & params_, time_t time_of_merge_); - - String getName() const override { return "GraphiteRollupSorted"; } - - ~GraphiteRollupSortedBlockInputStream() override - { - if (aggregate_state_created) - std::get<1>(current_rule)->function->destroy(place_for_aggregate_state.data()); - } - -protected: - Block readImpl() override; - -private: - Logger * log = &Logger::get("GraphiteRollupSortedBlockInputStream"); - - const Graphite::Params params; - - size_t path_column_num; - size_t time_column_num; - size_t value_column_num; - size_t version_column_num; - - /// All columns other than 'time', 'value', 'version'. They are unmodified during rollup. - ColumnNumbers unmodified_column_numbers; - - time_t time_of_merge; - - /// No data has been read. - bool is_first = true; - - /// All data has been read. - bool finished = false; - - /* | path | time | rounded_time | version | value | unmodified | - * ----------------------------------------------------------------------------------- - * | A | 11 | 10 | 1 | 1 | a | | - * | A | 11 | 10 | 3 | 2 | b |> subgroup(A, 11) | - * | A | 11 | 10 | 2 | 3 | c | |> group(A, 10) - * ----------------------------------------------------------------------------------|> - * | A | 12 | 10 | 0 | 4 | d | |> Outputs (A, 10, avg(2, 5), a) - * | A | 12 | 10 | 1 | 5 | e |> subgroup(A, 12) | - * ----------------------------------------------------------------------------------- - * | A | 21 | 20 | 1 | 6 | f | - * | B | 11 | 10 | 1 | 7 | g | - * ... - */ - - /// Path name of current bucket - StringRef current_group_path; - - /// Last row with maximum version for current primary key (time bucket). - SharedBlockRowRef current_subgroup_newest_row; - - /// Time of last read row - time_t current_time = 0; - time_t current_time_rounded = 0; - - Graphite::RollupRule current_rule = {nullptr, nullptr}; - AlignedBuffer place_for_aggregate_state; - bool aggregate_state_created = false; /// Invariant: if true then current_rule is not NULL. - - const Graphite::Pattern undef_pattern = - { /// temporary empty pattern for selectPatternForPath - nullptr, - "", - nullptr, - DB::Graphite::Retentions(), - undef_pattern.TypeUndef, - }; - Graphite::RollupRule selectPatternForPath(StringRef path) const; - UInt32 selectPrecision(const Graphite::Retentions & retentions, time_t time) const; - - - void merge(MutableColumns & merged_columns, SortingHeap & queue); - - /// Insert the values into the resulting columns, which will not be changed in the future. - template - void startNextGroup(MutableColumns & merged_columns, TSortCursor & cursor, Graphite::RollupRule next_rule); - - /// Insert the calculated `time`, `value`, `version` values into the resulting columns by the last group of rows. - void finishCurrentGroup(MutableColumns & merged_columns); - - /// Update the state of the aggregate function with the new `value`. - void accumulateRow(SharedBlockRowRef & row); -}; - -} diff --git a/src/DataStreams/IBlockInputStream.cpp b/src/DataStreams/IBlockInputStream.cpp index 7aaab6f0fc7..6929db45a94 100644 --- a/src/DataStreams/IBlockInputStream.cpp +++ b/src/DataStreams/IBlockInputStream.cpp @@ -202,7 +202,7 @@ void IBlockInputStream::updateExtremes(Block & block) } -bool IBlockInputStream::checkTimeLimit() +bool IBlockInputStream::checkTimeLimit() const { return limits.speed_limits.checkTimeLimit(info.total_stopwatch.elapsed(), limits.timeout_overflow_mode); } diff --git a/src/DataStreams/IBlockInputStream.h b/src/DataStreams/IBlockInputStream.h index 620de87a21d..66f3e68d601 100644 --- a/src/DataStreams/IBlockInputStream.h +++ b/src/DataStreams/IBlockInputStream.h @@ -259,7 +259,7 @@ protected: /** Check limits. * But only those that can be checked within each separate stream. */ - bool checkTimeLimit(); + bool checkTimeLimit() const; #ifndef NDEBUG bool read_prefix_is_called = false; diff --git a/src/DataStreams/InternalTextLogsRowOutputStream.cpp b/src/DataStreams/InternalTextLogsRowOutputStream.cpp index 270497ab814..96b36157cba 100644 --- a/src/DataStreams/InternalTextLogsRowOutputStream.cpp +++ b/src/DataStreams/InternalTextLogsRowOutputStream.cpp @@ -20,16 +20,16 @@ Block InternalTextLogsRowOutputStream::getHeader() const void InternalTextLogsRowOutputStream::write(const Block & block) { - auto & array_event_time = typeid_cast(*block.getByName("event_time").column).getData(); - auto & array_microseconds = typeid_cast(*block.getByName("event_time_microseconds").column).getData(); + const auto & array_event_time = typeid_cast(*block.getByName("event_time").column).getData(); + const auto & array_microseconds = typeid_cast(*block.getByName("event_time_microseconds").column).getData(); - auto & column_host_name = typeid_cast(*block.getByName("host_name").column); - auto & column_query_id = typeid_cast(*block.getByName("query_id").column); + const auto & column_host_name = typeid_cast(*block.getByName("host_name").column); + const auto & column_query_id = typeid_cast(*block.getByName("query_id").column); - auto & array_thread_id = typeid_cast(*block.getByName("thread_id").column).getData(); - auto & array_priority = typeid_cast(*block.getByName("priority").column).getData(); - auto & column_source = typeid_cast(*block.getByName("source").column); - auto & column_text = typeid_cast(*block.getByName("text").column); + const auto & array_thread_id = typeid_cast(*block.getByName("thread_id").column).getData(); + const auto & array_priority = typeid_cast(*block.getByName("priority").column).getData(); + const auto & column_source = typeid_cast(*block.getByName("source").column); + const auto & column_text = typeid_cast(*block.getByName("text").column); for (size_t row_num = 0; row_num < block.rows(); ++row_num) { @@ -88,7 +88,11 @@ void InternalTextLogsRowOutputStream::write(const Block & block) writeCString("> ", wb); auto source = column_source.getDataAt(row_num); - writeString(source, wb); + if (color) + writeString(setColor(StringRefHash()(source)), wb); + DB::writeString(source, wb); + if (color) + writeCString(resetColor(), wb); writeCString(": ", wb); auto text = column_text.getDataAt(row_num); diff --git a/src/DataStreams/LimitByBlockInputStream.cpp b/src/DataStreams/LimitByBlockInputStream.cpp index dddf3c21929..74973dfd618 100644 --- a/src/DataStreams/LimitByBlockInputStream.cpp +++ b/src/DataStreams/LimitByBlockInputStream.cpp @@ -35,7 +35,7 @@ Block LimitByBlockInputStream::readImpl() UInt128 key; SipHash hash; - for (auto & column : column_ptrs) + for (const auto & column : column_ptrs) column->updateHashWithValue(i, hash); hash.get128(key.low, key.high); diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index 10822485d52..1ed7c6dff22 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -18,10 +18,10 @@ namespace ErrorCodes MergingSortedBlockInputStream::MergingSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, - size_t max_block_size_, UInt64 limit_, WriteBuffer * out_row_sources_buf_, bool quiet_, bool average_block_sizes_) - : description(description_), max_block_size(max_block_size_), limit(limit_), quiet(quiet_) - , average_block_sizes(average_block_sizes_), source_blocks(inputs_.size()) + const BlockInputStreams & inputs_, SortDescription description_, + size_t max_block_size_, UInt64 limit_, WriteBuffer * out_row_sources_buf_, bool quiet_) + : description(std::move(description_)), max_block_size(max_block_size_), limit(limit_), quiet(quiet_) + , source_blocks(inputs_.size()) , cursors(inputs_.size()), out_row_sources_buf(out_row_sources_buf_) , log(&Logger::get("MergingSortedBlockInputStream")) { @@ -39,14 +39,14 @@ void MergingSortedBlockInputStream::init(MutableColumns & merged_columns) for (size_t i = 0; i < source_blocks.size(); ++i) { - SharedBlockPtr & shared_block_ptr = source_blocks[i]; + Block & block = source_blocks[i]; - if (shared_block_ptr.get()) + if (block) continue; - shared_block_ptr = new detail::SharedBlock(children[i]->read()); + block = children[i]->read(); - const size_t rows = shared_block_ptr->rows(); + const size_t rows = block.rows(); if (rows == 0) continue; @@ -54,9 +54,7 @@ void MergingSortedBlockInputStream::init(MutableColumns & merged_columns) if (expected_block_size < rows) expected_block_size = std::min(rows, max_block_size); - cursors[i] = SortCursorImpl(*shared_block_ptr, description, i); - shared_block_ptr->all_columns = cursors[i].all_columns; - shared_block_ptr->sort_columns = cursors[i].sort_columns; + cursors[i] = SortCursorImpl(block, description, i); has_collation |= cursors[i].has_collation; } @@ -67,12 +65,12 @@ void MergingSortedBlockInputStream::init(MutableColumns & merged_columns) } /// Let's check that all source blocks have the same structure. - for (const SharedBlockPtr & shared_block_ptr : source_blocks) + for (const auto & block : source_blocks) { - if (!*shared_block_ptr) + if (!block) continue; - assertBlocksHaveEqualStructure(*shared_block_ptr, header, getName()); + assertBlocksHaveEqualStructure(block, header, getName()); } merged_columns.resize(num_columns); @@ -118,51 +116,33 @@ void MergingSortedBlockInputStream::fetchNextBlock(const TSortCursor & current, while (true) { - source_blocks[order] = new detail::SharedBlock(children[order]->read()); /// intrusive ptr + source_blocks[order] = children[order]->read(); - if (!*source_blocks[order]) + if (!source_blocks[order]) { queue.removeTop(); break; } - if (source_blocks[order]->rows()) + if (source_blocks[order].rows()) { - cursors[order].reset(*source_blocks[order]); + cursors[order].reset(source_blocks[order]); queue.replaceTop(&cursors[order]); - - source_blocks[order]->all_columns = cursors[order].all_columns; - source_blocks[order]->sort_columns = cursors[order].sort_columns; break; } } } -bool MergingSortedBlockInputStream::MergeStopCondition::checkStop() const -{ - if (!count_average) - return sum_rows_count == max_block_size; - - if (sum_rows_count == 0) - return false; - - size_t average = sum_blocks_granularity / sum_rows_count; - return sum_rows_count >= average; -} - - template void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSortingHeap & queue) { size_t merged_rows = 0; - MergeStopCondition stop_condition(average_block_sizes, max_block_size); - /** Increase row counters. * Return true if it's time to finish generating the current data block. */ - auto count_row_and_check_limit = [&, this](size_t current_granularity) + auto count_row_and_check_limit = [&, this]() { ++total_merged_rows; if (limit && total_merged_rows == limit) @@ -174,15 +154,13 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSort } ++merged_rows; - stop_condition.addRowWithGranularity(current_granularity); - return stop_condition.checkStop(); + return merged_rows >= max_block_size; }; /// Take rows in required order and put them into `merged_columns`, while the number of rows are no more than `max_block_size` while (queue.isValid()) { auto current = queue.current(); - size_t current_block_granularity = current->rows; /** And what if the block is totally less or equal than the rest for the current cursor? * Or is there only one data source left in the queue? Then you can take the entire block on current cursor. @@ -207,7 +185,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSort throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR); for (size_t i = 0; i < num_columns; ++i) - merged_columns[i] = (*std::move(source_blocks[source_num]->getByPosition(i).column)).mutate(); + merged_columns[i] = (*std::move(source_blocks[source_num].getByPosition(i).column)).mutate(); // std::cerr << "copied columns\n"; @@ -267,7 +245,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSort fetchNextBlock(current, queue); } - if (count_row_and_check_limit(current_block_granularity)) + if (count_row_and_check_limit()) return; } diff --git a/src/DataStreams/MergingSortedBlockInputStream.h b/src/DataStreams/MergingSortedBlockInputStream.h index d9c9f1c26cc..9a732fff947 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.h +++ b/src/DataStreams/MergingSortedBlockInputStream.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include #include @@ -17,12 +15,6 @@ namespace Poco { class Logger; } namespace DB { -namespace ErrorCodes -{ - extern const int CORRUPTED_DATA; -} - - /** Merges several sorted streams into one sorted stream. */ class MergingSortedBlockInputStream : public IBlockInputStream @@ -33,8 +25,8 @@ public: * quiet - don't log profiling info */ MergingSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_, - UInt64 limit_ = 0, WriteBuffer * out_row_sources_buf_ = nullptr, bool quiet_ = false, bool average_block_sizes_ = false); + const BlockInputStreams & inputs_, SortDescription description_, size_t max_block_size_, + UInt64 limit_ = 0, WriteBuffer * out_row_sources_buf_ = nullptr, bool quiet_ = false); String getName() const override { return "MergingSorted"; } @@ -44,38 +36,6 @@ public: Block getHeader() const override { return header; } protected: - /// Simple class, which allows to check stop condition during merge process - /// in simple case it just compare amount of merged rows with max_block_size - /// in `count_average` case it compares amount of merged rows with linear combination - /// of block sizes from which these rows were taken. - struct MergeStopCondition - { - size_t sum_blocks_granularity = 0; - size_t sum_rows_count = 0; - bool count_average; - size_t max_block_size; - - MergeStopCondition(bool count_average_, size_t max_block_size_) - : count_average(count_average_) - , max_block_size(max_block_size_) - {} - - /// add single row from block size `granularity` - void addRowWithGranularity(size_t granularity) - { - sum_blocks_granularity += granularity; - sum_rows_count++; - } - - /// check that sum_rows_count is enough - bool checkStop() const; - - bool empty() const - { - return sum_blocks_granularity == 0; - } - }; - Block readImpl() override; void readSuffixImpl() override; @@ -87,7 +47,6 @@ protected: template void fetchNextBlock(const TSortCursor & current, SortingHeap & queue); - Block header; const SortDescription description; @@ -98,14 +57,13 @@ protected: bool first = true; bool has_collation = false; bool quiet = false; - bool average_block_sizes = false; /// May be smaller or equal to max_block_size. To do 'reserve' for columns. size_t expected_block_size = 0; /// Blocks currently being merged. size_t num_columns = 0; - std::vector source_blocks; + Blocks source_blocks; SortCursorImpls cursors; @@ -116,58 +74,6 @@ protected: /// If it is not nullptr then it should be populated during execution WriteBuffer * out_row_sources_buf; - - /// These methods are used in Collapsing/Summing/Aggregating... SortedBlockInputStream-s. - - /// Save the row pointed to by cursor in `row`. - template - void setRow(Row & row, TSortCursor & cursor) - { - for (size_t i = 0; i < num_columns; ++i) - { - try - { - cursor->all_columns[i]->get(cursor->pos, row[i]); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - - /// Find out the name of the column and throw more informative exception. - - String column_name; - for (const auto & block : source_blocks) - { - if (i < block->columns()) - { - column_name = block->safeGetByPosition(i).name; - break; - } - } - - throw Exception("MergingSortedBlockInputStream failed to read row " + toString(cursor->pos) - + " of column " + toString(i) + (column_name.empty() ? "" : " (" + column_name + ")"), - ErrorCodes::CORRUPTED_DATA); - } - } - } - - template - void setRowRef(SharedBlockRowRef & row_ref, TSortCursor & cursor) - { - row_ref.row_num = cursor.impl->pos; - row_ref.shared_block = source_blocks[cursor.impl->order]; - row_ref.columns = &row_ref.shared_block->all_columns; - } - - template - void setPrimaryKeyRef(SharedBlockRowRef & row_ref, TSortCursor & cursor) - { - row_ref.row_num = cursor.impl->pos; - row_ref.shared_block = source_blocks[cursor.impl->order]; - row_ref.columns = &row_ref.shared_block->sort_columns; - } - private: /** We support two different cursors - with Collation and without. diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 991d206777a..6bab4aba1f3 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -25,7 +25,8 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( * Although now any insertion into the table is done via PushingToViewsBlockOutputStream, * but it's clear that here is not the best place for this functionality. */ - addTableLock(storage->lockStructureForShare(true, context.getInitialQueryId())); + addTableLock( + storage->lockStructureForShare(true, context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout)); /// If the "root" table deduplactes blocks, there are no need to make deduplication for children /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks @@ -54,7 +55,9 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( if (auto * materialized_view = dynamic_cast(dependent_table.get())) { - addTableLock(materialized_view->lockStructureForShare(true, context.getInitialQueryId())); + addTableLock( + materialized_view->lockStructureForShare( + true, context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout)); StoragePtr inner_table = materialized_view->getTargetTable(); auto inner_table_id = inner_table->getStorageID(); @@ -247,7 +250,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n /// and two-level aggregation is triggered). in = std::make_shared( in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); - in = std::make_shared(context, in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Name); + in = std::make_shared(in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Name); } else in = std::make_shared(block); diff --git a/src/DataStreams/RemoteBlockInputStream.cpp b/src/DataStreams/RemoteBlockInputStream.cpp index 9d9f629d463..fc1578d1749 100644 --- a/src/DataStreams/RemoteBlockInputStream.cpp +++ b/src/DataStreams/RemoteBlockInputStream.cpp @@ -177,7 +177,7 @@ void RemoteBlockInputStream::sendExternalTables() /** If we receive a block with slightly different column types, or with excessive columns, * we will adapt it to expected structure. */ -static Block adaptBlockStructure(const Block & block, const Block & header, const Context & context) +static Block adaptBlockStructure(const Block & block, const Block & header) { /// Special case when reader doesn't care about result structure. Deprecated and used only in Benchmark, PerformanceTest. if (!header) @@ -204,7 +204,7 @@ static Block adaptBlockStructure(const Block & block, const Block & header, cons auto col = block.getByName(elem.name); col.column = block.getByName(elem.name).column->cut(0, 1); - column = castColumn(col, elem.type, context); + column = castColumn(col, elem.type); if (!isColumnConst(*column)) column = ColumnConst::create(column, block.rows()); @@ -216,7 +216,7 @@ static Block adaptBlockStructure(const Block & block, const Block & header, cons column = elem.column->cloneResized(block.rows()); } else - column = castColumn(block.getByName(elem.name), elem.type, context); + column = castColumn(block.getByName(elem.name), elem.type); res.insert({column, elem.type, elem.name}); } @@ -246,7 +246,7 @@ Block RemoteBlockInputStream::readImpl() case Protocol::Server::Data: /// If the block is not empty and is not a header block if (packet.block && (packet.block.rows() > 0)) - return adaptBlockStructure(packet.block, header, context); + return adaptBlockStructure(packet.block, header); break; /// If the block is empty - we will receive other packets before EndOfStream. case Protocol::Server::Exception: @@ -359,12 +359,17 @@ void RemoteBlockInputStream::sendQuery() void RemoteBlockInputStream::tryCancel(const char * reason) { - bool old_val = false; - if (!was_cancelled.compare_exchange_strong(old_val, true, std::memory_order_seq_cst, std::memory_order_relaxed)) - return; + { + std::lock_guard guard(was_cancelled_mutex); + + if (was_cancelled) + return; + + was_cancelled = true; + multiplexed_connections->sendCancel(); + } LOG_TRACE(log, "(" << multiplexed_connections->dumpAddresses() << ") " << reason); - multiplexed_connections->sendCancel(); } bool RemoteBlockInputStream::isQueryPending() const diff --git a/src/DataStreams/RemoteBlockInputStream.h b/src/DataStreams/RemoteBlockInputStream.h index 783811f2521..66b1ebbb6c3 100644 --- a/src/DataStreams/RemoteBlockInputStream.h +++ b/src/DataStreams/RemoteBlockInputStream.h @@ -135,7 +135,8 @@ private: * - data size is already satisfactory (when using LIMIT, for example) * - an exception was thrown from client side */ - std::atomic was_cancelled { false }; + bool was_cancelled { false }; + std::mutex was_cancelled_mutex; /** An exception from replica was received. No need in receiving more packets or * requesting to cancel query execution diff --git a/src/DataStreams/ReplacingSortedBlockInputStream.cpp b/src/DataStreams/ReplacingSortedBlockInputStream.cpp deleted file mode 100644 index 967b4ebb046..00000000000 --- a/src/DataStreams/ReplacingSortedBlockInputStream.cpp +++ /dev/null @@ -1,116 +0,0 @@ -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - - -void ReplacingSortedBlockInputStream::insertRow(MutableColumns & merged_columns) -{ - if (out_row_sources_buf) - { - /// true flag value means "skip row" - current_row_sources[max_pos].setSkipFlag(false); - - out_row_sources_buf->write(reinterpret_cast(current_row_sources.data()), - current_row_sources.size() * sizeof(RowSourcePart)); - current_row_sources.resize(0); - } - - for (size_t i = 0; i < num_columns; ++i) - merged_columns[i]->insertFrom(*(*selected_row.columns)[i], selected_row.row_num); -} - - -Block ReplacingSortedBlockInputStream::readImpl() -{ - if (finished) - return Block(); - - MutableColumns merged_columns; - init(merged_columns); - - if (has_collation) - throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR); - - if (merged_columns.empty()) - return Block(); - - merge(merged_columns, queue_without_collation); - return header.cloneWithColumns(std::move(merged_columns)); -} - - -void ReplacingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap & queue) -{ - MergeStopCondition stop_condition(average_block_sizes, max_block_size); - - /// Take the rows in needed order and put them into `merged_columns` until rows no more than `max_block_size` - while (queue.isValid()) - { - SortCursor current = queue.current(); - size_t current_block_granularity = current->rows; - - if (current_key.empty()) - setPrimaryKeyRef(current_key, current); - - setPrimaryKeyRef(next_key, current); - - bool key_differs = next_key != current_key; - - /// if there are enough rows and the last one is calculated completely - if (key_differs && stop_condition.checkStop()) - return; - - if (key_differs) - { - /// Write the data for the previous primary key. - insertRow(merged_columns); - stop_condition.addRowWithGranularity(current_block_granularity); - selected_row.reset(); - current_key.swap(next_key); - } - - /// Initially, skip all rows. Unskip last on insert. - size_t current_pos = current_row_sources.size(); - if (out_row_sources_buf) - current_row_sources.emplace_back(current.impl->order, true); - - /// A non-strict comparison, since we select the last row for the same version values. - if (version_column_number == -1 - || selected_row.empty() - || current->all_columns[version_column_number]->compareAt( - current->pos, selected_row.row_num, - *(*selected_row.columns)[version_column_number], - /* nan_direction_hint = */ 1) >= 0) - { - max_pos = current_pos; - setRowRef(selected_row, current); - } - - if (!current->isLast()) - { - queue.next(); - } - else - { - /// We get the next block from the corresponding source, if there is one. - fetchNextBlock(current, queue); - } - } - - /// We will write the data for the last primary key. - if (!selected_row.empty()) - insertRow(merged_columns); - - finished = true; -} - -} diff --git a/src/DataStreams/ReplacingSortedBlockInputStream.h b/src/DataStreams/ReplacingSortedBlockInputStream.h deleted file mode 100644 index 22920c2eb20..00000000000 --- a/src/DataStreams/ReplacingSortedBlockInputStream.h +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include - -#include -#include - - -namespace DB -{ - -/** Merges several sorted streams into one. - * For each group of consecutive identical values of the primary key (the columns by which the data is sorted), - * keeps row with max `version` value. - */ -class ReplacingSortedBlockInputStream : public MergingSortedBlockInputStream -{ -public: - ReplacingSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, - const String & version_column, size_t max_block_size_, WriteBuffer * out_row_sources_buf_ = nullptr, - bool average_block_sizes_ = false) - : MergingSortedBlockInputStream(inputs_, description_, max_block_size_, 0, out_row_sources_buf_, false, average_block_sizes_) - { - if (!version_column.empty()) - version_column_number = header.getPositionByName(version_column); - } - - String getName() const override { return "ReplacingSorted"; } - -protected: - /// Can return 1 more records than max_block_size. - Block readImpl() override; - -private: - ssize_t version_column_number = -1; - - Logger * log = &Logger::get("ReplacingSortedBlockInputStream"); - - /// All data has been read. - bool finished = false; - - /// Primary key of current row. - SharedBlockRowRef current_key; - /// Primary key of next row. - SharedBlockRowRef next_key; - /// Last row with maximum version for current primary key. - SharedBlockRowRef selected_row; - /// The position (into current_row_sources) of the row with the highest version. - size_t max_pos = 0; - - /// Sources of rows with the current primary key. - PODArray current_row_sources; - - void merge(MutableColumns & merged_columns, SortingHeap & queue); - - /// Output into result the rows for current primary key. - void insertRow(MutableColumns & merged_columns); -}; - -} diff --git a/src/DataStreams/SquashingBlockInputStream.cpp b/src/DataStreams/SquashingBlockInputStream.cpp index 12f6f23583f..e13dee37008 100644 --- a/src/DataStreams/SquashingBlockInputStream.cpp +++ b/src/DataStreams/SquashingBlockInputStream.cpp @@ -14,23 +14,19 @@ SquashingBlockInputStream::SquashingBlockInputStream( Block SquashingBlockInputStream::readImpl() { - if (all_read) - return {}; - - while (true) + while (!all_read) { Block block = children[0]->read(); if (!block) all_read = true; - SquashingTransform::Result result = transform.add(block.mutateColumns()); - if (result.ready) + auto squashed_block = transform.add(std::move(block)); + if (squashed_block) { - if (result.columns.empty()) - return {}; - return header.cloneWithColumns(std::move(result.columns)); + return squashed_block; } } + return {}; } } diff --git a/src/DataStreams/SquashingBlockOutputStream.cpp b/src/DataStreams/SquashingBlockOutputStream.cpp index 5d0638c1c57..ab12f66590f 100644 --- a/src/DataStreams/SquashingBlockOutputStream.cpp +++ b/src/DataStreams/SquashingBlockOutputStream.cpp @@ -12,9 +12,9 @@ SquashingBlockOutputStream::SquashingBlockOutputStream(BlockOutputStreamPtr dst, void SquashingBlockOutputStream::write(const Block & block) { - SquashingTransform::Result result = transform.add(Block(block).mutateColumns()); - if (result.ready) - output->write(header.cloneWithColumns(std::move(result.columns))); + auto squashed_block = transform.add(block); + if (squashed_block) + output->write(squashed_block); } @@ -25,9 +25,9 @@ void SquashingBlockOutputStream::finalize() all_written = true; - SquashingTransform::Result result = transform.add({}); - if (result.ready && !result.columns.empty()) - output->write(header.cloneWithColumns(std::move(result.columns))); + auto squashed_block = transform.add({}); + if (squashed_block) + output->write(squashed_block); } diff --git a/src/DataStreams/SquashingTransform.cpp b/src/DataStreams/SquashingTransform.cpp index 8ade6f5a926..8cbbebb75ab 100644 --- a/src/DataStreams/SquashingTransform.cpp +++ b/src/DataStreams/SquashingTransform.cpp @@ -15,71 +15,106 @@ SquashingTransform::SquashingTransform(size_t min_block_size_rows_, size_t min_b { } +Block SquashingTransform::add(Block && input_block) +{ + return addImpl(std::move(input_block)); +} -SquashingTransform::Result SquashingTransform::add(MutableColumns && columns) +Block SquashingTransform::add(const Block & input_block) +{ + return addImpl(input_block); +} + +/* + * To minimize copying, accept two types of argument: const reference for output + * stream, and rvalue reference for input stream, and decide whether to copy + * inside this function. This allows us not to copy Block unless we absolutely + * have to. + */ +template +Block SquashingTransform::addImpl(ReferenceType input_block) { /// End of input stream. - if (columns.empty()) - return Result(std::move(accumulated_columns)); + if (!input_block) + { + Block to_return; + std::swap(to_return, accumulated_block); + return to_return; + } /// Just read block is already enough. - if (isEnoughSize(columns)) + if (isEnoughSize(input_block)) { /// If no accumulated data, return just read block. - if (accumulated_columns.empty()) - return Result(std::move(columns)); + if (!accumulated_block) + { + return std::move(input_block); + } /// Return accumulated data (maybe it has small size) and place new block to accumulated data. - columns.swap(accumulated_columns); - return Result(std::move(columns)); + Block to_return = std::move(input_block); + std::swap(to_return, accumulated_block); + return to_return; } /// Accumulated block is already enough. - if (!accumulated_columns.empty() && isEnoughSize(accumulated_columns)) + if (isEnoughSize(accumulated_block)) { /// Return accumulated data and place new block to accumulated data. - columns.swap(accumulated_columns); - return Result(std::move(columns)); + Block to_return = std::move(input_block); + std::swap(to_return, accumulated_block); + return to_return; } - append(std::move(columns)); + append(std::move(input_block)); - if (isEnoughSize(accumulated_columns)) + if (isEnoughSize(accumulated_block)) { - MutableColumns res; - res.swap(accumulated_columns); - return Result(std::move(res)); + Block to_return; + std::swap(to_return, accumulated_block); + return to_return; } /// Squashed block is not ready. - return false; + return {}; } -void SquashingTransform::append(MutableColumns && columns) +template +void SquashingTransform::append(ReferenceType input_block) { - if (accumulated_columns.empty()) + if (!accumulated_block) { - accumulated_columns = std::move(columns); + accumulated_block = std::move(input_block); return; } - for (size_t i = 0, size = columns.size(); i < size; ++i) + assert(blocksHaveEqualStructure(input_block, accumulated_block)); + + for (size_t i = 0, size = accumulated_block.columns(); i < size; ++i) { - auto & column = accumulated_columns[i]; + const auto source_column = input_block.getByPosition(i).column; + + auto mutable_column = (*std::move( + accumulated_block.getByPosition(i).column)).mutate(); + if (reserve_memory) - column->reserve(min_block_size_bytes); - column->insertRangeFrom(*columns[i], 0, columns[i]->size()); + { + mutable_column->reserve(min_block_size_bytes); + } + mutable_column->insertRangeFrom(*source_column, 0, source_column->size()); + + accumulated_block.getByPosition(i).column = std::move(mutable_column); } } -bool SquashingTransform::isEnoughSize(const MutableColumns & columns) +bool SquashingTransform::isEnoughSize(const Block & block) { size_t rows = 0; size_t bytes = 0; - for (const auto & column : columns) + for (const auto & [column, type, name] : block) { if (!rows) rows = column->size(); diff --git a/src/DataStreams/SquashingTransform.h b/src/DataStreams/SquashingTransform.h index 836423dc90a..c26c3bd884a 100644 --- a/src/DataStreams/SquashingTransform.h +++ b/src/DataStreams/SquashingTransform.h @@ -25,31 +25,26 @@ public: /// Conditions on rows and bytes are OR-ed. If one of them is zero, then corresponding condition is ignored. SquashingTransform(size_t min_block_size_rows_, size_t min_block_size_bytes_, bool reserve_memory_ = false); - /// When not ready, you need to pass more blocks to add function. - struct Result - { - bool ready = false; - MutableColumns columns; - - Result(bool ready_) : ready(ready_) {} - Result(MutableColumns && columns_) : ready(true), columns(std::move(columns_)) {} - }; - /** Add next block and possibly returns squashed block. * At end, you need to pass empty block. As the result for last (empty) block, you will get last Result with ready = true. */ - Result add(MutableColumns && columns); + Block add(Block && block); + Block add(const Block & block); private: size_t min_block_size_rows; size_t min_block_size_bytes; bool reserve_memory; - MutableColumns accumulated_columns; + Block accumulated_block; - void append(MutableColumns && columns); + template + Block addImpl(ReferenceType block); - bool isEnoughSize(const MutableColumns & columns); + template + void append(ReferenceType block); + + bool isEnoughSize(const Block & block); bool isEnoughSize(size_t rows, size_t bytes) const; }; diff --git a/src/DataStreams/SummingSortedBlockInputStream.cpp b/src/DataStreams/SummingSortedBlockInputStream.cpp deleted file mode 100644 index ed5b1b820b4..00000000000 --- a/src/DataStreams/SummingSortedBlockInputStream.cpp +++ /dev/null @@ -1,522 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - - -namespace -{ - bool isInPrimaryKey(const SortDescription & description, const std::string & name, const size_t number) - { - for (auto & desc : description) - if (desc.column_name == name || (desc.column_name.empty() && desc.column_number == number)) - return true; - - return false; - } - - /// Returns true if merge result is not empty - bool mergeMap(const SummingSortedBlockInputStream::MapDescription & desc, Row & row, SortCursor & cursor) - { - /// Strongly non-optimal. - - Row & left = row; - Row right(left.size()); - - for (size_t col_num : desc.key_col_nums) - right[col_num] = (*cursor->all_columns[col_num])[cursor->pos].template get(); - - for (size_t col_num : desc.val_col_nums) - right[col_num] = (*cursor->all_columns[col_num])[cursor->pos].template get(); - - auto at_ith_column_jth_row = [&](const Row & matrix, size_t i, size_t j) -> const Field & - { - return matrix[i].get()[j]; - }; - - auto tuple_of_nth_columns_at_jth_row = [&](const Row & matrix, const ColumnNumbers & col_nums, size_t j) -> Array - { - size_t size = col_nums.size(); - Array res(size); - for (size_t col_num_index = 0; col_num_index < size; ++col_num_index) - res[col_num_index] = at_ith_column_jth_row(matrix, col_nums[col_num_index], j); - return res; - }; - - std::map merged; - - auto accumulate = [](Array & dst, const Array & src) - { - bool has_non_zero = false; - size_t size = dst.size(); - for (size_t i = 0; i < size; ++i) - if (applyVisitor(FieldVisitorSum(src[i]), dst[i])) - has_non_zero = true; - return has_non_zero; - }; - - auto merge = [&](const Row & matrix) - { - size_t rows = matrix[desc.key_col_nums[0]].get().size(); - - for (size_t j = 0; j < rows; ++j) - { - Array key = tuple_of_nth_columns_at_jth_row(matrix, desc.key_col_nums, j); - Array value = tuple_of_nth_columns_at_jth_row(matrix, desc.val_col_nums, j); - - auto it = merged.find(key); - if (merged.end() == it) - merged.emplace(std::move(key), std::move(value)); - else - { - if (!accumulate(it->second, value)) - merged.erase(it); - } - } - }; - - merge(left); - merge(right); - - for (size_t col_num : desc.key_col_nums) - row[col_num] = Array(merged.size()); - for (size_t col_num : desc.val_col_nums) - row[col_num] = Array(merged.size()); - - size_t row_num = 0; - for (const auto & key_value : merged) - { - for (size_t col_num_index = 0, size = desc.key_col_nums.size(); col_num_index < size; ++col_num_index) - row[desc.key_col_nums[col_num_index]].get()[row_num] = key_value.first[col_num_index]; - - for (size_t col_num_index = 0, size = desc.val_col_nums.size(); col_num_index < size; ++col_num_index) - row[desc.val_col_nums[col_num_index]].get()[row_num] = key_value.second[col_num_index]; - - ++row_num; - } - - return row_num != 0; - } -} - - -SummingSortedBlockInputStream::SummingSortedBlockInputStream( - const BlockInputStreams & inputs_, - const SortDescription & description_, - /// List of columns to be summed. If empty, all numeric columns that are not in the description are taken. - const Names & column_names_to_sum, - size_t max_block_size_) - : MergingSortedBlockInputStream(inputs_, description_, max_block_size_), - log(&Logger::get("SummingSortedBlockInputStream")) -{ - current_row.resize(num_columns); - - /// name of nested structure -> the column numbers that refer to it. - std::unordered_map> discovered_maps; - - /** Fill in the column numbers, which must be summed. - * This can only be numeric columns that are not part of the sort key. - * If a non-empty column_names_to_sum is specified, then we only take these columns. - * Some columns from column_names_to_sum may not be found. This is ignored. - */ - for (size_t i = 0; i < num_columns; ++i) - { - const ColumnWithTypeAndName & column = header.safeGetByPosition(i); - - /// Discover nested Maps and find columns for summation - if (typeid_cast(column.type.get())) - { - const auto map_name = Nested::extractTableName(column.name); - /// if nested table name ends with `Map` it is a possible candidate for special handling - if (map_name == column.name || !endsWith(map_name, "Map")) - { - column_numbers_not_to_aggregate.push_back(i); - continue; - } - - discovered_maps[map_name].emplace_back(i); - } - else - { - bool is_agg_func = WhichDataType(column.type).isAggregateFunction(); - - /// There are special const columns for example after prewere sections. - if ((!column.type->isSummable() && !is_agg_func) || isColumnConst(*column.column)) - { - column_numbers_not_to_aggregate.push_back(i); - continue; - } - - /// Are they inside the PK? - if (isInPrimaryKey(description, column.name, i)) - { - column_numbers_not_to_aggregate.push_back(i); - continue; - } - - if (column_names_to_sum.empty() - || column_names_to_sum.end() != - std::find(column_names_to_sum.begin(), column_names_to_sum.end(), column.name)) - { - // Create aggregator to sum this column - AggregateDescription desc; - desc.is_agg_func_type = is_agg_func; - desc.column_numbers = {i}; - - if (!is_agg_func) - { - desc.init("sumWithOverflow", {column.type}); - } - - columns_to_aggregate.emplace_back(std::move(desc)); - } - else - { - // Column is not going to be summed, use last value - column_numbers_not_to_aggregate.push_back(i); - } - } - } - - /// select actual nested Maps from list of candidates - for (const auto & map : discovered_maps) - { - /// map should contain at least two elements (key -> value) - if (map.second.size() < 2) - { - for (auto col : map.second) - column_numbers_not_to_aggregate.push_back(col); - continue; - } - - /// no elements of map could be in primary key - auto column_num_it = map.second.begin(); - for (; column_num_it != map.second.end(); ++column_num_it) - if (isInPrimaryKey(description, header.safeGetByPosition(*column_num_it).name, *column_num_it)) - break; - if (column_num_it != map.second.end()) - { - for (auto col : map.second) - column_numbers_not_to_aggregate.push_back(col); - continue; - } - - DataTypes argument_types; - AggregateDescription desc; - MapDescription map_desc; - - column_num_it = map.second.begin(); - for (; column_num_it != map.second.end(); ++column_num_it) - { - const ColumnWithTypeAndName & key_col = header.safeGetByPosition(*column_num_it); - const String & name = key_col.name; - const IDataType & nested_type = *static_cast(key_col.type.get())->getNestedType(); - - if (column_num_it == map.second.begin() - || endsWith(name, "ID") - || endsWith(name, "Key") - || endsWith(name, "Type")) - { - if (!nested_type.isValueRepresentedByInteger() && !isStringOrFixedString(nested_type)) - break; - - map_desc.key_col_nums.push_back(*column_num_it); - } - else - { - if (!nested_type.isSummable()) - break; - - map_desc.val_col_nums.push_back(*column_num_it); - } - - // Add column to function arguments - desc.column_numbers.push_back(*column_num_it); - argument_types.push_back(key_col.type); - } - - if (column_num_it != map.second.end()) - { - for (auto col : map.second) - column_numbers_not_to_aggregate.push_back(col); - continue; - } - - if (map_desc.key_col_nums.size() == 1) - { - // Create summation for all value columns in the map - desc.init("sumMapWithOverflow", argument_types); - columns_to_aggregate.emplace_back(std::move(desc)); - } - else - { - // Fall back to legacy mergeMaps for composite keys - for (auto col : map.second) - column_numbers_not_to_aggregate.push_back(col); - maps_to_sum.emplace_back(std::move(map_desc)); - } - } -} - - -void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & merged_columns) -{ - /// We have nothing to aggregate. It means that it could be non-zero, because we have columns_not_to_aggregate. - if (columns_to_aggregate.empty()) - current_row_is_zero = false; - - for (auto & desc : columns_to_aggregate) - { - // Do not insert if the aggregation state hasn't been created - if (desc.created) - { - if (desc.is_agg_func_type) - { - current_row_is_zero = false; - } - else - { - try - { - desc.function->insertResultInto(desc.state.data(), *desc.merged_column); - - /// Update zero status of current row - if (desc.column_numbers.size() == 1) - { - // Flag row as non-empty if at least one column number if non-zero - current_row_is_zero = current_row_is_zero && desc.merged_column->isDefaultAt(desc.merged_column->size() - 1); - } - else - { - /// It is sumMapWithOverflow aggregate function. - /// Assume that the row isn't empty in this case (just because it is compatible with previous version) - current_row_is_zero = false; - } - } - catch (...) - { - desc.destroyState(); - throw; - } - } - desc.destroyState(); - } - else - desc.merged_column->insertDefault(); - } - - /// If it is "zero" row, then rollback the insertion - /// (at this moment we need rollback only cols from columns_to_aggregate) - if (current_row_is_zero) - { - for (auto & desc : columns_to_aggregate) - desc.merged_column->popBack(1); - - return; - } - - for (auto i : column_numbers_not_to_aggregate) - merged_columns[i]->insert(current_row[i]); - - /// Update per-block and per-group flags - ++merged_rows; -} - - -Block SummingSortedBlockInputStream::readImpl() -{ - if (finished) - return Block(); - - MutableColumns merged_columns; - init(merged_columns); - - if (has_collation) - throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR); - - if (merged_columns.empty()) - return {}; - - /// Update aggregation result columns for current block - for (auto & desc : columns_to_aggregate) - { - // Wrap aggregated columns in a tuple to match function signature - if (!desc.is_agg_func_type && isTuple(desc.function->getReturnType())) - { - size_t tuple_size = desc.column_numbers.size(); - MutableColumns tuple_columns(tuple_size); - for (size_t i = 0; i < tuple_size; ++i) - tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column->cloneEmpty(); - - desc.merged_column = ColumnTuple::create(std::move(tuple_columns)); - } - else - desc.merged_column = header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty(); - } - - merge(merged_columns, queue_without_collation); - Block res = header.cloneWithColumns(std::move(merged_columns)); - - /// Place aggregation results into block. - for (auto & desc : columns_to_aggregate) - { - if (!desc.is_agg_func_type && isTuple(desc.function->getReturnType())) - { - /// Unpack tuple into block. - size_t tuple_size = desc.column_numbers.size(); - for (size_t i = 0; i < tuple_size; ++i) - res.getByPosition(desc.column_numbers[i]).column = assert_cast(*desc.merged_column).getColumnPtr(i); - } - else - res.getByPosition(desc.column_numbers[0]).column = std::move(desc.merged_column); - } - - return res; -} - - -void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap & queue) -{ - merged_rows = 0; - - /// Take the rows in needed order and put them in `merged_columns` until rows no more than `max_block_size` - while (queue.isValid()) - { - SortCursor current = queue.current(); - - setPrimaryKeyRef(next_key, current); - - bool key_differs; - - if (current_key.empty()) /// The first key encountered. - { - key_differs = true; - current_row_is_zero = true; - } - else - key_differs = next_key != current_key; - - if (key_differs) - { - if (!current_key.empty()) - /// Write the data for the previous group. - insertCurrentRowIfNeeded(merged_columns); - - if (merged_rows >= max_block_size) - { - /// The block is now full and the last row is calculated completely. - current_key.reset(); - return; - } - - current_key.swap(next_key); - - setRow(current_row, current); - - /// Reset aggregation states for next row - for (auto & desc : columns_to_aggregate) - desc.createState(); - - // Start aggregations with current row - addRow(current); - - if (maps_to_sum.empty()) - { - /// We have only columns_to_aggregate. The status of current row will be determined - /// in 'insertCurrentRowIfNeeded' method on the values of aggregate functions. - current_row_is_zero = true; // NOLINT - } - else - { - /// We have complex maps that will be summed with 'mergeMap' method. - /// The single row is considered non zero, and the status after merging with other rows - /// will be determined in the branch below (when key_differs == false). - current_row_is_zero = false; // NOLINT - } - } - else - { - addRow(current); - - // Merge maps only for same rows - for (const auto & desc : maps_to_sum) - if (mergeMap(desc, current_row, current)) - current_row_is_zero = false; - } - - if (!current->isLast()) - { - queue.next(); - } - else - { - /// We get the next block from the corresponding source, if there is one. - fetchNextBlock(current, queue); - } - } - - /// We will write the data for the last group, if it is non-zero. - /// If it is zero, and without it the output stream will be empty, we will write it anyway. - insertCurrentRowIfNeeded(merged_columns); - finished = true; -} - - -void SummingSortedBlockInputStream::addRow(SortCursor & cursor) -{ - for (auto & desc : columns_to_aggregate) - { - if (!desc.created) - throw Exception("Logical error in SummingSortedBlockInputStream, there are no description", ErrorCodes::LOGICAL_ERROR); - - if (desc.is_agg_func_type) - { - // desc.state is not used for AggregateFunction types - auto & col = cursor->all_columns[desc.column_numbers[0]]; - assert_cast(*desc.merged_column).insertMergeFrom(*col, cursor->pos); - } - else - { - // Specialized case for unary functions - if (desc.column_numbers.size() == 1) - { - auto & col = cursor->all_columns[desc.column_numbers[0]]; - desc.add_function(desc.function.get(), desc.state.data(), &col, cursor->pos, nullptr); - } - else - { - // Gather all source columns into a vector - ColumnRawPtrs columns(desc.column_numbers.size()); - for (size_t i = 0; i < desc.column_numbers.size(); ++i) - columns[i] = cursor->all_columns[desc.column_numbers[i]]; - - desc.add_function(desc.function.get(), desc.state.data(), columns.data(), cursor->pos, nullptr); - } - } - } -} - -} diff --git a/src/DataStreams/SummingSortedBlockInputStream.h b/src/DataStreams/SummingSortedBlockInputStream.h deleted file mode 100644 index 09bced85b1d..00000000000 --- a/src/DataStreams/SummingSortedBlockInputStream.h +++ /dev/null @@ -1,155 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - - -namespace Poco { class Logger; } - -namespace DB -{ - -namespace ErrorCodes -{ -} - - -/** Merges several sorted streams into one. - * For each group of consecutive identical values of the primary key (the columns by which the data is sorted), - * collapses them into one row, summing all the numeric columns except the primary key. - * If in all numeric columns, except for the primary key, the result is zero, it deletes the row. - */ -class SummingSortedBlockInputStream : public MergingSortedBlockInputStream -{ -public: - SummingSortedBlockInputStream( - const BlockInputStreams & inputs_, - const SortDescription & description_, - /// List of columns to be summed. If empty, all numeric columns that are not in the description are taken. - const Names & column_names_to_sum_, - size_t max_block_size_); - - String getName() const override { return "SummingSorted"; } - - /// Stores numbers of key-columns and value-columns. - struct MapDescription - { - std::vector key_col_nums; - std::vector val_col_nums; - }; - -protected: - /// Can return 1 more records than max_block_size. - Block readImpl() override; - -private: - Poco::Logger * log; - - /// Read up to the end. - bool finished = false; - - /// Columns with which values should be summed. - ColumnNumbers column_numbers_not_to_aggregate; - - /** A table can have nested tables that are treated in a special way. - * If the name of the nested table ends in `Map` and it contains at least two columns, - * satisfying the following criteria: - * - the first column, as well as all columns whose names end with `ID`, `Key` or `Type` - numeric ((U)IntN, Date, DateTime); - * (a tuple of such columns will be called `keys`) - * - the remaining columns are arithmetic ((U)IntN, Float32/64), called (`values`...). - * This nested table is treated as a mapping (keys...) => (values...) and when merge - * its rows, the merge of the elements of two sets by (keys...) with summing of corresponding (values...). - * - * Example: - * [(1, 100)] + [(2, 150)] -> [(1, 100), (2, 150)] - * [(1, 100)] + [(1, 150)] -> [(1, 250)] - * [(1, 100)] + [(1, 150), (2, 150)] -> [(1, 250), (2, 150)] - * [(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] - * - * This very unusual functionality is made exclusively for the banner system, - * is not supposed for use by anyone else, - * and can be deleted at any time. - */ - - /// Stores aggregation function, state, and columns to be used as function arguments - struct AggregateDescription - { - /// An aggregate function 'sumWithOverflow' or 'sumMapWithOverflow' for summing. - AggregateFunctionPtr function; - IAggregateFunction::AddFunc add_function = nullptr; - std::vector column_numbers; - MutableColumnPtr merged_column; - AlignedBuffer state; - bool created = false; - - /// In case when column has type AggregateFunction: use the aggregate function from itself instead of 'function' above. - bool is_agg_func_type = false; - - void init(const char * function_name, const DataTypes & argument_types) - { - function = AggregateFunctionFactory::instance().get(function_name, argument_types); - add_function = function->getAddressOfAddFunction(); - state.reset(function->sizeOfData(), function->alignOfData()); - } - - void createState() - { - if (created) - return; - if (is_agg_func_type) - merged_column->insertDefault(); - else - function->create(state.data()); - created = true; - } - - void destroyState() - { - if (!created) - return; - if (!is_agg_func_type) - function->destroy(state.data()); - created = false; - } - - /// Explicitly destroy aggregation state if the stream is terminated - ~AggregateDescription() - { - destroyState(); - } - - AggregateDescription() = default; - AggregateDescription(AggregateDescription &&) = default; - AggregateDescription(const AggregateDescription &) = delete; - }; - - std::vector columns_to_aggregate; - std::vector maps_to_sum; - - SharedBlockRowRef current_key; /// The current primary key. - SharedBlockRowRef next_key; /// The primary key of the next row. - - Row current_row; - bool current_row_is_zero = true; /// Are all summed columns zero (or empty)? It is updated incrementally. - - size_t merged_rows = 0; /// Number of rows merged into current result block - - /** We support two different cursors - with Collation and without. - * Templates are used instead of polymorphic SortCursor and calls to virtual functions. - */ - void merge(MutableColumns & merged_columns, SortingHeap & queue); - - /// Insert the summed row for the current group into the result and updates some of per-block flags if the row is not "zero". - void insertCurrentRowIfNeeded(MutableColumns & merged_columns); - - // Add the row under the cursor to the `row`. - void addRow(SortCursor & cursor); -}; - -} diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp index 5279ebca948..9f1771922eb 100644 --- a/src/DataStreams/TTLBlockInputStream.cpp +++ b/src/DataStreams/TTLBlockInputStream.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -62,7 +63,7 @@ TTLBlockInputStream::TTLBlockInputStream( } } -bool TTLBlockInputStream::isTTLExpired(time_t ttl) +bool TTLBlockInputStream::isTTLExpired(time_t ttl) const { return (ttl && (ttl <= current_time)); } diff --git a/src/DataStreams/TTLBlockInputStream.h b/src/DataStreams/TTLBlockInputStream.h index b2dd4ce901b..c071676b826 100644 --- a/src/DataStreams/TTLBlockInputStream.h +++ b/src/DataStreams/TTLBlockInputStream.h @@ -62,7 +62,7 @@ private: void updateMovesTTL(Block & block); UInt32 getTimestampByIndex(const IColumn * column, size_t ind); - bool isTTLExpired(time_t ttl); + bool isTTLExpired(time_t ttl) const; }; } diff --git a/src/DataStreams/VersionedCollapsingSortedBlockInputStream.cpp b/src/DataStreams/VersionedCollapsingSortedBlockInputStream.cpp deleted file mode 100644 index 30fa0460ac5..00000000000 --- a/src/DataStreams/VersionedCollapsingSortedBlockInputStream.cpp +++ /dev/null @@ -1,181 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; -} - - -VersionedCollapsingSortedBlockInputStream::VersionedCollapsingSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, - const String & sign_column_, size_t max_block_size_, - WriteBuffer * out_row_sources_buf_, bool average_block_sizes_) - : MergingSortedBlockInputStream(inputs_, description_, max_block_size_, 0, out_row_sources_buf_, false, average_block_sizes_) - , max_rows_in_queue(std::min(std::max(3, max_block_size_), MAX_ROWS_IN_MULTIVERSION_QUEUE) - 2) - , current_keys(max_rows_in_queue + 1) -{ - sign_column_number = header.getPositionByName(sign_column_); -} - - -inline ALWAYS_INLINE static void writeRowSourcePart(WriteBuffer & buffer, RowSourcePart row_source) -{ - if constexpr (sizeof(RowSourcePart) == 1) - buffer.write(*reinterpret_cast(&row_source)); - else - buffer.write(reinterpret_cast(&row_source), sizeof(RowSourcePart)); -} - -void VersionedCollapsingSortedBlockInputStream::insertGap(size_t gap_size) -{ - if (out_row_sources_buf) - { - for (size_t i = 0; i < gap_size; ++i) - { - writeRowSourcePart(*out_row_sources_buf, current_row_sources.front()); - current_row_sources.pop(); - } - } -} - -void VersionedCollapsingSortedBlockInputStream::insertRow(size_t skip_rows, const SharedBlockRowRef & row, MutableColumns & merged_columns) -{ - const auto & columns = row.shared_block->all_columns; - for (size_t i = 0; i < num_columns; ++i) - merged_columns[i]->insertFrom(*columns[i], row.row_num); - - insertGap(skip_rows); - - if (out_row_sources_buf) - { - current_row_sources.front().setSkipFlag(false); - writeRowSourcePart(*out_row_sources_buf, current_row_sources.front()); - current_row_sources.pop(); - } -} - -Block VersionedCollapsingSortedBlockInputStream::readImpl() -{ - if (finished) - return {}; - - MutableColumns merged_columns; - init(merged_columns); - - if (has_collation) - throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::NOT_IMPLEMENTED); - - if (merged_columns.empty()) - return {}; - - merge(merged_columns, queue_without_collation); - return header.cloneWithColumns(std::move(merged_columns)); -} - - -void VersionedCollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap & queue) -{ - MergeStopCondition stop_condition(average_block_sizes, max_block_size); - - auto update_queue = [this, & queue](SortCursor & cursor) - { - if (out_row_sources_buf) - current_row_sources.emplace(cursor->order, true); - - if (!cursor->isLast()) - { - queue.next(); - } - else - { - /// We take next block from the corresponding source, if there is one. - fetchNextBlock(cursor, queue); - } - }; - - /// Take rows in correct order and put them into `merged_columns` until the rows no more than `max_block_size` - while (queue.isValid()) - { - SortCursor current = queue.current(); - size_t current_block_granularity = current->rows; - - SharedBlockRowRef next_key; - - Int8 sign = assert_cast(*current->all_columns[sign_column_number]).getData()[current->pos]; - - setPrimaryKeyRef(next_key, current); - - size_t rows_to_merge = 0; - - /// Each branch either updates queue or increases rows_to_merge. - if (current_keys.empty()) - { - sign_in_queue = sign; - current_keys.pushBack(next_key); - update_queue(current); - } - else - { - if (current_keys.back() == next_key) - { - update_queue(current); - - if (sign == sign_in_queue) - current_keys.pushBack(next_key); - else - { - current_keys.popBack(); - current_keys.pushGap(2); - } - } - else - rows_to_merge = current_keys.size(); - } - - if (current_keys.size() > max_rows_in_queue) - rows_to_merge = std::max(rows_to_merge, current_keys.size() - max_rows_in_queue); - - while (rows_to_merge) - { - const auto & row = current_keys.front(); - auto gap = current_keys.frontGap(); - - insertRow(gap, row, merged_columns); - - current_keys.popFront(); - - stop_condition.addRowWithGranularity(current_block_granularity); - --rows_to_merge; - - if (stop_condition.checkStop()) - { - ++blocks_written; - return; - } - } - } - - while (!current_keys.empty()) - { - const auto & row = current_keys.front(); - auto gap = current_keys.frontGap(); - - insertRow(gap, row, merged_columns); - - current_keys.popFront(); - } - - /// Write information about last collapsed rows. - insertGap(current_keys.frontGap()); - - finished = true; -} - -} diff --git a/src/DataStreams/processConstants.cpp b/src/DataStreams/processConstants.cpp index 6129c600bdf..b2eb2b723a6 100644 --- a/src/DataStreams/processConstants.cpp +++ b/src/DataStreams/processConstants.cpp @@ -27,7 +27,7 @@ void removeConstantsFromSortDescription(const Block & header, SortDescription & description.erase(std::remove_if(description.begin(), description.end(), [&](const SortColumnDescription & elem) { - auto & column = !elem.column_name.empty() ? header.getByName(elem.column_name) + const auto & column = !elem.column_name.empty() ? header.getByName(elem.column_name) : header.safeGetByPosition(elem.column_number); return column.column && isColumnConst(*column.column); }), description.end()); diff --git a/src/DataStreams/tests/collapsing_sorted_stream.cpp b/src/DataStreams/tests/collapsing_sorted_stream.cpp index a34d6a2fbd2..fd7dc11add6 100644 --- a/src/DataStreams/tests/collapsing_sorted_stream.cpp +++ b/src/DataStreams/tests/collapsing_sorted_stream.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -67,7 +66,8 @@ try //CollapsingSortedBlockInputStream collapsed(inputs, descr, "Sign", 1048576); CollapsingFinalBlockInputStream collapsed(inputs, descr, "Sign"); - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); WriteBufferFromFileDescriptor out_buf(STDERR_FILENO); BlockOutputStreamPtr output = context.getOutputFormat("TabSeparated", out_buf, block1); diff --git a/src/DataStreams/tests/expression_stream.cpp b/src/DataStreams/tests/expression_stream.cpp index bd4117f5aab..84b35cc2d3d 100644 --- a/src/DataStreams/tests/expression_stream.cpp +++ b/src/DataStreams/tests/expression_stream.cpp @@ -33,9 +33,10 @@ try std::string input = "SELECT number, number / 3, number * number"; ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); NamesAndTypesList source_columns = {{"number", std::make_shared()}}; diff --git a/src/DataStreams/tests/filter_stream.cpp b/src/DataStreams/tests/filter_stream.cpp index 5e324251440..8c481e1f258 100644 --- a/src/DataStreams/tests/filter_stream.cpp +++ b/src/DataStreams/tests/filter_stream.cpp @@ -35,12 +35,13 @@ try std::string input = "SELECT number, number % 3 == 1"; ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); formatAST(*ast, std::cerr); std::cerr << std::endl; - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); NamesAndTypesList source_columns = {{"number", std::make_shared()}}; diff --git a/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp b/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp index ff89cebd156..65070596951 100644 --- a/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp +++ b/src/DataStreams/tests/gtest_blocks_size_merging_streams.cpp @@ -1,10 +1,13 @@ #include #include #include -#include #include #include #include +#include +#include +#include +#include using namespace DB; @@ -27,25 +30,25 @@ static Block getBlockWithSize(const std::vector & columns, size_t r } -static BlockInputStreams getInputStreams(const std::vector & column_names, const std::vector> & block_sizes) +static Pipes getInputStreams(const std::vector & column_names, const std::vector> & block_sizes) { - BlockInputStreams result; + Pipes pipes; for (auto [block_size_in_bytes, blocks_count, stride] : block_sizes) { BlocksList blocks; size_t start = stride; while (blocks_count--) blocks.push_back(getBlockWithSize(column_names, block_size_in_bytes, stride, start)); - result.push_back(std::make_shared(std::move(blocks))); + pipes.emplace_back(std::make_shared(std::make_shared(std::move(blocks)))); } - return result; + return pipes; } -static BlockInputStreams getInputStreamsEqualStride(const std::vector & column_names, const std::vector> & block_sizes) +static Pipes getInputStreamsEqualStride(const std::vector & column_names, const std::vector> & block_sizes) { - BlockInputStreams result; + Pipes pipes; size_t i = 0; for (auto [block_size_in_bytes, blocks_count, stride] : block_sizes) { @@ -53,10 +56,10 @@ static BlockInputStreams getInputStreamsEqualStride(const std::vector(std::move(blocks))); + pipes.emplace_back(std::make_shared(std::make_shared(std::move(blocks)))); i++; } - return result; + return pipes; } @@ -75,20 +78,23 @@ TEST(MergingSortedTest, SimpleBlockSizeTest) { std::vector key_columns{"K1", "K2", "K3"}; auto sort_description = getSortDescription(key_columns); - auto streams = getInputStreams(key_columns, {{5, 1, 1}, {10, 1, 2}, {21, 1, 3}}); + auto pipes = getInputStreams(key_columns, {{5, 1, 1}, {10, 1, 2}, {21, 1, 3}}); - EXPECT_EQ(streams.size(), 3); + EXPECT_EQ(pipes.size(), 3); - MergingSortedBlockInputStream stream(streams, sort_description, DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); + auto transform = std::make_shared(pipes.front().getHeader(), pipes.size(), sort_description, + DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); + + auto stream = std::make_shared(Pipe(std::move(pipes), std::move(transform))); size_t total_rows = 0; - auto block1 = stream.read(); - auto block2 = stream.read(); - auto block3 = stream.read(); + auto block1 = stream->read(); + auto block2 = stream->read(); + auto block3 = stream->read(); - EXPECT_EQ(stream.read(), Block()); + EXPECT_EQ(stream->read(), Block()); - for (auto & block : {block1, block2, block3}) + for (const auto & block : {block1, block2, block3}) total_rows += block.rows(); /** * First block consists of 1 row from block3 with 21 rows + 2 rows from block2 with 10 rows @@ -112,17 +118,20 @@ TEST(MergingSortedTest, MoreInterestingBlockSizes) { std::vector key_columns{"K1", "K2", "K3"}; auto sort_description = getSortDescription(key_columns); - auto streams = getInputStreamsEqualStride(key_columns, {{1000, 1, 3}, {1500, 1, 3}, {1400, 1, 3}}); + auto pipes = getInputStreamsEqualStride(key_columns, {{1000, 1, 3}, {1500, 1, 3}, {1400, 1, 3}}); - EXPECT_EQ(streams.size(), 3); + EXPECT_EQ(pipes.size(), 3); - MergingSortedBlockInputStream stream(streams, sort_description, DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); + auto transform = std::make_shared(pipes.front().getHeader(), pipes.size(), sort_description, + DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); - auto block1 = stream.read(); - auto block2 = stream.read(); - auto block3 = stream.read(); + auto stream = std::make_shared(Pipe(std::move(pipes), std::move(transform))); - EXPECT_EQ(stream.read(), Block()); + auto block1 = stream->read(); + auto block2 = stream->read(); + auto block3 = stream->read(); + + EXPECT_EQ(stream->read(), Block()); EXPECT_EQ(block1.rows(), (1000 + 1500 + 1400) / 3); EXPECT_EQ(block2.rows(), (1000 + 1500 + 1400) / 3); diff --git a/src/DataStreams/tests/union_stream2.cpp b/src/DataStreams/tests/union_stream2.cpp index f3778543a6d..6e0997e7f72 100644 --- a/src/DataStreams/tests/union_stream2.cpp +++ b/src/DataStreams/tests/union_stream2.cpp @@ -23,7 +23,8 @@ using namespace DB; int main(int, char **) try { - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); Settings settings = context.getSettings(); diff --git a/src/DataStreams/ya.make b/src/DataStreams/ya.make new file mode 100644 index 00000000000..14e1c319394 --- /dev/null +++ b/src/DataStreams/ya.make @@ -0,0 +1,65 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +NO_COMPILER_WARNINGS() + +SRCS( + AddingDefaultBlockOutputStream.cpp + AddingDefaultsBlockInputStream.cpp + AggregatingBlockInputStream.cpp + AsynchronousBlockInputStream.cpp + BlockIO.cpp + BlockStreamProfileInfo.cpp + CheckConstraintsBlockOutputStream.cpp + CheckSortedBlockInputStream.cpp + CollapsingFinalBlockInputStream.cpp + ColumnGathererStream.cpp + ConvertingBlockInputStream.cpp + copyData.cpp + CountingBlockOutputStream.cpp + CreatingSetsBlockInputStream.cpp + CubeBlockInputStream.cpp + DistinctBlockInputStream.cpp + DistinctSortedBlockInputStream.cpp + ExecutionSpeedLimits.cpp + ExpressionBlockInputStream.cpp + FillingBlockInputStream.cpp + FilterBlockInputStream.cpp + FilterColumnsBlockInputStream.cpp + finalizeBlock.cpp + FinishSortingBlockInputStream.cpp + IBlockInputStream.cpp + InputStreamFromASTInsertQuery.cpp + InternalTextLogsRowOutputStream.cpp + LimitBlockInputStream.cpp + LimitByBlockInputStream.cpp + materializeBlock.cpp + MaterializingBlockInputStream.cpp + MergeSortingBlockInputStream.cpp + MergingAggregatedBlockInputStream.cpp + MergingAggregatedMemoryEfficientBlockInputStream.cpp + MergingSortedBlockInputStream.cpp + narrowBlockInputStreams.cpp + NativeBlockInputStream.cpp + NativeBlockOutputStream.cpp + ParallelAggregatingBlockInputStream.cpp + ParallelParsingBlockInputStream.cpp + PartialSortingBlockInputStream.cpp + processConstants.cpp + PushingToViewsBlockOutputStream.cpp + RemoteBlockInputStream.cpp + RemoteBlockOutputStream.cpp + ReverseBlockInputStream.cpp + RollupBlockInputStream.cpp + SizeLimits.cpp + SquashingBlockInputStream.cpp + SquashingBlockOutputStream.cpp + SquashingTransform.cpp + TotalsHavingBlockInputStream.cpp + TTLBlockInputStream.cpp +) + +END() diff --git a/src/DataTypes/DataTypeArray.cpp b/src/DataTypes/DataTypeArray.cpp index 821579732b5..49666cca428 100644 --- a/src/DataTypes/DataTypeArray.cpp +++ b/src/DataTypes/DataTypeArray.cpp @@ -199,7 +199,7 @@ void DataTypeArray::serializeBinaryBulkWithMultipleStreams( /// First serialize array sizes. settings.path.push_back(Substream::ArraySizes); - if (auto stream = settings.getter(settings.path)) + if (auto * stream = settings.getter(settings.path)) { if (settings.position_independent_encoding) serializeArraySizesPositionIndependent(column, *stream, offset, limit); @@ -244,7 +244,7 @@ void DataTypeArray::deserializeBinaryBulkWithMultipleStreams( ColumnArray & column_array = typeid_cast(column); settings.path.push_back(Substream::ArraySizes); - if (auto stream = settings.getter(settings.path)) + if (auto * stream = settings.getter(settings.path)) { if (settings.position_independent_encoding) deserializeArraySizesPositionIndependent(column, *stream, limit); diff --git a/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp b/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp index 4b0ca163383..1ecf8e17e4a 100644 --- a/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp +++ b/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp @@ -24,7 +24,7 @@ class DataTypeCustomIPv4Serialization : public DataTypeCustomSimpleTextSerializa public: void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override { - const auto col = checkAndGetColumn(&column); + const auto * col = checkAndGetColumn(&column); if (!col) { throw Exception("IPv4 type can only serialize columns of type UInt32." + column.getName(), ErrorCodes::ILLEGAL_COLUMN); @@ -63,7 +63,7 @@ public: void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override { - const auto col = checkAndGetColumn(&column); + const auto * col = checkAndGetColumn(&column); if (!col) { throw Exception("IPv6 type domain can only serialize columns of type FixedString(16)." + column.getName(), ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/DataTypes/DataTypeDateTime64.cpp b/src/DataTypes/DataTypeDateTime64.cpp index 8f4ef3f52e4..9b4f16ab408 100644 --- a/src/DataTypes/DataTypeDateTime64.cpp +++ b/src/DataTypes/DataTypeDateTime64.cpp @@ -183,7 +183,7 @@ void DataTypeDateTime64::deserializeProtobuf(IColumn & column, ProtobufReader & bool DataTypeDateTime64::equals(const IDataType & rhs) const { - if (auto * ptype = typeid_cast(&rhs)) + if (const auto * ptype = typeid_cast(&rhs)) return this->scale == ptype->getScale(); return false; } diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp index 20f7681ec1b..f81adfe347c 100644 --- a/src/DataTypes/DataTypeFactory.cpp +++ b/src/DataTypes/DataTypeFactory.cpp @@ -9,6 +9,8 @@ #include #include #include +#include + namespace DB { @@ -26,7 +28,7 @@ namespace ErrorCodes DataTypePtr DataTypeFactory::get(const String & full_name) const { ParserIdentifierWithOptionalParameters parser; - ASTPtr ast = parseQuery(parser, full_name.data(), full_name.data() + full_name.size(), "data type", 0); + ASTPtr ast = parseQuery(parser, full_name.data(), full_name.data() + full_name.size(), "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); return get(ast); } diff --git a/src/DataTypes/DataTypeLowCardinality.cpp b/src/DataTypes/DataTypeLowCardinality.cpp index c271b06416b..95ef60bed6e 100644 --- a/src/DataTypes/DataTypeLowCardinality.cpp +++ b/src/DataTypes/DataTypeLowCardinality.cpp @@ -546,7 +546,7 @@ void DataTypeLowCardinality::serializeBinaryBulkWithMultipleStreams( ErrorCodes::LOGICAL_ERROR); } - if (auto * nullable_keys = checkAndGetColumn(*keys)) + if (const auto * nullable_keys = checkAndGetColumn(*keys)) keys = nullable_keys->getNestedColumnPtr(); bool need_additional_keys = !keys->empty(); @@ -835,7 +835,7 @@ template void DataTypeLowCardinality::serializeImpl( const IColumn & column, size_t row_num, DataTypeLowCardinality::SerializeFunctionPtr func, Args &&... args) const { - auto & low_cardinality_column = getColumnLowCardinality(column); + const auto & low_cardinality_column = getColumnLowCardinality(column); size_t unique_row_number = low_cardinality_column.getIndexes().getUInt(row_num); (dictionary_type.get()->*func)(*low_cardinality_column.getDictionary().getNestedColumn(), unique_row_number, std::forward(args)...); } @@ -879,8 +879,8 @@ template MutableColumnUniquePtr DataTypeLowCardinality::createColumnUniqueImpl(const IDataType & keys_type, const Creator & creator) { - auto * type = &keys_type; - if (auto * nullable_type = typeid_cast(&keys_type)) + const auto * type = &keys_type; + if (const auto * nullable_type = typeid_cast(&keys_type)) type = nullable_type->getNestedType().get(); if (isString(type)) @@ -944,7 +944,7 @@ bool DataTypeLowCardinality::equals(const IDataType & rhs) const if (typeid(rhs) != typeid(*this)) return false; - auto & low_cardinality_rhs= static_cast(rhs); + const auto & low_cardinality_rhs= static_cast(rhs); return dictionary_type->equals(*low_cardinality_rhs.dictionary_type); } @@ -966,7 +966,7 @@ void registerDataTypeLowCardinality(DataTypeFactory & factory) DataTypePtr removeLowCardinality(const DataTypePtr & type) { - if (auto * low_cardinality_type = typeid_cast(type.get())) + if (const auto * low_cardinality_type = typeid_cast(type.get())) return low_cardinality_type->getDictionaryType(); return type; } diff --git a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp index 0fddd15b404..673253500c4 100644 --- a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp +++ b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp @@ -52,7 +52,7 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column) if (const auto * column_array = typeid_cast(column.get())) { - auto & data = column_array->getDataPtr(); + const auto & data = column_array->getDataPtr(); auto data_no_lc = recursiveRemoveLowCardinality(data); if (data.get() == data_no_lc.get()) return column; @@ -62,7 +62,7 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column) if (const auto * column_const = typeid_cast(column.get())) { - auto & nested = column_const->getDataColumnPtr(); + const auto & nested = column_const->getDataColumnPtr(); auto nested_no_lc = recursiveRemoveLowCardinality(nested); if (nested.get() == nested_no_lc.get()) return column; @@ -98,7 +98,7 @@ ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & if (const auto * column_const = typeid_cast(column.get())) { - auto & nested = column_const->getDataColumnPtr(); + const auto & nested = column_const->getDataColumnPtr(); auto nested_no_lc = recursiveTypeConversion(nested, from_type, to_type); if (nested.get() == nested_no_lc.get()) return column; @@ -131,8 +131,8 @@ ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & throw Exception("Unexpected column " + column->getName() + " for type " + from_type->getName(), ErrorCodes::ILLEGAL_COLUMN); - auto & nested_from = from_array_type->getNestedType(); - auto & nested_to = to_array_type->getNestedType(); + const auto & nested_from = from_array_type->getNestedType(); + const auto & nested_to = to_array_type->getNestedType(); return ColumnArray::create( recursiveTypeConversion(column_array->getDataPtr(), nested_from, nested_to), @@ -150,8 +150,8 @@ ColumnPtr recursiveTypeConversion(const ColumnPtr & column, const DataTypePtr & ErrorCodes::ILLEGAL_COLUMN); auto columns = column_tuple->getColumns(); - auto & from_elements = from_tuple_type->getElements(); - auto & to_elements = to_tuple_type->getElements(); + const auto & from_elements = from_tuple_type->getElements(); + const auto & to_elements = to_tuple_type->getElements(); bool has_converted = false; diff --git a/src/DataTypes/DataTypeNullable.cpp b/src/DataTypes/DataTypeNullable.cpp index 7d4c1ed39b9..b5ffae9588c 100644 --- a/src/DataTypes/DataTypeNullable.cpp +++ b/src/DataTypes/DataTypeNullable.cpp @@ -93,7 +93,7 @@ void DataTypeNullable::serializeBinaryBulkWithMultipleStreams( /// First serialize null map. settings.path.push_back(Substream::NullMap); - if (auto stream = settings.getter(settings.path)) + if (auto * stream = settings.getter(settings.path)) DataTypeUInt8().serializeBinaryBulk(col.getNullMapColumn(), *stream, offset, limit); /// Then serialize contents of arrays. @@ -112,7 +112,7 @@ void DataTypeNullable::deserializeBinaryBulkWithMultipleStreams( ColumnNullable & col = assert_cast(column); settings.path.push_back(Substream::NullMap); - if (auto stream = settings.getter(settings.path)) + if (auto * stream = settings.getter(settings.path)) DataTypeUInt8().deserializeBinaryBulk(col.getNullMapColumn(), *stream, limit, 0); settings.path.back() = Substream::NullableElements; diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index 227fab95439..29db2a49b99 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -393,7 +393,7 @@ void DataTypeTuple::serializeBinaryBulkWithMultipleStreams( for (const auto i : ext::range(0, ext::size(elems))) { settings.path.back().tuple_element_name = names[i]; - auto & element_col = extractElementColumn(column, i); + const auto & element_col = extractElementColumn(column, i); elems[i]->serializeBinaryBulkWithMultipleStreams(element_col, offset, limit, settings, tuple_state->states[i]); } settings.path.pop_back(); diff --git a/src/DataTypes/FieldToDataType.cpp b/src/DataTypes/FieldToDataType.cpp index fcea9d53f39..de61230f8d4 100644 --- a/src/DataTypes/FieldToDataType.cpp +++ b/src/DataTypes/FieldToDataType.cpp @@ -106,7 +106,7 @@ DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) const { - auto & name = static_cast(x).name; + const auto & name = static_cast(x).name; return DataTypeFactory::instance().get(name); } diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index 3652e85e2d3..754365dfc87 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -1,23 +1,25 @@ #pragma once -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_EMBEDDED_COMPILER +# include +# include +# include +# include +# include +# include +# include +# include -#include -#include -#include -#include -#include -#include -#include -#include +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" +# include -#include - -#pragma GCC diagnostic pop +# pragma GCC diagnostic pop namespace DB diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index e988490a487..5f8b9f877bf 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -86,7 +86,8 @@ Block flatten(const Block & block) { if (const DataTypeArray * type_arr = typeid_cast(elem.type.get())) { - if (const DataTypeTuple * type_tuple = typeid_cast(type_arr->getNestedType().get())) + const DataTypeTuple * type_tuple = typeid_cast(type_arr->getNestedType().get()); + if (type_tuple && type_tuple->haveExplicitNames()) { const DataTypes & element_types = type_tuple->getElements(); const Strings & names = type_tuple->getElementNames(); diff --git a/src/DataTypes/NestedUtils.h b/src/DataTypes/NestedUtils.h index 098d98699e4..3039fd7f118 100644 --- a/src/DataTypes/NestedUtils.h +++ b/src/DataTypes/NestedUtils.h @@ -17,6 +17,7 @@ namespace Nested std::string extractTableName(const std::string & nested_name); /// Replace Array(Tuple(...)) columns to a multiple of Array columns in a form of `column_name.element_name`. + /// only for named tuples that actually represent Nested structures. Block flatten(const Block & block); /// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column. diff --git a/src/DataTypes/getLeastSupertype.cpp b/src/DataTypes/getLeastSupertype.cpp index cfe88987c2d..b11f1aa7454 100644 --- a/src/DataTypes/getLeastSupertype.cpp +++ b/src/DataTypes/getLeastSupertype.cpp @@ -232,7 +232,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types) UInt32 max_datetime64_whole_precision = 0; for (const auto & t : types) { - if (auto dt64 = typeid_cast(t.get())) + if (const auto * dt64 = typeid_cast(t.get())) { const auto whole_precision = dt64->getPrecision() - dt64->getScale(); max_datetime64_whole_precision = std::max(whole_precision, max_datetime64_whole_precision); diff --git a/src/DataTypes/getMostSubtype.cpp b/src/DataTypes/getMostSubtype.cpp index 54609219cf1..4a73f0e2de2 100644 --- a/src/DataTypes/getMostSubtype.cpp +++ b/src/DataTypes/getMostSubtype.cpp @@ -104,7 +104,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth for (const auto & type : types) { - if (const auto type_array = typeid_cast(type.get())) + if (const auto * type_array = typeid_cast(type.get())) { have_array = true; nested_types.emplace_back(type_array->getNestedType()); @@ -132,7 +132,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth for (const auto & type : types) { - if (const auto type_tuple = typeid_cast(type.get())) + if (const auto * type_tuple = typeid_cast(type.get())) { if (!have_tuple) { @@ -177,7 +177,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth for (const auto & type : types) { - if (const auto type_nullable = typeid_cast(type.get())) + if (const auto * type_nullable = typeid_cast(type.get())) { have_nullable = true; nested_types.emplace_back(type_nullable->getNestedType()); diff --git a/src/DataTypes/tests/gtest_data_type_get_common_type.cpp b/src/DataTypes/tests/gtest_data_type_get_common_type.cpp index f334ab222a4..fd511bfbbb4 100644 --- a/src/DataTypes/tests/gtest_data_type_get_common_type.cpp +++ b/src/DataTypes/tests/gtest_data_type_get_common_type.cpp @@ -71,7 +71,6 @@ public: expected_type.reset(); } -public: DataTypes from_types; DataTypePtr expected_type; }; diff --git a/src/DataTypes/ya.make b/src/DataTypes/ya.make new file mode 100644 index 00000000000..c3a1e452d0d --- /dev/null +++ b/src/DataTypes/ya.make @@ -0,0 +1,41 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + clickhouse/src/Formats +) + +SRCS( + convertMySQLDataType.cpp + DataTypeAggregateFunction.cpp + DataTypeArray.cpp + DataTypeCustomIPv4AndIPv6.cpp + DataTypeCustomSimpleAggregateFunction.cpp + DataTypeCustomSimpleTextSerialization.cpp + DataTypeDate.cpp + DataTypeDateTime.cpp + DataTypeDateTime64.cpp + DataTypeDecimalBase.cpp + DataTypeEnum.cpp + DataTypeFactory.cpp + DataTypeFixedString.cpp + DataTypeFunction.cpp + DataTypeInterval.cpp + DataTypeLowCardinality.cpp + DataTypeLowCardinalityHelpers.cpp + DataTypeNothing.cpp + DataTypeNullable.cpp + DataTypeNumberBase.cpp + DataTypesDecimal.cpp + DataTypesNumber.cpp + DataTypeString.cpp + DataTypeTuple.cpp + DataTypeUUID.cpp + FieldToDataType.cpp + getLeastSupertype.cpp + getMostSubtype.cpp + IDataType.cpp + NestedUtils.cpp +) + +END() diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp index 006eb1656a2..a0f52a8f39a 100644 --- a/src/Databases/DatabaseDictionary.cpp +++ b/src/Databases/DatabaseDictionary.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,18 @@ namespace DB namespace ErrorCodes { extern const int SYNTAX_ERROR; + extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY; +} + +namespace +{ + StoragePtr createStorageDictionary(const String & database_name, const ExternalLoader::LoadResult & load_result) + { + if (!load_result.config) + return nullptr; + DictionaryStructure dictionary_structure = ExternalDictionariesLoader::getDictionaryStructure(*load_result.config); + return StorageDictionary::create(StorageID(database_name, load_result.name), load_result.name, dictionary_structure); + } } DatabaseDictionary::DatabaseDictionary(const String & name_) @@ -26,29 +39,12 @@ DatabaseDictionary::DatabaseDictionary(const String & name_) Tables DatabaseDictionary::listTables(const Context & context, const FilterByNameFunction & filter_by_name) { Tables tables; - ExternalLoader::LoadResults load_results; - if (filter_by_name) + auto load_results = context.getExternalDictionariesLoader().getLoadResults(filter_by_name); + for (auto & load_result : load_results) { - /// If `filter_by_name` is set, we iterate through all dictionaries with such names. That's why we need to load all of them. - load_results = context.getExternalDictionariesLoader().tryLoad(filter_by_name); - } - else - { - /// If `filter_by_name` isn't set, we iterate through only already loaded dictionaries. We don't try to load all dictionaries in this case. - load_results = context.getExternalDictionariesLoader().getCurrentLoadResults(); - } - - for (const auto & load_result: load_results) - { - /// Load tables only from XML dictionaries, don't touch other - if (load_result.object && load_result.repository_name.empty()) - { - auto dict_ptr = std::static_pointer_cast(load_result.object); - auto dict_name = dict_ptr->getName(); - const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); - auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - tables[dict_name] = StorageDictionary::create(StorageID(getDatabaseName(), dict_name), ColumnsDescription{columns}, context, true, dict_name); - } + auto storage = createStorageDictionary(getDatabaseName(), load_result); + if (storage) + tables.emplace(storage->getStorageID().table_name, storage); } return tables; } @@ -64,15 +60,8 @@ StoragePtr DatabaseDictionary::tryGetTable( const Context & context, const String & table_name) const { - auto dict_ptr = context.getExternalDictionariesLoader().tryGetDictionary(table_name); - if (dict_ptr) - { - const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); - auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - return StorageDictionary::create(StorageID(getDatabaseName(), table_name), ColumnsDescription{columns}, context, true, table_name); - } - - return {}; + auto load_result = context.getExternalDictionariesLoader().getLoadResult(table_name); + return createStorageDictionary(getDatabaseName(), load_result); } DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) @@ -82,7 +71,7 @@ DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(const Context & bool DatabaseDictionary::empty(const Context & context) const { - return !context.getExternalDictionariesLoader().hasCurrentlyLoadedObjects(); + return !context.getExternalDictionariesLoader().hasObjects(); } ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context, @@ -92,15 +81,17 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context, { WriteBufferFromString buffer(query); - const auto & dictionaries = context.getExternalDictionariesLoader(); - auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name) - : dictionaries.tryGetDictionary(table_name); - if (!dictionary) + auto load_result = context.getExternalDictionariesLoader().getLoadResult(table_name); + if (!load_result.config) + { + if (throw_on_error) + throw Exception{"Dictionary " + backQuote(table_name) + " doesn't exist", ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY}; return {}; + } - auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure()); + auto names_and_types = StorageDictionary::getNamesAndTypes(ExternalDictionariesLoader::getDictionaryStructure(*load_result.config)); buffer << "CREATE TABLE " << backQuoteIfNeed(database_name) << '.' << backQuoteIfNeed(table_name) << " ("; - buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end()); + buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types); buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")"; } diff --git a/src/Databases/DatabaseDictionary.h b/src/Databases/DatabaseDictionary.h index b586cb1403f..0001a59efa6 100644 --- a/src/Databases/DatabaseDictionary.h +++ b/src/Databases/DatabaseDictionary.h @@ -43,6 +43,8 @@ public: ASTPtr getCreateDatabaseQuery(const Context & context) const override; + bool shouldBeEmptyOnDetach() const override { return false; } + void shutdown() override; protected: diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 40e5682565d..8a1783677ee 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -4,18 +4,21 @@ #include #include #include +#include #include #include #include #include -#include "config_core.h" #include "DatabaseFactory.h" #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_MYSQL - -#include - +# include +# include #endif @@ -33,22 +36,33 @@ namespace ErrorCodes DatabasePtr DatabaseFactory::get( const String & database_name, const String & metadata_path, const ASTStorage * engine_define, Context & context) { + bool created = false; + try { - Poco::File(metadata_path).createDirectory(); + created = Poco::File(metadata_path).createDirectory(); return getImpl(database_name, metadata_path, engine_define, context); } catch (...) { Poco::File metadata_dir(metadata_path); - if (metadata_dir.exists()) + if (created && metadata_dir.exists()) metadata_dir.remove(true); throw; } } +template +static inline ValueType safeGetLiteralValue(const ASTPtr &ast, const String &engine_name) +{ + if (!ast || !ast->as()) + throw Exception("Database engine " + engine_name + " requested literal argument.", ErrorCodes::BAD_ARGUMENTS); + + return ast->as()->value.safeGet(); +} + DatabasePtr DatabaseFactory::getImpl( const String & database_name, const String & metadata_path, const ASTStorage * engine_define, Context & context) { @@ -79,11 +93,14 @@ DatabasePtr DatabaseFactory::getImpl( throw Exception("MySQL Database require mysql_hostname, mysql_database_name, mysql_username, mysql_password arguments.", ErrorCodes::BAD_ARGUMENTS); - const auto & arguments = engine->arguments->children; - const auto & host_name_and_port = arguments[0]->as()->value.safeGet(); - const auto & database_name_in_mysql = arguments[1]->as()->value.safeGet(); - const auto & mysql_user_name = arguments[2]->as()->value.safeGet(); - const auto & mysql_user_password = arguments[3]->as()->value.safeGet(); + + ASTs & arguments = engine->arguments->children; + arguments[1] = evaluateConstantExpressionOrIdentifierAsLiteral(arguments[1], context); + + const auto & host_name_and_port = safeGetLiteralValue(arguments[0], "MySQL"); + const auto & database_name_in_mysql = safeGetLiteralValue(arguments[1], "MySQL"); + const auto & mysql_user_name = safeGetLiteralValue(arguments[2], "MySQL"); + const auto & mysql_user_password = safeGetLiteralValue(arguments[3], "MySQL"); try { @@ -114,7 +131,7 @@ DatabasePtr DatabaseFactory::getImpl( const auto & arguments = engine->arguments->children; - const auto cache_expiration_time_seconds = arguments[0]->as()->value.safeGet(); + const auto cache_expiration_time_seconds = safeGetLiteralValue(arguments[0], "Lazy"); return std::make_shared(database_name, metadata_path, cache_expiration_time_seconds, context); } diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index cfb002423bb..8ea2c602987 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -232,7 +232,7 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table auto ast = parseQueryFromMetadata(context, table_metadata_path, /*throw_on_error*/ true, /*remove_empty*/false); if (ast) { - auto & ast_create = ast->as(); + const auto & ast_create = ast->as(); String table_data_path_relative = getTableDataPath(ast_create); table = createTableFromAST(ast_create, database_name, table_data_path_relative, context_copy, false).second; } diff --git a/src/Databases/DatabaseMySQL.cpp b/src/Databases/DatabaseMySQL.cpp index 959121585ea..7528e3fa719 100644 --- a/src/Databases/DatabaseMySQL.cpp +++ b/src/Databases/DatabaseMySQL.cpp @@ -1,28 +1,30 @@ -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif #if USE_MYSQL +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include +# include +# include namespace DB @@ -40,6 +42,7 @@ namespace ErrorCodes constexpr static const auto suffix = ".remove_flag"; static constexpr const std::chrono::seconds cleaner_sleep_time{30}; +static const SettingSeconds lock_acquire_timeout{10}; static String toQueryStringWithQuote(const std::vector & quote_list) { @@ -358,7 +361,7 @@ void DatabaseMySQL::cleanOutdatedTables() ++iterator; else { - const auto table_lock = (*iterator)->lockAlterIntention(RWLockImpl::NO_QUERY); + const auto table_lock = (*iterator)->lockAlterIntention(RWLockImpl::NO_QUERY, lock_acquire_timeout); (*iterator)->shutdown(); (*iterator)->is_dropped = true; diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index a1f5ea1ae6f..af646e2bccd 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -74,18 +75,24 @@ namespace void tryAttachDictionary( - Context & context, - const ASTCreateQuery & query, - DatabaseOrdinary & database) + const ASTPtr & query, + DatabaseOrdinary & database, + const String & metadata_path) { - assert(query.is_dictionary); + auto & create_query = query->as(); + assert(create_query.is_dictionary); try { - database.attachDictionary(query.table, context); + Poco::File meta_file(metadata_path); + auto config = getDictionaryConfigurationFromAST(create_query, database.getDatabaseName()); + time_t modification_time = meta_file.getLastModified().epochTime(); + database.attachDictionary(create_query.table, DictionaryAttachInfo{query, config, modification_time}); } catch (Exception & e) { - e.addMessage("Cannot attach table '" + backQuote(query.table) + "' from query " + serializeAST(query)); + e.addMessage("Cannot attach dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(create_query.table) + + " from metadata file " + metadata_path + + " from query " + serializeAST(*query)); throw; } } @@ -173,12 +180,12 @@ void DatabaseOrdinary::loadStoredObjects( /// Attach dictionaries. attachToExternalDictionariesLoader(context); - for (const auto & name_with_query : file_names) + for (const auto & [name, query] : file_names) { - auto create_query = name_with_query.second->as(); + auto create_query = query->as(); if (create_query.is_dictionary) { - tryAttachDictionary(context, create_query, *this); + tryAttachDictionary(query, *this, getMetadataPath() + name); /// Messages, so that it's not boring to wait for the server to load for a long time. logAboutProgress(log, ++dictionaries_processed, total_dictionaries, watch); @@ -234,7 +241,8 @@ void DatabaseOrdinary::alterTable( } ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, 0); + ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, + 0, context.getSettingsRef().max_parser_depth); auto & ast_create_query = ast->as(); diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp index e849962aae3..aec8b66e572 100644 --- a/src/Databases/DatabaseWithDictionaries.cpp +++ b/src/Databases/DatabaseWithDictionaries.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -24,44 +26,80 @@ namespace ErrorCodes { extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY; extern const int TABLE_ALREADY_EXISTS; - extern const int UNKNOWN_TABLE; + extern const int UNKNOWN_DICTIONARY; extern const int DICTIONARY_ALREADY_EXISTS; + extern const int FILE_DOESNT_EXIST; } -void DatabaseWithDictionaries::attachDictionary(const String & dictionary_name, const Context & context) +void DatabaseWithDictionaries::attachDictionary(const String & dictionary_name, const DictionaryAttachInfo & attach_info) { String full_name = getDatabaseName() + "." + dictionary_name; { std::lock_guard lock(mutex); - if (!dictionaries.emplace(dictionary_name).second) + auto [it, inserted] = dictionaries.emplace(dictionary_name, attach_info); + if (!inserted) throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS); + + /// Attach the dictionary as table too. + try + { + attachTableUnlocked( + dictionary_name, + StorageDictionary::create( + StorageID(getDatabaseName(), dictionary_name), + full_name, + ExternalDictionariesLoader::getDictionaryStructure(*attach_info.config))); + } + catch (...) + { + dictionaries.erase(it); + throw; + } } CurrentStatusInfo::set(CurrentStatusInfo::DictionaryStatus, full_name, static_cast(ExternalLoaderStatus::NOT_LOADED)); + /// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added /// and in case `dictionaries_lazy_load == false` it will load the dictionary. - const auto & external_loader = context.getExternalDictionariesLoader(); - external_loader.reloadConfig(getDatabaseName(), full_name); + external_loader->reloadConfig(getDatabaseName(), full_name); } -void DatabaseWithDictionaries::detachDictionary(const String & dictionary_name, const Context & context) +void DatabaseWithDictionaries::detachDictionary(const String & dictionary_name) +{ + DictionaryAttachInfo attach_info; + detachDictionaryImpl(dictionary_name, attach_info); +} + +void DatabaseWithDictionaries::detachDictionaryImpl(const String & dictionary_name, DictionaryAttachInfo & attach_info) { String full_name = getDatabaseName() + "." + dictionary_name; + { std::lock_guard lock(mutex); auto it = dictionaries.find(dictionary_name); if (it == dictionaries.end()) - throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_DICTIONARY); + attach_info = std::move(it->second); dictionaries.erase(it); + + /// Detach the dictionary as table too. + try + { + detachTableUnlocked(dictionary_name); + } + catch (...) + { + dictionaries.emplace(dictionary_name, std::move(attach_info)); + throw; + } } CurrentStatusInfo::unset(CurrentStatusInfo::DictionaryStatus, full_name); + /// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed /// and therefore it will unload the dictionary. - const auto & external_loader = context.getExternalDictionariesLoader(); - external_loader.reloadConfig(getDatabaseName(), full_name); - + external_loader->reloadConfig(getDatabaseName(), full_name); } void DatabaseWithDictionaries::createDictionary(const Context & context, const String & dictionary_name, const ASTPtr & query) @@ -83,8 +121,7 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S /// A dictionary with the same full name could be defined in *.xml config files. String full_name = getDatabaseName() + "." + dictionary_name; - const auto & external_loader = context.getExternalDictionariesLoader(); - if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST) + if (external_loader->getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST) throw Exception( "Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS); @@ -115,23 +152,22 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S /// Add a temporary repository containing the dictionary. /// We need this temp repository to try loading the dictionary before actually attaching it to the database. - auto temp_repository - = const_cast(external_loader) /// the change of ExternalDictionariesLoader is temporary - .addConfigRepository(std::make_unique( - getDatabaseName(), dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as()))); + auto temp_repository = external_loader->addConfigRepository(std::make_unique( + getDatabaseName(), dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as()))); bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true); if (!lazy_load) { /// load() is called here to force loading the dictionary, wait until the loading is finished, /// and throw an exception if the loading is failed. - external_loader.load(full_name); + external_loader->load(full_name); } - attachDictionary(dictionary_name, context); + auto config = getDictionaryConfigurationFromAST(query->as()); + attachDictionary(dictionary_name, DictionaryAttachInfo{query, config, time(nullptr)}); SCOPE_EXIT({ if (!succeeded) - detachDictionary(dictionary_name, context); + detachDictionary(dictionary_name); }); /// If it was ATTACH query and file with dictionary metadata already exist @@ -140,70 +176,31 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S /// ExternalDictionariesLoader doesn't know we renamed the metadata path. /// So we have to manually call reloadConfig() here. - external_loader.reloadConfig(getDatabaseName(), full_name); + external_loader->reloadConfig(getDatabaseName(), full_name); /// Everything's ok. succeeded = true; } -void DatabaseWithDictionaries::removeDictionary(const Context & context, const String & dictionary_name) +void DatabaseWithDictionaries::removeDictionary(const Context &, const String & dictionary_name) { - detachDictionary(dictionary_name, context); - - String dictionary_metadata_path = getObjectMetadataPath(dictionary_name); + DictionaryAttachInfo attach_info; + detachDictionaryImpl(dictionary_name, attach_info); try { + String dictionary_metadata_path = getObjectMetadataPath(dictionary_name); Poco::File(dictionary_metadata_path).remove(); CurrentStatusInfo::unset(CurrentStatusInfo::DictionaryStatus, getDatabaseName() + "." + dictionary_name); } catch (...) { /// If remove was not possible for some reason - attachDictionary(dictionary_name, context); + attachDictionary(dictionary_name, attach_info); throw; } } -StoragePtr DatabaseWithDictionaries::tryGetTable(const Context & context, const String & table_name) const -{ - if (auto table_ptr = DatabaseWithOwnTablesBase::tryGetTable(context, table_name)) - return table_ptr; - - if (isDictionaryExist(context, table_name)) - /// We don't need lock database here, because database doesn't store dictionary itself - /// just metadata - return getDictionaryStorage(context, table_name); - - return {}; -} - -DatabaseTablesIteratorPtr DatabaseWithDictionaries::getTablesWithDictionaryTablesIterator( - const Context & context, const FilterByNameFunction & filter_by_dictionary_name) -{ - /// NOTE: it's not atomic - auto tables_it = getTablesIterator(context, filter_by_dictionary_name); - auto dictionaries_it = getDictionariesIterator(context, filter_by_dictionary_name); - - Tables result; - while (tables_it && tables_it->isValid()) - { - result.emplace(tables_it->name(), tables_it->table()); - tables_it->next(); - } - - while (dictionaries_it && dictionaries_it->isValid()) - { - auto table_name = dictionaries_it->name(); - auto table_ptr = getDictionaryStorage(context, table_name); - if (table_ptr) - result.emplace(table_name, table_ptr); - dictionaries_it->next(); - } - - return std::make_unique(result); -} - DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_dictionary_name) { std::lock_guard lock(mutex); @@ -211,9 +208,9 @@ DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterato return std::make_unique(dictionaries); Dictionaries filtered_dictionaries; - for (const auto & dictionary_name : dictionaries) + for (const auto & dictionary_name : dictionaries | boost::adaptors::map_keys) if (filter_by_dictionary_name(dictionary_name)) - filtered_dictionaries.emplace(dictionary_name); + filtered_dictionaries.emplace_back(dictionary_name); return std::make_unique(std::move(filtered_dictionaries)); } @@ -223,44 +220,84 @@ bool DatabaseWithDictionaries::isDictionaryExist(const Context & /*context*/, co return dictionaries.find(dictionary_name) != dictionaries.end(); } -StoragePtr DatabaseWithDictionaries::getDictionaryStorage(const Context & context, const String & table_name) const -{ - auto dict_name = database_name + "." + table_name; - const auto & external_loader = context.getExternalDictionariesLoader(); - auto dict_ptr = external_loader.tryGetDictionary(dict_name); - if (dict_ptr) - { - const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); - auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - return StorageDictionary::create(StorageID(database_name, table_name), ColumnsDescription{columns}, context, true, dict_name); - } - return nullptr; -} - ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl( const Context & context, const String & dictionary_name, bool throw_on_error) const { - ASTPtr ast; - - auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name); - ast = getCreateQueryFromMetadata(context, dictionary_metadata_path, throw_on_error); - if (!ast && throw_on_error) { - /// Handle system.* tables for which there are no table.sql files. - bool has_dictionary = isDictionaryExist(context, dictionary_name); - - auto msg = has_dictionary ? "There is no CREATE DICTIONARY query for table " : "There is no metadata file for dictionary "; - - throw Exception(msg + backQuote(dictionary_name), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY); + /// Try to get create query ifg for an attached dictionary. + std::lock_guard lock{mutex}; + auto it = dictionaries.find(dictionary_name); + if (it != dictionaries.end()) + { + ASTPtr ast = it->second.create_query->clone(); + auto & create_query = ast->as(); + create_query.attach = false; + create_query.database = getDatabaseName(); + return ast; + } } - return ast; + /// Try to get create query for non-attached dictionary. + ASTPtr ast; + try + { + auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name); + ast = getCreateQueryFromMetadata(context, dictionary_metadata_path, throw_on_error); + } + catch (const Exception & e) + { + if (throw_on_error && (e.code() != ErrorCodes::FILE_DOESNT_EXIST)) + throw; + } + + if (ast) + { + const auto * create_query = ast->as(); + if (create_query && create_query->is_dictionary) + return ast; + } + if (throw_on_error) + throw Exception{"Dictionary " + backQuote(dictionary_name) + " doesn't exist", + ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY}; + return nullptr; +} + + +Poco::AutoPtr DatabaseWithDictionaries::getDictionaryConfiguration(const String & dictionary_name) const +{ + std::lock_guard lock(mutex); + auto it = dictionaries.find(dictionary_name); + if (it != dictionaries.end()) + return it->second.config; + throw Exception("Dictionary " + backQuote(dictionary_name) + " doesn't exist", ErrorCodes::UNKNOWN_DICTIONARY); +} + +time_t DatabaseWithDictionaries::getObjectMetadataModificationTime(const String & object_name) const +{ + { + std::lock_guard lock(mutex); + auto it = dictionaries.find(object_name); + if (it != dictionaries.end()) + return it->second.modification_time; + } + return DatabaseOnDisk::getObjectMetadataModificationTime(object_name); +} + + +bool DatabaseWithDictionaries::empty(const Context &) const +{ + std::lock_guard lock{mutex}; + return tables.empty() && dictionaries.empty(); } void DatabaseWithDictionaries::shutdown() { + { + std::lock_guard lock(mutex); + dictionaries.clear(); + } detachFromExternalDictionariesLoader(); DatabaseOnDisk::shutdown(); } @@ -269,8 +306,9 @@ DatabaseWithDictionaries::~DatabaseWithDictionaries() = default; void DatabaseWithDictionaries::attachToExternalDictionariesLoader(Context & context) { - database_as_config_repo_for_external_loader = context.getExternalDictionariesLoader().addConfigRepository( - std::make_unique(*this, context)); + external_loader = &context.getExternalDictionariesLoader(); + database_as_config_repo_for_external_loader + = external_loader->addConfigRepository(std::make_unique(*this, context)); } void DatabaseWithDictionaries::detachFromExternalDictionariesLoader() diff --git a/src/Databases/DatabaseWithDictionaries.h b/src/Databases/DatabaseWithDictionaries.h index e47ab6206c5..fe78aa98a19 100644 --- a/src/Databases/DatabaseWithDictionaries.h +++ b/src/Databases/DatabaseWithDictionaries.h @@ -8,9 +8,9 @@ namespace DB class DatabaseWithDictionaries : public DatabaseOnDisk { public: - void attachDictionary(const String & name, const Context & context) override; + void attachDictionary(const String & dictionary_name, const DictionaryAttachInfo & attach_info) override; - void detachDictionary(const String & name, const Context & context) override; + void detachDictionary(const String & dictionary_name) override; void createDictionary(const Context & context, const String & dictionary_name, @@ -18,13 +18,15 @@ public: void removeDictionary(const Context & context, const String & dictionary_name) override; - StoragePtr tryGetTable(const Context & context, const String & table_name) const override; - - DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name) override; + bool isDictionaryExist(const Context & context, const String & dictionary_name) const override; DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name) override; - bool isDictionaryExist(const Context & context, const String & dictionary_name) const override; + Poco::AutoPtr getDictionaryConfiguration(const String & /*name*/) const override; + + time_t getObjectMetadataModificationTime(const String & object_name) const override; + + bool empty(const Context & context) const override; void shutdown() override; @@ -37,13 +39,16 @@ protected: void attachToExternalDictionariesLoader(Context & context); void detachFromExternalDictionariesLoader(); - StoragePtr getDictionaryStorage(const Context & context, const String & table_name) const; + void detachDictionaryImpl(const String & dictionary_name, DictionaryAttachInfo & attach_info); ASTPtr getCreateDictionaryQueryImpl(const Context & context, const String & dictionary_name, bool throw_on_error) const override; + std::unordered_map dictionaries; + private: + ExternalDictionariesLoader * external_loader = nullptr; ext::scope_guard database_as_config_repo_for_external_loader; }; diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index cd1b155853f..1b54424b484 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -27,7 +27,7 @@ bool DatabaseWithOwnTablesBase::isTableExist( const String & table_name) const { std::lock_guard lock(mutex); - return tables.find(table_name) != tables.end() || dictionaries.find(table_name) != dictionaries.end(); + return tables.find(table_name) != tables.end(); } StoragePtr DatabaseWithOwnTablesBase::tryGetTable( @@ -58,7 +58,7 @@ DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Con bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const { std::lock_guard lock(mutex); - return tables.empty() && dictionaries.empty(); + return tables.empty(); } StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) @@ -70,8 +70,6 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name) { StoragePtr res; - if (dictionaries.count(table_name)) - throw Exception("Cannot detach dictionary " + database_name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE); auto it = tables.find(table_name); if (it == tables.end()) @@ -127,7 +125,6 @@ void DatabaseWithOwnTablesBase::shutdown() std::lock_guard lock(mutex); tables.clear(); - dictionaries.clear(); } DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase() diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h index 3bf7460da01..0515c16deaf 100644 --- a/src/Databases/DatabasesCommon.h +++ b/src/Databases/DatabasesCommon.h @@ -42,7 +42,6 @@ public: protected: mutable std::mutex mutex; Tables tables; - Dictionaries dictionaries; Poco::Logger * log; DatabaseWithOwnTablesBase(const String & name_, const String & logger); diff --git a/src/Databases/DictionaryAttachInfo.h b/src/Databases/DictionaryAttachInfo.h new file mode 100644 index 00000000000..b2214d26f3c --- /dev/null +++ b/src/Databases/DictionaryAttachInfo.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +struct DictionaryAttachInfo +{ + ASTPtr create_query; + Poco::AutoPtr config; + time_t modification_time; +}; + +} diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 1dc1f9eb36b..9ef649dff58 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -5,8 +5,11 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -18,11 +21,10 @@ namespace DB class Context; struct Settings; struct ConstraintsDescription; -class ColumnsDescription; struct IndicesDescription; struct TableStructureWriteLockHolder; class ASTCreateQuery; -using Dictionaries = std::set; +using Dictionaries = std::vector; namespace ErrorCodes { @@ -74,9 +76,14 @@ private: public: DatabaseDictionariesSnapshotIterator() = default; DatabaseDictionariesSnapshotIterator(Dictionaries & dictionaries_) : dictionaries(dictionaries_), it(dictionaries.begin()) {} - DatabaseDictionariesSnapshotIterator(Dictionaries && dictionaries_) : dictionaries(dictionaries_), it(dictionaries.begin()) {} + DatabaseDictionariesSnapshotIterator(const std::unordered_map & dictionaries_) + { + boost::range::copy(dictionaries_ | boost::adaptors::map_keys, std::back_inserter(dictionaries)); + it = dictionaries.begin(); + } + void next() { ++it; } bool isValid() const { return !dictionaries.empty() && it != dictionaries.end(); } @@ -140,12 +147,6 @@ public: return std::make_unique(); } - /// Get an iterator to pass through all the tables and dictionary tables. - virtual DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name = {}) - { - return getTablesIterator(context, filter_by_name); - } - /// Is the database empty. virtual bool empty(const Context & context) const = 0; @@ -192,7 +193,7 @@ public: /// Add dictionary to the database, but do not add it to the metadata. The database may not support this method. /// If dictionaries_lazy_load is false it also starts loading the dictionary asynchronously. - virtual void attachDictionary(const String & /*name*/, const Context & /*context*/) + virtual void attachDictionary(const String & /* dictionary_name */, const DictionaryAttachInfo & /* attach_info */) { throw Exception("There is no ATTACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } @@ -204,7 +205,7 @@ public: } /// Forget about the dictionary without deleting it. The database may not support this method. - virtual void detachDictionary(const String & /*name*/, const Context & /*context*/) + virtual void detachDictionary(const String & /*name*/) { throw Exception("There is no DETACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } @@ -260,6 +261,11 @@ public: return getCreateDictionaryQueryImpl(context, name, true); } + virtual Poco::AutoPtr getDictionaryConfiguration(const String & /*name*/) const + { + throw Exception(getEngineName() + ": getDictionaryConfiguration() is not supported", ErrorCodes::NOT_IMPLEMENTED); + } + /// Get the CREATE DATABASE query for current database. virtual ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const = 0; @@ -276,6 +282,9 @@ public: /// Returns metadata path of a concrete table if the database supports it, empty string otherwise virtual String getObjectMetadataPath(const String & /*table_name*/) const { return {}; } + /// All tables and dictionaries should be detached before detaching the database. + virtual bool shouldBeEmptyOnDetach() const { return true; } + /// Ask all tables to complete the background threads they are using and delete all table objects. virtual void shutdown() = 0; diff --git a/src/Databases/ya.make b/src/Databases/ya.make new file mode 100644 index 00000000000..4aff1054023 --- /dev/null +++ b/src/Databases/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + DatabaseDictionary.cpp + DatabaseFactory.cpp + DatabaseLazy.cpp + DatabaseMemory.cpp + DatabaseMySQL.cpp + DatabaseOnDisk.cpp + DatabaseOrdinary.cpp + DatabasesCommon.cpp + DatabaseWithDictionaries.cpp +) + +END() diff --git a/src/Dictionaries/CMakeLists.txt b/src/Dictionaries/CMakeLists.txt index c7766fbc628..0a7693edc60 100644 --- a/src/Dictionaries/CMakeLists.txt +++ b/src/Dictionaries/CMakeLists.txt @@ -1,14 +1,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) -include(${ClickHouse_SOURCE_DIR}/cmake/generate_code.cmake) add_headers_and_sources(clickhouse_dictionaries .) -generate_code(ComplexKeyCacheDictionary_generate1 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) -generate_code(ComplexKeyCacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) -generate_code(ComplexKeyCacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) -generate_code(CacheDictionary_generate1 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) -generate_code(CacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) -generate_code(CacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) add_headers_and_sources(clickhouse_dictionaries ${CMAKE_CURRENT_BINARY_DIR}/generated/) list(REMOVE_ITEM clickhouse_dictionaries_sources DictionaryFactory.cpp DictionarySourceFactory.cpp DictionaryStructure.cpp getDictionaryConfigurationFromAST.cpp) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index 36a8c704f4f..241610db77b 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include "DictionaryBlockInputStream.h" #include "DictionaryFactory.h" + namespace ProfileEvents { extern const Event DictCacheKeysRequested; @@ -46,6 +48,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int UNSUPPORTED_METHOD; extern const int TOO_SMALL_BUFFER_SIZE; + extern const int TIMEOUT_EXCEEDED; } @@ -63,10 +66,12 @@ CacheDictionary::CacheDictionary( const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, DictionaryLifetime dict_lifetime_, + size_t strict_max_lifetime_seconds_, size_t size_, bool allow_read_expired_keys_, size_t max_update_queue_size_, size_t update_queue_push_timeout_milliseconds_, + size_t query_wait_timeout_milliseconds_, size_t max_threads_for_updates_) : database(database_) , name(name_) @@ -74,9 +79,11 @@ CacheDictionary::CacheDictionary( , dict_struct(dict_struct_) , source_ptr{std::move(source_ptr_)} , dict_lifetime(dict_lifetime_) + , strict_max_lifetime_seconds(strict_max_lifetime_seconds_) , allow_read_expired_keys(allow_read_expired_keys_) , max_update_queue_size(max_update_queue_size_) , update_queue_push_timeout_milliseconds(update_queue_push_timeout_milliseconds_) + , query_wait_timeout_milliseconds(query_wait_timeout_milliseconds_) , max_threads_for_updates(max_threads_for_updates_) , log(&Logger::get("ExternalDictionaries")) , size{roundUpToPowerOfTwoOrZero(std::max(size_, size_t(max_collision_length)))} @@ -139,7 +146,7 @@ void CacheDictionary::isInImpl(const PaddedPODArray & child_ids, const Ance PaddedPODArray children(out_size, 0); PaddedPODArray parents(child_ids.begin(), child_ids.end()); - while (true) + for (size_t i = 0; i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i) { size_t out_idx = 0; size_t parents_idx = 0; @@ -213,7 +220,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra std::vector ancestors(1, child_id); /// Iteratively find all ancestors for child. - while (true) + for (size_t i = 0; i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i) { toParent(child, parent); @@ -332,6 +339,13 @@ void CacheDictionary::has(const PaddedPODArray & ids, PaddedPODArray { if (find_result.outdated) { + /// Protection of reading very expired keys. + if (now > cells[find_result.cell_idx].strict_max) + { + cache_not_found_ids[id].push_back(row); + continue; + } + cache_expired_ids[id].push_back(row); if (allow_read_expired_keys) @@ -605,7 +619,7 @@ void CacheDictionary::setAttributeValue(Attribute & attribute, const Key idx, co const auto str_size = string.size(); if (str_size != 0) { - auto string_ptr = string_arena->alloc(str_size + 1); + auto * string_ptr = string_arena->alloc(str_size + 1); std::copy(string.data(), string.data() + str_size + 1, string_ptr); string_ref = StringRef{string_ptr, str_size}; } @@ -693,6 +707,9 @@ void registerDictionaryCache(DictionaryFactory & factory) const String name = config.getString(config_prefix + ".name"); const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; + const size_t strict_max_lifetime_seconds = + config.getUInt64(layout_prefix + ".cache.strict_max_lifetime_seconds", static_cast(dict_lifetime.max_sec)); + const size_t max_update_queue_size = config.getUInt64(layout_prefix + ".cache.max_update_queue_size", 100000); if (max_update_queue_size == 0) @@ -708,6 +725,9 @@ void registerDictionaryCache(DictionaryFactory & factory) throw Exception{name + ": dictionary of layout 'cache' have too little update_queue_push_timeout", ErrorCodes::BAD_ARGUMENTS}; + const size_t query_wait_timeout_milliseconds = + config.getUInt64(layout_prefix + ".cache.query_wait_timeout_milliseconds", 60000); + const size_t max_threads_for_updates = config.getUInt64(layout_prefix + ".max_threads_for_updates", 4); if (max_threads_for_updates == 0) @@ -715,8 +735,17 @@ void registerDictionaryCache(DictionaryFactory & factory) ErrorCodes::BAD_ARGUMENTS}; return std::make_unique( - database, name, dict_struct, std::move(source_ptr), dict_lifetime, size, - allow_read_expired_keys, max_update_queue_size, update_queue_push_timeout_milliseconds, + database, + name, + dict_struct, + std::move(source_ptr), + dict_lifetime, + strict_max_lifetime_seconds, + size, + allow_read_expired_keys, + max_update_queue_size, + update_queue_push_timeout_milliseconds, + query_wait_timeout_milliseconds, max_threads_for_updates); }; factory.registerLayout("cache", create_layout, false); @@ -782,20 +811,32 @@ void CacheDictionary::updateThreadFunction() void CacheDictionary::waitForCurrentUpdateFinish(UpdateUnitPtr & update_unit_ptr) const { - std::unique_lock lock(update_mutex); + std::unique_lock update_lock(update_mutex); - /* - * We wait here without any timeout to avoid SEGFAULT's. - * Consider timeout for wait had expired and main query's thread ended with exception - * or some other error. But the UpdateUnit with callbacks is left in the queue. - * It has these callback that capture god knows what from the current thread - * (most of the variables lies on the stack of finished thread) that - * intended to do a synchronous update. AsyncUpdate thread can touch deallocated memory and explode. - * */ - is_update_finished.wait( - lock, + size_t timeout_for_wait = 100000; + bool result = is_update_finished.wait_for( + update_lock, + std::chrono::milliseconds(timeout_for_wait), [&] {return update_unit_ptr->is_done || update_unit_ptr->current_exception; }); + if (!result) + { + std::lock_guard callback_lock(update_unit_ptr->callback_mutex); + /* + * We acquire a lock here and store false to special variable to avoid SEGFAULT's. + * Consider timeout for wait had expired and main query's thread ended with exception + * or some other error. But the UpdateUnit with callbacks is left in the queue. + * It has these callback that capture god knows what from the current thread + * (most of the variables lies on the stack of finished thread) that + * intended to do a synchronous update. AsyncUpdate thread can touch deallocated memory and explode. + * */ + update_unit_ptr->can_use_callback = false; + throw DB::Exception( + "Dictionary " + getName() + " source seems unavailable, because " + + toString(timeout_for_wait) + " timeout exceeded.", ErrorCodes::TIMEOUT_EXCEEDED); + } + + if (update_unit_ptr->current_exception) std::rethrow_exception(update_unit_ptr->current_exception); } @@ -855,7 +896,7 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const break; } - const auto id_column = typeid_cast(block.safeGetByPosition(0).column.get()); + const auto * id_column = typeid_cast(block.safeGetByPosition(0).column.get()); if (!id_column) throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH}; @@ -968,9 +1009,14 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const { std::uniform_int_distribution distribution{dict_lifetime.min_sec, dict_lifetime.max_sec}; cell.setExpiresAt(now + std::chrono::seconds{distribution(rnd_engine)}); + cell.strict_max = now + std::chrono::seconds{strict_max_lifetime_seconds}; } else + { cell.setExpiresAt(std::chrono::time_point::max()); + cell.strict_max = now + std::chrono::seconds{strict_max_lifetime_seconds}; + } + /// Set null_value for each attribute cell.setDefault(); diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index e425b9c391a..bb103c61107 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -55,10 +55,12 @@ public: const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, DictionaryLifetime dict_lifetime_, + size_t strict_max_lifetime_seconds, size_t size_, bool allow_read_expired_keys_, size_t max_update_queue_size_, size_t update_queue_push_timeout_milliseconds_, + size_t query_wait_timeout_milliseconds, size_t max_threads_for_updates); ~CacheDictionary() override; @@ -87,9 +89,18 @@ public: std::shared_ptr clone() const override { return std::make_shared( - database, name, dict_struct, source_ptr->clone(), dict_lifetime, size, - allow_read_expired_keys, max_update_queue_size, - update_queue_push_timeout_milliseconds, max_threads_for_updates); + database, + name, + dict_struct, + source_ptr->clone(), + dict_lifetime, + strict_max_lifetime_seconds, + size, + allow_read_expired_keys, + max_update_queue_size, + update_queue_push_timeout_milliseconds, + query_wait_timeout_milliseconds, + max_threads_for_updates); } const IDictionarySource * getSource() const override { return source_ptr.get(); } @@ -206,6 +217,8 @@ private: /// Stores both expiration time and `is_default` flag in the most significant bit time_point_urep_t data; + time_point_t strict_max; + /// Sets expiration time, resets `is_default` flag to false time_point_t expiresAt() const { return ext::safe_bit_cast(data & EXPIRES_AT_MASK); } void setExpiresAt(const time_point_t & t) { data = ext::safe_bit_cast(t); } @@ -294,9 +307,11 @@ private: const DictionaryStructure dict_struct; mutable DictionarySourcePtr source_ptr; const DictionaryLifetime dict_lifetime; + const size_t strict_max_lifetime_seconds; const bool allow_read_expired_keys; const size_t max_update_queue_size; const size_t update_queue_push_timeout_milliseconds; + const size_t query_wait_timeout_milliseconds; const size_t max_threads_for_updates; Logger * const log; @@ -366,6 +381,12 @@ private: alive_keys(CurrentMetrics::CacheDictionaryUpdateQueueKeys, requested_ids.size()){} std::vector requested_ids; + + /// It might seem that it is a leak of performance. + /// But aquiring a mutex without contention is rather cheap. + std::mutex callback_mutex; + bool can_use_callback{true}; + PresentIdHandler present_id_handler; AbsentIdHandler absent_id_handler; @@ -412,6 +433,7 @@ private: helper.push_back(unit_ptr->requested_ids.size() + helper.back()); present_id_handlers.emplace_back(unit_ptr->present_id_handler); absent_id_handlers.emplace_back(unit_ptr->absent_id_handler); + update_units.emplace_back(unit_ptr); } concatenated_requested_ids.reserve(total_requested_keys_count); @@ -428,31 +450,51 @@ private: void informCallersAboutPresentId(Key id, size_t cell_idx) { - for (size_t i = 0; i < concatenated_requested_ids.size(); ++i) + for (size_t position = 0; position < concatenated_requested_ids.size(); ++position) { - auto & curr = concatenated_requested_ids[i]; - if (curr == id) - getPresentIdHandlerForPosition(i)(id, cell_idx); + if (concatenated_requested_ids[position] == id) + { + auto unit_number = getUpdateUnitNumberForRequestedIdPosition(position); + auto lock = getLockToCurrentUnit(unit_number); + if (canUseCallback(unit_number)) + getPresentIdHandlerForPosition(unit_number)(id, cell_idx); + } } } void informCallersAboutAbsentId(Key id, size_t cell_idx) { - for (size_t i = 0; i < concatenated_requested_ids.size(); ++i) - if (concatenated_requested_ids[i] == id) - getAbsentIdHandlerForPosition(i)(id, cell_idx); + for (size_t position = 0; position < concatenated_requested_ids.size(); ++position) + if (concatenated_requested_ids[position] == id) + { + auto unit_number = getUpdateUnitNumberForRequestedIdPosition(position); + auto lock = getLockToCurrentUnit(unit_number); + if (canUseCallback(unit_number)) + getAbsentIdHandlerForPosition(unit_number)(id, cell_idx); + } } private: - PresentIdHandler & getPresentIdHandlerForPosition(size_t position) + /// Needed for control the usage of callback to avoid SEGFAULTs. + bool canUseCallback(size_t unit_number) { - return present_id_handlers[getUpdateUnitNumberForRequestedIdPosition(position)]; + return update_units[unit_number].get()->can_use_callback; } - AbsentIdHandler & getAbsentIdHandlerForPosition(size_t position) + std::unique_lock getLockToCurrentUnit(size_t unit_number) { - return absent_id_handlers[getUpdateUnitNumberForRequestedIdPosition((position))]; + return std::unique_lock(update_units[unit_number].get()->callback_mutex); + } + + PresentIdHandler & getPresentIdHandlerForPosition(size_t unit_number) + { + return update_units[unit_number].get()->present_id_handler; + } + + AbsentIdHandler & getAbsentIdHandlerForPosition(size_t unit_number) + { + return update_units[unit_number].get()->absent_id_handler; } size_t getUpdateUnitNumberForRequestedIdPosition(size_t position) @@ -464,6 +506,8 @@ private: std::vector present_id_handlers; std::vector absent_id_handlers; + std::vector> update_units; + std::vector helper; }; diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h index 7b108438f76..746b2609a36 100644 --- a/src/Dictionaries/CacheDictionary.inc.h +++ b/src/Dictionaries/CacheDictionary.inc.h @@ -75,6 +75,13 @@ void CacheDictionary::getItemsNumberImpl( if (find_result.outdated) { + /// Protection of reading very expired keys. + if (now > cells[find_result.cell_idx].strict_max) + { + cache_not_found_ids[id].push_back(row); + continue; + } + cache_expired_ids[id].push_back(row); if (allow_read_expired_keys) update_routine(); @@ -249,6 +256,13 @@ void CacheDictionary::getItemsString( { if (find_result.outdated) { + /// Protection of reading very expired keys. + if (now > cells[find_result.cell_idx].strict_max) + { + cache_not_found_ids[id].push_back(row); + continue; + } + cache_expired_ids[id].push_back(row); if (allow_read_expired_keys) diff --git a/src/Dictionaries/CacheDictionary_generate1.cpp b/src/Dictionaries/CacheDictionary_generate1.cpp new file mode 100644 index 00000000000..edb4b89d550 --- /dev/null +++ b/src/Dictionaries/CacheDictionary_generate1.cpp @@ -0,0 +1,32 @@ +#include +#include + +namespace DB +{ +#define DEFINE(TYPE) \ + void CacheDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray & ids, ResultArrayType & out) \ + const \ + { \ + auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ + const auto null_value = std::get(attribute.null_values); \ + getItemsNumberImpl(attribute, ids, out, [&](const size_t) { return null_value; }); \ + } + +DEFINE(UInt8) +DEFINE(UInt16) +DEFINE(UInt32) +DEFINE(UInt64) +DEFINE(UInt128) +DEFINE(Int8) +DEFINE(Int16) +DEFINE(Int32) +DEFINE(Int64) +DEFINE(Float32) +DEFINE(Float64) +DEFINE(Decimal32) +DEFINE(Decimal64) +DEFINE(Decimal128) + +#undef DEFINE +} diff --git a/src/Dictionaries/CacheDictionary_generate1.cpp.in b/src/Dictionaries/CacheDictionary_generate1.cpp.in deleted file mode 100644 index b870a0ed69b..00000000000 --- a/src/Dictionaries/CacheDictionary_generate1.cpp.in +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int TYPE_MISMATCH; -} - -using TYPE = @NAME@; -void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray & ids, ResultArrayType & out) const -{ - auto & attribute = getAttribute(attribute_name); - checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut@NAME@); - - const auto null_value = std::get(attribute.null_values); - - getItemsNumberImpl(attribute, ids, out, [&](const size_t) { return null_value; }); -} - -} diff --git a/src/Dictionaries/CacheDictionary_generate2.cpp b/src/Dictionaries/CacheDictionary_generate2.cpp new file mode 100644 index 00000000000..97fa9e1a365 --- /dev/null +++ b/src/Dictionaries/CacheDictionary_generate2.cpp @@ -0,0 +1,34 @@ +#include +#include + +namespace DB +{ +#define DEFINE(TYPE) \ + void CacheDictionary::get##TYPE( \ + const std::string & attribute_name, \ + const PaddedPODArray & ids, \ + const PaddedPODArray & def, \ + ResultArrayType & out) const \ + { \ + auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ + getItemsNumberImpl(attribute, ids, out, [&](const size_t row) { return def[row]; }); \ + } + +DEFINE(UInt8) +DEFINE(UInt16) +DEFINE(UInt32) +DEFINE(UInt64) +DEFINE(UInt128) +DEFINE(Int8) +DEFINE(Int16) +DEFINE(Int32) +DEFINE(Int64) +DEFINE(Float32) +DEFINE(Float64) +DEFINE(Decimal32) +DEFINE(Decimal64) +DEFINE(Decimal128) + +#undef DEFINE +} diff --git a/src/Dictionaries/CacheDictionary_generate2.cpp.in b/src/Dictionaries/CacheDictionary_generate2.cpp.in deleted file mode 100644 index 367e150c2cb..00000000000 --- a/src/Dictionaries/CacheDictionary_generate2.cpp.in +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int TYPE_MISMATCH; -} - -using TYPE = @NAME@; -void CacheDictionary::get@NAME@(const std::string & attribute_name, - const PaddedPODArray & ids, - const PaddedPODArray & def, - ResultArrayType & out) const -{ - auto & attribute = getAttribute(attribute_name); - checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut@NAME@); - - getItemsNumberImpl(attribute, ids, out, [&](const size_t row) { return def[row]; }); -} - -} diff --git a/src/Dictionaries/CacheDictionary_generate3.cpp b/src/Dictionaries/CacheDictionary_generate3.cpp new file mode 100644 index 00000000000..8a94ca6bc20 --- /dev/null +++ b/src/Dictionaries/CacheDictionary_generate3.cpp @@ -0,0 +1,31 @@ +#include +#include + +namespace DB +{ +#define DEFINE(TYPE) \ + void CacheDictionary::get##TYPE( \ + const std::string & attribute_name, const PaddedPODArray & ids, const TYPE def, ResultArrayType & out) const \ + { \ + auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ + getItemsNumberImpl(attribute, ids, out, [&](const size_t) { return def; }); \ + } + +DEFINE(UInt8) +DEFINE(UInt16) +DEFINE(UInt32) +DEFINE(UInt64) +DEFINE(UInt128) +DEFINE(Int8) +DEFINE(Int16) +DEFINE(Int32) +DEFINE(Int64) +DEFINE(Float32) +DEFINE(Float64) +DEFINE(Decimal32) +DEFINE(Decimal64) +DEFINE(Decimal128) + +#undef DEFINE +} diff --git a/src/Dictionaries/CacheDictionary_generate3.cpp.in b/src/Dictionaries/CacheDictionary_generate3.cpp.in deleted file mode 100644 index 8e2c26302e8..00000000000 --- a/src/Dictionaries/CacheDictionary_generate3.cpp.in +++ /dev/null @@ -1,20 +0,0 @@ -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int TYPE_MISMATCH; -} - -using TYPE = @NAME@; -void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray & ids, const TYPE def, ResultArrayType & out) const -{ - auto & attribute = getAttribute(attribute_name); - checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut@NAME@); - - getItemsNumberImpl(attribute, ids, out, [&](const size_t) { return def; }); -} - -} diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 015d3f499ee..98ed7985acb 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -13,7 +13,7 @@ #include "readInvalidateQuery.h" #include "writeParenthesisedString.h" #include "DictionaryFactory.h" - +#include "DictionarySourceHelpers.h" namespace DB { @@ -50,6 +50,7 @@ static ConnectionPoolWithFailoverPtr createPool( ClickHouseDictionarySource::ClickHouseDictionarySource( const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config, + const std::string & path_to_settings, const std::string & config_prefix, const Block & sample_block_, const Context & context_) @@ -74,8 +75,11 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( { /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0), {}); + context = copyContextAndApplySettings(path_to_settings, context, config); + /// Processors are not supported here yet. context.setSetting("experimental_use_processors", false); + /// Query context is needed because some code in executeQuery function may assume it exists. /// Current example is Context::getSampleBlockCache from InterpreterSelectWithUnionQuery::getSampleBlock. context.makeQueryContext(); @@ -132,7 +136,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadAll() { BlockIO res = executeQuery(load_all_query, context, true); /// FIXME res.in may implicitly use some objects owned be res, but them will be destructed after return - res.in = std::make_shared(context, res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); + res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; } return std::make_shared(pool, load_all_query, sample_block, context); @@ -144,7 +148,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadUpdatedAll() if (is_local) { auto res = executeQuery(load_update_query, context, true); - res.in = std::make_shared(context, res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); + res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; } return std::make_shared(pool, load_update_query, sample_block, context); @@ -192,7 +196,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::createStreamForSelectiveLoad(con { auto res = executeQuery(query, context, true); res.in = std::make_shared( - context, res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); + res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; } @@ -227,7 +231,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) const Context & context, bool /* check_config */) -> DictionarySourcePtr { - return std::make_unique(dict_struct, config, config_prefix + ".clickhouse", sample_block, context); + return std::make_unique(dict_struct, config, config_prefix, config_prefix + ".clickhouse", sample_block, context); }; factory.registerSource("clickhouse", create_table_source); } diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index 84bbd78b93a..13dc0323039 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -21,6 +21,7 @@ public: ClickHouseDictionarySource( const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config, + const std::string & path_to_settings, const std::string & config_prefix, const Block & sample_block_, const Context & context); diff --git a/src/Dictionaries/ComplexKeyCacheDictionary.cpp b/src/Dictionaries/ComplexKeyCacheDictionary.cpp index e16f389e1ce..d6a74191a79 100644 --- a/src/Dictionaries/ComplexKeyCacheDictionary.cpp +++ b/src/Dictionaries/ComplexKeyCacheDictionary.cpp @@ -345,8 +345,8 @@ template StringRef ComplexKeyCacheDictionary::placeKeysInPoolalloc(); - auto place = res; + auto * res = fixed_size_keys_pool->alloc(); + auto * place = res; for (const auto & key_column : key_columns) { @@ -367,7 +367,7 @@ StringRef ComplexKeyCacheDictionary::copyIntoArena(StringRef src, Arena & arena) StringRef ComplexKeyCacheDictionary::copyKey(const StringRef key) const { - const auto res = key_size_is_fixed ? fixed_size_keys_pool->alloc() : keys_pool->alloc(key.size); + auto * res = key_size_is_fixed ? fixed_size_keys_pool->alloc() : keys_pool->alloc(key.size); memcpy(res, key.data, key.size); return {res, key.size}; diff --git a/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp b/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp new file mode 100644 index 00000000000..6f9761cd064 --- /dev/null +++ b/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp @@ -0,0 +1,32 @@ +#include + +namespace DB +{ +#define DEFINE(TYPE) \ + void ComplexKeyCacheDictionary::get##TYPE( \ + const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType & out) const \ + { \ + dict_struct.validateKeyTypes(key_types); \ + auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ + const auto null_value = std::get(attribute.null_values); \ + getItemsNumberImpl(attribute, key_columns, out, [&](const size_t) { return null_value; }); \ + } + +DEFINE(UInt8) +DEFINE(UInt16) +DEFINE(UInt32) +DEFINE(UInt64) +DEFINE(UInt128) +DEFINE(Int8) +DEFINE(Int16) +DEFINE(Int32) +DEFINE(Int64) +DEFINE(Float32) +DEFINE(Float64) +DEFINE(Decimal32) +DEFINE(Decimal64) +DEFINE(Decimal128) + +#undef DEFINE +} diff --git a/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp.in b/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp.in deleted file mode 100644 index 5c0ed408a55..00000000000 --- a/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp.in +++ /dev/null @@ -1,22 +0,0 @@ -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int TYPE_MISMATCH; -} - -using TYPE = @NAME@; -void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType & out) const -{ - dict_struct.validateKeyTypes(key_types); - - auto & attribute = getAttribute(attribute_name); - checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut@NAME@); - - const auto null_value = std::get(attribute.null_values); - - getItemsNumberImpl(attribute, key_columns, out, [&](const size_t) { return null_value; }); -} -} diff --git a/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp b/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp new file mode 100644 index 00000000000..297da2c91a0 --- /dev/null +++ b/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp @@ -0,0 +1,35 @@ +#include + +namespace DB +{ +#define DEFINE(TYPE) \ + void ComplexKeyCacheDictionary::get##TYPE( \ + const std::string & attribute_name, \ + const Columns & key_columns, \ + const DataTypes & key_types, \ + const PaddedPODArray & def, \ + ResultArrayType & out) const \ + { \ + dict_struct.validateKeyTypes(key_types); \ + auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ + getItemsNumberImpl(attribute, key_columns, out, [&](const size_t row) { return def[row]; }); \ + } + +DEFINE(UInt8) +DEFINE(UInt16) +DEFINE(UInt32) +DEFINE(UInt64) +DEFINE(UInt128) +DEFINE(Int8) +DEFINE(Int16) +DEFINE(Int32) +DEFINE(Int64) +DEFINE(Float32) +DEFINE(Float64) +DEFINE(Decimal32) +DEFINE(Decimal64) +DEFINE(Decimal128) + +#undef DEFINE +} diff --git a/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp.in b/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp.in deleted file mode 100644 index b3233cd05e1..00000000000 --- a/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp.in +++ /dev/null @@ -1,25 +0,0 @@ -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int TYPE_MISMATCH; -} - -using TYPE = @NAME@; - -void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name, - const Columns & key_columns, - const DataTypes & key_types, - const PaddedPODArray & def, - ResultArrayType & out) const -{ - dict_struct.validateKeyTypes(key_types); - - auto & attribute = getAttribute(attribute_name); - checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut@NAME@); - - getItemsNumberImpl(attribute, key_columns, out, [&](const size_t row) { return def[row]; }); -} -} diff --git a/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp b/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp new file mode 100644 index 00000000000..222e9c8ee77 --- /dev/null +++ b/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp @@ -0,0 +1,35 @@ +#include + +namespace DB +{ +#define DEFINE(TYPE) \ + void ComplexKeyCacheDictionary::get##TYPE( \ + const std::string & attribute_name, \ + const Columns & key_columns, \ + const DataTypes & key_types, \ + const TYPE def, \ + ResultArrayType & out) const \ + { \ + dict_struct.validateKeyTypes(key_types); \ + auto & attribute = getAttribute(attribute_name); \ + checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ + getItemsNumberImpl(attribute, key_columns, out, [&](const size_t) { return def; }); \ + } + +DEFINE(UInt8) +DEFINE(UInt16) +DEFINE(UInt32) +DEFINE(UInt64) +DEFINE(UInt128) +DEFINE(Int8) +DEFINE(Int16) +DEFINE(Int32) +DEFINE(Int64) +DEFINE(Float32) +DEFINE(Float64) +DEFINE(Decimal32) +DEFINE(Decimal64) +DEFINE(Decimal128) + +#undef DEFINE +} diff --git a/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp.in b/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp.in deleted file mode 100644 index 02e77c01a4a..00000000000 --- a/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp.in +++ /dev/null @@ -1,25 +0,0 @@ -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int TYPE_MISMATCH; -} - -using TYPE = @NAME@; - -void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name, - const Columns & key_columns, - const DataTypes & key_types, - const TYPE def, - ResultArrayType & out) const -{ - dict_struct.validateKeyTypes(key_types); - - auto & attribute = getAttribute(attribute_name); - checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut@NAME@); - - getItemsNumberImpl(attribute, key_columns, out, [&](const size_t) { return def; }); -} -} diff --git a/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp b/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp index 24b7d83f14e..2df8f95bc0c 100644 --- a/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp +++ b/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp @@ -63,7 +63,7 @@ void ComplexKeyCacheDictionary::setAttributeValue(Attribute & attribute, const s const auto str_size = string.size(); if (str_size != 0) { - auto str_ptr = string_arena->alloc(str_size); + auto * str_ptr = string_arena->alloc(str_size); std::copy(string.data(), string.data() + str_size, str_ptr); string_ref = StringRef{str_ptr, str_size}; } diff --git a/src/Dictionaries/ComplexKeyHashedDictionary.cpp b/src/Dictionaries/ComplexKeyHashedDictionary.cpp index 485b0937be3..fc857254df4 100644 --- a/src/Dictionaries/ComplexKeyHashedDictionary.cpp +++ b/src/Dictionaries/ComplexKeyHashedDictionary.cpp @@ -358,7 +358,7 @@ void ComplexKeyHashedDictionary::updateData() for (size_t i = 0; i < saved_block->rows(); ++i) { const auto s_key = placeKeysInPool(i, saved_key_column_ptrs, keys, temp_key_pool); - auto it = update_key_hash.find(s_key); + auto * it = update_key_hash.find(s_key); if (it) filter[i] = 0; else @@ -619,7 +619,7 @@ bool ComplexKeyHashedDictionary::setAttributeValue(Attribute & attribute, const { auto & map = std::get>(attribute.maps); const auto & string = value.get(); - const auto string_in_arena = attribute.string_arena->insert(string.data(), string.size()); + const auto * string_in_arena = attribute.string_arena->insert(string.data(), string.size()); const auto pair = map.insert({key, StringRef{string_in_arena, string.size()}}); return pair.second; } @@ -649,7 +649,7 @@ StringRef ComplexKeyHashedDictionary::placeKeysInPool(const size_t row, const Co sum_keys_size += keys[j].size; } - auto key_start = block_start; + const auto * key_start = block_start; for (size_t j = 0; j < keys_size; ++j) { keys[j].data = key_start; diff --git a/src/Dictionaries/DictionarySourceFactory.cpp b/src/Dictionaries/DictionarySourceFactory.cpp index fa3b3017ad2..a9e90024745 100644 --- a/src/Dictionaries/DictionarySourceFactory.cpp +++ b/src/Dictionaries/DictionarySourceFactory.cpp @@ -84,11 +84,12 @@ DictionarySourcePtr DictionarySourceFactory::create( { Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_prefix, keys); - if (keys.size() != 1) - throw Exception{name + ": element dictionary.source should have exactly one child element", + + if (keys.empty() || keys.size() > 2) + throw Exception{name + ": element dictionary.source should have one or two child elements", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG}; - const auto & source_type = keys.front(); + const std::string & source_type = keys.front() == "settings" ? keys.back() : keys.front(); const auto found = registered_sources.find(source_type); if (found != registered_sources.end()) diff --git a/src/Dictionaries/DictionarySourceHelpers.cpp b/src/Dictionaries/DictionarySourceHelpers.cpp index d01c7560832..309bc64e179 100644 --- a/src/Dictionaries/DictionarySourceHelpers.cpp +++ b/src/Dictionaries/DictionarySourceHelpers.cpp @@ -6,7 +6,10 @@ #include #include #include "DictionaryStructure.h" - +#include +#include +#include +#include namespace DB { @@ -50,4 +53,30 @@ void formatKeys( out->flush(); } +Context copyContextAndApplySettings( + const std::string & config_prefix, + const Context & context, + const Poco::Util::AbstractConfiguration & config) +{ + Context local_context(context); + if (config.has(config_prefix + ".settings")) + { + const auto prefix = config_prefix + ".settings"; + + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(prefix, config_keys); + + SettingsChanges changes; + + for (const std::string & key : config_keys) + { + const auto value = config.getString(prefix + "." + key); + changes.emplace_back(key, value); + } + + local_context.applySettingsChanges(changes); + } + return local_context; +} + } diff --git a/src/Dictionaries/DictionarySourceHelpers.h b/src/Dictionaries/DictionarySourceHelpers.h index 99d5df5bda7..3f42700d336 100644 --- a/src/Dictionaries/DictionarySourceHelpers.h +++ b/src/Dictionaries/DictionarySourceHelpers.h @@ -3,7 +3,8 @@ #include #include #include - +#include +#include namespace DB { @@ -11,6 +12,7 @@ class IBlockOutputStream; using BlockOutputStreamPtr = std::shared_ptr; struct DictionaryStructure; +class Context; /// Write keys to block output stream. @@ -24,4 +26,14 @@ void formatKeys( const Columns & key_columns, const std::vector & requested_rows); +/// Used for applying settings to copied context in some register[...]Source functions +Context copyContextAndApplySettings( + const std::string & config_prefix, + const Context & context, + const Poco::Util::AbstractConfiguration & config); + +void applySettingsToContext( + const std::string & config_prefix, + Context & context, + const Poco::Util::AbstractConfiguration & config); } diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index f8b8fbd6aab..74bf67a2738 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -155,7 +155,7 @@ DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration if (id->name.empty()) throw Exception{"'id' cannot be empty", ErrorCodes::BAD_ARGUMENTS}; - const auto range_default_type = "Date"; + const char * range_default_type = "Date"; if (config.has(config_prefix + ".range_min")) range_min.emplace(makeDictionaryTypedSpecialAttribute(config, config_prefix + ".range_min", range_default_type)); diff --git a/src/Dictionaries/Embedded/RegionsHierarchy.h b/src/Dictionaries/Embedded/RegionsHierarchy.h index 25625284e32..7f42a76ecc2 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchy.h +++ b/src/Dictionaries/Embedded/RegionsHierarchy.h @@ -4,6 +4,7 @@ #include #include #include "GeodataProviders/IHierarchiesProvider.h" +#include class IRegionsHierarchyDataProvider; @@ -59,7 +60,7 @@ public: if (lhs >= parents.size()) return false; - while (lhs != 0 && lhs != rhs) + for (size_t i = 0; lhs != 0 && lhs != rhs && i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i) lhs = parents[lhs]; return lhs != 0; diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index d9903f759c0..34943d62b44 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -16,7 +16,6 @@ #include "DictionaryStructure.h" #include "registerDictionaries.h" - namespace DB { static const UInt64 max_block_size = 8192; @@ -232,9 +231,11 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) if (check_config) throw Exception("Dictionaries with Executable dictionary source is not allowed", ErrorCodes::DICTIONARY_ACCESS_DENIED); + Context context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + return std::make_unique( dict_struct, config, config_prefix + ".executable", - sample_block, context); + sample_block, context_local_copy); }; factory.registerSource("executable", create_table_source); } diff --git a/src/Dictionaries/ExecutableDictionarySource.h b/src/Dictionaries/ExecutableDictionarySource.h index 879248663dc..f28d71ca5e3 100644 --- a/src/Dictionaries/ExecutableDictionarySource.h +++ b/src/Dictionaries/ExecutableDictionarySource.h @@ -3,7 +3,7 @@ #include "DictionaryStructure.h" #include "IDictionarySource.h" #include - +#include namespace Poco { class Logger; } @@ -56,7 +56,7 @@ private: const std::string update_field; const std::string format; Block sample_block; - const Context & context; + Context context; }; } diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 5e7a784183c..0f20ab8edc4 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -7,6 +7,7 @@ #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" #include "registerDictionaries.h" +#include "DictionarySourceHelpers.h" namespace DB { @@ -83,7 +84,9 @@ void registerDictionarySourceFile(DictionarySourceFactory & factory) const auto filepath = config.getString(config_prefix + ".file.path"); const auto format = config.getString(config_prefix + ".file.format"); - return std::make_unique(filepath, format, sample_block, context, check_config); + Context context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + + return std::make_unique(filepath, format, sample_block, context_local_copy, check_config); }; factory.registerSource("file", create_table_source); diff --git a/src/Dictionaries/FileDictionarySource.h b/src/Dictionaries/FileDictionarySource.h index 3d00c026e07..fa47b280911 100644 --- a/src/Dictionaries/FileDictionarySource.h +++ b/src/Dictionaries/FileDictionarySource.h @@ -3,7 +3,7 @@ #include #include "IDictionarySource.h" #include - +#include namespace DB { @@ -11,7 +11,6 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; } -class Context; /// Allows loading dictionaries from a file with given format, does not support "random access" class FileDictionarySource final : public IDictionarySource @@ -62,7 +61,7 @@ private: const std::string filepath; const std::string format; Block sample_block; - const Context & context; + const Context context; Poco::Timestamp last_modification; }; diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index b70fe95346d..b6396d20f3a 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -2,6 +2,8 @@ #include #include "DictionaryBlockInputStream.h" #include "DictionaryFactory.h" +#include + namespace DB { @@ -77,7 +79,7 @@ void FlatDictionary::isInImpl(const ChildType & child_ids, const AncestorType & auto id = getAt(child_ids, row); const auto ancestor_id = getAt(ancestor_ids, row); - while (id < loaded_size && id != null_value && id != ancestor_id) + for (size_t i = 0; id < loaded_size && id != null_value && id != ancestor_id && i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i) id = attr[id]; out[row] = id != null_value && id == ancestor_id; @@ -603,7 +605,7 @@ template <> void FlatDictionary::setAttributeValueImpl(Attribute & attribute, const Key id, const String & value) { resize(attribute, id); - const auto string_in_arena = attribute.string_arena->insert(value.data(), value.size()); + const auto * string_in_arena = attribute.string_arena->insert(value.data(), value.size()); auto & array = std::get>(attribute.arrays); array[id] = StringRef{string_in_arena, value.size()}; loaded_ids[id] = true; diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 87d5381f30f..61f16797ce0 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -202,9 +202,11 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) if (dict_struct.has_expressions) throw Exception{"Dictionary source of type `http` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR}; + Context context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + return std::make_unique( dict_struct, config, config_prefix + ".http", - sample_block, context, check_config); + sample_block, context_local_copy, check_config); }; factory.registerSource("http", create_table_source); } diff --git a/src/Dictionaries/HTTPDictionarySource.h b/src/Dictionaries/HTTPDictionarySource.h index d04a6f1a789..e7920132e83 100644 --- a/src/Dictionaries/HTTPDictionarySource.h +++ b/src/Dictionaries/HTTPDictionarySource.h @@ -7,6 +7,7 @@ #include #include "DictionaryStructure.h" #include "IDictionarySource.h" +#include namespace Poco { @@ -64,7 +65,7 @@ private: std::string update_field; const std::string format; Block sample_block; - const Context & context; + Context context; ConnectionTimeouts timeouts; }; diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index c401142584c..62fa52acfe8 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -2,6 +2,8 @@ #include #include "DictionaryBlockInputStream.h" #include "DictionaryFactory.h" +#include + namespace { @@ -87,7 +89,7 @@ void HashedDictionary::isInAttrImpl(const AttrType & attr, const ChildType & chi auto id = getAt(child_ids, row); const auto ancestor_id = getAt(ancestor_ids, row); - while (id != null_value && id != ancestor_id) + for (size_t i = 0; id != null_value && id != ancestor_id && i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i) { auto it = attr.find(id); if (it != std::end(attr)) @@ -665,7 +667,7 @@ bool HashedDictionary::setAttributeValue(Attribute & attribute, const Key id, co case AttributeUnderlyingType::utString: { const auto & string = value.get(); - const auto string_in_arena = attribute.string_arena->insert(string.data(), string.size()); + const auto * string_in_arena = attribute.string_arena->insert(string.data(), string.size()); if (!sparse) { auto & map = *std::get>(attribute.maps); @@ -770,7 +772,7 @@ BlockInputStreamPtr HashedDictionary::getBlockInputStream(const Names & column_n void registerDictionaryHashed(DictionaryFactory & factory) { - auto create_layout = [=](const std::string & full_name, + auto create_layout = [](const std::string & full_name, const DictionaryStructure & dict_struct, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -793,8 +795,10 @@ void registerDictionaryHashed(DictionaryFactory & factory) return std::make_unique(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse); }; using namespace std::placeholders; - factory.registerLayout("hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ false), false); - factory.registerLayout("sparse_hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ true), false); + factory.registerLayout("hashed", + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e){ return create_layout(a, b, c, d, std::move(e), /* sparse = */ false); }, false); + factory.registerLayout("sparse_hashed", + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e){ return create_layout(a, b, c, d, std::move(e), /* sparse = */ true); }, false); } } diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h index 3f8eec979bb..a361352a8bd 100644 --- a/src/Dictionaries/HashedDictionary.h +++ b/src/Dictionaries/HashedDictionary.h @@ -156,8 +156,14 @@ private: template using CollectionPtrType = std::unique_ptr>; +#if !defined(ARCADIA_BUILD) template using SparseCollectionType = google::sparse_hash_map>; +#else + template + using SparseCollectionType = google::sparsehash::sparse_hash_map>; +#endif + template using SparseCollectionPtrType = std::unique_ptr>; diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 919411a28ea..79f90fcf22c 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -77,7 +77,7 @@ namespace if (!data) throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR); - auto columns_received = static_cast(data); + const auto * columns_received = static_cast(data); if (columns_received->error_code) throw Exception( "LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " " @@ -188,7 +188,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() ClickHouseLibrary::CStrings columns{static_cast(columns_holder.get()), dict_struct.attributes.size()}; size_t i = 0; - for (auto & a : dict_struct.attributes) + for (const auto & a : dict_struct.attributes) { columns.data[i] = a.name.c_str(); ++i; @@ -199,7 +199,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() auto func_load_all = library->getstrings), decltype(&columns))>("ClickHouseDictionary_v3_loadAll"); data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); - auto data = func_load_all(data_ptr, &settings->strings, &columns); + auto * data = func_load_all(data_ptr, &settings->strings, &columns); auto block = dataToBlock(description.sample_block, data); SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); return std::make_shared(block); @@ -214,7 +214,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & ClickHouseLibrary::CStrings columns_pass{static_cast(columns_holder.get()), dict_struct.attributes.size()}; size_t i = 0; - for (auto & a : dict_struct.attributes) + for (const auto & a : dict_struct.attributes) { columns_pass.data[i] = a.name.c_str(); ++i; @@ -226,7 +226,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & = library->getstrings), decltype(&columns_pass), decltype(&ids_data))>( "ClickHouseDictionary_v3_loadIds"); data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); - auto data = func_load_ids(data_ptr, &settings->strings, &columns_pass, &ids_data); + auto * data = func_load_ids(data_ptr, &settings->strings, &columns_pass, &ids_data); auto block = dataToBlock(description.sample_block, data); SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); return std::make_shared(block); @@ -259,7 +259,7 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column auto func_load_keys = library->getstrings), decltype(&request_cols))>( "ClickHouseDictionary_v3_loadKeys"); data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); - auto data = func_load_keys(data_ptr, &settings->strings, &request_cols); + auto * data = func_load_keys(data_ptr, &settings->strings, &request_cols); auto block = dataToBlock(description.sample_block, data); SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); return std::make_shared(block); diff --git a/src/Dictionaries/MongoDBBlockInputStream.cpp b/src/Dictionaries/MongoDBBlockInputStream.cpp index b1c2f67cc86..e0ff7cb6529 100644 --- a/src/Dictionaries/MongoDBBlockInputStream.cpp +++ b/src/Dictionaries/MongoDBBlockInputStream.cpp @@ -1,25 +1,27 @@ -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_POCO_MONGODB +# include +# include +# include -#include -#include -#include +# include +# include +# include +# include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include "DictionaryStructure.h" -#include "MongoDBBlockInputStream.h" +# include +# include +# include +# include +# include +# include +# include +# include +# include "DictionaryStructure.h" +# include "MongoDBBlockInputStream.h" namespace DB diff --git a/src/Dictionaries/MongoDBDictionarySource.h b/src/Dictionaries/MongoDBDictionarySource.h index bf4669248dc..622d61c5b09 100644 --- a/src/Dictionaries/MongoDBDictionarySource.h +++ b/src/Dictionaries/MongoDBDictionarySource.h @@ -1,9 +1,12 @@ #pragma once #include -#include "config_core.h" -#if USE_POCO_MONGODB +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + +#if USE_POCO_MONGODB # include "DictionaryStructure.h" # include "IDictionarySource.h" @@ -91,4 +94,5 @@ private: }; } + #endif diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 00527064d79..4ba16ca19de 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -1,8 +1,8 @@ #include "MySQLDictionarySource.h" + #include #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" -#include "config_core.h" #include "registerDictionaries.h" namespace DB diff --git a/src/Dictionaries/MySQLDictionarySource.h b/src/Dictionaries/MySQLDictionarySource.h index 95e660d220f..34f784cdfeb 100644 --- a/src/Dictionaries/MySQLDictionarySource.h +++ b/src/Dictionaries/MySQLDictionarySource.h @@ -2,9 +2,11 @@ #include -#include "config_core.h" -#if USE_MYSQL +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif +#if USE_MYSQL # include # include # include "DictionaryStructure.h" diff --git a/src/Dictionaries/PolygonDictionary.cpp b/src/Dictionaries/PolygonDictionary.cpp index b7d9ed41402..dd231b3f2d7 100644 --- a/src/Dictionaries/PolygonDictionary.cpp +++ b/src/Dictionaries/PolygonDictionary.cpp @@ -237,8 +237,8 @@ std::vector IPolygonDictionary::extractPoints(const C { if (key_columns.size() != 2) throw Exception{"Expected two columns of coordinates", ErrorCodes::BAD_ARGUMENTS}; - const auto column_x = typeid_cast*>(key_columns[0].get()); - const auto column_y = typeid_cast*>(key_columns[1].get()); + const auto * column_x = typeid_cast*>(key_columns[0].get()); + const auto * column_y = typeid_cast*>(key_columns[1].get()); if (!column_x || !column_y) throw Exception{"Expected columns of Float64", ErrorCodes::TYPE_MISMATCH}; const auto rows = key_columns.front()->size(); @@ -534,17 +534,17 @@ void addNewPoint(Float64 x, Float64 y, Data & data, Offset & offset) const IColumn * unrollMultiPolygons(const ColumnPtr & column, Offset & offset) { - const auto ptr_multi_polygons = typeid_cast(column.get()); + const auto * ptr_multi_polygons = typeid_cast(column.get()); if (!ptr_multi_polygons) throw Exception{"Expected a column containing arrays of polygons", ErrorCodes::TYPE_MISMATCH}; offset.multi_polygon_offsets.assign(ptr_multi_polygons->getOffsets()); - const auto ptr_polygons = typeid_cast(&ptr_multi_polygons->getData()); + const auto * ptr_polygons = typeid_cast(&ptr_multi_polygons->getData()); if (!ptr_polygons) throw Exception{"Expected a column containing arrays of rings when reading polygons", ErrorCodes::TYPE_MISMATCH}; offset.polygon_offsets.assign(ptr_polygons->getOffsets()); - const auto ptr_rings = typeid_cast(&ptr_polygons->getData()); + const auto * ptr_rings = typeid_cast(&ptr_polygons->getData()); if (!ptr_rings) throw Exception{"Expected a column containing arrays of points when reading rings", ErrorCodes::TYPE_MISMATCH}; offset.ring_offsets.assign(ptr_rings->getOffsets()); @@ -554,7 +554,7 @@ const IColumn * unrollMultiPolygons(const ColumnPtr & column, Offset & offset) const IColumn * unrollSimplePolygons(const ColumnPtr & column, Offset & offset) { - const auto ptr_polygons = typeid_cast(column.get()); + const auto * ptr_polygons = typeid_cast(column.get()); if (!ptr_polygons) throw Exception{"Expected a column containing arrays of points", ErrorCodes::TYPE_MISMATCH}; offset.ring_offsets.assign(ptr_polygons->getOffsets()); @@ -566,8 +566,8 @@ const IColumn * unrollSimplePolygons(const ColumnPtr & column, Offset & offset) void handlePointsReprByArrays(const IColumn * column, Data & data, Offset & offset) { - const auto ptr_points = typeid_cast(column); - const auto ptr_coord = typeid_cast*>(&ptr_points->getData()); + const auto * ptr_points = typeid_cast(column); + const auto * ptr_coord = typeid_cast*>(&ptr_points->getData()); if (!ptr_coord) throw Exception{"Expected coordinates to be of type Float64", ErrorCodes::TYPE_MISMATCH}; const auto & offsets = ptr_points->getOffsets(); @@ -583,13 +583,13 @@ void handlePointsReprByArrays(const IColumn * column, Data & data, Offset & offs void handlePointsReprByTuples(const IColumn * column, Data & data, Offset & offset) { - const auto ptr_points = typeid_cast(column); + const auto * ptr_points = typeid_cast(column); if (!ptr_points) throw Exception{"Expected a column of tuples representing points", ErrorCodes::TYPE_MISMATCH}; if (ptr_points->tupleSize() != 2) throw Exception{"Points should be two-dimensional", ErrorCodes::BAD_ARGUMENTS}; - const auto column_x = typeid_cast*>(&ptr_points->getColumn(0)); - const auto column_y = typeid_cast*>(&ptr_points->getColumn(1)); + const auto * column_x = typeid_cast*>(&ptr_points->getColumn(0)); + const auto * column_y = typeid_cast*>(&ptr_points->getColumn(1)); if (!column_x || !column_y) throw Exception{"Expected coordinates to be of type Float64", ErrorCodes::TYPE_MISMATCH}; for (size_t i = 0; i < column_x->size(); ++i) diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 7fab92fcde0..eed5815333a 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -127,7 +127,7 @@ void RangeHashedDictionary::getString( for (const auto i : ext::range(0, ids.size())) { - const auto it = attr.find(ids[i]); + const auto * it = attr.find(ids[i]); if (it) { const auto date = dates[i]; @@ -407,11 +407,11 @@ void RangeHashedDictionary::getItemsImpl( return v.range.contains(date); }); - out[i] = static_cast(val_it != std::end(ranges_and_values) ? val_it->value : null_value); + out[i] = static_cast(val_it != std::end(ranges_and_values) ? val_it->value : null_value); // NOLINT } else { - out[i] = static_cast(null_value); + out[i] = static_cast(null_value); // NOLINT } } @@ -493,10 +493,10 @@ void RangeHashedDictionary::setAttributeValue(Attribute & attribute, const Key i { auto & map = *std::get>(attribute.maps); const auto & string = value.get(); - const auto string_in_arena = attribute.string_arena->insert(string.data(), string.size()); + const auto * string_in_arena = attribute.string_arena->insert(string.data(), string.size()); const StringRef string_ref{string_in_arena, string.size()}; - const auto it = map.find(id); + auto * it = map.find(id); if (it) { @@ -649,7 +649,7 @@ struct RangeHashedDIctionaryCallGetBlockInputStreamImpl template void operator()() { - auto & type = dict->dict_struct.range_min->type; + const auto & type = dict->dict_struct.range_min->type; if (!stream && dynamic_cast *>(type.get())) stream = dict->getBlockInputStreamImpl(*column_names, max_block_size); } diff --git a/src/Dictionaries/RedisBlockInputStream.h b/src/Dictionaries/RedisBlockInputStream.h index 86448095787..6e350e7c4f1 100644 --- a/src/Dictionaries/RedisBlockInputStream.h +++ b/src/Dictionaries/RedisBlockInputStream.h @@ -1,8 +1,11 @@ #pragma once -#include "config_core.h" #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_POCO_REDIS # include # include diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index 51f5cdcafb5..6e32b7766b4 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -164,7 +164,7 @@ namespace DB RedisArray keys; auto key_type = storageTypeToKeyType(storage_type); - for (auto & key : all_keys) + for (const auto & key : all_keys) if (key_type == client->execute(RedisCommand("TYPE").addRedisType(key))) keys.addRedisType(std::move(key)); diff --git a/src/Dictionaries/RedisDictionarySource.h b/src/Dictionaries/RedisDictionarySource.h index dc62d7dc625..e42bd2678cb 100644 --- a/src/Dictionaries/RedisDictionarySource.h +++ b/src/Dictionaries/RedisDictionarySource.h @@ -1,10 +1,12 @@ #pragma once -#include "config_core.h" #include -#if USE_POCO_REDIS +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif +#if USE_POCO_REDIS # include "DictionaryStructure.h" # include "IDictionarySource.h" @@ -101,4 +103,5 @@ namespace ErrorCodes }; } + #endif diff --git a/src/Dictionaries/TrieDictionary.cpp b/src/Dictionaries/TrieDictionary.cpp index 83b52803325..8bc45069bd5 100644 --- a/src/Dictionaries/TrieDictionary.cpp +++ b/src/Dictionaries/TrieDictionary.cpp @@ -618,7 +618,7 @@ bool TrieDictionary::setAttributeValue(Attribute & attribute, const StringRef ke case AttributeUnderlyingType::utString: { const auto & string = value.get(); - const auto string_in_arena = attribute.string_arena->insert(string.data(), string.size()); + const auto * string_in_arena = attribute.string_arena->insert(string.data(), string.size()); setAttributeValueImpl(attribute, key, StringRef{string_in_arena, string.size()}); return true; } diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 8647c8ccac5..3fea0a024ef 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -16,9 +16,12 @@ #include "DictionaryStructure.h" #include "readInvalidateQuery.h" -#include #include "registerDictionaries.h" +#if !defined(ARCADIA_BUILD) +# include +#endif + #if USE_POCO_SQLODBC || USE_POCO_DATAODBC # include #endif diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index fe131b3d05a..e60a0641066 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -338,17 +338,17 @@ void buildConfigurationFromFunctionWithKeyValueArguments( AutoPtr current_xml_element(doc->createElement(pair->first)); root->appendChild(current_xml_element); - if (auto identifier = pair->second->as(); identifier) + if (const auto * identifier = pair->second->as(); identifier) { AutoPtr value(doc->createTextNode(identifier->name)); current_xml_element->appendChild(value); } - else if (auto literal = pair->second->as(); literal) + else if (const auto * literal = pair->second->as(); literal) { AutoPtr value(doc->createTextNode(getFieldAsString(literal->value))); current_xml_element->appendChild(value); } - else if (auto list = pair->second->as(); list) + else if (const auto * list = pair->second->as(); list) { buildConfigurationFromFunctionWithKeyValueArguments(doc, current_xml_element, list); } @@ -420,7 +420,7 @@ void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs } -DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query) +DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const std::string & database_) { checkAST(query); @@ -438,13 +438,13 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer AutoPtr database_element(xml_document->createElement("database")); current_dictionary->appendChild(database_element); - AutoPtr database(xml_document->createTextNode(query.database)); + AutoPtr database(xml_document->createTextNode(!database_.empty() ? database_ : query.database)); database_element->appendChild(database); AutoPtr structure_element(xml_document->createElement("structure")); current_dictionary->appendChild(structure_element); Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key); - auto dictionary_layout = query.dictionary->layout; + auto * dictionary_layout = query.dictionary->layout; bool complex = DictionaryFactory::instance().isComplex(dictionary_layout->layout_type); diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.h b/src/Dictionaries/getDictionaryConfigurationFromAST.h index bb48765c492..3038f450914 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.h +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.h @@ -10,6 +10,5 @@ using DictionaryConfigurationPtr = Poco::AutoPtras(); DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); @@ -120,7 +120,7 @@ TEST(ConvertDictionaryAST, TrickyAttributes) " SOURCE(CLICKHOUSE(HOST 'localhost'))"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); @@ -165,7 +165,7 @@ TEST(ConvertDictionaryAST, ComplexKeyAndLayoutWithParams) " LIFETIME(MIN 1 MAX 10)"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); @@ -216,7 +216,7 @@ TEST(ConvertDictionaryAST, ComplexSource) " RANGE(MIN second_column MAX third_column)"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); /// source diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make new file mode 100644 index 00000000000..3f831c3c9fe --- /dev/null +++ b/src/Dictionaries/ya.make @@ -0,0 +1,63 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/poco/Data + contrib/libs/poco/Data/ODBC + contrib/libs/poco/MongoDB + contrib/libs/poco/Redis + contrib/libs/sparsehash +) + +NO_COMPILER_WARNINGS() + +SRCS( + CacheDictionary_generate1.cpp + CacheDictionary_generate2.cpp + CacheDictionary_generate3.cpp + CacheDictionary.cpp + ClickHouseDictionarySource.cpp + ComplexKeyCacheDictionary_createAttributeWithType.cpp + ComplexKeyCacheDictionary_generate1.cpp + ComplexKeyCacheDictionary_generate2.cpp + ComplexKeyCacheDictionary_generate3.cpp + ComplexKeyCacheDictionary_setAttributeValue.cpp + ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp + ComplexKeyCacheDictionary.cpp + ComplexKeyHashedDictionary.cpp + DictionaryBlockInputStreamBase.cpp + DictionaryFactory.cpp + DictionarySourceFactory.cpp + DictionarySourceHelpers.cpp + DictionaryStructure.cpp + Embedded/GeodataProviders/HierarchiesProvider.cpp + Embedded/GeodataProviders/HierarchyFormatReader.cpp + Embedded/GeodataProviders/NamesFormatReader.cpp + Embedded/GeodataProviders/NamesProvider.cpp + Embedded/GeoDictionariesLoader.cpp + Embedded/RegionsHierarchies.cpp + Embedded/RegionsHierarchy.cpp + Embedded/RegionsNames.cpp + ExecutableDictionarySource.cpp + ExternalQueryBuilder.cpp + FileDictionarySource.cpp + FlatDictionary.cpp + getDictionaryConfigurationFromAST.cpp + HashedDictionary.cpp + HTTPDictionarySource.cpp + LibraryDictionarySource.cpp + LibraryDictionarySourceExternal.cpp + MongoDBBlockInputStream.cpp + MongoDBDictionarySource.cpp + MySQLDictionarySource.cpp + PolygonDictionary.cpp + RangeHashedDictionary.cpp + readInvalidateQuery.cpp + RedisBlockInputStream.cpp + RedisDictionarySource.cpp + registerDictionaries.cpp + writeParenthesisedString.cpp + XDBCDictionarySource.cpp +) + +END() diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index 05ae59600ca..3e43d159ba5 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -131,7 +131,6 @@ private: impl.write(working_buffer.begin(), offset()); } -private: WriteBufferFromOwnString impl; DiskMemory * disk; const String path; diff --git a/src/Disks/DiskS3.cpp b/src/Disks/DiskS3.cpp index 6efe6cf8c4d..081f0334398 100644 --- a/src/Disks/DiskS3.cpp +++ b/src/Disks/DiskS3.cpp @@ -252,7 +252,6 @@ namespace return true; } - private: std::shared_ptr client_ptr; const String & bucket; Metadata metadata; @@ -328,7 +327,6 @@ namespace impl.swap(*this); } - private: WriteBufferFromS3 impl; bool finalized = false; Metadata metadata; diff --git a/src/Disks/DiskS3.h b/src/Disks/DiskS3.h index 1b61ed1cde3..c89fde4b103 100644 --- a/src/Disks/DiskS3.h +++ b/src/Disks/DiskS3.h @@ -1,6 +1,8 @@ #pragma once -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_AWS_S3 # include "DiskFactory.h" diff --git a/src/Disks/registerDisks.cpp b/src/Disks/registerDisks.cpp index f28918cdbd0..2da39e62b19 100644 --- a/src/Disks/registerDisks.cpp +++ b/src/Disks/registerDisks.cpp @@ -2,7 +2,9 @@ #include "DiskFactory.h" -#include +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB { diff --git a/src/Disks/ya.make b/src/Disks/ya.make new file mode 100644 index 00000000000..71eb47c489c --- /dev/null +++ b/src/Disks/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + DiskFactory.cpp + DiskLocal.cpp + DiskMemory.cpp + DiskS3.cpp + DiskSpaceMonitor.cpp + IDisk.cpp + registerDisks.cpp +) + +END() diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index c2b890ec631..91cfb7eade9 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -1,12 +1,12 @@ +#include + #include -#include #include #include #include #include #include #include -#include #include #include #include @@ -16,6 +16,10 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + namespace DB { @@ -347,15 +351,19 @@ FormatFactory::FormatFactory() registerOutputFormatProcessorJSONCompactEachRow(*this); registerInputFormatProcessorProtobuf(*this); registerOutputFormatProcessorProtobuf(*this); +#if !defined(ARCADIA_BUILD) registerInputFormatProcessorCapnProto(*this); registerInputFormatProcessorORC(*this); registerInputFormatProcessorParquet(*this); registerOutputFormatProcessorParquet(*this); registerInputFormatProcessorAvro(*this); registerOutputFormatProcessorAvro(*this); +#endif registerInputFormatProcessorTemplate(*this); registerOutputFormatProcessorTemplate(*this); registerInputFormatProcessorRegexp(*this); + registerInputFormatProcessorMsgPack(*this); + registerOutputFormatProcessorMsgPack(*this); registerFileSegmentationEngineTabSeparated(*this); registerFileSegmentationEngineCSV(*this); @@ -375,6 +383,7 @@ FormatFactory::FormatFactory() registerOutputFormatProcessorODBCDriver2(*this); registerOutputFormatProcessorNull(*this); registerOutputFormatProcessorMySQLWrite(*this); + registerOutputFormatProcessorMarkdown(*this); } FormatFactory & FormatFactory::instance() diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 7c515dbce90..39d6eb0cc15 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -171,7 +171,9 @@ void registerOutputFormatProcessorProtobuf(FormatFactory & factory); void registerInputFormatProcessorAvro(FormatFactory & factory); void registerOutputFormatProcessorAvro(FormatFactory & factory); void registerInputFormatProcessorTemplate(FormatFactory & factory); -void registerOutputFormatProcessorTemplate(FormatFactory &factory); +void registerOutputFormatProcessorTemplate(FormatFactory & factory); +void registerInputFormatProcessorMsgPack(FormatFactory & factory); +void registerOutputFormatProcessorMsgPack(FormatFactory & factory); /// File Segmentation Engines for parallel reading @@ -196,6 +198,7 @@ void registerOutputFormatProcessorODBCDriver(FormatFactory & factory); void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory); void registerOutputFormatProcessorNull(FormatFactory & factory); void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory); +void registerOutputFormatProcessorMarkdown(FormatFactory & factory); /// Input only formats. void registerInputFormatProcessorCapnProto(FormatFactory & factory); diff --git a/src/Formats/IRowOutputStream.cpp b/src/Formats/IRowOutputStream.cpp index 881a467dfe0..f84d810b8e8 100644 --- a/src/Formats/IRowOutputStream.cpp +++ b/src/Formats/IRowOutputStream.cpp @@ -22,7 +22,7 @@ void IRowOutputStream::write(const Block & block, size_t row_num) if (i != 0) writeFieldDelimiter(); - auto & col = block.getByPosition(i); + const auto & col = block.getByPosition(i); writeField(*col.column, *col.type, row_num); } diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index 4f8291ffebe..17c09cdc14d 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -1,15 +1,17 @@ -#include "config_core.h" -#if USE_MYSQL +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include "MySQLBlockInputStream.h" +#if USE_MYSQL +# include +# include +# include +# include +# include +# include +# include +# include +# include "MySQLBlockInputStream.h" namespace DB diff --git a/src/Formats/ProtobufColumnMatcher.h b/src/Formats/ProtobufColumnMatcher.h index ed6c632f8e8..03c5ec40fc6 100644 --- a/src/Formats/ProtobufColumnMatcher.h +++ b/src/Formats/ProtobufColumnMatcher.h @@ -1,15 +1,17 @@ #pragma once -#include "config_formats.h" -#if USE_PROTOBUF +#if !defined(ARCADIA_BUILD) +# include "config_formats.h" +#endif -#include -#include -#include -#include -#include -#include -#include +#if USE_PROTOBUF +# include +# include +# include +# include +# include +# include +# include namespace google { @@ -190,4 +192,5 @@ namespace ProtobufColumnMatcher } } + #endif diff --git a/src/Formats/ProtobufReader.cpp b/src/Formats/ProtobufReader.cpp index 5426e8fac62..d8302771002 100644 --- a/src/Formats/ProtobufReader.cpp +++ b/src/Formats/ProtobufReader.cpp @@ -1,16 +1,14 @@ -#include "config_formats.h" -#if USE_PROTOBUF - #include "ProtobufReader.h" -#include -#include -#include -#include -#include -#include -#include -#include +#if USE_PROTOBUF +# include +# include +# include +# include +# include +# include +# include +# include namespace DB @@ -272,25 +270,25 @@ UInt64 ProtobufReader::SimpleReader::continueReadingVarint(UInt64 first_byte) UInt64 result = (first_byte & ~static_cast(0x80)); char c; -#define PROTOBUF_READER_READ_VARINT_BYTE(byteNo) \ - do \ - { \ - in.readStrict(c); \ - ++cursor; \ - if constexpr ((byteNo) < 10) \ +# define PROTOBUF_READER_READ_VARINT_BYTE(byteNo) \ + do \ { \ - result |= static_cast(static_cast(c)) << (7 * ((byteNo) - 1)); \ - if (likely(!(c & 0x80))) \ - return result; \ - } \ - else \ - { \ - if (likely(c == 1)) \ - return result; \ - } \ - if constexpr ((byteNo) < 9) \ - result &= ~(static_cast(0x80) << (7 * ((byteNo) - 1))); \ - } while (false) + in.readStrict(c); \ + ++cursor; \ + if constexpr ((byteNo) < 10) \ + { \ + result |= static_cast(static_cast(c)) << (7 * ((byteNo)-1)); \ + if (likely(!(c & 0x80))) \ + return result; \ + } \ + else \ + { \ + if (likely(c == 1)) \ + return result; \ + } \ + if constexpr ((byteNo) < 9) \ + result &= ~(static_cast(0x80) << (7 * ((byteNo)-1))); \ + } while (false) PROTOBUF_READER_READ_VARINT_BYTE(2); PROTOBUF_READER_READ_VARINT_BYTE(3); @@ -302,7 +300,7 @@ UInt64 ProtobufReader::SimpleReader::continueReadingVarint(UInt64 first_byte) PROTOBUF_READER_READ_VARINT_BYTE(9); PROTOBUF_READER_READ_VARINT_BYTE(10); -#undef PROTOBUF_READER_READ_VARINT_BYTE +# undef PROTOBUF_READER_READ_VARINT_BYTE throwUnknownFormat(); } @@ -311,22 +309,22 @@ void ProtobufReader::SimpleReader::ignoreVarint() { char c; -#define PROTOBUF_READER_IGNORE_VARINT_BYTE(byteNo) \ - do \ - { \ - in.readStrict(c); \ - ++cursor; \ - if constexpr ((byteNo) < 10) \ +# define PROTOBUF_READER_IGNORE_VARINT_BYTE(byteNo) \ + do \ { \ - if (likely(!(c & 0x80))) \ - return; \ - } \ - else \ - { \ - if (likely(c == 1)) \ - return; \ - } \ - } while (false) + in.readStrict(c); \ + ++cursor; \ + if constexpr ((byteNo) < 10) \ + { \ + if (likely(!(c & 0x80))) \ + return; \ + } \ + else \ + { \ + if (likely(c == 1)) \ + return; \ + } \ + } while (false) PROTOBUF_READER_IGNORE_VARINT_BYTE(1); PROTOBUF_READER_IGNORE_VARINT_BYTE(2); @@ -338,7 +336,8 @@ void ProtobufReader::SimpleReader::ignoreVarint() PROTOBUF_READER_IGNORE_VARINT_BYTE(8); PROTOBUF_READER_IGNORE_VARINT_BYTE(9); PROTOBUF_READER_IGNORE_VARINT_BYTE(10); -#undef PROTOBUF_READER_IGNORE_VARINT_BYTE + +# undef PROTOBUF_READER_IGNORE_VARINT_BYTE throwUnknownFormat(); } @@ -694,16 +693,17 @@ private: std::optional> enum_name_to_value_map; }; -#define PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(field_type_id) \ - template <> \ - std::unique_ptr ProtobufReader::createConverter( \ - const google::protobuf::FieldDescriptor * field) \ - { \ - return std::make_unique(simple_reader, field); \ - } +# define PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(field_type_id) \ + template <> \ + std::unique_ptr ProtobufReader::createConverter( \ + const google::protobuf::FieldDescriptor * field) \ + { \ + return std::make_unique(simple_reader, field); \ + } PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(google::protobuf::FieldDescriptor::TYPE_STRING) PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(google::protobuf::FieldDescriptor::TYPE_BYTES) -#undef PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS + +# undef PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS template @@ -850,13 +850,14 @@ private: std::optional> set_of_enum_values; }; -#define PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(field_type_id, field_type) \ - template <> \ - std::unique_ptr ProtobufReader::createConverter( \ - const google::protobuf::FieldDescriptor * field) \ - { \ - return std::make_unique>(simple_reader, field); /* NOLINT */ \ - } +# define PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(field_type_id, field_type) \ + template <> \ + std::unique_ptr ProtobufReader::createConverter( \ + const google::protobuf::FieldDescriptor * field) \ + { \ + return std::make_unique>(simple_reader, field); /* NOLINT */ \ + } + PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_INT32, Int64); PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_SINT32, Int64); PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_UINT32, UInt64); @@ -869,7 +870,8 @@ PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::Fi PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_SFIXED64, Int64); PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_FLOAT, float); PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_DOUBLE, double); -#undef PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS + +# undef PROTOBUF_READER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS class ProtobufReader::ConverterFromBool : public ConverterBaseImpl @@ -1021,7 +1023,7 @@ private: enum_pbnumber_always_equals_value = true; for (const auto & name_value_pair : name_value_pairs) { - Int16 value = name_value_pair.second; + Int16 value = name_value_pair.second; // NOLINT const auto * enum_descriptor = field->enum_type()->FindValueByName(name_value_pair.first); if (enum_descriptor) { @@ -1073,10 +1075,10 @@ void ProtobufReader::setTraitsDataAfterMatchingColumns(Message * message) } switch (field.field_descriptor->type()) { -#define PROTOBUF_READER_CONVERTER_CREATING_CASE(field_type_id) \ - case field_type_id: \ - field.data.converter = createConverter(field.field_descriptor); \ - break +# define PROTOBUF_READER_CONVERTER_CREATING_CASE(field_type_id) \ + case field_type_id: \ + field.data.converter = createConverter(field.field_descriptor); \ + break PROTOBUF_READER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_STRING); PROTOBUF_READER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_BYTES); PROTOBUF_READER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_INT32); @@ -1093,8 +1095,9 @@ void ProtobufReader::setTraitsDataAfterMatchingColumns(Message * message) PROTOBUF_READER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_DOUBLE); PROTOBUF_READER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_BOOL); PROTOBUF_READER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_ENUM); -#undef PROTOBUF_READER_CONVERTER_CREATING_CASE - default: __builtin_unreachable(); +# undef PROTOBUF_READER_CONVERTER_CREATING_CASE + default: + __builtin_unreachable(); } message->data.field_number_to_field_map.emplace(field.field_number, &field); } @@ -1171,4 +1174,5 @@ bool ProtobufReader::readColumnIndex(size_t & column_index) } } + #endif diff --git a/src/Formats/ProtobufReader.h b/src/Formats/ProtobufReader.h index 393b15eb343..a50c2f202f0 100644 --- a/src/Formats/ProtobufReader.h +++ b/src/Formats/ProtobufReader.h @@ -5,13 +5,15 @@ #include #include -#include "config_formats.h" -#if USE_PROTOBUF +#if !defined(ARCADIA_BUILD) +# include "config_formats.h" +#endif -#include -#include "ProtobufColumnMatcher.h" -#include -#include +#if USE_PROTOBUF +# include +# include +# include +# include "ProtobufColumnMatcher.h" namespace google { diff --git a/src/Formats/ProtobufSchemas.cpp b/src/Formats/ProtobufSchemas.cpp index f4973263bc8..9c6ed76ef27 100644 --- a/src/Formats/ProtobufSchemas.cpp +++ b/src/Formats/ProtobufSchemas.cpp @@ -1,10 +1,12 @@ -#include "config_formats.h" -#if USE_PROTOBUF +#if !defined(ARCADIA_BUILD) +# include "config_formats.h" +#endif -#include -#include -#include -#include +#if USE_PROTOBUF +# include +# include +# include +# include namespace DB diff --git a/src/Formats/ProtobufWriter.cpp b/src/Formats/ProtobufWriter.cpp index 8ee309d41c9..0422416ecb5 100644 --- a/src/Formats/ProtobufWriter.cpp +++ b/src/Formats/ProtobufWriter.cpp @@ -1,18 +1,16 @@ -#include "config_formats.h" -#if USE_PROTOBUF - #include "ProtobufWriter.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#if USE_PROTOBUF +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include namespace DB @@ -496,19 +494,19 @@ private: std::optional> enum_value_to_name_map; }; -#define PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(field_type_id) \ - template <> \ - std::unique_ptr ProtobufWriter::createConverter( \ - const google::protobuf::FieldDescriptor * field) \ - { \ - if (shouldSkipNullValue(field)) \ - return std::make_unique>(simple_writer, field); \ - else \ - return std::make_unique>(simple_writer, field); \ - } +# define PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(field_type_id) \ + template <> \ + std::unique_ptr ProtobufWriter::createConverter( \ + const google::protobuf::FieldDescriptor * field) \ + { \ + if (shouldSkipNullValue(field)) \ + return std::make_unique>(simple_writer, field); \ + else \ + return std::make_unique>(simple_writer, field); \ + } PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(google::protobuf::FieldDescriptor::TYPE_STRING) PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS(google::protobuf::FieldDescriptor::TYPE_BYTES) -#undef PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS +# undef PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_STRINGS template @@ -606,18 +604,19 @@ private: } }; -#define PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(field_type_id, field_type) \ - template <> \ - std::unique_ptr ProtobufWriter::createConverter( \ - const google::protobuf::FieldDescriptor * field) \ - { \ - if (shouldSkipNullValue(field)) \ - return std::make_unique>(simple_writer, field); \ - else if (shouldPackRepeated(field)) \ - return std::make_unique>(simple_writer, field); \ - else \ - return std::make_unique>(simple_writer, field); \ - } +# define PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(field_type_id, field_type) \ + template <> \ + std::unique_ptr ProtobufWriter::createConverter( \ + const google::protobuf::FieldDescriptor * field) \ + { \ + if (shouldSkipNullValue(field)) \ + return std::make_unique>(simple_writer, field); \ + else if (shouldPackRepeated(field)) \ + return std::make_unique>(simple_writer, field); \ + else \ + return std::make_unique>(simple_writer, field); \ + } + PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_INT32, Int32); PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_SINT32, Int32); PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_UINT32, UInt32); @@ -630,7 +629,7 @@ PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::Fi PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_SFIXED64, Int64); PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_FLOAT, float); PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS(google::protobuf::FieldDescriptor::TYPE_DOUBLE, double); -#undef PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS +# undef PROTOBUF_WRITER_CREATE_CONVERTER_SPECIALIZATION_FOR_NUMBERS template @@ -779,7 +778,7 @@ private: enum_value_always_equals_pbnumber = true; for (const auto & name_value_pair : name_value_pairs) { - Int16 value = name_value_pair.second; + Int16 value = name_value_pair.second; // NOLINT const auto * enum_descriptor = field->enum_type()->FindValueByName(name_value_pair.first); if (enum_descriptor) { @@ -871,10 +870,10 @@ void ProtobufWriter::setTraitsDataAfterMatchingColumns(Message * message) } switch (field.field_descriptor->type()) { -#define PROTOBUF_WRITER_CONVERTER_CREATING_CASE(field_type_id) \ - case field_type_id: \ - field.data.converter = createConverter(field.field_descriptor); \ - break +# define PROTOBUF_WRITER_CONVERTER_CREATING_CASE(field_type_id) \ + case field_type_id: \ + field.data.converter = createConverter(field.field_descriptor); \ + break PROTOBUF_WRITER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_STRING); PROTOBUF_WRITER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_BYTES); PROTOBUF_WRITER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_INT32); @@ -891,7 +890,7 @@ void ProtobufWriter::setTraitsDataAfterMatchingColumns(Message * message) PROTOBUF_WRITER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_DOUBLE); PROTOBUF_WRITER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_BOOL); PROTOBUF_WRITER_CONVERTER_CREATING_CASE(google::protobuf::FieldDescriptor::TYPE_ENUM); -#undef PROTOBUF_WRITER_CONVERTER_CREATING_CASE +# undef PROTOBUF_WRITER_CONVERTER_CREATING_CASE default: throw Exception( String("Protobuf type '") + field.field_descriptor->type_name() + "' isn't supported", ErrorCodes::NOT_IMPLEMENTED); @@ -991,4 +990,5 @@ void ProtobufWriter::setNestedMessageNeedsRepeat() } } + #endif diff --git a/src/Formats/ProtobufWriter.h b/src/Formats/ProtobufWriter.h index 3862ed88cdb..1778f2adb21 100644 --- a/src/Formats/ProtobufWriter.h +++ b/src/Formats/ProtobufWriter.h @@ -3,17 +3,17 @@ #include #include #include - -#include "config_formats.h" - #include -#if USE_PROTOBUF +#if !defined(ARCADIA_BUILD) +# include "config_formats.h" +#endif -#include "ProtobufColumnMatcher.h" -#include -#include -#include +#if USE_PROTOBUF +# include +# include +# include +# include "ProtobufColumnMatcher.h" namespace google @@ -233,6 +233,8 @@ private: } #else +# include + namespace DB { diff --git a/src/Formats/verbosePrintString.cpp b/src/Formats/verbosePrintString.cpp index 1fdaf0cb7b7..2f3e09ed75f 100644 --- a/src/Formats/verbosePrintString.cpp +++ b/src/Formats/verbosePrintString.cpp @@ -16,7 +16,7 @@ void verbosePrintString(const char * begin, const char * end, WriteBuffer & out) out << "\""; - for (auto pos = begin; pos < end; ++pos) + for (const char * pos = begin; pos < end; ++pos) { switch (*pos) { diff --git a/src/Formats/ya.make b/src/Formats/ya.make new file mode 100644 index 00000000000..de61820e58d --- /dev/null +++ b/src/Formats/ya.make @@ -0,0 +1,24 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/protobuf_std +) + +SRCS( + FormatFactory.cpp + FormatSchemaInfo.cpp + IRowInputStream.cpp + IRowOutputStream.cpp + MySQLBlockInputStream.cpp + NativeFormat.cpp + NullFormat.cpp + ParsedTemplateFormatString.cpp + ProtobufColumnMatcher.cpp + ProtobufReader.cpp + ProtobufSchemas.cpp + ProtobufWriter.cpp + verbosePrintString.cpp +) + +END() diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index df3b86f721d..5a0c1b8232a 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -2,9 +2,11 @@ #include #include -#include #include +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB { diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index 27a873f69fa..5b100bf68ce 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -1,13 +1,16 @@ -#include "config_functions.h" +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + #if USE_BASE64 -#include -#include -#include -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include +# include +# include namespace DB diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 2ad52d69f09..30b6da8b696 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -27,13 +27,16 @@ #include "FunctionFactory.h" #include #include -#include + +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_EMBEDDED_COMPILER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#include -#pragma GCC diagnostic pop +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-parameter" +# include +# pragma GCC diagnostic pop #endif diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index 9e1bc22e6c7..b03abd4c1cf 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -69,12 +69,12 @@ static Block createBlockWithNestedColumnsImpl(const Block & block, const std::un { res.insert({nullptr, nested_type, col.name}); } - else if (auto * nullable = checkAndGetColumn(*col.column)) + else if (const auto * nullable = checkAndGetColumn(*col.column)) { const auto & nested_col = nullable->getNestedColumnPtr(); res.insert({nested_col, nested_type, col.name}); } - else if (auto * const_column = checkAndGetColumn(*col.column)) + else if (const auto * const_column = checkAndGetColumn(*col.column)) { const auto & nested_col = checkAndGetColumn(const_column->getDataColumn())->getNestedColumnPtr(); res.insert({ ColumnConst::create(nested_col, col.column->size()), nested_type, col.name}); diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h index 00957935448..34aa0add6e1 100644 --- a/src/Functions/FunctionHelpers.h +++ b/src/Functions/FunctionHelpers.h @@ -24,6 +24,12 @@ const Type * checkAndGetDataType(const IDataType * data_type) return typeid_cast(data_type); } +template +bool checkDataTypes(const IDataType * data_type) +{ + return (... || typeid_cast(data_type)); +} + template const ColumnConst * checkAndGetColumnConst(const IColumn * column) { diff --git a/src/Functions/FunctionIfBase.h b/src/Functions/FunctionIfBase.h index 48aa1c0b6b2..bdb883ee342 100644 --- a/src/Functions/FunctionIfBase.h +++ b/src/Functions/FunctionIfBase.h @@ -1,9 +1,11 @@ #pragma once -#include #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB { diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 1745343cc33..d6be2469759 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -22,7 +22,7 @@ static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & co throw Exception{"Function joinGet takes 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; String join_name; - if (auto name_col = checkAndGetColumnConst(arguments[0].column.get())) + if (const auto * name_col = checkAndGetColumnConst(arguments[0].column.get())) { join_name = name_col->getValue(); } @@ -49,7 +49,7 @@ static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & co throw Exception{"Table " + join_name + " should have engine StorageJoin", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; String attr_name; - if (auto name_col = checkAndGetColumnConst(arguments[1].column.get())) + if (const auto * name_col = checkAndGetColumnConst(arguments[1].column.get())) { attr_name = name_col->getValue(); } @@ -60,50 +60,64 @@ static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & co return std::make_pair(storage_join, attr_name); } -FunctionBaseImplPtr JoinGetOverloadResolver::build(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const +template +FunctionBaseImplPtr JoinGetOverloadResolver::build(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const { auto [storage_join, attr_name] = getJoin(arguments, context); auto join = storage_join->getJoin(); DataTypes data_types(arguments.size()); - auto table_lock = storage_join->lockStructureForShare(false, context.getInitialQueryId()); + auto table_lock = storage_join->lockStructureForShare( + false, context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout); for (size_t i = 0; i < arguments.size(); ++i) data_types[i] = arguments[i].type; - auto return_type = join->joinGetReturnType(attr_name); - return std::make_unique(table_lock, storage_join, join, attr_name, data_types, return_type); + auto return_type = join->joinGetReturnType(attr_name, or_null); + return std::make_unique>(table_lock, storage_join, join, attr_name, data_types, return_type); } -DataTypePtr JoinGetOverloadResolver::getReturnType(const ColumnsWithTypeAndName & arguments) const +template +DataTypePtr JoinGetOverloadResolver::getReturnType(const ColumnsWithTypeAndName & arguments) const { auto [storage_join, attr_name] = getJoin(arguments, context); auto join = storage_join->getJoin(); - return join->joinGetReturnType(attr_name); + return join->joinGetReturnType(attr_name, or_null); } -void ExecutableFunctionJoinGet::execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) +template +void ExecutableFunctionJoinGet::execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) { auto ctn = block.getByPosition(arguments[2]); if (isColumnConst(*ctn.column)) ctn.column = ctn.column->cloneResized(1); ctn.name = ""; // make sure the key name never collide with the join columns Block key_block = {ctn}; - join->joinGet(key_block, attr_name); + join->joinGet(key_block, attr_name, or_null); auto & result_ctn = key_block.getByPosition(1); if (isColumnConst(*ctn.column)) result_ctn.column = ColumnConst::create(result_ctn.column, input_rows_count); block.getByPosition(result) = result_ctn; } -ExecutableFunctionImplPtr FunctionJoinGet::prepare(const Block &, const ColumnNumbers &, size_t) const +template +ExecutableFunctionImplPtr FunctionJoinGet::prepare(const Block &, const ColumnNumbers &, size_t) const { - return std::make_unique(join, attr_name); + return std::make_unique>(join, attr_name); } void registerFunctionJoinGet(FunctionFactory & factory) { - factory.registerFunction(); + // joinGet + factory.registerFunction>(); + // joinGetOrNull + factory.registerFunction>(); } +template class ExecutableFunctionJoinGet; +template class ExecutableFunctionJoinGet; +template class FunctionJoinGet; +template class FunctionJoinGet; +template class JoinGetOverloadResolver; +template class JoinGetOverloadResolver; } diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index 42ff2b16217..f233ccd8a4f 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -9,13 +9,14 @@ class Context; class HashJoin; using HashJoinPtr = std::shared_ptr; +template class ExecutableFunctionJoinGet final : public IExecutableFunctionImpl { public: ExecutableFunctionJoinGet(HashJoinPtr join_, String attr_name_) : join(std::move(join_)), attr_name(std::move(attr_name_)) {} - static constexpr auto name = "joinGet"; + static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet"; bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } @@ -30,10 +31,11 @@ private: const String attr_name; }; +template class FunctionJoinGet final : public IFunctionBaseImpl { public: - static constexpr auto name = "joinGet"; + static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet"; FunctionJoinGet(TableStructureReadLockHolder table_lock_, StoragePtr storage_join_, HashJoinPtr join_, String attr_name_, @@ -63,10 +65,11 @@ private: DataTypePtr return_type; }; +template class JoinGetOverloadResolver final : public IFunctionOverloadResolverImpl { public: - static constexpr auto name = "joinGet"; + static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet"; static FunctionOverloadResolverImplPtr create(const Context & context) { return std::make_unique(context); } explicit JoinGetOverloadResolver(const Context & context_) : context(context_) {} diff --git a/src/Functions/FunctionMathBinaryFloat64.h b/src/Functions/FunctionMathBinaryFloat64.h index 2927d4ef228..f696e4728fb 100644 --- a/src/Functions/FunctionMathBinaryFloat64.h +++ b/src/Functions/FunctionMathBinaryFloat64.h @@ -8,7 +8,10 @@ #include #include #include -#include "config_functions.h" + +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif /** More efficient implementations of mathematical functions are possible when using a separate library. * Disabled due to license compatibility limitations. diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h index 716b52b40c5..b048b994951 100644 --- a/src/Functions/FunctionMathUnary.h +++ b/src/Functions/FunctionMathUnary.h @@ -7,7 +7,10 @@ #include #include #include -#include "config_functions.h" + +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif /** More efficient implementations of mathematical functions are possible when using a separate library. * Disabled due to license compatibility limitations. @@ -20,7 +23,7 @@ * Enabled by default. */ #if USE_FASTOPS -#include +# include #endif diff --git a/src/Functions/FunctionStringToString.h b/src/Functions/FunctionStringToString.h index 1f712a7a785..8bc8ad8575d 100644 --- a/src/Functions/FunctionStringToString.h +++ b/src/Functions/FunctionStringToString.h @@ -35,7 +35,7 @@ public: return 1; } - bool isInjective(const Block &) override + bool isInjective(const Block &) const override { return is_injective; } diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 89687b5b23e..94cd2c538d6 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -10,13 +10,16 @@ #include #include #include -#include + +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_EMBEDDED_COMPILER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#include -#pragma GCC diagnostic pop +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-parameter" +# include +# pragma GCC diagnostic pop #endif @@ -112,7 +115,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return is_injective; } + bool isInjective(const Block &) const override { return is_injective; } bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 597caec91d2..6a3d6db546c 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -72,7 +72,7 @@ public: String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -326,7 +326,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return mask_tail_octets == 0; } + bool isInjective(const Block &) const override { return mask_tail_octets == 0; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -447,7 +447,7 @@ public: String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -546,7 +546,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -739,7 +739,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -837,7 +837,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -941,7 +941,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -1224,7 +1224,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -1313,7 +1313,7 @@ public: } bool isVariadic() const override { return true; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } size_t getNumberOfArguments() const override { return 0; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -1408,7 +1408,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 824f64a6b45..5671a8674e8 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -160,7 +160,7 @@ struct StringComparisonImpl for (size_t i = 0; i < size; ++i) { - c[i] = Op::apply(memcmpSmallAllowOverflow15( + c[i] = Op::apply(memcmpSmallLikeZeroPaddedAllowOverflow15( a_data.data() + prev_a_offset, a_offsets[i] - prev_a_offset - 1, b_data.data() + i * b_n, b_n), 0); @@ -168,7 +168,7 @@ struct StringComparisonImpl } } - static void NO_INLINE string_vectorConstant( + static void NO_INLINE string_vector_constant( const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -239,11 +239,11 @@ struct StringComparisonImpl size_t size = a_data.size() / a_n; for (size_t i = 0; i < size; ++i) - c[i] = Op::apply(memcmpSmallAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data() + i * b_n, b_n), 0); + c[i] = Op::apply(memcmpSmallLikeZeroPaddedAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data() + i * b_n, b_n), 0); } } - static void NO_INLINE fixed_string_vectorConstant( + static void NO_INLINE fixed_string_vector_constant( const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -262,7 +262,7 @@ struct StringComparisonImpl { size_t size = a_data.size(); for (size_t i = 0, j = 0; i < size; i += a_n, ++j) - c[j] = Op::apply(memcmpSmallAllowOverflow15(a_data.data() + i, a_n, b_data.data(), b_size), 0); + c[j] = Op::apply(0, memcmpSmallLikeZeroPaddedAllowOverflow15(a_data.data() + i, a_n, b_data.data(), b_size)); } } @@ -271,7 +271,7 @@ struct StringComparisonImpl const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) { - StringComparisonImpl::string_vectorConstant(b_data, b_offsets, a_data, a_size, c); + StringComparisonImpl::string_vector_constant(b_data, b_offsets, a_data, a_size, c); } static void constant_fixed_string_vector( @@ -279,15 +279,7 @@ struct StringComparisonImpl const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) { - StringComparisonImpl::fixed_string_vectorConstant(b_data, b_n, a_data, a_size, c); - } - - static void constantConstant( - const ColumnString::Chars & a_data, ColumnString::Offset a_size, - const ColumnString::Chars & b_data, ColumnString::Offset b_size, - UInt8 & c) - { - c = Op::apply(memcmpSmallAllowOverflow15(a_data.data(), a_size, b_data.data(), b_size), 0); + StringComparisonImpl::fixed_string_vector_constant(b_data, b_n, a_data, a_size, c); } }; @@ -331,7 +323,7 @@ struct StringEqualsImpl { auto a_size = a_offsets[i] - prev_a_offset - 1; - c[i] = positive == memequalSmallAllowOverflow15( + c[i] = positive == memequalSmallLikeZeroPaddedAllowOverflow15( a_data.data() + prev_a_offset, a_size, b_data.data() + b_n * i, b_n); @@ -339,7 +331,7 @@ struct StringEqualsImpl } } - static void NO_INLINE string_vectorConstant( + static void NO_INLINE string_vector_constant( const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -397,15 +389,21 @@ struct StringEqualsImpl { fixed_string_vector_fixed_string_vector_16(a_data, b_data, c); } + else if (a_n == b_n) + { + size_t size = a_data.size() / a_n; + for (size_t i = 0; i < size; ++i) + c[i] = positive == memequalSmallAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data() + i * a_n, a_n); + } else { size_t size = a_data.size() / a_n; for (size_t i = 0; i < size; ++i) - c[i] = positive == memequalSmallAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data() + i * b_n, b_n); + c[i] = positive == memequalSmallLikeZeroPaddedAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data() + i * b_n, b_n); } } - static void NO_INLINE fixed_string_vectorConstant( + static void NO_INLINE fixed_string_vector_constant( const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -418,7 +416,7 @@ struct StringEqualsImpl { size_t size = a_data.size() / a_n; for (size_t i = 0; i < size; ++i) - c[i] = positive == memequalSmallAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data(), b_size); + c[i] = positive == memequalSmallLikeZeroPaddedAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data(), b_size); } } @@ -435,7 +433,7 @@ struct StringEqualsImpl const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) { - string_vectorConstant(b_data, b_offsets, a_data, a_size, c); + string_vector_constant(b_data, b_offsets, a_data, a_size, c); } static void constant_fixed_string_vector( @@ -443,15 +441,7 @@ struct StringEqualsImpl const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) { - fixed_string_vectorConstant(b_data, b_n, a_data, a_size, c); - } - - static void constantConstant( - const ColumnString::Chars & a_data, ColumnString::Offset a_size, - const ColumnString::Chars & b_data, ColumnString::Offset b_size, - UInt8 & c) - { - c = positive == memequalSmallAllowOverflow15(a_data.data(), a_size, b_data.data(), b_size); + fixed_string_vector_constant(b_data, b_n, a_data, a_size, c); } }; @@ -758,9 +748,11 @@ private: if (c0_const && c1_const) { - UInt8 res = 0; - StringImpl::constantConstant(*c0_const_chars, c0_const_size, *c1_const_chars, c1_const_size, res); - block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(c0_const->size(), toField(res)); + auto res = executeString(block, result, &c0_const->getDataColumn(), &c1_const->getDataColumn()); + if (!res) + return false; + + block.getByPosition(result).column = ColumnConst::create(block.getByPosition(result).column, c0_const->size()); return true; } else @@ -780,7 +772,7 @@ private: c1_fixed_string->getChars(), c1_fixed_string->getN(), c_res->getData()); else if (c0_string && c1_const) - StringImpl::string_vectorConstant( + StringImpl::string_vector_constant( c0_string->getChars(), c0_string->getOffsets(), *c1_const_chars, c1_const_size, c_res->getData()); @@ -795,7 +787,7 @@ private: c1_fixed_string->getChars(), c1_fixed_string->getN(), c_res->getData()); else if (c0_fixed_string && c1_const) - StringImpl::fixed_string_vectorConstant( + StringImpl::fixed_string_vector_constant( c0_fixed_string->getChars(), c0_fixed_string->getN(), *c1_const_chars, c1_const_size, c_res->getData()); @@ -1105,8 +1097,8 @@ private: { DataTypePtr common_type = getLeastSupertype({c0.type, c1.type}); - ColumnPtr c0_converted = castColumn(c0, common_type, context); - ColumnPtr c1_converted = castColumn(c1, common_type, context); + ColumnPtr c0_converted = castColumn(c0, common_type); + ColumnPtr c1_converted = castColumn(c1, common_type); executeGenericIdenticalTypes(block, result, c0_converted.get(), c1_converted.get()); } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 100737b43c7..64708f45598 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -913,7 +913,7 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - bool isInjective(const Block &) override { return std::is_same_v; } + bool isInjective(const Block &) const override { return std::is_same_v; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -1268,7 +1268,7 @@ public: } size_t getNumberOfArguments() const override { return 2; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -1496,10 +1496,12 @@ struct ToStringMonotonicity IFunction::Monotonicity positive(true, true); IFunction::Monotonicity not_monotonic; - /// `toString` function is monotonous if the argument is Date or DateTime, or non-negative numbers with the same number of symbols. + auto type_ptr = &type; + if (auto * low_cardinality_type = checkAndGetDataType(type_ptr)) + type_ptr = low_cardinality_type->getDictionaryType().get(); - if (checkAndGetDataType(&type) - || typeid_cast(&type)) + /// `toString` function is monotonous if the argument is Date or DateTime or String, or non-negative numbers with the same number of symbols. + if (checkDataTypes(type_ptr)) return positive; if (left.isNull() || right.isNull()) @@ -2392,10 +2394,17 @@ protected: DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments) const override { - const auto type_col = checkAndGetColumnConst(arguments.back().column.get()); + const auto & column = arguments.back().column; + if (!column) + throw Exception("Second argument to " + getName() + " must be a constant string describing type." + " Instead there is non-constant column of type " + arguments.back().type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const auto type_col = checkAndGetColumnConst(column.get()); if (!type_col) - throw Exception("Second argument to " + getName() + " must be a constant string describing type", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Second argument to " + getName() + " must be a constant string describing type." + " Instead there is a column with the following structure: " + column->dumpStructure(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return DataTypeFactory::instance().get(type_col->getValue()); } diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index 12b478a26b6..d337f9ea363 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -15,8 +15,12 @@ #include #include #include -#include #include +#include + +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB @@ -447,7 +451,7 @@ public: for (size_t i = 0; i < size; ++i) { T cur = vec_from[i]; - while (cur) + for (size_t depth = 0; cur && depth < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++depth) { res_values.push_back(cur); cur = Transform::toParent(cur, dict); @@ -589,7 +593,7 @@ public: /// For the purpose of query optimization, we assume this function to be injective /// even in face of fact that there are many different cities named Moscow. - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index fc3c2c583a9..93ed1b75029 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -135,7 +135,9 @@ private: !executeDispatchSimple(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && +#if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && +#endif !executeDispatchComplex(block, arguments, result, dict_ptr)) throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } @@ -241,7 +243,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) override + bool isInjective(const Block & sample_block) const override { return isDictGetFunctionInjective(dictionaries_loader, sample_block); } @@ -306,7 +308,9 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && +#if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && +#endif !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchRange(block, arguments, result, dict_ptr)) throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; @@ -488,8 +492,10 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr)) +#if !defined(ARCADIA_BUILD) + !executeDispatchComplex(block, arguments, result, dict_ptr) && +#endif + !executeDispatchComplex(block, arguments, result, dict_ptr)) throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } @@ -763,7 +769,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) override + bool isInjective(const Block & sample_block) const override { return isDictGetFunctionInjective(dictionaries_loader, sample_block); } @@ -826,7 +832,9 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && +#if !defined(ARCADIA_BUILD) !executeDispatchComplex(block, arguments, result, dict_ptr) && +#endif !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchRange(block, arguments, result, dict_ptr)) throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; @@ -1086,8 +1094,10 @@ private: !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr)) +#if !defined(ARCADIA_BUILD) + !executeDispatchComplex(block, arguments, result, dict_ptr) && +#endif + !executeDispatchComplex(block, arguments, result, dict_ptr)) throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } @@ -1328,7 +1338,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) override + bool isInjective(const Block & sample_block) const override { return isDictGetFunctionInjective(dictionaries_loader, sample_block); } @@ -1476,7 +1486,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) override + bool isInjective(const Block & sample_block) const override { return isDictGetFunctionInjective(dictionaries_loader, sample_block); } @@ -1617,7 +1627,7 @@ public: private: size_t getNumberOfArguments() const override { return 2; } - bool isInjective(const Block & /*sample_block*/) override { return true; } + bool isInjective(const Block & /*sample_block*/) const override { return true; } bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } diff --git a/src/Functions/FunctionsExternalModels.cpp b/src/Functions/FunctionsExternalModels.cpp index df9c438d4ca..a7ec5947c4f 100644 --- a/src/Functions/FunctionsExternalModels.cpp +++ b/src/Functions/FunctionsExternalModels.cpp @@ -42,7 +42,7 @@ DataTypePtr FunctionModelEvaluate::getReturnTypeImpl(const ColumnsWithTypeAndNam throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - const auto name_col = checkAndGetColumnConst(arguments[0].column.get()); + const auto * name_col = checkAndGetColumnConst(arguments[0].column.get()); if (!name_col) throw Exception("First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN); @@ -56,7 +56,7 @@ DataTypePtr FunctionModelEvaluate::getReturnTypeImpl(const ColumnsWithTypeAndNam if (has_nullable) { - if (auto * tuple = typeid_cast(type.get())) + if (const auto * tuple = typeid_cast(type.get())) { auto elements = tuple->getElements(); for (auto & element : elements) @@ -73,7 +73,7 @@ DataTypePtr FunctionModelEvaluate::getReturnTypeImpl(const ColumnsWithTypeAndNam void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { - const auto name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); + const auto * name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); if (!name_col) throw Exception("First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN); @@ -94,7 +94,7 @@ void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arg materialized_columns.push_back(full_column); columns.back() = full_column.get(); } - if (auto * col_nullable = checkAndGetColumn(*columns.back())) + if (const auto * col_nullable = checkAndGetColumn(*columns.back())) { if (!null_map) null_map = col_nullable->getNullMapColumnPtr(); @@ -120,7 +120,7 @@ void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arg if (null_map) { - if (auto * tuple = typeid_cast(res.get())) + if (const auto * tuple = typeid_cast(res.get())) { auto nested = tuple->getColumns(); for (auto & col : nested) diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index 0a789b1223a..a5c5635e0ad 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -42,7 +42,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) override { return true; } + bool isInjective(const Block &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { diff --git a/src/Functions/FunctionsHashing.cpp b/src/Functions/FunctionsHashing.cpp index 8705e6bfaa3..900455a1015 100644 --- a/src/Functions/FunctionsHashing.cpp +++ b/src/Functions/FunctionsHashing.cpp @@ -26,12 +26,14 @@ void registerFunctionsHashing(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); +#if !defined(ARCADIA_BUILD) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); +#endif #if USE_XXHASH factory.registerFunction(); diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index a589b220351..f647390e1c8 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -3,22 +3,24 @@ #include #include #include -#include -#include +#if !defined(ARCADIA_BUILD) +# include +# include +# include "config_functions.h" +# include "config_core.h" +#endif #include #include #include -#include "config_functions.h" #if USE_XXHASH -# include +# include #endif -#include "config_core.h" #if USE_SSL -# include -# include +# include +# include #endif #include @@ -219,7 +221,7 @@ struct SipHash128Impl } }; - +#if !defined(ARCADIA_BUILD) /** Why we need MurmurHash2? * MurmurHash2 is an outdated hash function, superseded by MurmurHash3 and subsequently by CityHash, xxHash, HighwayHash. * Usually there is no reason to use MurmurHash. @@ -331,6 +333,18 @@ struct MurmurHash3Impl64 static constexpr bool use_int_hash_for_pods = false; }; +struct MurmurHash3Impl128 +{ + static constexpr auto name = "murmurHash3_128"; + enum { length = 16 }; + + static void apply(const char * begin, const size_t size, unsigned char * out_char_data) + { + MurmurHash3_x64_128(begin, size, 0, out_char_data); + } +}; +#endif + /// http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452 /// Care should be taken to do all calculation in unsigned integers (to avoid undefined behaviour on overflow) /// but obtain the same result as it is done in singed integers with two's complement arithmetic. @@ -411,17 +425,6 @@ struct HiveHashImpl static constexpr bool use_int_hash_for_pods = false; }; -struct MurmurHash3Impl128 -{ - static constexpr auto name = "murmurHash3_128"; - enum { length = 16 }; - - static void apply(const char * begin, const size_t size, unsigned char * out_char_data) - { - MurmurHash3_x64_128(begin, size, 0, out_char_data); - } -}; - struct ImplCityHash64 { static constexpr auto name = "cityHash64"; @@ -1145,12 +1148,16 @@ using FunctionSipHash128 = FunctionStringHashFixedString; using FunctionCityHash64 = FunctionAnyHash; using FunctionFarmHash64 = FunctionAnyHash; using FunctionMetroHash64 = FunctionAnyHash; + +#if !defined(ARCADIA_BUILD) using FunctionMurmurHash2_32 = FunctionAnyHash; using FunctionMurmurHash2_64 = FunctionAnyHash; using FunctionGccMurmurHash = FunctionAnyHash; using FunctionMurmurHash3_32 = FunctionAnyHash; using FunctionMurmurHash3_64 = FunctionAnyHash; using FunctionMurmurHash3_128 = FunctionStringHashFixedString; +#endif + using FunctionJavaHash = FunctionAnyHash; using FunctionJavaHashUTF16LE = FunctionAnyHash; using FunctionHiveHash = FunctionAnyHash; diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 79dea768f61..e7598104102 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -21,6 +21,7 @@ void registerFunctionsJSON(FunctionFactory & factory) factory.registerFunction>(); factory.registerFunction>(); factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 468b499b78c..7bd69ea770d 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -5,7 +5,6 @@ #include #include #include -#include "config_functions.h" #include #include #include @@ -27,6 +26,10 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + namespace DB { @@ -292,6 +295,7 @@ struct NameJSONExtract { static constexpr auto name{"JSONExtract"}; }; struct NameJSONExtractKeysAndValues { static constexpr auto name{"JSONExtractKeysAndValues"}; }; struct NameJSONExtractRaw { static constexpr auto name{"JSONExtractRaw"}; }; struct NameJSONExtractArrayRaw { static constexpr auto name{"JSONExtractArrayRaw"}; }; +struct NameJSONExtractKeysAndValuesRaw { static constexpr auto name{"JSONExtractKeysAndValuesRaw"}; }; template @@ -696,8 +700,8 @@ struct JSONExtractTree if (!JSONParser::isArray(it)) return false; - Iterator it2 = it; - if (!JSONParser::firstArrayElement(it2)) + Iterator array_it = it; + if (!JSONParser::firstArrayElement(array_it)) return false; ColumnArray & col_arr = assert_cast(dest); @@ -707,12 +711,12 @@ struct JSONExtractTree do { - if (nested->addValueToColumn(data, it2)) + if (nested->addValueToColumn(data, array_it)) were_valid_elements = true; else data.insertDefault(); } - while (JSONParser::nextArrayElement(it2)); + while (JSONParser::nextArrayElement(array_it)); if (!were_valid_elements) { @@ -761,17 +765,17 @@ struct JSONExtractTree if (JSONParser::isArray(it)) { - Iterator it2 = it; - if (!JSONParser::firstArrayElement(it2)) + Iterator array_it = it; + if (!JSONParser::firstArrayElement(array_it)) return false; for (size_t index = 0; index != nested.size(); ++index) { - if (nested[index]->addValueToColumn(tuple.getColumn(index), it2)) + if (nested[index]->addValueToColumn(tuple.getColumn(index), array_it)) were_valid_elements = true; else tuple.getColumn(index).insertDefault(); - if (!JSONParser::nextArrayElement(it2)) + if (!JSONParser::nextArrayElement(array_it)) break; } @@ -783,25 +787,25 @@ struct JSONExtractTree { if (name_to_index_map.empty()) { - Iterator it2 = it; - if (!JSONParser::firstObjectMember(it2)) + Iterator object_it = it; + if (!JSONParser::firstObjectMember(object_it)) return false; for (size_t index = 0; index != nested.size(); ++index) { - if (nested[index]->addValueToColumn(tuple.getColumn(index), it2)) + if (nested[index]->addValueToColumn(tuple.getColumn(index), object_it)) were_valid_elements = true; else tuple.getColumn(index).insertDefault(); - if (!JSONParser::nextObjectMember(it2)) + if (!JSONParser::nextObjectMember(object_it)) break; } } else { - Iterator it2 = it; + Iterator object_it = it; StringRef key; - if (!JSONParser::firstObjectMember(it2, key)) + if (!JSONParser::firstObjectMember(object_it, key)) return false; do @@ -809,11 +813,11 @@ struct JSONExtractTree auto index = name_to_index_map.find(key); if (index != name_to_index_map.end()) { - if (nested[index->second]->addValueToColumn(tuple.getColumn(index->second), it2)) + if (nested[index->second]->addValueToColumn(tuple.getColumn(index->second), object_it)) were_valid_elements = true; } } - while (JSONParser::nextObjectMember(it2, key)); + while (JSONParser::nextObjectMember(object_it, key)); } set_size(old_size + static_cast(were_valid_elements)); @@ -872,6 +876,7 @@ struct JSONExtractTree } }; + template class JSONExtractImpl { @@ -927,8 +932,8 @@ public: + " should be a constant string specifying the values' data type, illegal value: " + col.column->getName(), ErrorCodes::ILLEGAL_COLUMN}; - DataTypePtr value_type = DataTypeFactory::instance().get(col_type_const->getValue()); DataTypePtr key_type = std::make_unique(); + DataTypePtr value_type = DataTypeFactory::instance().get(col_type_const->getValue()); DataTypePtr tuple_type = std::make_unique(DataTypes{key_type, value_type}); return std::make_unique(tuple_type); } @@ -954,16 +959,16 @@ public: auto & col_value = col_tuple.getColumn(1); StringRef key; - Iterator it2 = it; - if (!JSONParser::firstObjectMember(it2, key)) + Iterator object_it = it; + if (!JSONParser::firstObjectMember(object_it, key)) return false; do { - if (extract_tree->addValueToColumn(col_value, it2)) + if (extract_tree->addValueToColumn(col_value, object_it)) col_key.insertData(key.data, key.size); } - while (JSONParser::nextObjectMember(it2, key)); + while (JSONParser::nextObjectMember(object_it, key)); if (col_tuple.size() == old_size) return false; @@ -1036,14 +1041,14 @@ private: if (JSONParser::isArray(it)) { writeChar('[', buf); - Iterator it2 = it; - if (JSONParser::firstArrayElement(it2)) + Iterator array_it = it; + if (JSONParser::firstArrayElement(array_it)) { - traverse(it2, buf); - while (JSONParser::nextArrayElement(it2)) + traverse(array_it, buf); + while (JSONParser::nextArrayElement(array_it)) { writeChar(',', buf); - traverse(it2, buf); + traverse(array_it, buf); } } writeChar(']', buf); @@ -1052,19 +1057,19 @@ private: if (JSONParser::isObject(it)) { writeChar('{', buf); - Iterator it2 = it; + Iterator object_it = it; StringRef key; - if (JSONParser::firstObjectMember(it2, key)) + if (JSONParser::firstObjectMember(object_it, key)) { writeJSONString(key, buf, format_settings()); writeChar(':', buf); - traverse(it2, buf); - while (JSONParser::nextObjectMember(it2, key)) + traverse(object_it, buf); + while (JSONParser::nextObjectMember(object_it, key)) { writeChar(',', buf); writeJSONString(key, buf, format_settings()); writeChar(':', buf); - traverse(it2, buf); + traverse(object_it, buf); } } writeChar('}', buf); @@ -1089,6 +1094,7 @@ private: } }; + template class JSONExtractArrayRawImpl { @@ -1102,9 +1108,8 @@ public: static bool addValueToColumn(IColumn & dest, const Iterator & it) { if (!JSONParser::isArray(it)) - { return false; - } + ColumnArray & col_res = assert_cast(dest); Iterator array_it = it; size_t size = 0; @@ -1124,4 +1129,50 @@ public: static constexpr size_t num_extra_arguments = 0; static void prepare(const char *, const Block &, const ColumnNumbers &, size_t) {} }; + + +template +class JSONExtractKeysAndValuesRawImpl +{ +public: + + static DataTypePtr getType(const char *, const ColumnsWithTypeAndName &) + { + DataTypePtr string_type = std::make_unique(); + DataTypePtr tuple_type = std::make_unique(DataTypes{string_type, string_type}); + return std::make_unique(tuple_type); + } + + using Iterator = typename JSONParser::Iterator; + bool addValueToColumn(IColumn & dest, const Iterator & it) + { + if (!JSONParser::isObject(it)) + return false; + + auto & col_arr = assert_cast(dest); + auto & col_tuple = assert_cast(col_arr.getData()); + auto & col_key = assert_cast(col_tuple.getColumn(0)); + auto & col_value = assert_cast(col_tuple.getColumn(1)); + + Iterator object_it = it; + StringRef key; + size_t size = 0; + if (JSONParser::firstObjectMember(object_it, key)) + { + do + { + col_key.insertData(key.data, key.size); + JSONExtractRawImpl::addValueToColumn(col_value, object_it); + ++size; + } while (JSONParser::nextObjectMember(object_it, key)); + } + + col_arr.getOffsets().push_back(col_arr.getOffsets().back() + size); + return true; + } + + static constexpr size_t num_extra_arguments = 0; + static void prepare(const char *, const Block &, const ColumnNumbers &, size_t) {} +}; + } diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index c2a277a478d..ade2fe960b0 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -198,9 +198,9 @@ struct ValueGetterBuilderImpl { static ValueGetter build(const IColumn * x) { - if (const auto nullable_column = typeid_cast(x)) + if (const auto * nullable_column = typeid_cast(x)) { - if (const auto nested_column = typeid_cast *>(nullable_column->getNestedColumnPtr().get())) + if (const auto * nested_column = typeid_cast *>(nullable_column->getNestedColumnPtr().get())) { return [&null_data = nullable_column->getNullMapData(), &column_data = nested_column->getData()](size_t i) { return Ternary::makeValue(column_data[i], null_data[i]); }; @@ -442,7 +442,7 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re Columns converted_columns_holder; for (const IColumn * column : arguments) { - if (auto uint8_column = checkAndGetColumn(column)) + if (const auto * uint8_column = checkAndGetColumn(column)) uint8_args.push_back(uint8_column); else { diff --git a/src/Functions/FunctionsRound.h b/src/Functions/FunctionsRound.h index 6d1afe5480c..99fa1d9441a 100644 --- a/src/Functions/FunctionsRound.h +++ b/src/Functions/FunctionsRound.h @@ -596,8 +596,7 @@ class FunctionRoundDown : public IFunction { public: static constexpr auto name = "roundDown"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - FunctionRoundDown(const Context & context_) : context(context_) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } public: String getName() const override { return name; } @@ -645,10 +644,10 @@ public: auto out = column_result.get(); if (!in_type->equals(*return_type)) - in_column = castColumn(block.getByPosition(arguments[0]), return_type, context); + in_column = castColumn(block.getByPosition(arguments[0]), return_type); if (!array_type->equals(*return_type)) - array_column = castColumn(block.getByPosition(arguments[1]), std::make_shared(return_type), context); + array_column = castColumn(block.getByPosition(arguments[1]), std::make_shared(return_type)); const auto in = in_column.get(); auto boundaries = typeid_cast(*array_column).getValue(); @@ -764,9 +763,6 @@ private: } } } - -private: - const Context & context; }; diff --git a/src/Functions/FunctionsStringRegex.cpp b/src/Functions/FunctionsStringRegex.cpp index 856852ae197..2f4c2432d54 100644 --- a/src/Functions/FunctionsStringRegex.cpp +++ b/src/Functions/FunctionsStringRegex.cpp @@ -17,16 +17,15 @@ #include #include -#include "config_functions.h" -#if USE_HYPERSCAN -# if __has_include() -# include -# else -# include -# endif +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +# include +#endif + +#if USE_HYPERSCAN +# include #endif -#include #if USE_RE2_ST # include #else diff --git a/src/Functions/GatherUtils/createArraySink.cpp b/src/Functions/GatherUtils/createArraySink.cpp index c68b10747d1..dbf75b2cf60 100644 --- a/src/Functions/GatherUtils/createArraySink.cpp +++ b/src/Functions/GatherUtils/createArraySink.cpp @@ -42,7 +42,7 @@ struct ArraySinkCreator<> std::unique_ptr createArraySink(ColumnArray & col, size_t column_size) { using Creator = ApplyTypeListForClass::Type; - if (auto column_nullable = typeid_cast(&col.getData())) + if (auto * column_nullable = typeid_cast(&col.getData())) { auto column = ColumnArray::create(column_nullable->getNestedColumnPtr()->assumeMutable(), col.getOffsetsPtr()->assumeMutable()); return Creator::create(*column, &column_nullable->getNullMapData(), column_size); diff --git a/src/Functions/GatherUtils/createArraySource.cpp b/src/Functions/GatherUtils/createArraySource.cpp index 7d816315d04..0ad9f88c819 100644 --- a/src/Functions/GatherUtils/createArraySource.cpp +++ b/src/Functions/GatherUtils/createArraySource.cpp @@ -54,7 +54,7 @@ struct ArraySourceCreator<> std::unique_ptr createArraySource(const ColumnArray & col, bool is_const, size_t total_rows) { using Creator = typename ApplyTypeListForClass::Type; - if (auto column_nullable = typeid_cast(&col.getData())) + if (const auto * column_nullable = typeid_cast(&col.getData())) { auto column = ColumnArray::create(column_nullable->getNestedColumnPtr(), col.getOffsetsPtr()); return Creator::create(*column, &column_nullable->getNullMapData(), is_const, total_rows); diff --git a/src/Functions/GatherUtils/createValueSource.cpp b/src/Functions/GatherUtils/createValueSource.cpp index 77eeda1bbdd..8f06ab86333 100644 --- a/src/Functions/GatherUtils/createValueSource.cpp +++ b/src/Functions/GatherUtils/createValueSource.cpp @@ -54,7 +54,7 @@ struct ValueSourceCreator<> std::unique_ptr createValueSource(const IColumn & col, bool is_const, size_t total_rows) { using Creator = typename ApplyTypeListForClass::Type; - if (auto column_nullable = typeid_cast(&col)) + if (const auto * column_nullable = typeid_cast(&col)) { return Creator::create(column_nullable->getNestedColumn(), &column_nullable->getNullMapData(), is_const, total_rows); } diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 7355b5df9ad..95294378e4f 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -24,11 +23,15 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + #if USE_EMBEDDED_COMPILER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#include -#pragma GCC diagnostic pop +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-parameter" +# include +# pragma GCC diagnostic pop #endif @@ -113,7 +116,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, const Block & block, const Colum if (src->onlyNull()) return src; - else if (auto * nullable = checkAndGetColumn(*src)) + else if (const auto * nullable = checkAndGetColumn(*src)) { src_not_nullable = nullable->getNestedColumnPtr(); result_null_map_column = nullable->getNullMapColumnPtr(); @@ -136,7 +139,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, const Block & block, const Colum if (isColumnConst(*elem.column)) continue; - if (auto * nullable = checkAndGetColumn(*elem.column)) + if (const auto * nullable = checkAndGetColumn(*elem.column)) { const ColumnPtr & null_map_column = nullable->getNullMapColumnPtr(); if (!result_null_map_column) @@ -330,7 +333,7 @@ static const ColumnLowCardinality * findLowCardinalityArgument(const Block & blo for (auto arg : args) { const ColumnWithTypeAndName & column = block.getByPosition(arg); - if (auto * low_cardinality_column = checkAndGetColumn(column.column.get())) + if (const auto * low_cardinality_column = checkAndGetColumn(column.column.get())) { if (result_column) throw Exception("Expected single dictionary argument for function.", ErrorCodes::LOGICAL_ERROR); @@ -352,13 +355,13 @@ static ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes( for (auto arg : args) { ColumnWithTypeAndName & column = block.getByPosition(arg); - if (auto * low_cardinality_column = checkAndGetColumn(column.column.get())) + if (const auto * low_cardinality_column = checkAndGetColumn(column.column.get())) { /// Single LowCardinality column is supported now. if (indexes) throw Exception("Expected single dictionary argument for function.", ErrorCodes::LOGICAL_ERROR); - auto * low_cardinality_type = checkAndGetDataType(column.type.get()); + const auto * low_cardinality_type = checkAndGetDataType(column.type.get()); if (!low_cardinality_type) throw Exception("Incompatible type for low cardinality column: " + column.type->getName(), @@ -388,7 +391,7 @@ static ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes( for (auto arg : args) { ColumnWithTypeAndName & column = block.getByPosition(arg); - if (auto * column_const = checkAndGetColumn(column.column.get())) + if (const auto * column_const = checkAndGetColumn(column.column.get())) { column.column = column_const->removeLowCardinality()->cloneResized(num_rows); column.type = removeLowCardinality(column.type); @@ -423,7 +426,7 @@ void ExecutableFunctionAdaptor::execute(Block & block, const ColumnNumbers & arg for (auto arg : arguments) block_without_low_cardinality.safeGetByPosition(arg).column = block.safeGetByPosition(arg).column; - if (auto * res_low_cardinality_type = typeid_cast(res.type.get())) + if (const auto * res_low_cardinality_type = typeid_cast(res.type.get())) { const auto * low_cardinality_column = findLowCardinalityArgument(block, arguments); bool can_be_executed_on_default_arguments = impl->canBeExecutedOnDefaultArguments(); @@ -608,7 +611,7 @@ DataTypePtr FunctionOverloadResolverAdaptor::getReturnType(const ColumnsWithType if (is_const) arg.column = assert_cast(*arg.column).removeLowCardinality(); - if (auto * low_cardinality_type = typeid_cast(arg.type.get())) + if (const auto * low_cardinality_type = typeid_cast(arg.type.get())) { arg.type = low_cardinality_type->getDictionaryType(); has_low_cardinality = true; diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 3887f3f1669..b8873ea2671 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -2,12 +2,15 @@ #include -#include "config_core.h" #include #include #include #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + /// This file contains user interface for functions. /// For developer interface (in case you need to implement a new function) see IFunctionImpl.h @@ -131,7 +134,7 @@ public: * * sample_block should contain data types of arguments and values of constants, if relevant. */ - virtual bool isInjective(const Block & /*sample_block*/) { return false; } + virtual bool isInjective(const Block & /*sample_block*/) const { return false; } /** Function is called "deterministic", if it returns same result for same values of arguments. * Most of functions are deterministic. Notable counterexample is rand(). @@ -186,6 +189,7 @@ public: /// See the comment for the same method in IFunctionBase virtual bool isDeterministic() const = 0; virtual bool isDeterministicInScopeOfQuery() const = 0; + virtual bool isInjective(const Block &) const = 0; /// Override and return true if function needs to depend on the state of the data. virtual bool isStateful() const = 0; diff --git a/src/Functions/IFunctionAdaptors.h b/src/Functions/IFunctionAdaptors.h index 123faa859e9..82afaad4c27 100644 --- a/src/Functions/IFunctionAdaptors.h +++ b/src/Functions/IFunctionAdaptors.h @@ -68,7 +68,7 @@ public: return impl->getResultIfAlwaysReturnsConstantAndHasArguments(block, arguments); } - bool isInjective(const Block & sample_block) final { return impl->isInjective(sample_block); } + bool isInjective(const Block & sample_block) const final { return impl->isInjective(sample_block); } bool isDeterministic() const final { return impl->isDeterministic(); } bool isDeterministicInScopeOfQuery() const final { return impl->isDeterministicInScopeOfQuery(); } bool hasInformationAboutMonotonicity() const final { return impl->hasInformationAboutMonotonicity(); } @@ -96,6 +96,8 @@ public: bool isDeterministicInScopeOfQuery() const final { return impl->isDeterministicInScopeOfQuery(); } + bool isInjective(const Block & block) const final { return impl->isInjective(block); } + bool isStateful() const final { return impl->isStateful(); } bool isVariadic() const final { return impl->isVariadic(); } @@ -195,7 +197,7 @@ public: bool isStateful() const override { return function->isStateful(); } - bool isInjective(const Block & sample_block) override { return function->isInjective(sample_block); } + bool isInjective(const Block & sample_block) const override { return function->isInjective(sample_block); } bool isDeterministic() const override { return function->isDeterministic(); } @@ -226,6 +228,7 @@ public: bool isDeterministic() const override { return function->isDeterministic(); } bool isDeterministicInScopeOfQuery() const override { return function->isDeterministicInScopeOfQuery(); } + bool isInjective(const Block &block) const override { return function->isInjective(block); } String getName() const override { return function->getName(); } bool isStateful() const override { return function->isStateful(); } diff --git a/src/Functions/IFunctionImpl.h b/src/Functions/IFunctionImpl.h index 66196070afe..116363705de 100644 --- a/src/Functions/IFunctionImpl.h +++ b/src/Functions/IFunctionImpl.h @@ -107,7 +107,7 @@ public: virtual bool isSuitableForConstantFolding() const { return true; } virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const Block & /*block*/, const ColumnNumbers & /*arguments*/) const { return nullptr; } - virtual bool isInjective(const Block & /*sample_block*/) { return false; } + virtual bool isInjective(const Block & /*sample_block*/) const { return false; } virtual bool isDeterministic() const { return true; } virtual bool isDeterministicInScopeOfQuery() const { return true; } virtual bool hasInformationAboutMonotonicity() const { return false; } @@ -152,6 +152,7 @@ public: /// Properties from IFunctionOverloadResolver. See comments in IFunction.h virtual bool isDeterministic() const { return true; } virtual bool isDeterministicInScopeOfQuery() const { return true; } + virtual bool isInjective(const Block &) const { return false; } virtual bool isStateful() const { return false; } virtual bool isVariadic() const { return false; } @@ -256,7 +257,7 @@ public: /// Properties from IFunctionBase (see IFunction.h) virtual bool isSuitableForConstantFolding() const { return true; } virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const Block & /*block*/, const ColumnNumbers & /*arguments*/) const { return nullptr; } - virtual bool isInjective(const Block & /*sample_block*/) { return false; } + virtual bool isInjective(const Block & /*sample_block*/) const { return false; } virtual bool isDeterministic() const { return true; } virtual bool isDeterministicInScopeOfQuery() const { return true; } virtual bool isStateful() const { return false; } diff --git a/src/Functions/LeastGreatestGeneric.h b/src/Functions/LeastGreatestGeneric.h new file mode 100644 index 00000000000..2d6d71b20c7 --- /dev/null +++ b/src/Functions/LeastGreatestGeneric.h @@ -0,0 +1,140 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +enum class LeastGreatest +{ + Least, + Greatest +}; + + +template +class FunctionLeastGreatestGeneric : public IFunction +{ +public: + static constexpr auto name = kind == LeastGreatest::Least ? "least" : "greatest"; + static FunctionPtr create(const Context &) { return std::make_shared>(); } + +private: + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & types) const override + { + if (types.empty()) + throw Exception("Function " + getName() + " cannot be called without arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return getLeastSupertype(types); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + size_t num_arguments = arguments.size(); + if (1 == num_arguments) + { + block.getByPosition(result).column = block.getByPosition(arguments[0]).column; + return; + } + + auto result_type = block.getByPosition(result).type; + + Columns converted_columns(num_arguments); + for (size_t arg = 0; arg < num_arguments; ++arg) + converted_columns[arg] = castColumn(block.getByPosition(arguments[arg]), result_type)->convertToFullColumnIfConst(); + + auto result_column = result_type->createColumn(); + result_column->reserve(input_rows_count); + + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + size_t best_arg = 0; + for (size_t arg = 1; arg < num_arguments; ++arg) + { + auto cmp_result = converted_columns[arg]->compareAt(row_num, row_num, *converted_columns[best_arg], 1); + + if constexpr (kind == LeastGreatest::Least) + { + if (cmp_result < 0) + best_arg = arg; + } + else + { + if (cmp_result > 0) + best_arg = arg; + } + } + + result_column->insertFrom(*converted_columns[best_arg], row_num); + } + + block.getByPosition(result).column = std::move(result_column); + } +}; + + +template +class LeastGreatestOverloadResolver : public IFunctionOverloadResolverImpl +{ +public: + static constexpr auto name = kind == LeastGreatest::Least ? "least" : "greatest"; + + static FunctionOverloadResolverImplPtr create(const Context & context) + { + return std::make_unique>(context); + } + + explicit LeastGreatestOverloadResolver(const Context & context_) : context(context_) {} + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + + FunctionBaseImplPtr build(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override + { + DataTypes argument_types; + + /// More efficient specialization for two numeric arguments. + if (arguments.size() == 2 && isNumber(arguments[0].type) && isNumber(arguments[1].type)) + return std::make_unique(SpecializedFunction::create(context), argument_types, return_type); + + return std::make_unique( + FunctionLeastGreatestGeneric::create(context), argument_types, return_type); + } + + DataTypePtr getReturnType(const DataTypes & types) const override + { + if (types.empty()) + throw Exception("Function " + getName() + " cannot be called without arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (types.size() == 2 && isNumber(types[0]) && isNumber(types[1])) + return SpecializedFunction::create(context)->getReturnTypeImpl(types); + + return getLeastSupertype(types); + } + +private: + const Context & context; +}; + +} + + diff --git a/src/Functions/RapidJSONParser.h b/src/Functions/RapidJSONParser.h index ff4ecd506fd..07b0aea3543 100644 --- a/src/Functions/RapidJSONParser.h +++ b/src/Functions/RapidJSONParser.h @@ -1,13 +1,15 @@ #pragma once -#include "config_functions.h" +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + #if USE_RAPIDJSON +# include +# include +# include -#include -#include -#include - -#include +# include namespace DB diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 5d93e823419..433e416a18f 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -14,14 +14,12 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif -#include "config_functions.h" #if USE_HYPERSCAN -# if __has_include() -# include -# else -# include -# endif +# include #endif namespace ProfileEvents diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index a12119c5c6d..d5630cf1cf1 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -1,13 +1,15 @@ #pragma once -#include "config_functions.h" +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + #if USE_SIMDJSON +# include +# include +# include -#include -#include -#include - -#include +# include namespace DB @@ -137,4 +139,5 @@ private: }; } + #endif diff --git a/src/Functions/URL/path.cpp b/src/Functions/URL/path.cpp index 43e21ece00f..2260604c1fc 100644 --- a/src/Functions/URL/path.cpp +++ b/src/Functions/URL/path.cpp @@ -1,35 +1,15 @@ #include #include #include "FunctionsURL.h" +#include "path.h" #include + namespace DB { -struct ExtractPath -{ - static size_t getReserveLengthForElement() { return 25; } - - static void execute(Pos data, size_t size, Pos & res_data, size_t & res_size) - { - res_data = data; - res_size = 0; - - Pos pos = data; - Pos end = pos + size; - - if (end != (pos = find_first_symbols<'/'>(pos, end)) && pos[1] == '/' && end != (pos = find_first_symbols<'/'>(pos + 2, end))) - { - Pos query_string_or_fragment = find_first_symbols<'?', '#'>(pos, end); - - res_data = pos; - res_size = query_string_or_fragment - res_data; - } - } -}; - struct NamePath { static constexpr auto name = "path"; }; -using FunctionPath = FunctionStringToString, NamePath>; +using FunctionPath = FunctionStringToString>, NamePath>; void registerFunctionPath(FunctionFactory & factory) { diff --git a/src/Functions/URL/path.h b/src/Functions/URL/path.h new file mode 100644 index 00000000000..f2c5d31a0b0 --- /dev/null +++ b/src/Functions/URL/path.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +template +struct ExtractPath +{ + static size_t getReserveLengthForElement() { return 25; } + + static void execute(Pos data, size_t size, Pos & res_data, size_t & res_size) + { + res_data = data; + res_size = 0; + + Pos pos = data; + Pos end = pos + size; + + /// We support URLs with and without schema: + /// 1. http://host/path + /// 2. host/path + /// We search for first slash and if there is subsequent slash, then skip and repeat search for the next slash. + + pos = find_first_symbols<'/'>(pos, end); + if (end == pos) + return; + + /// Note that strings are zero-terminated. + bool has_subsequent_slash = pos[1] == '/'; + if (has_subsequent_slash) + { + /// Search for next slash. + pos = find_first_symbols<'/'>(pos + 2, end); + if (end == pos) + return; + } + + res_data = pos; + + if constexpr (with_query_string) + { + res_size = end - res_data; + } + else + { + Pos query_string_or_fragment = find_first_symbols<'?', '#'>(pos, end); + res_size = query_string_or_fragment - res_data; + } + } +}; + +} diff --git a/src/Functions/URL/pathFull.cpp b/src/Functions/URL/pathFull.cpp index da31737c0f9..661fb298c04 100644 --- a/src/Functions/URL/pathFull.cpp +++ b/src/Functions/URL/pathFull.cpp @@ -1,33 +1,14 @@ #include #include #include "FunctionsURL.h" +#include "path.h" #include namespace DB { -struct ExtractPathFull -{ - static size_t getReserveLengthForElement() { return 30; } - - static void execute(const Pos data, const size_t size, Pos & res_data, size_t & res_size) - { - res_data = data; - res_size = 0; - - Pos pos = data; - Pos end = pos + size; - - if (end != (pos = find_first_symbols<'/'>(pos, end)) && pos[1] == '/' && end != (pos = find_first_symbols<'/'>(pos + 2, end))) - { - res_data = pos; - res_size = end - res_data; - } - } -}; - struct NamePathFull { static constexpr auto name = "pathFull"; }; -using FunctionPathFull = FunctionStringToString, NamePathFull>; +using FunctionPathFull = FunctionStringToString>, NamePathFull>; void registerFunctionPathFull(FunctionFactory & factory) { diff --git a/src/Functions/URL/topLevelDomain.cpp b/src/Functions/URL/topLevelDomain.cpp index 589a9bbd1e7..802c5a388d7 100644 --- a/src/Functions/URL/topLevelDomain.cpp +++ b/src/Functions/URL/topLevelDomain.cpp @@ -21,7 +21,7 @@ struct ExtractTopLevelDomain if (host.data[host.size - 1] == '.') host.size -= 1; - auto host_end = host.data + host.size; + const auto * host_end = host.data + host.size; Pos last_dot = find_last_symbols_or_null<'.'>(host.data, host_end); if (!last_dot) diff --git a/src/Functions/addressToLine.cpp b/src/Functions/addressToLine.cpp index f2fb544cfdc..6489fac3371 100644 --- a/src/Functions/addressToLine.cpp +++ b/src/Functions/addressToLine.cpp @@ -102,7 +102,7 @@ private: { const SymbolIndex & symbol_index = SymbolIndex::instance(); - if (auto object = symbol_index.findObject(reinterpret_cast(addr))) + if (const auto * object = symbol_index.findObject(reinterpret_cast(addr))) { auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; if (!std::filesystem::exists(object->name)) diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp index 1c3267343ca..5513bc8aa56 100644 --- a/src/Functions/appendTrailingCharIfAbsent.cpp +++ b/src/Functions/appendTrailingCharIfAbsent.cpp @@ -66,7 +66,7 @@ private: if (trailing_char_str.size() != 1) throw Exception{"Second argument of function " + getName() + " must be a one-character string", ErrorCodes::BAD_ARGUMENTS}; - if (const auto col = checkAndGetColumn(column.get())) + if (const auto * col = checkAndGetColumn(column.get())) { auto col_res = ColumnString::create(); diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index aa4b945055a..5073dc061be 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -14,14 +14,9 @@ class FunctionArray : public IFunction { public: static constexpr auto name = "array"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context); - } - - explicit FunctionArray(const Context & context_) - : context(context_) + static FunctionPtr create(const Context &) { + return std::make_shared(); } bool useDefaultImplementationForNulls() const override { return false; } @@ -67,7 +62,7 @@ public: ColumnPtr preprocessed_column = arg.column; if (!arg.type->equals(*elem_type)) - preprocessed_column = castColumn(arg, elem_type, context); + preprocessed_column = castColumn(arg, elem_type); preprocessed_column = preprocessed_column->convertToFullColumnIfConst(); @@ -104,9 +99,6 @@ private: } bool addField(DataTypePtr type_res, const Field & f, Array & arr) const; - -private: - const Context & context; }; diff --git a/src/Functions/array/arrayAll.cpp b/src/Functions/array/arrayAll.cpp index 3083a8d425c..34deafdffdf 100644 --- a/src/Functions/array/arrayAll.cpp +++ b/src/Functions/array/arrayAll.cpp @@ -31,7 +31,7 @@ struct ArrayAllImpl if (!column_filter) { - auto column_filter_const = checkAndGetColumnConst(&*mapped); + const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) throw Exception("Unexpected type of filter column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayConcat.cpp b/src/Functions/array/arrayConcat.cpp index f96584e3f54..093bda9019b 100644 --- a/src/Functions/array/arrayConcat.cpp +++ b/src/Functions/array/arrayConcat.cpp @@ -26,8 +26,7 @@ class FunctionArrayConcat : public IFunction { public: static constexpr auto name = "arrayConcat"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionArrayConcat(const Context & context_) : context(context_) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } String getName() const override { return name; } @@ -41,7 +40,7 @@ public: for (auto i : ext::range(0, arguments.size())) { - auto array_type = typeid_cast(arguments[i].get()); + const auto * array_type = typeid_cast(arguments[i].get()); if (!array_type) throw Exception("Argument " + std::to_string(i) + " for function " + getName() + " must be an array but it has type " + arguments[i]->getName() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -73,7 +72,7 @@ public: ColumnPtr preprocessed_column = arg.column; if (!arg.type->equals(*return_type)) - preprocessed_column = castColumn(arg, return_type, context); + preprocessed_column = castColumn(arg, return_type); preprocessed_columns[i] = std::move(preprocessed_column); } @@ -84,13 +83,13 @@ public: { bool is_const = false; - if (auto argument_column_const = typeid_cast(argument_column.get())) + if (const auto * argument_column_const = typeid_cast(argument_column.get())) { is_const = true; argument_column = argument_column_const->getDataColumnPtr(); } - if (auto argument_column_array = typeid_cast(argument_column.get())) + if (const auto * argument_column_array = typeid_cast(argument_column.get())) sources.emplace_back(GatherUtils::createArraySource(*argument_column_array, is_const, rows)); else throw Exception{"Arguments for function " + getName() + " must be arrays.", ErrorCodes::LOGICAL_ERROR}; @@ -103,9 +102,6 @@ public: } bool useDefaultImplementationForConstants() const override { return true; } - -private: - const Context & context; }; diff --git a/src/Functions/array/arrayCount.cpp b/src/Functions/array/arrayCount.cpp index 05afca64945..377a6eb8fb1 100644 --- a/src/Functions/array/arrayCount.cpp +++ b/src/Functions/array/arrayCount.cpp @@ -31,7 +31,7 @@ struct ArrayCountImpl if (!column_filter) { - auto column_filter_const = checkAndGetColumnConst(&*mapped); + const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) throw Exception("Unexpected type of filter column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayCumSum.cpp b/src/Functions/array/arrayCumSum.cpp index a15ca998366..40c0cd4ade2 100644 --- a/src/Functions/array/arrayCumSum.cpp +++ b/src/Functions/array/arrayCumSum.cpp @@ -82,7 +82,7 @@ struct ArrayCumSumImpl // skip empty arrays if (pos < offset) { - res_values[pos++] = x; + res_values[pos++] = x; // NOLINT for (; pos < offset; ++pos) res_values[pos] = res_values[pos - 1] + x; } @@ -110,7 +110,7 @@ struct ArrayCumSumImpl // skip empty arrays if (pos < offset) { - res_values[pos] = data[pos]; + res_values[pos] = data[pos]; // NOLINT for (++pos; pos < offset; ++pos) res_values[pos] = res_values[pos - 1] + data[pos]; } diff --git a/src/Functions/array/arrayCumSumNonNegative.cpp b/src/Functions/array/arrayCumSumNonNegative.cpp index 87c7172aaec..ff0f081d70b 100644 --- a/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/src/Functions/array/arrayCumSumNonNegative.cpp @@ -78,7 +78,7 @@ struct ArrayCumSumNonNegativeImpl // skip empty arrays if (pos < offset) { - accum_sum = data[pos] > 0 ? data[pos] : Element(0); + accum_sum = data[pos] > 0 ? data[pos] : Element(0); // NOLINT res_values[pos] = accum_sum; for (++pos; pos < offset; ++pos) { diff --git a/src/Functions/array/arrayExists.cpp b/src/Functions/array/arrayExists.cpp index 5ebc6a35d02..34ea71af259 100644 --- a/src/Functions/array/arrayExists.cpp +++ b/src/Functions/array/arrayExists.cpp @@ -31,7 +31,7 @@ struct ArrayExistsImpl if (!column_filter) { - auto column_filter_const = checkAndGetColumnConst(&*mapped); + const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) throw Exception("Unexpected type of filter column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayFill.cpp b/src/Functions/array/arrayFill.cpp index 0a463f908a3..4cc2caaaafa 100644 --- a/src/Functions/array/arrayFill.cpp +++ b/src/Functions/array/arrayFill.cpp @@ -80,7 +80,7 @@ struct ArrayFillImpl } else { - auto column_fill_const = checkAndGetColumnConst(&*mapped); + const auto * column_fill_const = checkAndGetColumnConst(&*mapped); if (!column_fill_const) throw Exception("Unexpected type of cut column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayFilter.cpp b/src/Functions/array/arrayFilter.cpp index 60e6f948b56..1291989f9a2 100644 --- a/src/Functions/array/arrayFilter.cpp +++ b/src/Functions/array/arrayFilter.cpp @@ -31,7 +31,7 @@ struct ArrayFilterImpl if (!column_filter) { - auto column_filter_const = checkAndGetColumnConst(&*mapped); + const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) throw Exception("Unexpected type of filter column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayFirst.cpp b/src/Functions/array/arrayFirst.cpp index a635e0d9230..dbe545ea387 100644 --- a/src/Functions/array/arrayFirst.cpp +++ b/src/Functions/array/arrayFirst.cpp @@ -24,11 +24,11 @@ struct ArrayFirstImpl static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped) { - auto column_filter = typeid_cast(&*mapped); + const auto * column_filter = typeid_cast(&*mapped); if (!column_filter) { - auto column_filter_const = checkAndGetColumnConst(&*mapped); + const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) throw Exception("Unexpected type of filter column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayFirstIndex.cpp b/src/Functions/array/arrayFirstIndex.cpp index f5c660a63fb..d229687774e 100644 --- a/src/Functions/array/arrayFirstIndex.cpp +++ b/src/Functions/array/arrayFirstIndex.cpp @@ -24,11 +24,11 @@ struct ArrayFirstIndexImpl static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped) { - auto column_filter = typeid_cast(&*mapped); + const auto * column_filter = typeid_cast(&*mapped); if (!column_filter) { - auto column_filter_const = checkAndGetColumnConst(&*mapped); + const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) throw Exception("Unexpected type of filter column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index 1b1ac172f24..fab1332cbda 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -19,7 +19,6 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } @@ -225,21 +224,6 @@ public: } }; -/// Specialization that catches internal errors. -template -struct ArrayIndexNumImpl -{ - template - static void vector( - const PaddedPODArray &, const ColumnArray::Offsets &, - const ScalarOrVector &, - PaddedPODArray &, - const PaddedPODArray *, - const PaddedPODArray *) - { - throw Exception{"Logical error in implementation of a function that returns array index", ErrorCodes::LOGICAL_ERROR}; - } -}; /// Implementation for arrays of numbers when the 2nd function argument /// is a NULL value. @@ -623,8 +607,7 @@ private: || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result); + || executeNumberNumber(block, arguments, result); } template diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index ffeb6e99222..cccad7c7a03 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -88,8 +88,8 @@ private: ColumnsWithTypeAndName casted; }; - CastArgumentsResult castColumns(Block & block, const ColumnNumbers & arguments, - const DataTypePtr & return_type, const DataTypePtr & return_type_with_nulls) const; + static CastArgumentsResult castColumns(Block & block, const ColumnNumbers & arguments, + const DataTypePtr & return_type, const DataTypePtr & return_type_with_nulls); UnpackedArrays prepareArrays(const ColumnsWithTypeAndName & columns, ColumnsWithTypeAndName & initial_columns) const; template @@ -135,7 +135,7 @@ DataTypePtr FunctionArrayIntersect::getReturnTypeImpl(const DataTypes & argument for (auto i : ext::range(0, arguments.size())) { - auto array_type = typeid_cast(arguments[i].get()); + const auto * array_type = typeid_cast(arguments[i].get()); if (!array_type) throw Exception("Argument " + std::to_string(i) + " for function " + getName() + " must be an array but it has type " + arguments[i]->getName() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -161,9 +161,9 @@ DataTypePtr FunctionArrayIntersect::getReturnTypeImpl(const DataTypes & argument ColumnPtr FunctionArrayIntersect::castRemoveNullable(const ColumnPtr & column, const DataTypePtr & data_type) const { - if (auto column_nullable = checkAndGetColumn(column.get())) + if (const auto * column_nullable = checkAndGetColumn(column.get())) { - auto nullable_type = checkAndGetDataType(data_type.get()); + const auto * nullable_type = checkAndGetDataType(data_type.get()); const auto & nested = column_nullable->getNestedColumnPtr(); if (nullable_type) { @@ -172,9 +172,9 @@ ColumnPtr FunctionArrayIntersect::castRemoveNullable(const ColumnPtr & column, c } return castRemoveNullable(nested, data_type); } - else if (auto column_array = checkAndGetColumn(column.get())) + else if (const auto * column_array = checkAndGetColumn(column.get())) { - auto array_type = checkAndGetDataType(data_type.get()); + const auto * array_type = checkAndGetDataType(data_type.get()); if (!array_type) throw Exception{"Cannot cast array column to column with type " + data_type->getName() + " in function " + getName(), ErrorCodes::LOGICAL_ERROR}; @@ -182,9 +182,9 @@ ColumnPtr FunctionArrayIntersect::castRemoveNullable(const ColumnPtr & column, c auto casted_column = castRemoveNullable(column_array->getDataPtr(), array_type->getNestedType()); return ColumnArray::create(casted_column, column_array->getOffsetsPtr()); } - else if (auto column_tuple = checkAndGetColumn(column.get())) + else if (const auto * column_tuple = checkAndGetColumn(column.get())) { - auto tuple_type = checkAndGetDataType(data_type.get()); + const auto * tuple_type = checkAndGetDataType(data_type.get()); if (!tuple_type) throw Exception{"Cannot cast tuple column to type " @@ -207,14 +207,14 @@ ColumnPtr FunctionArrayIntersect::castRemoveNullable(const ColumnPtr & column, c FunctionArrayIntersect::CastArgumentsResult FunctionArrayIntersect::castColumns( Block & block, const ColumnNumbers & arguments, const DataTypePtr & return_type, - const DataTypePtr & return_type_with_nulls) const + const DataTypePtr & return_type_with_nulls) { size_t num_args = arguments.size(); ColumnsWithTypeAndName initial_columns(num_args); ColumnsWithTypeAndName columns(num_args); - auto type_array = checkAndGetDataType(return_type.get()); - auto & type_nested = type_array->getNestedType(); + const auto * type_array = checkAndGetDataType(return_type.get()); + const auto & type_nested = type_array->getNestedType(); auto type_not_nullable_nested = removeNullable(type_nested); const bool is_numeric_or_string = isNativeNumber(type_not_nullable_nested) @@ -245,7 +245,7 @@ FunctionArrayIntersect::CastArgumentsResult FunctionArrayIntersect::castColumns( { if (!arg.type->equals(*return_type)) { - column.column = castColumn(arg, return_type, context); + column.column = castColumn(arg, return_type); column.type = return_type; } } @@ -258,12 +258,12 @@ FunctionArrayIntersect::CastArgumentsResult FunctionArrayIntersect::castColumns( /// because cannot cast Nullable(T) to T. if (static_cast(*arg.type).getNestedType()->isNullable()) { - column.column = castColumn(arg, nullable_return_type, context); + column.column = castColumn(arg, nullable_return_type); column.type = nullable_return_type; } else { - column.column = castColumn(arg, return_type, context); + column.column = castColumn(arg, return_type); column.type = return_type; } } @@ -274,7 +274,7 @@ FunctionArrayIntersect::CastArgumentsResult FunctionArrayIntersect::castColumns( /// return_type_with_nulls is the most common subtype with possible nullable parts. if (!arg.type->equals(*return_type_with_nulls)) { - column.column = castColumn(arg, return_type_with_nulls, context); + column.column = castColumn(arg, return_type_with_nulls); column.type = return_type_with_nulls; } } @@ -313,17 +313,17 @@ FunctionArrayIntersect::UnpackedArrays FunctionArrayIntersect::prepareArrays( for (auto i : ext::range(0, columns_number)) { auto & arg = arrays.args[i]; - auto argument_column = columns[i].column.get(); - auto initial_column = initial_columns[i].column.get(); + const auto * argument_column = columns[i].column.get(); + const auto * initial_column = initial_columns[i].column.get(); - if (auto argument_column_const = typeid_cast(argument_column)) + if (const auto * argument_column_const = typeid_cast(argument_column)) { arg.is_const = true; argument_column = argument_column_const->getDataColumnPtr().get(); initial_column = typeid_cast(initial_column)->getDataColumnPtr().get(); } - if (auto argument_column_array = typeid_cast(argument_column)) + if (const auto * argument_column_array = typeid_cast(argument_column)) { if (!arg.is_const) all_const = false; @@ -333,7 +333,7 @@ FunctionArrayIntersect::UnpackedArrays FunctionArrayIntersect::prepareArrays( initial_column = &typeid_cast(initial_column)->getData(); - if (auto column_nullable = typeid_cast(arg.nested_column)) + if (const auto * column_nullable = typeid_cast(arg.nested_column)) { arg.null_map = &column_nullable->getNullMapData(); arg.nested_column = &column_nullable->getNestedColumn(); @@ -343,8 +343,8 @@ FunctionArrayIntersect::UnpackedArrays FunctionArrayIntersect::prepareArrays( /// In case column was casted need to create overflow mask for integer types. if (arg.nested_column != initial_column) { - auto & nested_init_type = typeid_cast(removeNullable(initial_columns[i].type).get())->getNestedType(); - auto & nested_cast_type = typeid_cast(removeNullable(columns[i].type).get())->getNestedType(); + const auto & nested_init_type = typeid_cast(removeNullable(initial_columns[i].type).get())->getNestedType(); + const auto & nested_cast_type = typeid_cast(removeNullable(columns[i].type).get())->getNestedType(); if (isInteger(nested_init_type) || isDateOrDateTime(nested_init_type)) { @@ -388,7 +388,7 @@ FunctionArrayIntersect::UnpackedArrays FunctionArrayIntersect::prepareArrays( void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) { const auto & return_type = block.getByPosition(result).type; - auto return_type_array = checkAndGetDataType(return_type.get()); + const auto * return_type_array = checkAndGetDataType(return_type.get()); if (!return_type_array) throw Exception{"Return type for function " + getName() + " must be array.", ErrorCodes::LOGICAL_ERROR}; @@ -483,7 +483,7 @@ ColumnPtr FunctionArrayIntersect::execute(const UnpackedArrays & arrays, Mutable std::vector columns; columns.reserve(args); - for (auto & arg : arrays.args) + for (const auto & arg : arrays.args) { if constexpr (std::is_same::value) columns.push_back(arg.nested_column); @@ -516,7 +516,7 @@ ColumnPtr FunctionArrayIntersect::execute(const UnpackedArrays & arrays, Mutable for (auto arg_num : ext::range(0, args)) { - auto & arg = arrays.args[arg_num]; + const auto & arg = arrays.args[arg_num]; bool current_has_nullable = false; size_t off; diff --git a/src/Functions/array/arrayPush.h b/src/Functions/array/arrayPush.h index 1b20a9a1d74..61ed420b099 100644 --- a/src/Functions/array/arrayPush.h +++ b/src/Functions/array/arrayPush.h @@ -21,8 +21,8 @@ namespace ErrorCodes class FunctionArrayPush : public IFunction { public: - FunctionArrayPush(const Context & context_, bool push_front_, const char * name_) - : context(context_), push_front(push_front_), name(name_) {} + FunctionArrayPush(bool push_front_, const char * name_) + : push_front(push_front_), name(name_) {} String getName() const override { return name; } @@ -62,11 +62,11 @@ public: auto appended_column = block.getByPosition(arguments[1]).column; if (!block.getByPosition(arguments[0]).type->equals(*return_type)) - array_column = castColumn(block.getByPosition(arguments[0]), return_type, context); + array_column = castColumn(block.getByPosition(arguments[0]), return_type); const DataTypePtr & return_nested_type = typeid_cast(*return_type).getNestedType(); if (!block.getByPosition(arguments[1]).type->equals(*return_nested_type)) - appended_column = castColumn(block.getByPosition(arguments[1]), return_nested_type, context); + appended_column = castColumn(block.getByPosition(arguments[1]), return_nested_type); std::unique_ptr array_source; std::unique_ptr value_source; @@ -106,7 +106,6 @@ public: bool useDefaultImplementationForNulls() const override { return false; } private: - const Context & context; bool push_front; const char * name; }; diff --git a/src/Functions/array/arrayPushBack.cpp b/src/Functions/array/arrayPushBack.cpp index 74d9596dcd2..3c6d4113b88 100644 --- a/src/Functions/array/arrayPushBack.cpp +++ b/src/Functions/array/arrayPushBack.cpp @@ -9,8 +9,8 @@ class FunctionArrayPushBack : public FunctionArrayPush { public: static constexpr auto name = "arrayPushBack"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionArrayPushBack(const Context & context_) : FunctionArrayPush(context_, false, name) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } + FunctionArrayPushBack() : FunctionArrayPush(false, name) {} }; void registerFunctionArrayPushBack(FunctionFactory & factory) diff --git a/src/Functions/array/arrayPushFront.cpp b/src/Functions/array/arrayPushFront.cpp index ab8535b6672..49f47170f3c 100644 --- a/src/Functions/array/arrayPushFront.cpp +++ b/src/Functions/array/arrayPushFront.cpp @@ -10,8 +10,8 @@ class FunctionArrayPushFront : public FunctionArrayPush { public: static constexpr auto name = "arrayPushFront"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionArrayPushFront(const Context & context_) : FunctionArrayPush(context_, true, name) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } + FunctionArrayPushFront() : FunctionArrayPush(true, name) {} }; diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index 3df07303eba..8d44acc82f5 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -149,7 +149,7 @@ void FunctionArrayReduce::executeImpl(Block & block, const ColumnNumbers & argum IColumn & res_col = *result_holder; /// AggregateFunction's states should be inserted into column using specific way - auto res_col_aggregate_function = typeid_cast(&res_col); + auto * res_col_aggregate_function = typeid_cast(&res_col); if (!res_col_aggregate_function && agg_func.isState()) throw Exception("State function " + agg_func.getName() + " inserts results into non-state column " @@ -177,9 +177,9 @@ void FunctionArrayReduce::executeImpl(Block & block, const ColumnNumbers & argum }); { - auto that = &agg_func; + auto * that = &agg_func; /// Unnest consecutive trailing -State combinators - while (auto func = typeid_cast(that)) + while (auto * func = typeid_cast(that)) that = func->getNestedFunction().get(); that->addBatchArray(input_rows_count, places.data(), 0, aggregate_arguments, offsets->data(), arena.get()); diff --git a/src/Functions/array/arrayReduceInRanges.cpp b/src/Functions/array/arrayReduceInRanges.cpp index 18f1086ea6a..2dd0cd56343 100644 --- a/src/Functions/array/arrayReduceInRanges.cpp +++ b/src/Functions/array/arrayReduceInRanges.cpp @@ -198,7 +198,7 @@ void FunctionArrayReduceInRanges::executeImpl(Block & block, const ColumnNumbers result_arr->getOffsets().insert(ranges_offsets->begin(), ranges_offsets->end()); /// AggregateFunction's states should be inserted into column using specific way - auto res_col_aggregate_function = typeid_cast(&result_data); + auto * res_col_aggregate_function = typeid_cast(&result_data); if (!res_col_aggregate_function && agg_func.isState()) throw Exception("State function " + agg_func.getName() + " inserts results into non-state column " @@ -256,9 +256,9 @@ void FunctionArrayReduceInRanges::executeImpl(Block & block, const ColumnNumbers agg_func.destroy(places[j]); }); - auto true_func = &agg_func; + auto * true_func = &agg_func; /// Unnest consecutive trailing -State combinators - while (auto func = typeid_cast(true_func)) + while (auto * func = typeid_cast(true_func)) true_func = func->getNestedFunction().get(); /// Pre-aggregate to the initial level diff --git a/src/Functions/array/arrayResize.cpp b/src/Functions/array/arrayResize.cpp index 9e34e7ccd92..85536b3c18a 100644 --- a/src/Functions/array/arrayResize.cpp +++ b/src/Functions/array/arrayResize.cpp @@ -25,8 +25,7 @@ class FunctionArrayResize : public IFunction { public: static constexpr auto name = "arrayResize"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionArrayResize(const Context & context_) : context(context_) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } String getName() const override { return name; } @@ -45,7 +44,7 @@ public: if (arguments[0]->onlyNull()) return arguments[0]; - auto array_type = typeid_cast(arguments[0].get()); + const auto * array_type = typeid_cast(arguments[0].get()); if (!array_type) throw Exception("First argument for function " + getName() + " must be an array but it has type " + arguments[0]->getName() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -80,7 +79,7 @@ public: auto size_column = block.getByPosition(arguments[1]).column; if (!block.getByPosition(arguments[0]).type->equals(*return_type)) - array_column = castColumn(block.getByPosition(arguments[0]), return_type, context); + array_column = castColumn(block.getByPosition(arguments[0]), return_type); const DataTypePtr & return_nested_type = typeid_cast(*return_type).getNestedType(); size_t size = array_column->size(); @@ -90,7 +89,7 @@ public: { appended_column = block.getByPosition(arguments[2]).column; if (!block.getByPosition(arguments[2]).type->equals(*return_nested_type)) - appended_column = castColumn(block.getByPosition(arguments[2]), return_nested_type, context); + appended_column = castColumn(block.getByPosition(arguments[2]), return_nested_type); } else appended_column = return_nested_type->createColumnConstWithDefaultValue(size); @@ -100,20 +99,20 @@ public: bool is_const = false; - if (auto const_array_column = typeid_cast(array_column.get())) + if (const auto * const_array_column = typeid_cast(array_column.get())) { is_const = true; array_column = const_array_column->getDataColumnPtr(); } - if (auto argument_column_array = typeid_cast(array_column.get())) + if (const auto * argument_column_array = typeid_cast(array_column.get())) array_source = GatherUtils::createArraySource(*argument_column_array, is_const, size); else throw Exception{"First arguments for function " + getName() + " must be array.", ErrorCodes::LOGICAL_ERROR}; bool is_appended_const = false; - if (auto const_appended_column = typeid_cast(appended_column.get())) + if (const auto * const_appended_column = typeid_cast(appended_column.get())) { is_appended_const = true; appended_column = const_appended_column->getDataColumnPtr(); @@ -133,9 +132,6 @@ public: bool useDefaultImplementationForConstants() const override { return true; } bool useDefaultImplementationForNulls() const override { return false; } - -private: - const Context & context; }; diff --git a/src/Functions/array/arrayReverse.cpp b/src/Functions/array/arrayReverse.cpp index 8913b27db4c..5fba7015ae4 100644 --- a/src/Functions/array/arrayReverse.cpp +++ b/src/Functions/array/arrayReverse.cpp @@ -142,7 +142,7 @@ bool FunctionArrayReverse::executeNumber(const IColumn & src_data, const ColumnA if (src == src_end) continue; - auto dst = &res_vec[src_offsets[i] - 1]; + auto * dst = &res_vec[src_offsets[i] - 1]; while (src < src_end) { diff --git a/src/Functions/array/arraySlice.cpp b/src/Functions/array/arraySlice.cpp index a952aa72d2e..cfd75c633c4 100644 --- a/src/Functions/array/arraySlice.cpp +++ b/src/Functions/array/arraySlice.cpp @@ -53,7 +53,7 @@ public: if (arguments[0]->onlyNull()) return arguments[0]; - auto array_type = typeid_cast(arguments[0].get()); + const auto * array_type = typeid_cast(arguments[0].get()); if (!array_type) throw Exception("First argument for function " + getName() + " must be an array but it has type " + arguments[0]->getName() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -90,13 +90,13 @@ public: size_t size = array_column->size(); bool is_const = false; - if (auto const_array_column = typeid_cast(array_column.get())) + if (const auto * const_array_column = typeid_cast(array_column.get())) { is_const = true; array_column = const_array_column->getDataColumnPtr(); } - if (auto argument_column_array = typeid_cast(array_column.get())) + if (const auto * argument_column_array = typeid_cast(array_column.get())) source = GatherUtils::createArraySource(*argument_column_array, is_const, size); else throw Exception{"First arguments for function " + getName() + " must be array.", ErrorCodes::LOGICAL_ERROR}; diff --git a/src/Functions/array/arraySplit.cpp b/src/Functions/array/arraySplit.cpp index 5190bd1c724..2e5f2d8432e 100644 --- a/src/Functions/array/arraySplit.cpp +++ b/src/Functions/array/arraySplit.cpp @@ -62,7 +62,7 @@ struct ArraySplitImpl } else { - auto column_cut_const = checkAndGetColumnConst(&*mapped); + const auto * column_cut_const = checkAndGetColumnConst(&*mapped); if (!column_cut_const) throw Exception("Unexpected type of cut column", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index d94efc47970..4b09e725f25 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -146,7 +146,7 @@ void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & argumen throw Exception("Lengths of all arrays passed to " + getName() + " must be equal.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); - auto * array_data = &array->getData(); + const auto * array_data = &array->getData(); data_columns[i] = array_data; } @@ -154,7 +154,7 @@ void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & argumen for (size_t i = 0; i < num_arguments; ++i) { - if (auto * nullable_col = checkAndGetColumn(*data_columns[i])) + if (const auto * nullable_col = checkAndGetColumn(*data_columns[i])) { if (num_arguments == 1) data_columns[i] = &nullable_col->getNestedColumn(); diff --git a/src/Functions/array/emptyArrayToSingle.cpp b/src/Functions/array/emptyArrayToSingle.cpp index 27f4e01c547..273276706ef 100644 --- a/src/Functions/array/emptyArrayToSingle.cpp +++ b/src/Functions/array/emptyArrayToSingle.cpp @@ -145,7 +145,7 @@ namespace const size_t n = src_data_concrete->getN(); const ColumnFixedString::Chars & src_data_vec = src_data_concrete->getChars(); - auto concrete_res_data = typeid_cast(&res_data_col); + auto * concrete_res_data = typeid_cast(&res_data_col); if (!concrete_res_data) throw Exception{"Internal error", ErrorCodes::LOGICAL_ERROR}; @@ -211,14 +211,14 @@ namespace { const ColumnString::Offsets & src_string_offsets = src_data_concrete->getOffsets(); - auto concrete_res_string_offsets = typeid_cast(&res_data_col); + auto * concrete_res_string_offsets = typeid_cast(&res_data_col); if (!concrete_res_string_offsets) throw Exception{"Internal error", ErrorCodes::LOGICAL_ERROR}; ColumnString::Offsets & res_string_offsets = concrete_res_string_offsets->getOffsets(); const ColumnString::Chars & src_data_vec = src_data_concrete->getChars(); - auto concrete_res_data = typeid_cast(&res_data_col); + auto * concrete_res_data = typeid_cast(&res_data_col); if (!concrete_res_data) throw Exception{"Internal error", ErrorCodes::LOGICAL_ERROR}; ColumnString::Chars & res_data = concrete_res_data->getChars(); @@ -393,7 +393,7 @@ void FunctionEmptyArrayToSingle::executeImpl(Block & block, const ColumnNumbers const IColumn * inner_col; IColumn * inner_res_col; - auto nullable_col = checkAndGetColumn(src_data); + const auto * nullable_col = checkAndGetColumn(src_data); if (nullable_col) { inner_col = &nullable_col->getNestedColumn(); diff --git a/src/Functions/array/hasAll.cpp b/src/Functions/array/hasAll.cpp index 8d833adb5f5..b325a74f015 100644 --- a/src/Functions/array/hasAll.cpp +++ b/src/Functions/array/hasAll.cpp @@ -9,8 +9,8 @@ class FunctionArrayHasAll : public FunctionArrayHasAllAny { public: static constexpr auto name = "hasAll"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionArrayHasAll(const Context & context_) : FunctionArrayHasAllAny(context_, true, name) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } + FunctionArrayHasAll() : FunctionArrayHasAllAny(true, name) {} }; void registerFunctionHasAll(FunctionFactory & factory) diff --git a/src/Functions/array/hasAllAny.h b/src/Functions/array/hasAllAny.h index fe6c026aecd..ea42e182c8c 100644 --- a/src/Functions/array/hasAllAny.h +++ b/src/Functions/array/hasAllAny.h @@ -27,8 +27,8 @@ namespace ErrorCodes class FunctionArrayHasAllAny : public IFunction { public: - FunctionArrayHasAllAny(const Context & context_, bool all_, const char * name_) - : context(context_), all(all_), name(name_) {} + FunctionArrayHasAllAny(bool all_, const char * name_) + : all(all_), name(name_) {} String getName() const override { return name; } @@ -81,7 +81,7 @@ public: /// Converts Array(Nothing) or Array(Nullable(Nothing) to common type. Example: hasAll([Null, 1], [Null]) -> 1 if (typeid_cast(removeNullable(nested_type).get())) - preprocessed_column = castColumn(argument, commonType(), context); + preprocessed_column = castColumn(argument, commonType()); preprocessed_columns[i] = std::move(preprocessed_column); } @@ -114,7 +114,6 @@ public: bool useDefaultImplementationForConstants() const override { return true; } private: - const Context & context; bool all; const char * name; }; diff --git a/src/Functions/array/hasAny.cpp b/src/Functions/array/hasAny.cpp index 84a3a736364..ace86ce10c4 100644 --- a/src/Functions/array/hasAny.cpp +++ b/src/Functions/array/hasAny.cpp @@ -9,8 +9,8 @@ class FunctionArrayHasAny : public FunctionArrayHasAllAny { public: static constexpr auto name = "hasAny"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionArrayHasAny(const Context & context_) : FunctionArrayHasAllAny(context_, false, name) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } + FunctionArrayHasAny() : FunctionArrayHasAllAny(false, name) {} }; void registerFunctionHasAny(FunctionFactory & factory) diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index 283eb760fcf..b98deb151f1 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -27,11 +27,9 @@ class FunctionRange : public IFunction public: static constexpr auto name = "range"; static constexpr size_t max_elements = 100'000'000; - static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } - explicit FunctionRange(const Context & context_) : context(context_) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } private: - const Context & context; String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -328,7 +326,7 @@ private: { if (arguments.size() == 1) { - const auto col = block.getByPosition(arguments[0]).column.get(); + const auto * col = block.getByPosition(arguments[0]).column.get(); if (!executeInternal(block, col, result) && !executeInternal(block, col, result) && !executeInternal(block, col, result) && @@ -347,9 +345,9 @@ private: for (size_t i = 0; i < arguments.size(); ++i) { if (i == 1) - columns_holder[i] = castColumn(block.getByPosition(arguments[i]), return_type, context)->convertToFullColumnIfConst(); + columns_holder[i] = castColumn(block.getByPosition(arguments[i]), return_type)->convertToFullColumnIfConst(); else - columns_holder[i] = castColumn(block.getByPosition(arguments[i]), return_type, context); + columns_holder[i] = castColumn(block.getByPosition(arguments[i]), return_type); columns[i] = columns_holder[i].get(); } diff --git a/src/Functions/assumeNotNull.cpp b/src/Functions/assumeNotNull.cpp index 4fc98e43b12..cb28d655b03 100644 --- a/src/Functions/assumeNotNull.cpp +++ b/src/Functions/assumeNotNull.cpp @@ -43,7 +43,7 @@ public: const ColumnPtr & col = block.getByPosition(arguments[0]).column; ColumnPtr & res_col = block.getByPosition(result).column; - if (auto * nullable_col = checkAndGetColumn(*col)) + if (const auto * nullable_col = checkAndGetColumn(*col)) res_col = nullable_col->getNestedColumnPtr(); else res_col = col; diff --git a/src/Functions/base64Encode.cpp b/src/Functions/base64Encode.cpp index f76fedda71e..8cf54eca768 100644 --- a/src/Functions/base64Encode.cpp +++ b/src/Functions/base64Encode.cpp @@ -1,9 +1,12 @@ #include #include -#include "config_functions.h" + +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif #if USE_BASE64 -#include +# include namespace DB { diff --git a/src/Functions/caseWithExpression.cpp b/src/Functions/caseWithExpression.cpp index dc48536b4a7..0fe5ce5bbc3 100644 --- a/src/Functions/caseWithExpression.cpp +++ b/src/Functions/caseWithExpression.cpp @@ -20,7 +20,6 @@ public: static constexpr auto name = "caseWithExpression"; static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } -public: explicit FunctionCaseWithExpression(const Context & context_) : context(context_) {} bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } diff --git a/src/Functions/coalesce.cpp b/src/Functions/coalesce.cpp index 9d5d4df4a7f..9dd6a2d5214 100644 --- a/src/Functions/coalesce.cpp +++ b/src/Functions/coalesce.cpp @@ -160,9 +160,9 @@ public: /// if last argument is not nullable, result should be also not nullable if (!block.getByPosition(multi_if_args.back()).column->isNullable() && res->isNullable()) { - if (auto * column_lc = checkAndGetColumn(*res)) + if (const auto * column_lc = checkAndGetColumn(*res)) res = checkAndGetColumn(*column_lc->convertToFullColumn())->getNestedColumnPtr(); - else if (auto * column_const = checkAndGetColumn(*res)) + else if (const auto * column_const = checkAndGetColumn(*res)) res = checkAndGetColumn(column_const->getDataColumn())->getNestedColumnPtr(); else res = checkAndGetColumn(*res)->getNestedColumnPtr(); diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 7cf0b2ce891..d83fbed30e9 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -41,7 +41,7 @@ public: size_t getNumberOfArguments() const override { return 0; } - bool isInjective(const Block &) override { return is_injective; } + bool isInjective(const Block &) const override { return is_injective; } bool useDefaultImplementationForConstants() const override { return true; } @@ -61,7 +61,7 @@ public: for (const auto arg_idx : ext::range(0, arguments.size())) { - const auto arg = arguments[arg_idx].get(); + const auto * arg = arguments[arg_idx].get(); if (!isStringOrFixedString(arg)) throw Exception{"Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx + 1) + " of function " + getName(), diff --git a/src/Functions/convertCharset.cpp b/src/Functions/convertCharset.cpp index 372bf08a0d2..9bd1f1b1cf6 100644 --- a/src/Functions/convertCharset.cpp +++ b/src/Functions/convertCharset.cpp @@ -1,20 +1,22 @@ -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_ICU +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include +# include +# include +# include namespace DB diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index 6d7574a2eed..245b737725d 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -82,7 +82,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - auto * unit_column = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); + const auto * unit_column = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); if (!unit_column) throw Exception("First argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_COLUMN); @@ -126,13 +126,13 @@ private: const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * x_vec_16 = checkAndGetColumn(&x)) + if (const auto * x_vec_16 = checkAndGetColumn(&x)) dispatchForSecondColumn(*x_vec_16, y, timezone_x, timezone_y, result); - else if (auto * x_vec_32 = checkAndGetColumn(&x)) + else if (const auto * x_vec_32 = checkAndGetColumn(&x)) dispatchForSecondColumn(*x_vec_32, y, timezone_x, timezone_y, result); - else if (auto * x_const_16 = checkAndGetColumnConst(&x)) + else if (const auto * x_const_16 = checkAndGetColumnConst(&x)) dispatchConstForSecondColumn(x_const_16->getValue(), y, timezone_x, timezone_y, result); - else if (auto * x_const_32 = checkAndGetColumnConst(&x)) + else if (const auto * x_const_32 = checkAndGetColumnConst(&x)) dispatchConstForSecondColumn(x_const_32->getValue(), y, timezone_x, timezone_y, result); else throw Exception("Illegal column for first argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); @@ -144,13 +144,13 @@ private: const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * y_vec_16 = checkAndGetColumn(&y)) + if (const auto * y_vec_16 = checkAndGetColumn(&y)) vectorVector(x, *y_vec_16, timezone_x, timezone_y, result); - else if (auto * y_vec_32 = checkAndGetColumn(&y)) + else if (const auto * y_vec_32 = checkAndGetColumn(&y)) vectorVector(x, *y_vec_32, timezone_x, timezone_y, result); - else if (auto * y_const_16 = checkAndGetColumnConst(&y)) + else if (const auto * y_const_16 = checkAndGetColumnConst(&y)) vectorConstant(x, y_const_16->getValue(), timezone_x, timezone_y, result); - else if (auto * y_const_32 = checkAndGetColumnConst(&y)) + else if (const auto * y_const_32 = checkAndGetColumnConst(&y)) vectorConstant(x, y_const_32->getValue(), timezone_x, timezone_y, result); else throw Exception("Illegal column for second argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); @@ -162,9 +162,9 @@ private: const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * y_vec_16 = checkAndGetColumn(&y)) + if (const auto * y_vec_16 = checkAndGetColumn(&y)) constantVector(x, *y_vec_16, timezone_x, timezone_y, result); - else if (auto * y_vec_32 = checkAndGetColumn(&y)) + else if (const auto * y_vec_32 = checkAndGetColumn(&y)) constantVector(x, *y_vec_32, timezone_x, timezone_y, result); else throw Exception("Illegal column for second argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index cace9d4a3ef..080c1108deb 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -356,8 +356,8 @@ public: } } - auto begin = reinterpret_cast(dst_data.data()); - auto pos = begin; + auto * begin = reinterpret_cast(dst_data.data()); + auto * pos = begin; for (size_t i = 0; i < vec.size(); ++i) { diff --git a/src/Functions/formatString.cpp b/src/Functions/formatString.cpp index 0fefe3f95cb..eca245314b3 100644 --- a/src/Functions/formatString.cpp +++ b/src/Functions/formatString.cpp @@ -54,7 +54,7 @@ public: for (const auto arg_idx : ext::range(0, arguments.size())) { - const auto arg = arguments[arg_idx].get(); + const auto * arg = arguments[arg_idx].get(); if (!isStringOrFixedString(arg)) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx + 1) + " of function " + getName(), diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index efd5e535afa..dcf3e306d68 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -40,7 +40,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isFloat64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be Float64", @@ -63,9 +63,9 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_lon = block.getByPosition(arguments[0]).column.get(); - const auto col_lat = block.getByPosition(arguments[1]).column.get(); - const auto col_res = block.getByPosition(arguments[2]).column.get(); + const auto * col_lon = block.getByPosition(arguments[0]).column.get(); + const auto * col_lat = block.getByPosition(arguments[1]).column.get(); + const auto * col_res = block.getByPosition(arguments[2]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/getSizeOfEnumType.cpp b/src/Functions/getSizeOfEnumType.cpp index 87ad24bc3f0..04afb0ab812 100644 --- a/src/Functions/getSizeOfEnumType.cpp +++ b/src/Functions/getSizeOfEnumType.cpp @@ -55,9 +55,9 @@ public: ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const Block & block, const ColumnNumbers & arguments) const override { - if (auto type8 = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) + if (const auto * type8 = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) return DataTypeUInt8().createColumnConst(1, type8->getValues().size()); - else if (auto type16 = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) + else if (const auto * type16 = checkAndGetDataType(block.getByPosition(arguments[0]).type.get())) return DataTypeUInt16().createColumnConst(1, type16->getValues().size()); else throw Exception("The argument for function " + getName() + " must be Enum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index cdaf1589074..238499f8def 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -242,7 +242,7 @@ private: { for (const auto arg_idx : ext::range(0, arguments.size())) { - const auto arg = arguments[arg_idx].get(); + const auto * arg = arguments[arg_idx].get(); if (!isNumber(WhichDataType(arg))) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx + 1) + " of function " + getName() + ". Must be numeric", diff --git a/src/Functions/greatest.cpp b/src/Functions/greatest.cpp index 6eb123708a4..63f08d0affe 100644 --- a/src/Functions/greatest.cpp +++ b/src/Functions/greatest.cpp @@ -1,6 +1,8 @@ #include #include #include +#include + namespace DB { @@ -57,7 +59,7 @@ using FunctionGreatest = FunctionBinaryArithmetic; void registerFunctionGreatest(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(FunctionFactory::CaseInsensitive); } } diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index bcb55b3f346..c645cddec54 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -34,7 +34,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8", @@ -45,7 +45,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 969f4d7e68b..50e197be57b 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -45,7 +45,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8", @@ -56,7 +56,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index abc1d1e629c..eadf00e4876 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -30,7 +30,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -41,7 +41,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index 7decd6f51ba..81d0cf5d9af 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -34,7 +34,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -45,7 +45,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index ad9a3f29381..43d6e0ce881 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -41,7 +41,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8", @@ -52,7 +52,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index 699b093f22e..e9f3499b171 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -30,7 +30,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -47,8 +47,8 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex_origin = block.getByPosition(arguments[0]).column.get(); - const auto col_hindex_dest = block.getByPosition(arguments[1]).column.get(); + const auto * col_hindex_origin = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex_dest = block.getByPosition(arguments[1]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index 01e4f538f17..8b1a9ae1a34 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -34,7 +34,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -45,7 +45,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index 595919dded3..ebbf7c0956a 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -32,7 +32,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -49,8 +49,8 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); - const auto col_resolution = block.getByPosition(arguments[1]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_resolution = block.getByPosition(arguments[1]).column.get(); auto dst = ColumnArray::create(ColumnUInt64::create()); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index b87e862282f..7232a6734cc 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -30,7 +30,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -47,8 +47,8 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); - const auto col_resolution = block.getByPosition(arguments[1]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_resolution = block.getByPosition(arguments[1]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index f692ab129c6..fb0ee26acac 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -31,7 +31,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -42,7 +42,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto col_res = ColumnString::create(); auto & vec_res = col_res->getChars(); diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index fade5ef8155..7761d24c7b2 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -38,7 +38,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", @@ -55,8 +55,8 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); - const auto col_k = block.getByPosition(arguments[1]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_k = block.getByPosition(arguments[1]).column.get(); auto dst = ColumnArray::create(ColumnUInt64::create()); auto & dst_data = dst->getData(); diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index f16033b1f52..58dbf74af5e 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -172,8 +172,7 @@ class FunctionIf : public FunctionIfBase { public: static constexpr auto name = "if"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionIf(const Context & context_) : context(context_) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } private: template @@ -207,13 +206,13 @@ private: const IColumn * col_right_untyped = block.getByPosition(arguments[2]).column.get(); UInt32 scale = decimalScale(block, arguments); - if (auto col_right_vec = checkAndGetColumn(col_right_untyped)) + if (const auto * col_right_vec = checkAndGetColumn(col_right_untyped)) { NumIfImpl::vectorVector( cond_col->getData(), col_left->getData(), col_right_vec->getData(), block, result, scale); return true; } - else if (auto col_right_const = checkAndGetColumnConst(col_right_untyped)) + else if (const auto * col_right_const = checkAndGetColumnConst(col_right_untyped)) { NumIfImpl::vectorConstant( cond_col->getData(), col_left->getData(), col_right_const->template getValue(), block, result, scale); @@ -236,13 +235,13 @@ private: const IColumn * col_right_untyped = block.getByPosition(arguments[2]).column.get(); UInt32 scale = decimalScale(block, arguments); - if (auto col_right_vec = checkAndGetColumn(col_right_untyped)) + if (const auto * col_right_vec = checkAndGetColumn(col_right_untyped)) { NumIfImpl::constantVector( cond_col->getData(), col_left->template getValue(), col_right_vec->getData(), block, result, scale); return true; } - else if (auto col_right_const = checkAndGetColumnConst(col_right_untyped)) + else if (const auto * col_right_const = checkAndGetColumnConst(col_right_untyped)) { NumIfImpl::constantConstant( cond_col->getData(), col_left->template getValue(), col_right_const->template getValue(), block, result, scale); @@ -269,7 +268,7 @@ private: const IColumn * col_right_untyped = block.getByPosition(arguments[2]).column.get(); - if (auto col_right_array = checkAndGetColumn(col_right_untyped)) + if (const auto * col_right_array = checkAndGetColumn(col_right_untyped)) { const ColVecT1 * col_right_vec = checkAndGetColumn(&col_right_array->getData()); if (!col_right_vec) @@ -286,7 +285,7 @@ private: block.getByPosition(result).column = std::move(res); return true; } - else if (auto col_right_const_array = checkAndGetColumnConst(col_right_untyped)) + else if (const auto * col_right_const_array = checkAndGetColumnConst(col_right_untyped)) { const ColumnArray * col_right_const_array_data = checkAndGetColumn(&col_right_const_array->getDataColumn()); if (!checkColumn(&col_right_const_array_data->getData())) @@ -325,7 +324,7 @@ private: const IColumn * col_right_untyped = block.getByPosition(arguments[2]).column.get(); - if (auto col_right_array = checkAndGetColumn(col_right_untyped)) + if (const auto * col_right_array = checkAndGetColumn(col_right_untyped)) { const ColVecT1 * col_right_vec = checkAndGetColumn(&col_right_array->getData()); @@ -343,7 +342,7 @@ private: block.getByPosition(result).column = std::move(res); return true; } - else if (auto col_right_const_array = checkAndGetColumnConst(col_right_untyped)) + else if (const auto * col_right_const_array = checkAndGetColumnConst(col_right_untyped)) { const ColumnArray * col_right_const_array_data = checkAndGetColumn(&col_right_const_array->getDataColumn()); if (!checkColumn(&col_right_const_array_data->getData())) @@ -376,17 +375,17 @@ private: bool left_ok = false; bool right_ok = false; - if (auto col_left = checkAndGetColumn(col_left_untyped)) + if (const auto * col_left = checkAndGetColumn(col_left_untyped)) { left_ok = true; right_ok = executeRightType(cond_col, block, arguments, result, col_left); } - else if (auto col_const_left = checkAndGetColumnConst(col_left_untyped)) + else if (const auto * col_const_left = checkAndGetColumnConst(col_left_untyped)) { left_ok = true; right_ok = executeConstRightType(cond_col, block, arguments, result, col_const_left); } - else if (auto col_arr_left = checkAndGetColumn(col_left_untyped)) + else if (const auto * col_arr_left = checkAndGetColumn(col_left_untyped)) { if (auto col_arr_left_elems = checkAndGetColumn(&col_arr_left->getData())) { @@ -395,7 +394,7 @@ private: cond_col, block, arguments, result, col_arr_left, input_rows_count); } } - else if (auto col_const_arr_left = checkAndGetColumnConst(col_left_untyped)) + else if (const auto * col_const_arr_left = checkAndGetColumnConst(col_left_untyped)) { if (checkColumn(&assert_cast(col_const_arr_left->getDataColumn()).getData())) { @@ -515,7 +514,7 @@ private: && (col_arr_else || col_arr_else_const)) { auto res = block.getByPosition(result).type->createColumn(); - auto col_res = assert_cast(res.get()); + auto * col_res = assert_cast(res.get()); if (col_arr_then && col_arr_else) conditional(GenericArraySource(*col_arr_then), GenericArraySource(*col_arr_else), GenericArraySink(*col_res, rows), cond_data); @@ -591,7 +590,7 @@ private: return true; } - void executeGeneric(const ColumnUInt8 * cond_col, Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) + static void executeGeneric(const ColumnUInt8 * cond_col, Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) { /// Convert both columns to the common type (if needed). @@ -600,8 +599,8 @@ private: DataTypePtr common_type = getLeastSupertype({arg1.type, arg2.type}); - ColumnPtr col_then = castColumn(arg1, common_type, context); - ColumnPtr col_else = castColumn(arg2, common_type, context); + ColumnPtr col_then = castColumn(arg1, common_type); + ColumnPtr col_else = castColumn(arg2, common_type); MutableColumnPtr result_column = common_type->createColumn(); result_column->reserve(input_rows_count); @@ -668,7 +667,7 @@ private: return true; } - if (auto * nullable = checkAndGetColumn(*arg_cond.column)) + if (const auto * nullable = checkAndGetColumn(*arg_cond.column)) { Block temporary_block { @@ -703,7 +702,7 @@ private: static ColumnPtr getNestedColumn(const ColumnPtr & column) { - if (auto * nullable = checkAndGetColumn(*column)) + if (const auto * nullable = checkAndGetColumn(*column)) return nullable->getNestedColumnPtr(); return column; @@ -715,8 +714,8 @@ private: const ColumnWithTypeAndName & arg_then = block.getByPosition(arguments[1]); const ColumnWithTypeAndName & arg_else = block.getByPosition(arguments[2]); - auto * then_is_nullable = checkAndGetColumn(*arg_then.column); - auto * else_is_nullable = checkAndGetColumn(*arg_else.column); + const auto * then_is_nullable = checkAndGetColumn(*arg_then.column); + const auto * else_is_nullable = checkAndGetColumn(*arg_else.column); if (!then_is_nullable && !else_is_nullable) return false; @@ -788,7 +787,7 @@ private: return true; } - bool executeForNullThenElse(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) + bool executeForNullThenElse(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const { const ColumnWithTypeAndName & arg_cond = block.getByPosition(arguments[0]); const ColumnWithTypeAndName & arg_then = block.getByPosition(arguments[1]); @@ -847,7 +846,7 @@ private: if (cond_col) { size_t size = input_rows_count; - auto & null_map_data = cond_col->getData(); + const auto & null_map_data = cond_col->getData(); auto negated_null_map = ColumnUInt8::create(); auto & negated_null_map_data = negated_null_map->getData(); @@ -973,11 +972,11 @@ public: TypeIndex left_id = arg_then.type->getTypeId(); TypeIndex right_id = arg_else.type->getTypeId(); - if (auto left_array = checkAndGetDataType(arg_then.type.get())) + if (const auto * left_array = checkAndGetDataType(arg_then.type.get())) left_id = left_array->getNestedType()->getTypeId(); - if (auto rigth_array = checkAndGetDataType(arg_else.type.get())) - right_id = rigth_array->getNestedType()->getTypeId(); + if (const auto * right_array = checkAndGetDataType(arg_else.type.get())) + right_id = right_array->getNestedType()->getTypeId(); if (!(callOnBasicTypes(left_id, right_id, call) || executeTyped(cond_col, block, arguments, result, input_rows_count) @@ -988,8 +987,6 @@ public: executeGeneric(cond_col, block, arguments, result, input_rows_count); } } - - const Context & context; }; void registerFunctionIf(FunctionFactory & factory) diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 0b25ca201bb..a89535c675a 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -21,38 +21,62 @@ namespace ErrorCodes * notIn(x, set) - and NOT IN. */ -template +template struct FunctionInName; template <> -struct FunctionInName +struct FunctionInName { static constexpr auto name = "in"; }; template <> -struct FunctionInName +struct FunctionInName { static constexpr auto name = "globalIn"; }; template <> -struct FunctionInName +struct FunctionInName { static constexpr auto name = "notIn"; }; template <> -struct FunctionInName +struct FunctionInName { static constexpr auto name = "globalNotIn"; }; -template +template <> +struct FunctionInName +{ + static constexpr auto name = "nullIn"; +}; + +template <> +struct FunctionInName +{ + static constexpr auto name = "globalNullIn"; +}; + +template <> +struct FunctionInName +{ + static constexpr auto name = "notNullIn"; +}; + +template <> +struct FunctionInName +{ + static constexpr auto name = "globalNotNullIn"; +}; + +template class FunctionIn : public IFunction { public: - static constexpr auto name = FunctionInName::name; + static constexpr auto name = FunctionInName::name; static FunctionPtr create(const Context &) { return std::make_shared(); @@ -75,6 +99,8 @@ public: bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForNulls() const override { return null_is_skipped; } + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override { /// NOTE: after updating this code, check that FunctionIgnoreExceptNull returns the same type of column. @@ -123,10 +149,14 @@ public: void registerFunctionsIn(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/intDiv.cpp b/src/Functions/intDiv.cpp index 0b6734c0136..d21e779045f 100644 --- a/src/Functions/intDiv.cpp +++ b/src/Functions/intDiv.cpp @@ -1,8 +1,8 @@ #include #include -#ifdef __SSE2__ - #define LIBDIVIDE_USE_SSE2 1 +#if defined(__SSE2__) +# define LIBDIVIDE_SSE2 1 #endif #include @@ -45,7 +45,7 @@ struct DivideIntegralByConstantImpl const A * a_end = a_pos + size; -#ifdef __SSE2__ +#if defined(__SSE2__) static constexpr size_t values_per_sse_register = 16 / sizeof(A); const A * a_end_sse = a_pos + size / values_per_sse_register * values_per_sse_register; diff --git a/src/Functions/isConstant.cpp b/src/Functions/isConstant.cpp new file mode 100644 index 00000000000..5416fbd2d3e --- /dev/null +++ b/src/Functions/isConstant.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include + + +namespace DB +{ + +/// Returns 1 if and only if the argument is constant expression. +/// This function exists for development, debugging and demonstration purposes. +class FunctionIsConstant : public IFunction +{ +public: + static constexpr auto name = "isConstant"; + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool useDefaultImplementationForNulls() const override { return false; } + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + const auto & elem = block.getByPosition(arguments[0]); + block.getByPosition(result).column = ColumnUInt8::create(input_rows_count, isColumnConst(*elem.column)); + } +}; + + +void registerFunctionIsConstant(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/src/Functions/isNotNull.cpp b/src/Functions/isNotNull.cpp index e8d242ab496..18e0caac16e 100644 --- a/src/Functions/isNotNull.cpp +++ b/src/Functions/isNotNull.cpp @@ -40,7 +40,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { const ColumnWithTypeAndName & elem = block.getByPosition(arguments[0]); - if (auto * nullable = checkAndGetColumn(*elem.column)) + if (const auto * nullable = checkAndGetColumn(*elem.column)) { /// Return the negated null map. auto res_column = ColumnUInt8::create(input_rows_count); diff --git a/src/Functions/isNull.cpp b/src/Functions/isNull.cpp index 254d5378dab..014fd01da97 100644 --- a/src/Functions/isNull.cpp +++ b/src/Functions/isNull.cpp @@ -39,7 +39,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t) override { const ColumnWithTypeAndName & elem = block.getByPosition(arguments[0]); - if (auto * nullable = checkAndGetColumn(*elem.column)) + if (const auto * nullable = checkAndGetColumn(*elem.column)) { /// Merely return the embedded null map. block.getByPosition(result).column = nullable->getNullMapColumnPtr(); diff --git a/src/Functions/isValidUTF8.cpp b/src/Functions/isValidUTF8.cpp index 354dc24e3d8..52420d5b7bc 100644 --- a/src/Functions/isValidUTF8.cpp +++ b/src/Functions/isValidUTF8.cpp @@ -277,7 +277,7 @@ SOFTWARE. len -= 16; }; - while (len >= 16) + while (len >= 16) // NOLINT check_packed(_mm_loadu_si128(reinterpret_cast(data))); /// 0 <= len <= 15 for now. Reading data from data - 1 because of right padding of 15 and left padding diff --git a/src/Functions/least.cpp b/src/Functions/least.cpp index 47af759c956..ba87e4bd7e4 100644 --- a/src/Functions/least.cpp +++ b/src/Functions/least.cpp @@ -1,6 +1,8 @@ #include #include #include +#include + namespace DB { @@ -57,7 +59,7 @@ using FunctionLeast = FunctionBinaryArithmetic; void registerFunctionLeast(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(FunctionFactory::CaseInsensitive); } } diff --git a/src/Functions/lowCardinalityIndices.cpp b/src/Functions/lowCardinalityIndices.cpp index c54ee9de34e..d69a84c7d49 100644 --- a/src/Functions/lowCardinalityIndices.cpp +++ b/src/Functions/lowCardinalityIndices.cpp @@ -31,7 +31,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto * type = typeid_cast(arguments[0].get()); + const auto * type = typeid_cast(arguments[0].get()); if (!type) throw Exception("First first argument of function lowCardinalityIndexes must be ColumnLowCardinality, but got" + arguments[0]->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Functions/lowCardinalityKeys.cpp b/src/Functions/lowCardinalityKeys.cpp index f17e9bcad01..c96bba4f37f 100644 --- a/src/Functions/lowCardinalityKeys.cpp +++ b/src/Functions/lowCardinalityKeys.cpp @@ -30,7 +30,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto * type = typeid_cast(arguments[0].get()); + const auto * type = typeid_cast(arguments[0].get()); if (!type) throw Exception("First first argument of function lowCardinalityKeys must be ColumnLowCardinality, but got" + arguments[0]->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Functions/modulo.cpp b/src/Functions/modulo.cpp index 9e4409ca91b..631b7d12263 100644 --- a/src/Functions/modulo.cpp +++ b/src/Functions/modulo.cpp @@ -1,8 +1,8 @@ #include #include -#ifdef __SSE2__ - #define LIBDIVIDE_USE_SSE2 1 +#if defined(__SSE2__) +# define LIBDIVIDE_SSE2 1 #endif #include diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 2340f7826c7..b57c9f6316a 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -33,10 +33,8 @@ class FunctionMultiIf final : public FunctionIfBase { public: static constexpr auto name = "multiIf"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - explicit FunctionMultiIf(const Context & context_) : context(context_) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } -public: String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -176,7 +174,7 @@ public: else { /// Cast all columns to result type. - converted_columns_holder.emplace_back(castColumn(source_col, return_type, context)); + converted_columns_holder.emplace_back(castColumn(source_col, return_type)); instruction.source = converted_columns_holder.back().get(); } @@ -225,9 +223,6 @@ public: block.getByPosition(result).column = std::move(res); } - -private: - const Context & context; }; void registerFunctionMultiIf(FunctionFactory & factory) diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index dc83aeb0d31..7c26693f7e8 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -27,9 +27,7 @@ class FunctionNeighbor : public IFunction { public: static constexpr auto name = "neighbor"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } - - explicit FunctionNeighbor(const Context & context_) : context(context_) {} + static FunctionPtr create(const Context &) { return std::make_shared(); } /// Get the name of the function. String getName() const override { return name; } @@ -83,14 +81,14 @@ public: const ColumnWithTypeAndName & offset_elem = block.getByPosition(arguments[1]); bool has_defaults = arguments.size() == 3; - ColumnPtr source_column_casted = castColumn(source_elem, result_type, context); + ColumnPtr source_column_casted = castColumn(source_elem, result_type); ColumnPtr offset_column = offset_elem.column; ColumnPtr default_column_casted; if (has_defaults) { const ColumnWithTypeAndName & default_elem = block.getByPosition(arguments[2]); - default_column_casted = castColumn(default_elem, result_type, context); + default_column_casted = castColumn(default_elem, result_type); } bool source_is_constant = isColumnConst(*source_column_casted); @@ -181,9 +179,6 @@ public: block.getByPosition(result).column = std::move(result_column); } } - -private: - const Context & context; }; void registerFunctionNeighbor(FunctionFactory & factory) diff --git a/src/Functions/pointInEllipses.cpp b/src/Functions/pointInEllipses.cpp index c3091185429..2445ce87296 100644 --- a/src/Functions/pointInEllipses.cpp +++ b/src/Functions/pointInEllipses.cpp @@ -73,7 +73,7 @@ private: for (const auto arg_idx : ext::range(0, arguments.size())) { - const auto arg = arguments[arg_idx].get(); + const auto * arg = arguments[arg_idx].get(); if (!WhichDataType(arg).isFloat64()) { throw Exception( @@ -99,8 +99,8 @@ private: for (const auto idx : ext::range(0, 4)) { int arg_idx = 2 + 4 * ellipse_idx + idx; - const auto column = block.getByPosition(arguments[arg_idx]).column.get(); - if (const auto col = checkAndGetColumnConst>(column)) + const auto * column = block.getByPosition(arguments[arg_idx]).column.get(); + if (const auto * col = checkAndGetColumnConst>(column)) { ellipse_data[idx] = col->getValue(); } @@ -117,7 +117,7 @@ private: int const_cnt = 0; for (const auto idx : ext::range(0, 2)) { - const auto column = block.getByPosition(arguments[idx]).column.get(); + const auto * column = block.getByPosition(arguments[idx]).column.get(); if (typeid_cast (column)) { ++const_cnt; @@ -129,12 +129,12 @@ private: } } - const auto col_x = block.getByPosition(arguments[0]).column.get(); - const auto col_y = block.getByPosition(arguments[1]).column.get(); + const auto * col_x = block.getByPosition(arguments[0]).column.get(); + const auto * col_y = block.getByPosition(arguments[1]).column.get(); if (const_cnt == 0) { - const auto col_vec_x = assert_cast *> (col_x); - const auto col_vec_y = assert_cast *> (col_y); + const auto * col_vec_x = assert_cast *> (col_x); + const auto * col_vec_y = assert_cast *> (col_y); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); @@ -150,8 +150,8 @@ private: } else if (const_cnt == 2) { - const auto col_const_x = assert_cast (col_x); - const auto col_const_y = assert_cast (col_y); + const auto * col_const_x = assert_cast (col_x); + const auto * col_const_y = assert_cast (col_y); size_t start_index = 0; UInt8 res = isPointInEllipses(col_const_x->getValue(), col_const_y->getValue(), ellipses.data(), ellipses_count, start_index); block.getByPosition(result).column = DataTypeUInt8().createColumnConst(size, res); diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 6a0f2c8ac2f..460c60d6e4c 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -120,11 +120,11 @@ public: for (size_t i = 1; i < arguments.size(); ++i) { - auto * array = checkAndGetDataType(arguments[i].get()); + const auto * array = checkAndGetDataType(arguments[i].get()); if (array == nullptr && i != 1) throw Exception(get_message_prefix(i) + " must be array of tuples.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - auto * tuple = checkAndGetDataType(array ? array->getNestedType().get() : arguments[i].get()); + const auto * tuple = checkAndGetDataType(array ? array->getNestedType().get() : arguments[i].get()); if (tuple == nullptr) throw Exception(get_message_prefix(i) + " must contains tuple.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -149,10 +149,10 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override { const IColumn * point_col = block.getByPosition(arguments[0]).column.get(); - auto const_tuple_col = checkAndGetColumn(point_col); + const auto * const_tuple_col = checkAndGetColumn(point_col); if (const_tuple_col) point_col = &const_tuple_col->getDataColumn(); - auto tuple_col = checkAndGetColumn(point_col); + const auto * tuple_col = checkAndGetColumn(point_col); if (!tuple_col) throw Exception("First argument for function " + getName() + " must be constant array of tuples.", @@ -178,9 +178,9 @@ private: for (size_t i = 1; i < arguments.size(); ++i) { - auto const_col = checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); - auto array_col = const_col ? checkAndGetColumn(&const_col->getDataColumn()) : nullptr; - auto tuple_col = array_col ? checkAndGetColumn(&array_col->getData()) : nullptr; + const auto * const_col = checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + const auto * array_col = const_col ? checkAndGetColumn(&const_col->getDataColumn()) : nullptr; + const auto * tuple_col = array_col ? checkAndGetColumn(&array_col->getData()) : nullptr; if (!tuple_col) throw Exception(get_message_prefix(i) + " must be constant array of tuples.", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/regexpQuoteMeta.cpp b/src/Functions/regexpQuoteMeta.cpp index 0ae0a0da97d..57e86beabcc 100644 --- a/src/Functions/regexpQuoteMeta.cpp +++ b/src/Functions/regexpQuoteMeta.cpp @@ -67,8 +67,8 @@ public: const ColumnString::Offsets & src_offsets = input->getOffsets(); - auto src_begin = reinterpret_cast(input->getChars().data()); - auto src_pos = src_begin; + const auto * src_begin = reinterpret_cast(input->getChars().data()); + const auto * src_pos = src_begin; for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx) { diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 233018c7f16..02013e33d16 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -46,7 +46,9 @@ void registerFunctions() registerFunctionsArithmetic(factory); registerFunctionsArray(factory); registerFunctionsTuple(factory); +#if !defined(ARCADIA_BUILD) registerFunctionsBitmap(factory); +#endif registerFunctionsCoding(factory); registerFunctionsComparison(factory); registerFunctionsConditional(factory); diff --git a/src/Functions/registerFunctionsConsistentHashing.cpp b/src/Functions/registerFunctionsConsistentHashing.cpp index 95a856b6d3c..ceec6dca5e6 100644 --- a/src/Functions/registerFunctionsConsistentHashing.cpp +++ b/src/Functions/registerFunctionsConsistentHashing.cpp @@ -5,14 +5,18 @@ class FunctionFactory; void registerFunctionYandexConsistentHash(FunctionFactory & factory); void registerFunctionJumpConsistentHash(FunctionFactory & factory); +#if !defined(ARCADIA_BUILD) void registerFunctionSumburConsistentHash(FunctionFactory & factory); +#endif void registerFunctionsConsistentHashing(FunctionFactory & factory) { registerFunctionYandexConsistentHash(factory); registerFunctionJumpConsistentHash(factory); +#if !defined(ARCADIA_BUILD) registerFunctionSumburConsistentHash(factory); +#endif } } diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index cb3c268a19a..2ed02bbd73b 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -1,4 +1,6 @@ -#include "config_functions.h" +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif namespace DB { diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 44e26542c7d..221e14fcce1 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB { @@ -56,6 +58,7 @@ void registerFunctionBasename(FunctionFactory &); void registerFunctionTransform(FunctionFactory &); void registerFunctionGetMacro(FunctionFactory &); void registerFunctionGetScalar(FunctionFactory &); +void registerFunctionIsConstant(FunctionFactory &); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -114,6 +117,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionTransform(factory); registerFunctionGetMacro(factory); registerFunctionGetScalar(factory); + registerFunctionIsConstant(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index cc94e877bbf..5493b98d3fc 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -1,4 +1,6 @@ -#include "config_functions.h" +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif namespace DB { diff --git a/src/Functions/replicate.cpp b/src/Functions/replicate.cpp index bea1f883496..57a4c60611c 100644 --- a/src/Functions/replicate.cpp +++ b/src/Functions/replicate.cpp @@ -53,7 +53,7 @@ public: ColumnPtr temp_column; if (!array_column) { - auto const_array_column = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); + const auto * const_array_column = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); if (!const_array_column) throw Exception("Unexpected column for replicate", ErrorCodes::ILLEGAL_COLUMN); temp_column = const_array_column->convertToFullColumn(); diff --git a/src/Functions/reverse.cpp b/src/Functions/reverse.cpp index 2c135cf3d7d..bc1237fc457 100644 --- a/src/Functions/reverse.cpp +++ b/src/Functions/reverse.cpp @@ -71,7 +71,7 @@ public: return 1; } - bool isInjective(const Block &) override + bool isInjective(const Block &) const override { return true; } diff --git a/src/Functions/stringToH3.cpp b/src/Functions/stringToH3.cpp index 1f9f4004b1c..2e09a371520 100644 --- a/src/Functions/stringToH3.cpp +++ b/src/Functions/stringToH3.cpp @@ -36,7 +36,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - auto arg = arguments[0].get(); + const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isStringOrFixedString()) throw Exception( "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be String or FixedString", @@ -47,15 +47,15 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto col_hindex = block.getByPosition(arguments[0]).column.get(); + const auto * col_hindex = block.getByPosition(arguments[0]).column.get(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); dst_data.resize(input_rows_count); - if (auto * h3index = checkAndGetColumn(col_hindex)) + if (const auto * h3index = checkAndGetColumn(col_hindex)) execute(StringSource(*h3index), dst_data); - else if (auto * h3index_fixed = checkAndGetColumn(col_hindex)) + else if (const auto * h3index_fixed = checkAndGetColumn(col_hindex)) execute(FixedStringSource(*h3index_fixed), dst_data); else if (const ColumnConst * h3index_const = checkAndGetColumnConst(col_hindex)) execute>(ConstSource(*h3index_const), dst_data); diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index 1bdf7358821..2d1bcad14c0 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -69,13 +69,13 @@ public: std::optional custom_message; if (arguments.size() == 2) { - auto * msg_column = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); + const auto * msg_column = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); if (!msg_column) throw Exception{"Second argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_COLUMN}; custom_message = msg_column->getValue(); } - const auto in = block.getByPosition(arguments.front()).column.get(); + const auto * in = block.getByPosition(arguments.front()).column.get(); if ( !execute(block, in, result, custom_message) && !execute(block, in, result, custom_message) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index b908a9c00ef..a87a160aa0e 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -145,11 +145,11 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t) override { - auto starts = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); - auto const_starts = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); + const auto * starts = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); + const auto * const_starts = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - auto durations = checkAndGetColumn(block.getByPosition(arguments[1]).column.get()); - auto const_durations = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); + const auto * durations = checkAndGetColumn(block.getByPosition(arguments[1]).column.get()); + const auto * const_durations = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); auto res = ColumnArray::create(ColumnUInt32::create()); ColumnUInt32::Container & res_values = typeid_cast(res->getData()).getData(); @@ -158,7 +158,7 @@ public: if (arguments.size() == 3) { - auto time_slot_column = checkAndGetColumn(block.getByPosition(arguments[2]).column.get()); + const auto * time_slot_column = checkAndGetColumn(block.getByPosition(arguments[2]).column.get()); if (!time_slot_column) throw Exception("Third argument for function " + getName() + " must be constant UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index 0d1131c5106..a47307acbe9 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -74,7 +74,7 @@ public: { DataTypes types; types.reserve(arguments.size()); - for (auto & elem : arguments) + for (const auto & elem : arguments) types.emplace_back(elem.type); return std::make_unique(types, return_type); diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index 0ad3c6874bd..08d5cd5237e 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -154,7 +154,7 @@ public: initialize(array_from->getValue(), array_to->getValue(), block, arguments); - const auto in = block.getByPosition(arguments.front()).column.get(); + const auto * in = block.getByPosition(arguments.front()).column.get(); if (isColumnConst(*in)) { @@ -167,7 +167,7 @@ public: default_column = block.getByPosition(arguments[3]).column.get(); auto column_result = block.getByPosition(result).type->createColumn(); - auto out = column_result.get(); + auto * out = column_result.get(); if (!executeNum(in, out, default_column) && !executeNum(in, out, default_column) @@ -277,7 +277,7 @@ private: bool executeString(const IColumn * in_untyped, IColumn * out_untyped, const IColumn * default_untyped) { - if (const auto in = checkAndGetColumn(in_untyped)) + if (const auto * in = checkAndGetColumn(in_untyped)) { if (!default_untyped) { @@ -379,7 +379,7 @@ private: template bool executeNumToStringWithConstDefault(const ColumnVector * in, IColumn * out_untyped) { - auto out = typeid_cast(out_untyped); + auto * out = typeid_cast(out_untyped); if (!out) return false; @@ -392,11 +392,11 @@ private: template bool executeNumToStringWithNonConstDefault(const ColumnVector * in, IColumn * out_untyped, const IColumn * default_untyped) { - auto out = typeid_cast(out_untyped); + auto * out = typeid_cast(out_untyped); if (!out) return false; - auto default_col = checkAndGetColumn(default_untyped); + const auto * default_col = checkAndGetColumn(default_untyped); if (!default_col) { throw Exception{"Illegal column " + default_untyped->getName() + " of fourth argument of function " + getName(), @@ -460,7 +460,7 @@ private: bool executeStringToString(const ColumnString * in, IColumn * out_untyped) { - auto out = typeid_cast(out_untyped); + auto * out = typeid_cast(out_untyped); if (!out) return false; @@ -470,7 +470,7 @@ private: bool executeStringToStringWithConstDefault(const ColumnString * in, IColumn * out_untyped) { - auto out = typeid_cast(out_untyped); + auto * out = typeid_cast(out_untyped); if (!out) return false; @@ -482,11 +482,11 @@ private: bool executeStringToStringWithNonConstDefault(const ColumnString * in, IColumn * out_untyped, const IColumn * default_untyped) { - auto out = typeid_cast(out_untyped); + auto * out = typeid_cast(out_untyped); if (!out) return false; - auto default_col = checkAndGetColumn(default_untyped); + const auto * default_col = checkAndGetColumn(default_untyped); if (!default_col) { throw Exception{"Illegal column " + default_untyped->getName() + " of fourth argument of function " + getName(), @@ -530,7 +530,7 @@ private: if (it) memcpy(&dst[i], &it->getMapped(), sizeof(dst[i])); /// little endian. else - dst[i] = dst_default[i]; + dst[i] = dst_default[i]; // NOLINT } } @@ -613,7 +613,7 @@ private: { StringRef ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; current_src_offset = src_offsets[i]; - auto it = table.find(ref); + const auto * it = table.find(ref); if (it) memcpy(&dst[i], &it->getMapped(), sizeof(dst[i])); else @@ -634,11 +634,11 @@ private: { StringRef ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; current_src_offset = src_offsets[i]; - auto it = table.find(ref); + const auto * it = table.find(ref); if (it) memcpy(&dst[i], &it->getMapped(), sizeof(dst[i])); else - dst[i] = dst_default[i]; + dst[i] = dst_default[i]; // NOLINT } } @@ -657,7 +657,7 @@ private: StringRef src_ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; current_src_offset = src_offsets[i]; - auto it = table.find(src_ref); + const auto * it = table.find(src_ref); StringRef dst_ref = it ? it->getMapped() : (with_default ? dst_default : src_ref); dst_data.resize(current_dst_offset + dst_ref.size); @@ -697,7 +697,7 @@ private: StringRef src_ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; current_src_offset = src_offsets[i]; - auto it = table.find(src_ref); + const auto * it = table.find(src_ref); StringRef dst_ref; if (it) diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 451f732c869..772cb4e3c07 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -43,7 +43,7 @@ public: return 0; } - bool isInjective(const Block &) override + bool isInjective(const Block &) const override { return true; } diff --git a/src/Functions/tupleElement.cpp b/src/Functions/tupleElement.cpp index 8922ac98f8d..f7ff81e59d4 100644 --- a/src/Functions/tupleElement.cpp +++ b/src/Functions/tupleElement.cpp @@ -113,7 +113,7 @@ public: private: size_t getElementNum(const ColumnPtr & index_column, const DataTypeTuple & tuple) const { - if (auto index_col = checkAndGetColumnConst(index_column.get())) + if (const auto * index_col = checkAndGetColumnConst(index_column.get())) { size_t index = index_col->getValue(); @@ -125,7 +125,7 @@ private: return index - 1; } - else if (auto name_col = checkAndGetColumnConst(index_column.get())) + else if (const auto * name_col = checkAndGetColumnConst(index_column.get())) { return tuple.getPositionByName(name_col->getValue()); } diff --git a/src/Functions/version.cpp b/src/Functions/version.cpp index 34eabd30336..e11f45f160a 100644 --- a/src/Functions/version.cpp +++ b/src/Functions/version.cpp @@ -1,9 +1,11 @@ #include #include #include -#include #include +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB { diff --git a/src/Functions/visitParamExtractRaw.cpp b/src/Functions/visitParamExtractRaw.cpp index e6e89f1c7ba..7a02d29e446 100644 --- a/src/Functions/visitParamExtractRaw.cpp +++ b/src/Functions/visitParamExtractRaw.cpp @@ -15,7 +15,7 @@ struct ExtractRaw ExpectChars expects_end; UInt8 current_expect_end = 0; - for (auto extract_begin = pos; pos != end; ++pos) + for (const auto * extract_begin = pos; pos != end; ++pos) { if (current_expect_end && *pos == current_expect_end) { diff --git a/src/Functions/ya.make b/src/Functions/ya.make new file mode 100644 index 00000000000..efcae6bf73b --- /dev/null +++ b/src/Functions/ya.make @@ -0,0 +1,418 @@ +LIBRARY() + +ADDINCL( + library/consistent_hashing + contrib/libs/farmhash + contrib/libs/hyperscan/src + contrib/libs/icu/common + contrib/libs/libdivide + contrib/libs/rapidjson/include + contrib/libs/xxhash +) + +PEERDIR( + clickhouse/src/Common + clickhouse/src/Dictionaries + contrib/libs/farmhash + contrib/libs/fastops/fastops + contrib/libs/hyperscan + contrib/libs/icu + contrib/libs/libdivide + contrib/libs/metrohash + contrib/libs/rapidjson + contrib/libs/xxhash + library/consistent_hashing +) + +SRCS( + array/array.cpp + array/arrayAll.cpp + array/arrayAUC.cpp + array/arrayCompact.cpp + array/arrayConcat.cpp + array/arrayCount.cpp + array/arrayCumSum.cpp + array/arrayCumSumNonNegative.cpp + array/arrayDifference.cpp + array/arrayDistinct.cpp + array/arrayElement.cpp + array/arrayEnumerate.cpp + array/arrayEnumerateDense.cpp + array/arrayEnumerateDenseRanked.cpp + array/arrayEnumerateRanked.cpp + array/arrayEnumerateUniq.cpp + array/arrayEnumerateUniqRanked.cpp + array/arrayExists.cpp + array/arrayFill.cpp + array/arrayFilter.cpp + array/arrayFirst.cpp + array/arrayFirstIndex.cpp + array/arrayFlatten.cpp + array/arrayIntersect.cpp + array/arrayJoin.cpp + array/arrayMap.cpp + array/arrayPopBack.cpp + array/arrayPopFront.cpp + array/arrayPushBack.cpp + array/arrayPushFront.cpp + array/arrayReduce.cpp + array/arrayReduceInRanges.cpp + array/arrayResize.cpp + array/arrayReverse.cpp + array/arraySlice.cpp + array/arraySort.cpp + array/arraySplit.cpp + array/arraySum.cpp + array/arrayUniq.cpp + array/arrayWithConstant.cpp + array/arrayZip.cpp + array/countEqual.cpp + array/emptyArray.cpp + array/emptyArrayToSingle.cpp + array/has.cpp + array/hasAll.cpp + array/hasAny.cpp + array/indexOf.cpp + array/length.cpp + array/range.cpp + array/registerFunctionsArray.cpp + GatherUtils/concat.cpp + GatherUtils/createArraySink.cpp + GatherUtils/createArraySource.cpp + GatherUtils/createValueSource.cpp + GatherUtils/has.cpp + GatherUtils/push.cpp + GatherUtils/resizeConstantSize.cpp + GatherUtils/resizeDynamicSize.cpp + GatherUtils/sliceDynamicOffsetBounded.cpp + GatherUtils/sliceDynamicOffsetUnbounded.cpp + GatherUtils/sliceFromLeftConstantOffsetBounded.cpp + GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp + GatherUtils/sliceFromRightConstantOffsetBounded.cpp + GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp + URL/basename.cpp + URL/cutFragment.cpp + URL/cutQueryString.cpp + URL/cutQueryStringAndFragment.cpp + URL/cutToFirstSignificantSubdomain.cpp + URL/cutURLParameter.cpp + URL/cutWWW.cpp + URL/decodeURLComponent.cpp + URL/domain.cpp + URL/domainWithoutWWW.cpp + URL/extractURLParameter.cpp + URL/extractURLParameterNames.cpp + URL/extractURLParameters.cpp + URL/firstSignificantSubdomain.cpp + URL/fragment.cpp + URL/path.cpp + URL/pathFull.cpp + URL/protocol.cpp + URL/queryString.cpp + URL/queryStringAndFragment.cpp + URL/registerFunctionsURL.cpp + URL/tldLookup.generated.cpp + URL/topLevelDomain.cpp + URL/URLHierarchy.cpp + URL/URLPathHierarchy.cpp + abs.cpp + acos.cpp + addDays.cpp + addHours.cpp + addMinutes.cpp + addMonths.cpp + addQuarters.cpp + addressToLine.cpp + addressToSymbol.cpp + addSeconds.cpp + addWeeks.cpp + addYears.cpp + appendTrailingCharIfAbsent.cpp + asin.cpp + assumeNotNull.cpp + atan.cpp + bar.cpp + base64Decode.cpp + base64Encode.cpp + bitAnd.cpp + bitBoolMaskAnd.cpp + bitBoolMaskOr.cpp + bitCount.cpp + bitNot.cpp + bitOr.cpp + bitRotateLeft.cpp + bitRotateRight.cpp + bitShiftLeft.cpp + bitShiftRight.cpp + bitSwapLastTwo.cpp + bitTest.cpp + bitTestAll.cpp + bitTestAny.cpp + bitWrapperFunc.cpp + bitXor.cpp + blockNumber.cpp + blockSerializedSize.cpp + blockSize.cpp + caseWithExpression.cpp + cbrt.cpp + coalesce.cpp + concat.cpp + convertCharset.cpp + cos.cpp + CRC.cpp + currentDatabase.cpp + currentQuota.cpp + currentRowPolicies.cpp + currentUser.cpp + dateDiff.cpp + defaultValueOfArgumentType.cpp + demange.cpp + divide.cpp + dumpColumnStructure.cpp + e.cpp + empty.cpp + endsWith.cpp + equals.cpp + erf.cpp + erfc.cpp + evalMLMethod.cpp + exp.cpp + exp2.cpp + exp10.cpp + extractTimeZoneFromFunctionArguments.cpp + filesystem.cpp + finalizeAggregation.cpp + formatDateTime.cpp + formatString.cpp + FunctionFactory.cpp + FunctionFQDN.cpp + FunctionHelpers.cpp + FunctionJoinGet.cpp + FunctionsCoding.cpp + FunctionsConversion.cpp + FunctionsEmbeddedDictionaries.cpp + FunctionsExternalDictionaries.cpp + FunctionsExternalModels.cpp + FunctionsFormatting.cpp + FunctionsHashing.cpp + FunctionsJSON.cpp + FunctionsLogical.cpp + FunctionsRandom.cpp + FunctionsRound.cpp + FunctionsStringArray.cpp + FunctionsStringRegex.cpp + FunctionsStringSimilarity.cpp + gcd.cpp + generateUUIDv4.cpp + GeoHash.cpp + geohashDecode.cpp + geohashEncode.cpp + geohashesInBox.cpp + getMacro.cpp + getScalar.cpp + getSizeOfEnumType.cpp + greatCircleDistance.cpp + greater.cpp + greaterOrEquals.cpp + greatest.cpp + hasColumnInTable.cpp + hasToken.cpp + hasTokenCaseInsensitive.cpp + hostName.cpp + identity.cpp + if.cpp + ifNotFinite.cpp + ifNull.cpp + IFunction.cpp + ignore.cpp + ignoreExceptNull.cpp + in.cpp + intDiv.cpp + intDivOrZero.cpp + intExp2.cpp + intExp10.cpp + isConstant.cpp + isFinite.cpp + isInfinite.cpp + isNaN.cpp + isNotNull.cpp + isNull.cpp + isValidUTF8.cpp + jumpConsistentHash.cpp + lcm.cpp + least.cpp + lengthUTF8.cpp + less.cpp + lessOrEquals.cpp + lgamma.cpp + log.cpp + log2.cpp + log10.cpp + lowCardinalityIndices.cpp + lowCardinalityKeys.cpp + lower.cpp + lowerUTF8.cpp + materialize.cpp + minus.cpp + modulo.cpp + moduloOrZero.cpp + multiIf.cpp + multiply.cpp + multiSearchAllPositions.cpp + multiSearchAllPositionsCaseInsensitive.cpp + multiSearchAllPositionsCaseInsensitiveUTF8.cpp + multiSearchAllPositionsUTF8.cpp + multiSearchAny.cpp + multiSearchAnyCaseInsensitive.cpp + multiSearchAnyCaseInsensitiveUTF8.cpp + multiSearchAnyUTF8.cpp + multiSearchFirstIndex.cpp + multiSearchFirstIndexCaseInsensitive.cpp + multiSearchFirstIndexCaseInsensitiveUTF8.cpp + multiSearchFirstIndexUTF8.cpp + multiSearchFirstPosition.cpp + multiSearchFirstPositionCaseInsensitive.cpp + multiSearchFirstPositionCaseInsensitiveUTF8.cpp + multiSearchFirstPositionUTF8.cpp + negate.cpp + neighbor.cpp + notEmpty.cpp + notEquals.cpp + now.cpp + now64.cpp + nullIf.cpp + pi.cpp + plus.cpp + pointInEllipses.cpp + pointInPolygon.cpp + position.cpp + positionCaseInsensitive.cpp + positionCaseInsensitiveUTF8.cpp + positionUTF8.cpp + pow.cpp + rand.cpp + rand64.cpp + randConstant.cpp + randomPrintableASCII.cpp + regexpQuoteMeta.cpp + registerFunctions.cpp + registerFunctionsArithmetic.cpp + registerFunctionsComparison.cpp + registerFunctionsConditional.cpp + registerFunctionsConsistentHashing.cpp + registerFunctionsDateTime.cpp + registerFunctionsGeo.cpp + registerFunctionsHigherOrder.cpp + registerFunctionsIntrospection.cpp + registerFunctionsMath.cpp + registerFunctionsMiscellaneous.cpp + registerFunctionsNull.cpp + registerFunctionsRandom.cpp + registerFunctionsReinterpret.cpp + registerFunctionsString.cpp + registerFunctionsStringSearch.cpp + registerFunctionsTuple.cpp + registerFunctionsVisitParam.cpp + reinterpretAsFixedString.cpp + reinterpretAsString.cpp + reinterpretStringAs.cpp + repeat.cpp + replicate.cpp + reverse.cpp + reverseUTF8.cpp + roundAge.cpp + roundDuration.cpp + roundToExp2.cpp + rowNumberInAllBlocks.cpp + rowNumberInBlock.cpp + runningAccumulate.cpp + runningDifference.cpp + runningDifferenceStartingWithFirstValue.cpp + sigmoid.cpp + sin.cpp + sleep.cpp + sleepEachRow.cpp + sqrt.cpp + startsWith.cpp + substring.cpp + subtractDays.cpp + subtractHours.cpp + subtractMinutes.cpp + subtractMonths.cpp + subtractQuarters.cpp + subtractSeconds.cpp + subtractWeeks.cpp + subtractYears.cpp + tan.cpp + tanh.cpp + tgamma.cpp + throwIf.cpp + timeSlot.cpp + timeSlots.cpp + timezone.cpp + toColumnTypeName.cpp + toCustomWeek.cpp + today.cpp + toDayOfMonth.cpp + toDayOfWeek.cpp + toDayOfYear.cpp + toHour.cpp + toISOWeek.cpp + toISOYear.cpp + toLowCardinality.cpp + toMinute.cpp + toMonday.cpp + toMonth.cpp + toNullable.cpp + toQuarter.cpp + toRelativeDayNum.cpp + toRelativeHourNum.cpp + toRelativeMinuteNum.cpp + toRelativeMonthNum.cpp + toRelativeQuarterNum.cpp + toRelativeSecondNum.cpp + toRelativeWeekNum.cpp + toRelativeYearNum.cpp + toSecond.cpp + toStartOfDay.cpp + toStartOfFifteenMinutes.cpp + toStartOfFiveMinute.cpp + toStartOfHour.cpp + toStartOfInterval.cpp + toStartOfISOYear.cpp + toStartOfMinute.cpp + toStartOfMonth.cpp + toStartOfQuarter.cpp + toStartOfTenMinutes.cpp + toStartOfYear.cpp + toTime.cpp + toTimeZone.cpp + toTypeName.cpp + toValidUTF8.cpp + toYear.cpp + toYYYYMM.cpp + toYYYYMMDD.cpp + toYYYYMMDDhhmmss.cpp + transform.cpp + trap.cpp + trim.cpp + tryBase64Decode.cpp + tuple.cpp + tupleElement.cpp + upper.cpp + upperUTF8.cpp + uptime.cpp + version.cpp + visibleWidth.cpp + visitParamExtractBool.cpp + visitParamExtractFloat.cpp + visitParamExtractInt.cpp + visitParamExtractRaw.cpp + visitParamExtractString.cpp + visitParamExtractUInt.cpp + visitParamHas.cpp + yandexConsistentHash.cpp + yesterday.cpp +) + +END() diff --git a/src/IO/AIOContextPool.cpp b/src/IO/AIOContextPool.cpp index e173de246f7..31d0948fedc 100644 --- a/src/IO/AIOContextPool.cpp +++ b/src/IO/AIOContextPool.cpp @@ -59,7 +59,7 @@ void AIOContextPool::waitForCompletion() } -int AIOContextPool::getCompletionEvents(io_event events[], const int max_events) +int AIOContextPool::getCompletionEvents(io_event events[], const int max_events) const { timespec timeout{timeout_sec, 0}; @@ -146,11 +146,10 @@ std::future AIOContextPool::post(struct iocb & iocb) /// store id in AIO request for further identification iocb.aio_data = request_id; - auto num_requests = 0; struct iocb * requests[] { &iocb }; /// submit a request - while ((num_requests = io_submit(aio_context.ctx, 1, requests)) < 0) + while (io_submit(aio_context.ctx, 1, requests) < 0) { if (errno == EAGAIN) /// wait until at least one event has been completed (or a spurious wakeup) and try again diff --git a/src/IO/AIOContextPool.h b/src/IO/AIOContextPool.h index abbfa52725a..15160bc7fb6 100644 --- a/src/IO/AIOContextPool.h +++ b/src/IO/AIOContextPool.h @@ -36,7 +36,7 @@ class AIOContextPool : private boost::noncopyable void doMonitor(); void waitForCompletion(); - int getCompletionEvents(io_event events[], const int max_events); + int getCompletionEvents(io_event events[], const int max_events) const; void fulfillPromises(const io_event events[], const int num_events); void notifyProducers(const int num_producers) const; void reportExceptionToAnyProducer(); diff --git a/src/IO/BrotliReadBuffer.cpp b/src/IO/BrotliReadBuffer.cpp index f68424cd54e..70d3a76e629 100644 --- a/src/IO/BrotliReadBuffer.cpp +++ b/src/IO/BrotliReadBuffer.cpp @@ -1,8 +1,10 @@ -#include -#if USE_BROTLI +#if !defined(ARCADIA_BUILD) +# include +#endif -#include "BrotliReadBuffer.h" -#include +#if USE_BROTLI +# include +# include "BrotliReadBuffer.h" namespace DB { @@ -27,7 +29,6 @@ public: BrotliDecoderDestroyInstance(state); } -public: BrotliDecoderState * state; BrotliDecoderResult result; }; diff --git a/src/IO/BrotliWriteBuffer.cpp b/src/IO/BrotliWriteBuffer.cpp index ac1e2b3c188..e4e3713d379 100644 --- a/src/IO/BrotliWriteBuffer.cpp +++ b/src/IO/BrotliWriteBuffer.cpp @@ -1,8 +1,10 @@ -#include -#if USE_BROTLI +#if !defined(ARCADIA_BUILD) +# include +#endif -#include -#include +#if USE_BROTLI +# include +# include namespace DB { @@ -26,7 +28,6 @@ public: BrotliEncoderDestroyInstance(state); } -public: BrotliEncoderState * state; }; diff --git a/src/IO/CompressionMethod.cpp b/src/IO/CompressionMethod.cpp index 20f1ea44301..a0a5e19f4fa 100644 --- a/src/IO/CompressionMethod.cpp +++ b/src/IO/CompressionMethod.cpp @@ -7,7 +7,9 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB diff --git a/src/IO/HDFSCommon.cpp b/src/IO/HDFSCommon.cpp index d952d81ac89..4789dd2b0f4 100644 --- a/src/IO/HDFSCommon.cpp +++ b/src/IO/HDFSCommon.cpp @@ -15,7 +15,7 @@ extern const int NETWORK_ERROR; HDFSBuilderPtr createHDFSBuilder(const std::string & uri_str) { const Poco::URI uri(uri_str); - auto & host = uri.getHost(); + const auto & host = uri.getHost(); auto port = uri.getPort(); const std::string path = "//"; if (host.empty()) diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index d8ce3427c91..97a0525a9b8 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -9,14 +8,18 @@ #include +#if !defined(ARCADIA_BUILD) +# include +#endif + #if USE_POCO_NETSSL -#include -#include -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include +# include #endif #include diff --git a/src/IO/ReadBufferFromHDFS.cpp b/src/IO/ReadBufferFromHDFS.cpp index 6d40f8326c2..d6dfd12bd82 100644 --- a/src/IO/ReadBufferFromHDFS.cpp +++ b/src/IO/ReadBufferFromHDFS.cpp @@ -36,7 +36,7 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl ErrorCodes::CANNOT_OPEN_FILE); } - int read(char * start, size_t size) + int read(char * start, size_t size) const { int bytes_read = hdfsRead(fs.get(), fin, start, size); if (bytes_read < 0) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index f5e6f91537d..2d4e3f0472e 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -14,10 +14,13 @@ #include #include #include -#include #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + #define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800 #define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1 @@ -302,4 +305,3 @@ public: }; } - diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 19eb2b42360..2a892446904 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -38,13 +38,13 @@ public: void Log(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, ...) final // NOLINT { - auto & [level, prio] = convertLogLevel(log_level); + const auto & [level, prio] = convertLogLevel(log_level); LOG_SIMPLE(log, std::string(tag) + ": " + format_str, level, prio); } void LogStream(Aws::Utils::Logging::LogLevel log_level, const char * tag, const Aws::OStringStream & message_stream) final { - auto & [level, prio] = convertLogLevel(log_level); + const auto & [level, prio] = convertLogLevel(log_level); LOG_SIMPLE(log, std::string(tag) + ": " + message_stream.str(), level, prio); } @@ -108,7 +108,7 @@ namespace S3 /// Case when bucket name represented in domain name of S3 URL. /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access - static const RE2 virtual_hosted_style_pattern("(.+\\.)?s3[.\\-][a-z0-9\\-.]+"); + static const RE2 virtual_hosted_style_pattern(R"((.+\.)?s3[.\-][a-z0-9\-.]+)"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access diff --git a/src/IO/UseSSL.cpp b/src/IO/UseSSL.cpp index 2aad0ef49e2..6770de2cbad 100644 --- a/src/IO/UseSSL.cpp +++ b/src/IO/UseSSL.cpp @@ -1,9 +1,11 @@ #include "UseSSL.h" -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_POCO_NETSSL -#include +# include #endif namespace DB diff --git a/src/IO/WriteBufferFromHDFS.cpp b/src/IO/WriteBufferFromHDFS.cpp index 0793a966559..2176c77b914 100644 --- a/src/IO/WriteBufferFromHDFS.cpp +++ b/src/IO/WriteBufferFromHDFS.cpp @@ -54,7 +54,7 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl } - int write(const char * start, size_t size) + int write(const char * start, size_t size) const { int bytes_written = hdfsWrite(fs.get(), fout, start, size); if (bytes_written < 0) @@ -63,7 +63,7 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl return bytes_written; } - void sync() + void sync() const { int result = hdfsSync(fs.get(), fout); if (result < 0) diff --git a/src/IO/WriteBufferFromHTTPServerResponse.cpp b/src/IO/WriteBufferFromHTTPServerResponse.cpp index 77198ca93ea..0f30f1352e3 100644 --- a/src/IO/WriteBufferFromHTTPServerResponse.cpp +++ b/src/IO/WriteBufferFromHTTPServerResponse.cpp @@ -7,7 +7,10 @@ #include #include #include -#include + +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB diff --git a/src/IO/WriteBufferFromHTTPServerResponse.h b/src/IO/WriteBufferFromHTTPServerResponse.h index 1307c007656..ffa36c11c5b 100644 --- a/src/IO/WriteBufferFromHTTPServerResponse.h +++ b/src/IO/WriteBufferFromHTTPServerResponse.h @@ -13,7 +13,10 @@ #include #include #include -#include + +#if !defined(ARCADIA_BUILD) +# include +#endif namespace Poco diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 6e747b13b3f..7e40909226c 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -69,7 +69,6 @@ template inline void readDecimalNumber(T & res, size_t num_digits, const char * src) { #define READ_DECIMAL_NUMBER(N) do { res *= common::exp10_i32(N); readDecimalNumber(res, src); src += (N); num_digits -= (N); } while (false) - while (num_digits) { switch (num_digits) @@ -80,7 +79,7 @@ inline void readDecimalNumber(T & res, size_t num_digits, const char * src) default: READ_DECIMAL_NUMBER(4); break; } } -#undef DECIMAL_NUMBER_CASE +#undef READ_DECIMAL_NUMBER } struct DateTimeSubsecondPart @@ -90,7 +89,12 @@ struct DateTimeSubsecondPart }; template -ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeSubsecondPart * fractional = nullptr) +ReturnType parseDateTimeBestEffortImpl( + time_t & res, + ReadBuffer & in, + const DateLUTImpl & local_time_zone, + const DateLUTImpl & utc_time_zone, + DateTimeSubsecondPart * fractional) { auto on_error = [](const std::string & message [[maybe_unused]], int code [[maybe_unused]]) { @@ -367,7 +371,10 @@ ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const Date { char c = *in.position(); - if (c == ' ' || c == 'T') + /// 'T' is a separator between date and time according to ISO 8601. + /// But don't skip it if we didn't read the date part yet, because 'T' is also a prefix for 'Tue' and 'Thu'. + + if (c == ' ' || (c == 'T' && year && !has_time)) { ++in.position(); } @@ -582,12 +589,12 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone); + parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); } bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone); + return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); } void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index 5b2a3f76481..6edc300eac8 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -10,7 +10,10 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; } - +/// Try to read Decimal into underlying type T from ReadBuffer. Throws if 'digits_only' is set and there's unexpected symbol in input. +/// Returns integer 'exponent' factor that x should be muntiplyed by to get correct Decimal value: result = x * 10^exponent. +/// Use 'digits' input as max allowed meaning decimal digits in result. Place actual meanin digits in 'digits' output. +/// Do not care about decimal scale, only about meaning digits in decimal text representation. template inline bool readDigits(ReadBuffer & buf, T & x, unsigned int & digits, int & exponent, bool digits_only = false) { diff --git a/src/IO/readFloatText.h b/src/IO/readFloatText.h index fc3ffc43a91..4e0825222a7 100644 --- a/src/IO/readFloatText.h +++ b/src/IO/readFloatText.h @@ -156,6 +156,9 @@ ReturnType readFloatTextPreciseImpl(T & x, ReadBuffer & buf) { switch (*buf.position()) { + case '+': + continue; + case '-': { negative = true; @@ -335,6 +338,7 @@ ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) ++in.position(); } + auto count_after_sign = in.count(); constexpr int significant_digits = std::numeric_limits::digits10; @@ -380,7 +384,7 @@ ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) if (in.eof()) { if constexpr (throw_exception) - throw Exception("Cannot read floating point value", ErrorCodes::CANNOT_PARSE_NUMBER); + throw Exception("Cannot read floating point value: nothing after exponent", ErrorCodes::CANNOT_PARSE_NUMBER); else return false; } @@ -418,11 +422,30 @@ ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) if (in.eof()) { if constexpr (throw_exception) - throw Exception("Cannot read floating point value", ErrorCodes::CANNOT_PARSE_NUMBER); + throw Exception("Cannot read floating point value: no digits read", ErrorCodes::CANNOT_PARSE_NUMBER); else return false; } + if (*in.position() == '+') + { + ++in.position(); + if (in.eof()) + { + if constexpr (throw_exception) + throw Exception("Cannot read floating point value: nothing after plus sign", ErrorCodes::CANNOT_PARSE_NUMBER); + else + return false; + } + else if (negative) + { + if constexpr (throw_exception) + throw Exception("Cannot read floating point value: plus after minus sign", ErrorCodes::CANNOT_PARSE_NUMBER); + else + return false; + } + } + if (*in.position() == 'i' || *in.position() == 'I') { if (assertOrParseInfinity(in)) diff --git a/src/IO/tests/gtest_DateTime64_parsing_and_writing.cpp b/src/IO/tests/gtest_DateTime64_parsing_and_writing.cpp index 3e5a1998380..c6208af2d5e 100644 --- a/src/IO/tests/gtest_DateTime64_parsing_and_writing.cpp +++ b/src/IO/tests/gtest_DateTime64_parsing_and_writing.cpp @@ -104,7 +104,7 @@ INSTANTIATE_TEST_SUITE_P(Basic, DateLUT::instance("Europe/Minsk") }, { - "When scale is 0, subsecond part is 0 despite beeing present in string.", + "When scale is 0, subsecond part is 0 despite being present in string.", "2019-09-16 19:20:17.123", 1568650817ULL, 0, diff --git a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp index 1dd62682f36..2c90fad3106 100644 --- a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp +++ b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp @@ -42,7 +42,7 @@ static void testCascadeBufferRedability( if (!wbuf) continue; - auto wbuf_readable = dynamic_cast(wbuf.get()); + auto * wbuf_readable = dynamic_cast(wbuf.get()); ASSERT_FALSE(!wbuf_readable); auto rbuf = wbuf_readable->tryGetReadBuffer(); @@ -133,7 +133,7 @@ static void checkHTTPHandlerCase(size_t input_size, size_t memory_buffer_size) { [res_buf] (const WriteBufferPtr & prev_buf) { - auto prev_memory_buffer = typeid_cast(prev_buf.get()); + auto * prev_memory_buffer = typeid_cast(prev_buf.get()); if (prev_memory_buffer != nullptr) { auto rdbuf = prev_memory_buffer->tryGetReadBuffer(); diff --git a/dbms/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp similarity index 100% rename from dbms/src/IO/tests/gtest_s3_uri.cpp rename to src/IO/tests/gtest_s3_uri.cpp diff --git a/src/IO/ya.make b/src/IO/ya.make new file mode 100644 index 00000000000..81bf6060a94 --- /dev/null +++ b/src/IO/ya.make @@ -0,0 +1,59 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/brotli/dec + contrib/libs/brotli/enc + contrib/libs/poco/NetSSL_OpenSSL +) + +SRCS( + AIO.cpp + AIOContextPool.cpp + BrotliReadBuffer.cpp + BrotliWriteBuffer.cpp + CascadeWriteBuffer.cpp + CompressionMethod.cpp + copyData.cpp + createReadBufferFromFileBase.cpp + createWriteBufferFromFileBase.cpp + DoubleConverter.cpp + HashingWriteBuffer.cpp + HexWriteBuffer.cpp + HTTPCommon.cpp + LimitReadBuffer.cpp + MemoryReadWriteBuffer.cpp + MMapReadBufferFromFile.cpp + MMapReadBufferFromFileDescriptor.cpp + NullWriteBuffer.cpp + parseDateTimeBestEffort.cpp + PeekableReadBuffer.cpp + Progress.cpp + ReadBufferAIO.cpp + ReadBufferFromFile.cpp + ReadBufferFromFileBase.cpp + ReadBufferFromFileDescriptor.cpp + ReadBufferFromIStream.cpp + ReadBufferFromMemory.cpp + ReadBufferFromPocoSocket.cpp + readFloatText.cpp + ReadHelpers.cpp + ReadWriteBufferFromHTTP.cpp + UseSSL.cpp + WriteBufferAIO.cpp + WriteBufferFromFile.cpp + WriteBufferFromFileBase.cpp + WriteBufferFromFileDescriptor.cpp + WriteBufferFromFileDescriptorDiscardOnFailure.cpp + WriteBufferFromHTTP.cpp + WriteBufferFromHTTPServerResponse.cpp + WriteBufferFromOStream.cpp + WriteBufferFromPocoSocket.cpp + WriteBufferFromTemporaryFile.cpp + WriteBufferValidUTF8.cpp + WriteHelpers.cpp + ZlibDeflatingWriteBuffer.cpp + ZlibInflatingReadBuffer.cpp +) + +END() diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 4e008a81973..4d032e552ca 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -64,7 +64,7 @@ static NamesAndTypesList::iterator findColumn(const String & name, NamesAndTypes } template -static Block createBlockFromCollection(const Collection & collection, const DataTypes & types) +static Block createBlockFromCollection(const Collection & collection, const DataTypes & types, const Context & context) { size_t columns_num = types.size(); MutableColumns columns(columns_num); @@ -77,7 +77,7 @@ static Block createBlockFromCollection(const Collection & collection, const Data if (columns_num == 1) { auto field = convertFieldToType(value, *types[0]); - if (!field.isNull()) + if (!field.isNull() || context.getSettingsRef().transform_null_in) columns[0]->insert(std::move(field)); } else @@ -100,7 +100,7 @@ static Block createBlockFromCollection(const Collection & collection, const Data for (; i < tuple_size; ++i) { tuple_values[i] = convertFieldToType(tuple[i], *types[i]); - if (tuple_values[i].isNull()) + if (tuple_values[i].isNull() && !context.getSettingsRef().transform_null_in) break; } @@ -131,7 +131,7 @@ SetPtr makeExplicitSet( const DataTypePtr & left_arg_type = sample_block.getByName(left_arg->getColumnName()).type; DataTypes set_element_types = {left_arg_type}; - auto left_tuple_type = typeid_cast(left_arg_type.get()); + const auto * left_tuple_type = typeid_cast(left_arg_type.get()); if (left_tuple_type && left_tuple_type->getElements().size() != 1) set_element_types = left_tuple_type->getElements(); @@ -148,9 +148,9 @@ SetPtr makeExplicitSet( std::function get_type_depth; get_type_depth = [&get_type_depth](const DataTypePtr & type) -> size_t { - if (auto array_type = typeid_cast(type.get())) + if (const auto * array_type = typeid_cast(type.get())) return 1 + get_type_depth(array_type->getNestedType()); - else if (auto tuple_type = typeid_cast(type.get())) + else if (const auto * tuple_type = typeid_cast(type.get())) return 1 + (tuple_type->getElements().empty() ? 0 : get_type_depth(tuple_type->getElements().at(0))); return 0; @@ -170,23 +170,23 @@ SetPtr makeExplicitSet( if (left_type_depth == right_type_depth) { Array array{right_arg_value}; - block = createBlockFromCollection(array, set_element_types); + block = createBlockFromCollection(array, set_element_types, context); } /// 1 in (1, 2); (1, 2) in ((1, 2), (3, 4)); etc. else if (left_type_depth + 1 == right_type_depth) { auto type_index = right_arg_type->getTypeId(); if (type_index == TypeIndex::Tuple) - block = createBlockFromCollection(DB::get(right_arg_value), set_element_types); + block = createBlockFromCollection(DB::get(right_arg_value), set_element_types, context); else if (type_index == TypeIndex::Array) - block = createBlockFromCollection(DB::get(right_arg_value), set_element_types); + block = createBlockFromCollection(DB::get(right_arg_value), set_element_types, context); else throw_unsupported_type(right_arg_type); } else throw_unsupported_type(right_arg_type); - SetPtr set = std::make_shared(size_limits, create_ordered_set); + SetPtr set = std::make_shared(size_limits, create_ordered_set, context.getSettingsRef().transform_null_in); set->setHeader(block); set->insertFromBlock(block); @@ -195,14 +195,6 @@ SetPtr makeExplicitSet( return set; } -static String getUniqueName(const Block & block, const String & prefix) -{ - int i = 1; - while (block.has(prefix + toString(i))) - ++i; - return prefix + toString(i); -} - ScopeStack::ScopeStack(const ExpressionActionsPtr & actions, const Context & context_) : context(context_) { @@ -431,7 +423,6 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & for (size_t arg = 0; arg < node.arguments->children.size(); ++arg) { auto & child = node.arguments->children[arg]; - auto child_column_name = child->getColumnName(); const auto * lambda = child->as(); const auto * identifier = child->as(); @@ -459,9 +450,9 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// If the argument is a set given by an enumeration of values (so, the set was already built), give it a unique name, /// so that sets with the same literal representation do not fuse together (they can have different types). if (!prepared_set->empty()) - column.name = getUniqueName(data.getSampleBlock(), "__set"); + column.name = data.getUniqueName("__set"); else - column.name = child_column_name; + column.name = child->getColumnName(); if (!data.hasColumn(column.name)) { @@ -487,7 +478,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & ColumnWithTypeAndName column( ColumnConst::create(std::move(column_string), 1), std::make_shared(), - getUniqueName(data.getSampleBlock(), "__joinGet")); + data.getUniqueName("__joinGet")); data.addAction(ExpressionAction::addColumn(column)); argument_types.push_back(column.type); argument_names.push_back(column.name); @@ -496,6 +487,18 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { /// If the argument is not a lambda expression, call it recursively and find out its type. visit(child, data); + + // In the above visit() call, if the argument is a literal, we + // generated a unique column name for it. Use it instead of a generic + // display name. + auto child_column_name = child->getColumnName(); + const auto * as_literal = child->as(); + if (as_literal) + { + assert(!as_literal->unique_column_name.empty()); + child_column_name = as_literal->unique_column_name; + } + if (data.hasColumn(child_column_name)) { argument_types.push_back(data.getSampleBlock().getByName(child_column_name).type); @@ -556,7 +559,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// We can not name `getColumnName()`, /// because it does not uniquely define the expression (the types of arguments can be different). - String lambda_name = getUniqueName(data.getSampleBlock(), "__lambda"); + String lambda_name = data.getUniqueName("__lambda"); auto function_capture = std::make_unique( lambda_actions, captured, lambda_arguments, result_type, result_name); @@ -587,18 +590,53 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & } } -void ActionsMatcher::visit(const ASTLiteral & literal, const ASTPtr & ast, Data & data) +void ActionsMatcher::visit(const ASTLiteral & literal, const ASTPtr & /* ast */, + Data & data) { - CachedColumnName column_name; - if (data.hasColumn(column_name.get(ast))) - return; - DataTypePtr type = applyVisitor(FieldToDataType(), literal.value); + const auto value = convertFieldToType(literal.value, *type); + + // FIXME why do we have a second pass with a clean sample block over the same + // AST here? Anyway, do not modify the column name if it is set already. + if (literal.unique_column_name.empty()) + { + const auto default_name = literal.getColumnName(); + const auto & block = data.getSampleBlock(); + const auto * existing_column = block.findByName(default_name); + + /* + * To approximate CSE, bind all identical literals to a single temporary + * columns. We try to find the column by its default name, but after that + * we have to check that it contains the correct data. This might not be + * the case if it is a user-supplied column, or it is from under a join, + * etc. + * Overall, this is a hack around a generally poor name-based notion of + * column identity we currently use. + */ + if (existing_column + && existing_column->column + && isColumnConst(*existing_column->column) + && existing_column->column->size() == 1 + && existing_column->column->operator[](0) == value) + { + const_cast(literal).unique_column_name = default_name; + } + else + { + const_cast(literal).unique_column_name + = data.getUniqueName(default_name); + } + } + + if (data.hasColumn(literal.unique_column_name)) + { + return; + } ColumnWithTypeAndName column; - column.column = type->createColumnConst(1, convertFieldToType(literal.value, *type)); + column.name = literal.unique_column_name; + column.column = type->createColumnConst(1, value); column.type = type; - column.name = column_name.get(ast); data.addAction(ExpressionAction::addColumn(column)); } @@ -654,7 +692,7 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su return subquery_for_set.set; } - SetPtr set = std::make_shared(data.set_size_limit, false); + SetPtr set = std::make_shared(data.set_size_limit, false, data.context.getSettingsRef().transform_null_in); /** The following happens for GLOBAL INs: * - in the addExternalStorage function, the IN (SELECT ...) subquery is replaced with IN _data1, diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index f6db551ff33..dbcc54c01d6 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -80,6 +80,13 @@ public: size_t visit_depth; ScopeStack actions_stack; + /* + * Remember the last unique column suffix to avoid quadratic behavior + * when we add lots of column with same prefix. One counter for all + * prefixes is good enough. + */ + int next_unique_suffix; + Data(const Context & context_, SizeLimits set_size_limit_, size_t subquery_depth_, const NamesAndTypesList & source_columns_, const ExpressionActionsPtr & actions, PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, @@ -95,7 +102,8 @@ public: only_consts(only_consts_), no_storage_or_local(no_storage_or_local_), visit_depth(0), - actions_stack(actions, context) + actions_stack(actions, context), + next_unique_suffix(actions_stack.getSampleBlock().columns() + 1) {} void updateActions(ExpressionActionsPtr & actions) @@ -118,6 +126,26 @@ public: { return actions_stack.getSampleBlock().has(columnName); } + + /* + * Generate a column name that is not present in the sample block, using + * the given prefix and an optional numeric suffix. + */ + String getUniqueName(const String & prefix) + { + const auto & block = getSampleBlock(); + auto result = prefix; + + // First, try the name without any suffix, because it is currently + // used both as a display name and a column id. + while (block.has(result)) + { + result = prefix + "_" + toString(next_unique_suffix); + ++next_unique_suffix; + } + + return result; + } }; static void visit(const ASTPtr & ast, Data & data); diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 07c1d7476ad..8a85ddb52f0 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -25,11 +25,14 @@ #include #include #include -#include #include #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + namespace ProfileEvents { @@ -147,8 +150,8 @@ Aggregator::Aggregator(const Params & params_) isCancelled([]() { return false; }) { /// Use query-level memory tracker - if (auto memory_tracker_child = CurrentThread::getMemoryTracker()) - if (auto memory_tracker = memory_tracker_child->getParent()) + if (auto * memory_tracker_child = CurrentThread::getMemoryTracker()) + if (auto * memory_tracker = memory_tracker_child->getParent()) memory_usage_before_aggregation = memory_tracker->get(); aggregate_functions.resize(params.aggregates_size); @@ -604,18 +607,18 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData aggregate_functions_instructions[i].arguments = aggregate_columns[i].data(); aggregate_functions_instructions[i].state_offset = offsets_of_aggregate_states[i]; - auto that = aggregate_functions[i]; + auto * that = aggregate_functions[i]; /// Unnest consecutive trailing -State combinators - while (auto func = typeid_cast(that)) + while (const auto * func = typeid_cast(that)) that = func->getNestedFunction().get(); aggregate_functions_instructions[i].that = that; aggregate_functions_instructions[i].func = that->getAddressOfAddFunction(); - if (auto func = typeid_cast(that)) + if (const auto * func = typeid_cast(that)) { /// Unnest consecutive -State combinators before -Array that = func->getNestedFunction().get(); - while (auto nested_func = typeid_cast(that)) + while (const auto * nested_func = typeid_cast(that)) that = nested_func->getNestedFunction().get(); auto [nested_columns, offsets] = checkAndGetNestedArrayOffset(aggregate_columns[i].data(), that->getArgumentTypes().size()); nested_columns_holder.push_back(std::move(nested_columns)); @@ -662,8 +665,8 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData size_t result_size = result.sizeWithoutOverflowRow(); Int64 current_memory_usage = 0; - if (auto memory_tracker_child = CurrentThread::getMemoryTracker()) - if (auto memory_tracker = memory_tracker_child->getParent()) + if (auto * memory_tracker_child = CurrentThread::getMemoryTracker()) + if (auto * memory_tracker = memory_tracker_child->getParent()) current_memory_usage = memory_tracker->get(); auto result_size_bytes = current_memory_usage - memory_usage_before_aggregation; /// Here all the results in the sum are taken into account, from different threads. @@ -691,11 +694,20 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData && worth_convert_to_two_level) { size_t size = current_memory_usage + params.min_free_disk_space; - auto reservation = params.tmp_volume->reserve(size); - if (!reservation) - throw Exception("Not enough space for external aggregation in temporary storage", ErrorCodes::NOT_ENOUGH_SPACE); + const std::string tmp_path = params.tmp_volume->getNextDisk()->getPath(); + + // enoughSpaceInDirectory() is not enough to make it right, since + // another process (or another thread of aggregator) can consume all + // space. + // + // But true reservation (IVolume::reserve()) cannot be used here since + // current_memory_usage does not takes compression into account and + // will reserve way more that actually will be used. + // + // Hence let's do a simple check. + if (!enoughSpaceInDirectory(tmp_path, size)) + throw Exception("Not enough space for external aggregation in " + tmp_path, ErrorCodes::NOT_ENOUGH_SPACE); - const std::string tmp_path(reservation->getDisk()->getPath()); writeToTemporaryFile(result, tmp_path); } @@ -1208,7 +1220,7 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( if (method.data.impls[bucket].empty()) continue; - tasks[bucket] = std::packaged_task(std::bind(converter, bucket, CurrentThread::getGroup())); + tasks[bucket] = std::packaged_task([group = CurrentThread::getGroup(), bucket, &converter]{ return converter(bucket, group); }); if (thread_pool) thread_pool->scheduleOrThrowOnError([bucket, &tasks] { tasks[bucket](); }); @@ -1670,8 +1682,9 @@ private: if (max_scheduled_bucket_num >= NUM_BUCKETS) return; - parallel_merge_data->pool.scheduleOrThrowOnError(std::bind(&MergingAndConvertingBlockInputStream::thread, this, - max_scheduled_bucket_num, CurrentThread::getGroup())); + parallel_merge_data->pool.scheduleOrThrowOnError( + [this, max_scheduled_bucket_num = max_scheduled_bucket_num, group = CurrentThread::getGroup()] + { return thread(max_scheduled_bucket_num, group); }); } void thread(Int32 bucket_num, ThreadGroupStatusPtr thread_group) @@ -2021,7 +2034,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari result.aggregates_pools.push_back(std::make_shared()); Arena * aggregates_pool = result.aggregates_pools.back().get(); - auto task = std::bind(merge_bucket, bucket, aggregates_pool, CurrentThread::getGroup()); + auto task = [group = CurrentThread::getGroup(), bucket, &merge_bucket, aggregates_pool]{ return merge_bucket(bucket, aggregates_pool, group); }; if (thread_pool) thread_pool->scheduleOrThrowOnError(task); diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 35a352b67b6..3d7d1e26b0e 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -5,7 +5,6 @@ #include #include #include -#include "config_core.h" #include #include #include @@ -13,12 +12,13 @@ #include #include -#if __has_include() -#include + +#if !defined(ARCADIA_BUILD) +# include "config_core.h" #endif #if USE_JEMALLOC - #include +# include #endif @@ -131,6 +131,24 @@ void AsynchronousMetrics::update() set("Uptime", context.getUptimeSeconds()); + /// Process memory usage according to OS +#if defined(OS_LINUX) + { + MemoryStatisticsOS::Data data = memory_stat.get(); + + set("MemoryVirtual", data.virt); + set("MemoryResident", data.resident); + set("MemoryShared", data.shared); + set("MemoryCode", data.code); + set("MemoryDataAndStack", data.data_and_stack); + + /// We must update the value of total_memory_tracker periodically. + /// Otherwise it might be calculated incorrectly - it can include a "drift" of memory amount. + /// See https://github.com/ClickHouse/ClickHouse/issues/10293 + total_memory_tracker.set(data.resident); + } +#endif + { auto databases = DatabaseCatalog::instance().getDatabases(); @@ -158,7 +176,7 @@ void AsynchronousMetrics::update() for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { ++total_number_of_tables; - auto & table = iterator->table(); + const auto & table = iterator->table(); StorageMergeTree * table_merge_tree = dynamic_cast(table.get()); StorageReplicatedMergeTree * table_replicated_merge_tree = dynamic_cast(table.get()); @@ -216,9 +234,9 @@ void AsynchronousMetrics::update() set("NumberOfTables", total_number_of_tables); } -#if USE_JEMALLOC +#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4 { - #define FOR_EACH_METRIC(M) \ +# define FOR_EACH_METRIC(M) \ M("allocated", size_t) \ M("active", size_t) \ M("metadata", size_t) \ @@ -228,9 +246,9 @@ void AsynchronousMetrics::update() M("retained", size_t) \ M("background_thread.num_threads", size_t) \ M("background_thread.num_runs", uint64_t) \ - M("background_thread.run_interval", uint64_t) \ + M("background_thread.run_interval", uint64_t) - #define GET_METRIC(NAME, TYPE) \ +# define GET_METRIC(NAME, TYPE) \ do \ { \ TYPE value{}; \ @@ -241,13 +259,12 @@ void AsynchronousMetrics::update() FOR_EACH_METRIC(GET_METRIC) - #undef GET_METRIC - #undef FOR_EACH_METRIC +# undef GET_METRIC +# undef FOR_EACH_METRIC } #endif /// Add more metrics as you wish. } - } diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 8ccefb9e930..ce6c0aae552 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -44,6 +45,10 @@ private: Container container; mutable std::mutex container_mutex; +#if defined(OS_LINUX) + MemoryStatisticsOS memory_stat; +#endif + ThreadFromGlobalPool thread; void run(); diff --git a/src/Interpreters/CatBoostModel.cpp b/src/Interpreters/CatBoostModel.cpp index 67251859298..3e4329c6080 100644 --- a/src/Interpreters/CatBoostModel.cpp +++ b/src/Interpreters/CatBoostModel.cpp @@ -139,7 +139,7 @@ public: bool cat_features_are_strings = true; for (size_t i = float_features_count; i < float_features_count + cat_features_count; ++i) { - auto column = columns[i]; + const auto * column = columns[i]; if (column->isNumeric()) cat_features_are_strings = false; else if (!(typeid_cast(column) @@ -160,7 +160,7 @@ public: return result; size_t column_size = columns.front()->size(); - auto result_buf = result->getData().data(); + auto * result_buf = result->getData().data(); /// Multiple trees case. Copy data to several columns. MutableColumns mutable_columns(tree_count); @@ -262,7 +262,7 @@ private: T * data = data_column->getData().data(); for (size_t i = 0; i < size; ++i) { - auto column = columns[offset + i]; + const auto * column = columns[offset + i]; if (column->isNumeric()) placeColumnAsNumber(column, data + i, size); } @@ -288,10 +288,10 @@ private: std::vector> data; for (size_t i = 0; i < size; ++i) { - auto column = columns[offset + i]; - if (auto column_string = typeid_cast(column)) + const auto * column = columns[offset + i]; + if (const auto * column_string = typeid_cast(column)) placeStringColumn(*column_string, buffer + i, size); - else if (auto column_fixed_string = typeid_cast(column)) + else if (const auto * column_fixed_string = typeid_cast(column)) data.push_back(placeFixedStringColumn(*column_fixed_string, buffer + i, size)); else throw Exception("Cannot place string column.", ErrorCodes::LOGICAL_ERROR); @@ -335,10 +335,10 @@ private: std::vector> data; for (size_t i = 0; i < size; ++i) { - auto column = columns[offset + i]; - if (auto column_string = typeid_cast(column)) + const auto * column = columns[offset + i]; + if (const auto * column_string = typeid_cast(column)) calcStringHashes(column_string, i, buffer); - else if (auto column_fixed_string = typeid_cast(column)) + else if (const auto * column_fixed_string = typeid_cast(column)) calcStringHashes(column_fixed_string, i, buffer); else calcIntHashes(column_size, i, buffer); @@ -369,14 +369,14 @@ private: size_t column_size = columns.front()->size(); auto result = ColumnFloat64::create(column_size * tree_count); - auto result_buf = result->getData().data(); + auto * result_buf = result->getData().data(); if (!column_size) return result; /// Prepare float features. PODArray float_features(column_size); - auto float_features_buf = float_features.data(); + auto * float_features_buf = float_features.data(); /// Store all float data into single column. float_features is a list of pointers to it. auto float_features_col = placeNumericColumns(columns, 0, float_features_count, float_features_buf); @@ -398,7 +398,7 @@ private: /// cat_features_holder stores pointers to ColumnString data or fixed_strings_data. PODArray cat_features_holder(cat_features_count * column_size); PODArray cat_features(column_size); - auto cat_features_buf = cat_features.data(); + auto * cat_features_buf = cat_features.data(); fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size); /// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data. @@ -416,7 +416,7 @@ private: else { PODArray cat_features(column_size); - auto cat_features_buf = cat_features.data(); + auto * cat_features_buf = cat_features.data(); auto cat_features_col = placeNumericColumns(columns, float_features_count, cat_features_count, cat_features_buf); calcHashes(columns, float_features_count, cat_features_count, cat_features_buf); diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index ed806e5ad57..909582aa308 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -6,9 +6,12 @@ #include #include #include -#include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + namespace DB { diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index f1790249cc0..151dfc5c9bb 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -407,7 +407,7 @@ Cluster::Cluster(const Settings & settings, const std::vector #include -#include #include #include #include @@ -71,7 +70,9 @@ SelectStreamFactory::SelectStreamFactory( namespace { -Pipe createLocalStream(const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage, bool force_tree_shaped_pipeline) +Pipe createLocalStream( + const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage, + bool add_totals_port, bool add_extremes_port, bool force_tree_shaped_pipeline) { checkStackSize(); @@ -82,10 +83,17 @@ Pipe createLocalStream(const ASTPtr & query_ast, const Block & header, const Con /// This flag means that pipeline must be tree-shaped, /// so we can't enable processors for InterpreterSelectQuery here. auto stream = interpreter.execute().in; - Pipe pipe(std::make_shared(std::move(stream))); + auto source = std::make_shared(std::move(stream)); + + if (add_totals_port) + source->addTotalsPort(); + if (add_extremes_port) + source->addExtremesPort(); + + Pipe pipe(std::move(source)); pipe.addSimpleTransform(std::make_shared( - pipe.getHeader(), header, ConvertingTransform::MatchColumnsMode::Name, context)); + pipe.getHeader(), header, ConvertingTransform::MatchColumnsMode::Name)); return pipe; } @@ -95,7 +103,7 @@ Pipe createLocalStream(const ASTPtr & query_ast, const Block & header, const Con pipeline.addSimpleTransform([&](const Block & source_header) { return std::make_shared( - source_header, header, ConvertingTransform::MatchColumnsMode::Name, context); + source_header, header, ConvertingTransform::MatchColumnsMode::Name); }); /** Materialization is needed, since from remote servers the constants come materialized. @@ -130,7 +138,13 @@ void SelectStreamFactory::createForShard( Pipes & res) { bool force_add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState; - bool add_totals_port = processed_stage == QueryProcessingStage::Complete; + bool add_totals_port = false; + bool add_extremes_port = false; + if (processed_stage == QueryProcessingStage::Complete) + { + add_totals_port = query_ast->as().group_by_with_totals; + add_extremes_port = context.getSettingsRef().extremes; + } auto modified_query_ast = query_ast->clone(); if (has_virtual_shard_num_column) @@ -138,7 +152,8 @@ void SelectStreamFactory::createForShard( auto emplace_local_stream = [&]() { - res.emplace_back(createLocalStream(modified_query_ast, header, context, processed_stage, query_info.force_tree_shaped_pipeline)); + res.emplace_back(createLocalStream(modified_query_ast, header, context, processed_stage, + add_totals_port, add_extremes_port, query_info.force_tree_shaped_pipeline)); }; String modified_query = formattedAST(modified_query_ast); @@ -155,6 +170,8 @@ void SelectStreamFactory::createForShard( if (add_totals_port) source->addTotalsPort(); + if (add_extremes_port) + source->addExtremesPort(); res.emplace_back(std::move(source)); }; @@ -254,7 +271,7 @@ void SelectStreamFactory::createForShard( auto lazily_create_stream = [ pool = shard_info.pool, shard_num = shard_info.shard_num, modified_query, header = header, modified_query_ast, context, throttler, main_table = main_table, table_func_ptr = table_func_ptr, scalars = scalars, external_tables = external_tables, - stage = processed_stage, local_delay]() + stage = processed_stage, local_delay, add_totals_port, add_extremes_port]() -> BlockInputStreamPtr { auto current_settings = context.getSettingsRef(); @@ -287,7 +304,8 @@ void SelectStreamFactory::createForShard( } if (try_results.empty() || local_delay < max_remote_delay) - return std::make_shared(createLocalStream(modified_query_ast, header, context, stage, true)); + return std::make_shared( + createLocalStream(modified_query_ast, header, context, stage, add_totals_port, add_extremes_port, true)); else { std::vector connections; @@ -305,6 +323,8 @@ void SelectStreamFactory::createForShard( if (add_totals_port) source->addTotalsPort(); + if (add_extremes_port) + source->addExtremesPort(); res.emplace_back(std::move(source)); } diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 07802f468c9..fa977249eaa 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -23,13 +23,10 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin /// Does not matter on remote servers, because queries are sent under different user. new_settings.max_concurrent_queries_for_user = 0; new_settings.max_memory_usage_for_user = 0; - /// This setting is really not for user and should not be sent to remote server. - new_settings.max_memory_usage_for_all_queries = 0; /// Set as unchanged to avoid sending to remote server. new_settings.max_concurrent_queries_for_user.changed = false; new_settings.max_memory_usage_for_user.changed = false; - new_settings.max_memory_usage_for_all_queries.changed = false; if (settings.force_optimize_skip_unused_shards_no_nested) { @@ -54,7 +51,7 @@ Pipes executeQuery( Context new_context = removeUserRestrictionsFromSettings(context, settings); ThrottlerPtr user_level_throttler; - if (auto process_list_element = context.getProcessListElement()) + if (auto * process_list_element = context.getProcessListElement()) user_level_throttler = process_list_element->getUserNetworkThrottler(); /// Network bandwidth limit, if needed. diff --git a/src/Interpreters/ColumnNamesContext.h b/src/Interpreters/ColumnNamesContext.h deleted file mode 100644 index c30102cf8d7..00000000000 --- a/src/Interpreters/ColumnNamesContext.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include - -namespace DB -{ - -/// Information about table and column names extracted from ASTSelectQuery block. Do not include info from subselects. -struct ColumnNamesContext -{ - struct JoinedTable - { - const ASTTableExpression * expr = nullptr; - const ASTTableJoin * join = nullptr; - - std::optional alias() const - { - String alias; - if (expr) - { - if (expr->database_and_table_name) - alias = expr->database_and_table_name->tryGetAlias(); - else if (expr->table_function) - alias = expr->table_function->tryGetAlias(); - else if (expr->subquery) - alias = expr->subquery->tryGetAlias(); - } - if (!alias.empty()) - return alias; - return {}; - } - - std::optional name() const - { - if (expr) - return tryGetIdentifierName(expr->database_and_table_name); - return {}; - } - - std::optional joinKind() const - { - if (join) - return join->kind; - return {}; - } - }; - - struct NameInfo - { - std::set aliases; - size_t appears = 0; - - void addInclusion(const String & alias) - { - if (!alias.empty()) - aliases.insert(alias); - ++appears; - } - }; - - std::unordered_map required_names; - NameSet table_aliases; - NameSet private_aliases; - NameSet complex_aliases; - NameSet masked_columns; - NameSet array_join_columns; - std::vector tables; /// ordered list of visited tables in FROM section with joins - bool has_table_join = false; - bool has_array_join = false; - - bool addTableAliasIfAny(const IAST & ast); - bool addColumnAliasIfAny(const IAST & ast); - void addColumnIdentifier(const ASTIdentifier & node); - bool addArrayJoinAliasIfAny(const IAST & ast); - void addArrayJoinIdentifier(const ASTIdentifier & node); - - NameSet requiredColumns() const; - size_t nameInclusion(const String & name) const; -}; - -std::ostream & operator << (std::ostream & os, const ColumnNamesContext & cols); - -} diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 5e98a0267ca..96dc914dcda 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -317,9 +317,11 @@ struct ContextShared MergeList merge_list; /// The list of executable merge (for (Replicated)?MergeTree) ConfigurationPtr users_config; /// Config with the users, profiles and quotas sections. InterserverIOHandler interserver_io_handler; /// Handler for interserver communication. + std::optional buffer_flush_schedule_pool; /// A thread pool that can do background flush for Buffer tables. std::optional background_pool; /// The thread pool for the background work performed by the tables. std::optional background_move_pool; /// The thread pool for the background moves performed by the tables. std::optional schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables) + std::optional distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends) MultiVersion macros; /// Substitutions extracted from config. std::unique_ptr ddl_worker; /// Process ddl commands from zk. /// Rules for selecting the compression settings, depending on the size of the part. @@ -413,9 +415,11 @@ struct ContextShared embedded_dictionaries.reset(); external_dictionaries_loader.reset(); external_models_loader.reset(); + buffer_flush_schedule_pool.reset(); background_pool.reset(); background_move_pool.reset(); schedule_pool.reset(); + distributed_schedule_pool.reset(); ddl_worker.reset(); /// Stop trace collector if any @@ -441,14 +445,28 @@ Context::Context() = default; Context::Context(const Context &) = default; Context & Context::operator=(const Context &) = default; +SharedContextHolder::SharedContextHolder(SharedContextHolder &&) noexcept = default; +SharedContextHolder & SharedContextHolder::operator=(SharedContextHolder &&) = default; +SharedContextHolder::SharedContextHolder() = default; +SharedContextHolder::~SharedContextHolder() = default; +SharedContextHolder::SharedContextHolder(std::unique_ptr shared_context) + : shared(std::move(shared_context)) {} -Context Context::createGlobal() +void SharedContextHolder::reset() { shared.reset(); } + + +Context Context::createGlobal(ContextShared * shared) { Context res; - res.shared = std::make_shared(); + res.shared = shared; return res; } +SharedContextHolder Context::createShared() +{ + return SharedContextHolder(std::make_unique()); +} + Context::~Context() = default; @@ -802,7 +820,7 @@ Tables Context::getExternalTables() const auto lock = getLock(); Tables res; - for (auto & table : external_tables_mapping) + for (const auto & table : external_tables_mapping) res[table.first] = table.second->getTable(); if (query_context && query_context != this) @@ -907,7 +925,6 @@ void Context::setSettings(const Settings & settings_) auto old_allow_introspection_functions = settings.allow_introspection_functions; settings = settings_; - active_default_settings = nullptr; if ((settings.readonly != old_readonly) || (settings.allow_ddl != old_allow_ddl) || (settings.allow_introspection_functions != old_allow_introspection_functions)) calculateAccessRights(); @@ -917,7 +934,6 @@ void Context::setSettings(const Settings & settings_) void Context::setSetting(const StringRef & name, const String & value) { auto lock = getLock(); - active_default_settings = nullptr; if (name == "profile") { setProfile(value); @@ -933,7 +949,6 @@ void Context::setSetting(const StringRef & name, const String & value) void Context::setSetting(const StringRef & name, const Field & value) { auto lock = getLock(); - active_default_settings = nullptr; if (name == "profile") { setProfile(value.safeGet()); @@ -960,20 +975,6 @@ void Context::applySettingsChanges(const SettingsChanges & changes) } -void Context::resetSettingsToDefault() -{ - auto lock = getLock(); - auto default_settings = getAccess()->getDefaultSettings(); - if (default_settings && (default_settings == active_default_settings)) - return; - if (default_settings) - setSettings(*default_settings); - else - setSettings(Settings{}); - active_default_settings = default_settings; -} - - void Context::checkSettingsConstraints(const SettingChange & change) const { if (auto settings_constraints = getSettingsConstraints()) @@ -1313,7 +1314,7 @@ BackgroundProcessingPool & Context::getBackgroundPool() if (!shared->background_pool) { BackgroundProcessingPool::PoolSettings pool_settings; - auto & config = getConfigRef(); + const auto & config = getConfigRef(); pool_settings.thread_sleep_seconds = config.getDouble("background_processing_pool_thread_sleep_seconds", 10); pool_settings.thread_sleep_seconds_random_part = config.getDouble("background_processing_pool_thread_sleep_seconds_random_part", 1.0); pool_settings.thread_sleep_seconds_if_nothing_to_do = config.getDouble("background_processing_pool_thread_sleep_seconds_if_nothing_to_do", 0.1); @@ -1332,7 +1333,7 @@ BackgroundProcessingPool & Context::getBackgroundMovePool() if (!shared->background_move_pool) { BackgroundProcessingPool::PoolSettings pool_settings; - auto & config = getConfigRef(); + const auto & config = getConfigRef(); pool_settings.thread_sleep_seconds = config.getDouble("background_move_processing_pool_thread_sleep_seconds", 10); pool_settings.thread_sleep_seconds_random_part = config.getDouble("background_move_processing_pool_thread_sleep_seconds_random_part", 1.0); pool_settings.thread_sleep_seconds_if_nothing_to_do = config.getDouble("background_move_processing_pool_thread_sleep_seconds_if_nothing_to_do", 0.1); @@ -1347,6 +1348,14 @@ BackgroundProcessingPool & Context::getBackgroundMovePool() return *shared->background_move_pool; } +BackgroundSchedulePool & Context::getBufferFlushSchedulePool() +{ + auto lock = getLock(); + if (!shared->buffer_flush_schedule_pool) + shared->buffer_flush_schedule_pool.emplace(settings.background_buffer_flush_schedule_pool_size); + return *shared->buffer_flush_schedule_pool; +} + BackgroundSchedulePool & Context::getSchedulePool() { auto lock = getLock(); @@ -1355,6 +1364,14 @@ BackgroundSchedulePool & Context::getSchedulePool() return *shared->schedule_pool; } +BackgroundSchedulePool & Context::getDistributedSchedulePool() +{ + auto lock = getLock(); + if (!shared->distributed_schedule_pool) + shared->distributed_schedule_pool.emplace(settings.background_distributed_schedule_pool_size); + return *shared->distributed_schedule_pool; +} + void Context::setDDLWorker(std::unique_ptr ddl_worker) { auto lock = getLock(); @@ -1445,7 +1462,7 @@ UInt16 Context::getTCPPort() const { auto lock = getLock(); - auto & config = getConfigRef(); + const auto & config = getConfigRef(); return config.getInt("tcp_port", DBMS_DEFAULT_PORT); } @@ -1453,7 +1470,7 @@ std::optional Context::getTCPPortSecure() const { auto lock = getLock(); - auto & config = getConfigRef(); + const auto & config = getConfigRef(); if (config.has("tcp_port_secure")) return config.getInt("tcp_port_secure"); return {}; @@ -1486,7 +1503,7 @@ void Context::reloadClusterConfig() cluster_config = shared->clusters_config; } - auto & config = cluster_config ? *cluster_config : getConfigRef(); + const auto & config = cluster_config ? *cluster_config : getConfigRef(); auto new_clusters = std::make_unique(config, settings); { @@ -1508,7 +1525,7 @@ Clusters & Context::getClusters() const std::lock_guard lock(shared->clusters_mutex); if (!shared->clusters) { - auto & config = shared->clusters_config ? *shared->clusters_config : getConfigRef(); + const auto & config = shared->clusters_config ? *shared->clusters_config : getConfigRef(); shared->clusters = std::make_unique(config, settings); } @@ -1638,7 +1655,7 @@ CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double par if (!shared->compression_codec_selector) { constexpr auto config_name = "compression"; - auto & config = getConfigRef(); + const auto & config = getConfigRef(); if (config.has(config_name)) shared->compression_codec_selector = std::make_unique(config, "compression"); @@ -1667,7 +1684,7 @@ DiskSelectorPtr Context::getDiskSelector() const if (!shared->merge_tree_disk_selector) { constexpr auto config_name = "storage_configuration.disks"; - auto & config = getConfigRef(); + const auto & config = getConfigRef(); shared->merge_tree_disk_selector = std::make_shared(config, config_name, *this); } @@ -1692,7 +1709,7 @@ StoragePolicySelectorPtr Context::getStoragePolicySelector() const if (!shared->merge_tree_storage_policy_selector) { constexpr auto config_name = "storage_configuration.policies"; - auto & config = getConfigRef(); + const auto & config = getConfigRef(); shared->merge_tree_storage_policy_selector = std::make_shared(config, config_name, getDiskSelector()); } @@ -1727,7 +1744,7 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const if (!shared->merge_tree_settings) { - auto & config = getConfigRef(); + const auto & config = getConfigRef(); MergeTreeSettings mt_settings; mt_settings.loadFromConfig("merge_tree", config); shared->merge_tree_settings.emplace(mt_settings); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index b34a0e0c542..86c6ebfa0f6 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -13,7 +13,6 @@ #include #include #include -#include "config_core.h" #include #include #include @@ -25,6 +24,10 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + namespace Poco { @@ -128,6 +131,23 @@ struct IHostContext using IHostContextPtr = std::shared_ptr; +/// A small class which owns ContextShared. +/// We don't use something like unique_ptr directly to allow ContextShared type to be incomplete. +struct SharedContextHolder +{ + ~SharedContextHolder(); + SharedContextHolder(); + SharedContextHolder(std::unique_ptr shared_context); + SharedContextHolder(SharedContextHolder &&) noexcept; + + SharedContextHolder & operator=(SharedContextHolder &&); + + ContextShared * get() const { return shared.get(); } + void reset(); +private: + std::unique_ptr shared; +}; + /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) * and copied part (which can be its own for each session or query). @@ -137,8 +157,7 @@ using IHostContextPtr = std::shared_ptr; class Context { private: - using Shared = std::shared_ptr; - Shared shared; + ContextShared * shared; ClientInfo client_info; ExternalTablesInitializer external_tables_initializer_callback; @@ -151,7 +170,6 @@ private: bool use_default_roles = false; std::shared_ptr access; std::shared_ptr initial_row_policy; - std::shared_ptr active_default_settings; String current_database; Settings settings; /// Setting for query execution. using ProgressCallback = std::function; @@ -191,7 +209,8 @@ private: public: /// Create initial Context with ContextShared and etc. - static Context createGlobal(); + static Context createGlobal(ContextShared * shared); + static SharedContextHolder createShared(); Context(const Context &); Context & operator=(const Context &); @@ -346,9 +365,6 @@ public: void applySettingChange(const SettingChange & change); void applySettingsChanges(const SettingsChanges & changes); - /// Reset settings to the default values for the current user. - void resetSettingsToDefault(); - /// Checks the constraints. void checkSettingsConstraints(const SettingChange & change) const; void checkSettingsConstraints(const SettingsChanges & changes) const; @@ -472,9 +488,11 @@ public: */ void dropCaches() const; + BackgroundSchedulePool & getBufferFlushSchedulePool(); BackgroundProcessingPool & getBackgroundPool(); BackgroundProcessingPool & getBackgroundMovePool(); BackgroundSchedulePool & getSchedulePool(); + BackgroundSchedulePool & getDistributedSchedulePool(); void setDDLWorker(std::unique_ptr ddl_worker); DDLWorker & getDDLWorker() const; diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index 62bbd18c060..49b52b0fb7f 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -52,17 +52,21 @@ struct JoinedElement void rewriteCommaToCross() { - if (join) + if (join && join->kind == ASTTableJoin::Kind::Comma) join->kind = ASTTableJoin::Kind::Cross; } - void rewriteCrossToInner(ASTPtr on_expression) + bool rewriteCrossToInner(ASTPtr on_expression) { + if (join->kind != ASTTableJoin::Kind::Cross) + return false; + join->kind = ASTTableJoin::Kind::Inner; join->strictness = ASTTableJoin::Strictness::All; join->on_expression = on_expression; join->children.push_back(join->on_expression); + return true; } ASTPtr arrayJoin() const { return element.array_join; } @@ -214,7 +218,7 @@ private: } }; -using CheckExpressionMatcher = ConstOneTypeMatcher; +using CheckExpressionMatcher = ConstOneTypeMatcher; using CheckExpressionVisitor = ConstInDepthNodeVisitor; @@ -235,9 +239,9 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl size_t num_array_join = 0; size_t num_using = 0; - for (auto & child : tables->children) + for (const auto & child : tables->children) { - auto table_element = child->as(); + auto * table_element = child->as(); if (!table_element) throw Exception("Logical error: TablesInSelectQueryElement expected", ErrorCodes::LOGICAL_ERROR); @@ -256,7 +260,7 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl continue; } - if (auto * join = t.tableJoin()) + if (const auto * join = t.tableJoin()) { if (join->kind == ASTTableJoin::Kind::Cross || join->kind == ASTTableJoin::Kind::Comma) @@ -329,8 +333,8 @@ void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr &, Data & da { if (visitor_data.matchAny(i)) { - joined_tables[i].rewriteCrossToInner(visitor_data.makeOnExpression(i)); - data.done = true; + if (joined_tables[i].rewriteCrossToInner(visitor_data.makeOnExpression(i))) + data.done = true; } } } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 4a39cc6b8a1..9829e967350 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -463,7 +463,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) ParserQuery parser_query(end); String description; - task.query = parseQuery(parser_query, begin, end, description, 0); + task.query = parseQuery(parser_query, begin, end, description, 0, context.getSettingsRef().max_parser_depth); } // XXX: serious design flaw since `ASTQueryWithOnCluster` is not inherited from `IAST`! @@ -632,7 +632,7 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) String rewritten_query = queryToString(rewritten_ast); LOG_DEBUG(log, "Executing query: " << rewritten_query); - if (auto query_with_table = dynamic_cast(rewritten_ast.get()); query_with_table) + if (auto * query_with_table = dynamic_cast(rewritten_ast.get()); query_with_table) { StoragePtr storage; if (!query_with_table->table.empty()) @@ -683,7 +683,7 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) bool DDLWorker::taskShouldBeExecutedOnLeader(const ASTPtr ast_ddl, const StoragePtr storage) { /// Pure DROP queries have to be executed on each node separately - if (auto query = ast_ddl->as(); query && query->kind != ASTDropQuery::Kind::Truncate) + if (auto * query = ast_ddl->as(); query && query->kind != ASTDropQuery::Kind::Truncate) return false; if (!ast_ddl->as() && !ast_ddl->as() && !ast_ddl->as()) @@ -1239,7 +1239,6 @@ private: return diff; } -private: String node_path; const Context & context; Stopwatch watch; diff --git a/src/Interpreters/DatabaseAndTableWithAlias.cpp b/src/Interpreters/DatabaseAndTableWithAlias.cpp index aeee0766799..e99c6b78086 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.cpp +++ b/src/Interpreters/DatabaseAndTableWithAlias.cpp @@ -49,7 +49,7 @@ DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableExpression & throw Exception("Logical error: no known elements in ASTTableExpression", ErrorCodes::LOGICAL_ERROR); } -bool DatabaseAndTableWithAlias::satisfies(const DatabaseAndTableWithAlias & db_table, bool table_may_be_an_alias) +bool DatabaseAndTableWithAlias::satisfies(const DatabaseAndTableWithAlias & db_table, bool table_may_be_an_alias) const { /// table.*, alias.* or database.table.* diff --git a/src/Interpreters/DatabaseAndTableWithAlias.h b/src/Interpreters/DatabaseAndTableWithAlias.h index 92d6d40b455..adb0829a54e 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.h +++ b/src/Interpreters/DatabaseAndTableWithAlias.h @@ -36,7 +36,7 @@ struct DatabaseAndTableWithAlias String getQualifiedNamePrefix(bool with_dot = true) const; /// Check if it satisfies another db_table name. @note opterion is not symmetric. - bool satisfies(const DatabaseAndTableWithAlias & table, bool table_may_be_an_alias); + bool satisfies(const DatabaseAndTableWithAlias & table, bool table_may_be_an_alias) const; /// Exactly the same table name bool same(const DatabaseAndTableWithAlias & db_table) const @@ -49,61 +49,51 @@ struct TableWithColumnNames { DatabaseAndTableWithAlias table; Names columns; - Names hidden_columns; + Names hidden_columns; /// Not general columns like MATERIALIZED and ALIAS. They are omitted in * and t.* results. TableWithColumnNames(const DatabaseAndTableWithAlias & table_, const Names & columns_) : table(table_) , columns(columns_) - {} + { + columns_set.insert(columns.begin(), columns.end()); + } TableWithColumnNames(const DatabaseAndTableWithAlias table_, Names && columns_, Names && hidden_columns_) : table(table_) , columns(columns_) , hidden_columns(hidden_columns_) - {} - - bool hasColumn(const String & name) const { - if (columns_set.empty()) - { - columns_set.insert(columns.begin(), columns.end()); - columns_set.insert(hidden_columns.begin(), hidden_columns.end()); - } - - return columns_set.count(name); + columns_set.insert(columns.begin(), columns.end()); + columns_set.insert(hidden_columns.begin(), hidden_columns.end()); } + bool hasColumn(const String & name) const { return columns_set.count(name); } + private: - mutable NameSet columns_set; + NameSet columns_set; }; struct TableWithColumnNamesAndTypes { DatabaseAndTableWithAlias table; NamesAndTypesList columns; - NamesAndTypesList hidden_columns; + NamesAndTypesList hidden_columns; /// Not general columns like MATERIALIZED and ALIAS. They are omitted in * and t.* results. TableWithColumnNamesAndTypes(const DatabaseAndTableWithAlias & table_, const NamesAndTypesList & columns_) : table(table_) , columns(columns_) - {} - - bool hasColumn(const String & name) const { - if (names.empty()) - { - for (auto & col : columns) - names.insert(col.name); - for (auto & col : hidden_columns) - names.insert(col.name); - } - - return names.count(name); + for (auto & col : columns) + names.insert(col.name); } + bool hasColumn(const String & name) const { return names.count(name); } + void addHiddenColumns(const NamesAndTypesList & addition) { hidden_columns.insert(hidden_columns.end(), addition.begin(), addition.end()); + for (auto & col : addition) + names.insert(col.name); } TableWithColumnNames removeTypes() const @@ -122,7 +112,7 @@ struct TableWithColumnNamesAndTypes } private: - mutable NameSet names; + NameSet names; }; std::vector getDatabaseAndTables(const ASTSelectQuery & select_query, const String & current_database); diff --git a/src/Interpreters/DictionaryReader.cpp b/src/Interpreters/DictionaryReader.cpp new file mode 100644 index 00000000000..e996be65095 --- /dev/null +++ b/src/Interpreters/DictionaryReader.cpp @@ -0,0 +1,167 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; + extern const int TYPE_MISMATCH; +} + + +DictionaryReader::FunctionWrapper::FunctionWrapper(FunctionOverloadResolverPtr resolver, const ColumnsWithTypeAndName & arguments, + Block & block, const ColumnNumbers & arg_positions_, const String & column_name, + TypeIndex expected_type) + : arg_positions(arg_positions_) + , result_pos(block.columns()) +{ + FunctionBasePtr prepared_function = resolver->build(arguments); + + ColumnWithTypeAndName result; + result.name = "get_" + column_name; + result.type = prepared_function->getReturnType(); + if (result.type->getTypeId() != expected_type) + throw Exception("Type mismatch in dictionary reader for: " + column_name, ErrorCodes::TYPE_MISMATCH); + block.insert(result); + + function = prepared_function->prepare(block, arg_positions, result_pos); +} + +static constexpr const size_t key_size = 1; + +DictionaryReader::DictionaryReader(const String & dictionary_name, const Names & src_column_names, const NamesAndTypesList & result_columns, + const Context & context) + : result_header(makeResultBlock(result_columns)) + , key_position(key_size + result_header.columns()) +{ + if (src_column_names.size() != result_columns.size()) + throw Exception("Columns number mismatch in dictionary reader", ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH); + + ColumnWithTypeAndName dict_name; + ColumnWithTypeAndName key; + ColumnWithTypeAndName column_name; + + { + dict_name.name = "dict"; + dict_name.type = std::make_shared(); + dict_name.column = dict_name.type->createColumnConst(1, dictionary_name); + + /// TODO: composite key (key_size > 1) + key.name = "key"; + key.type = std::make_shared(); + + column_name.name = "column"; + column_name.type = std::make_shared(); + } + + /// dictHas('dict_name', id) + ColumnsWithTypeAndName arguments_has; + arguments_has.push_back(dict_name); + arguments_has.push_back(key); + + /// dictGet('dict_name', 'attr_name', id) + ColumnsWithTypeAndName arguments_get; + arguments_get.push_back(dict_name); + arguments_get.push_back(column_name); + arguments_get.push_back(key); + + sample_block.insert(dict_name); + + for (const auto & columns_name : src_column_names) + { + ColumnWithTypeAndName name; + name.name = "col_" + columns_name; + name.type = std::make_shared(); + name.column = name.type->createColumnConst(1, columns_name); + + sample_block.insert(name); + } + + sample_block.insert(key); + + ColumnNumbers positions_has{0, key_position}; + function_has = std::make_unique(FunctionFactory::instance().get("dictHas", context), + arguments_has, sample_block, positions_has, "has", DataTypeUInt8().getTypeId()); + functions_get.reserve(result_header.columns()); + + for (size_t i = 0; i < result_header.columns(); ++i) + { + size_t column_name_pos = key_size + i; + auto & column = result_header.getByPosition(i); + arguments_get[1].column = DataTypeString().createColumnConst(1, src_column_names[i]); + ColumnNumbers positions_get{0, column_name_pos, key_position}; + functions_get.emplace_back( + FunctionWrapper(FunctionFactory::instance().get("dictGet", context), + arguments_get, sample_block, positions_get, column.name, column.type->getTypeId())); + } +} + +void DictionaryReader::readKeys(const IColumn & keys, Block & out_block, ColumnVector::Container & found, + std::vector & positions) const +{ + Block working_block = sample_block; + size_t has_position = key_position + 1; + size_t size = keys.size(); + + /// set keys for dictHas() + ColumnWithTypeAndName & key_column = working_block.getByPosition(key_position); + key_column.column = keys.cloneResized(size); /// just a copy we cannot avoid + + /// calculate and extract dictHas() + function_has->execute(working_block, size); + ColumnWithTypeAndName & has_column = working_block.getByPosition(has_position); + auto mutable_has = (*std::move(has_column.column)).mutate(); + found.swap(typeid_cast &>(*mutable_has).getData()); + has_column.column = nullptr; + + /// set mapping form source keys to resulting rows in output block + positions.clear(); + positions.resize(size, 0); + size_t pos = 0; + for (size_t i = 0; i < size; ++i) + if (found[i]) + positions[i] = pos++; + + /// set keys for dictGet(): remove not found keys + key_column.column = key_column.column->filter(found, -1); + size_t rows = key_column.column->size(); + + /// calculate dictGet() + for (const auto & func : functions_get) + func.execute(working_block, rows); + + /// make result: copy header block with correct names and move data columns + out_block = result_header.cloneEmpty(); + size_t first_get_position = has_position + 1; + for (size_t i = 0; i < out_block.columns(); ++i) + { + auto & src_column = working_block.getByPosition(first_get_position + i); + auto & dst_column = out_block.getByPosition(i); + dst_column.column = src_column.column; + src_column.column = nullptr; + } +} + +Block DictionaryReader::makeResultBlock(const NamesAndTypesList & names) +{ + Block block; + for (const auto & nm : names) + { + ColumnWithTypeAndName column{nullptr, nm.type, nm.name}; + if (column.type->isNullable()) + column.type = typeid_cast(*column.type).getNestedType(); + block.insert(std::move(column)); + } + return block; +} + +} diff --git a/src/Interpreters/DictionaryReader.h b/src/Interpreters/DictionaryReader.h new file mode 100644 index 00000000000..92e4924ae80 --- /dev/null +++ b/src/Interpreters/DictionaryReader.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class Context; + +/// Read block of required columns from Dictionary by UInt64 key column. Rename columns if needed. +/// Current implementation uses dictHas() + N * dictGet() functions. +class DictionaryReader +{ +public: + struct FunctionWrapper + { + ExecutableFunctionPtr function; + ColumnNumbers arg_positions; + size_t result_pos = 0; + + FunctionWrapper(FunctionOverloadResolverPtr resolver, const ColumnsWithTypeAndName & arguments, Block & block, + const ColumnNumbers & arg_positions_, const String & column_name, TypeIndex expected_type); + + void execute(Block & block, size_t rows) const + { + function->execute(block, arg_positions, result_pos, rows, false); + } + }; + + DictionaryReader(const String & dictionary_name, const Names & src_column_names, const NamesAndTypesList & result_columns, + const Context & context); + void readKeys(const IColumn & keys, Block & out_block, ColumnVector::Container & found, std::vector & positions) const; + +private: + Block result_header; + Block sample_block; /// dictionary name, column names, key, dictHas() result, dictGet() results + size_t key_position; + std::unique_ptr function_has; + std::vector functions_get; + + static Block makeResultBlock(const NamesAndTypesList & names); +}; + +} diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 6c5b3ed6425..5009db420e6 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -64,7 +64,7 @@ void ExecuteScalarSubqueriesMatcher::visit(ASTPtr & ast, Data & data) /// blacklist them here. static bool worthConvertingToLiteral(const Block & scalar) { - auto scalar_type_name = scalar.safeGetByPosition(0).type->getFamilyName(); + const auto * scalar_type_name = scalar.safeGetByPosition(0).type->getFamilyName(); std::set useless_literal_types = {"Array", "Tuple", "AggregateFunction", "Function", "Set", "LowCardinality"}; return !useless_literal_types.count(scalar_type_name); } diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 435e493ffa9..cb65ec35b9e 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -1,4 +1,3 @@ -#include "config_core.h" #include #include #include @@ -15,6 +14,10 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + namespace ProfileEvents { @@ -1052,7 +1055,7 @@ bool ExpressionActions::resultIsAlwaysEmpty() const { /// Check that has join which returns empty result. - for (auto & action : actions) + for (const auto & action : actions) { if (action.type == action.JOIN && action.join && action.join->alwaysReturnsEmptySet()) return true; @@ -1069,7 +1072,7 @@ bool ExpressionActions::checkColumnIsAlwaysFalse(const String & column_name) con for (auto it = actions.rbegin(); it != actions.rend(); ++it) { - auto & action = *it; + const auto & action = *it; if (action.type == action.APPLY_FUNCTION && action.function_base) { auto name = action.function_base->getName(); @@ -1085,12 +1088,12 @@ bool ExpressionActions::checkColumnIsAlwaysFalse(const String & column_name) con if (!set_to_check.empty()) { - for (auto & action : actions) + for (const auto & action : actions) { if (action.type == action.ADD_COLUMN && action.result_name == set_to_check) { // Constant ColumnSet cannot be empty, so we only need to check non-constant ones. - if (auto * column_set = checkAndGetColumn(action.added_column.get())) + if (const auto * column_set = checkAndGetColumn(action.added_column.get())) { if (column_set->getData()->isCreated() && column_set->getData()->getTotalRowCount() == 0) return true; diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 0c3027dfbab..5a29eaaab9e 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -7,12 +7,15 @@ #include #include #include -#include "config_core.h" #include #include #include #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + namespace DB { diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 6494918c532..0cbfb5c6ac5 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -31,17 +31,20 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include #include #include @@ -85,7 +88,7 @@ bool allowEarlyConstantFolding(const ExpressionActions & actions, const Settings if (!settings.enable_early_constant_folding) return false; - for (auto & action : actions.getActions()) + for (const auto & action : actions.getActions()) { if (action.type == action.APPLY_FUNCTION && action.function_base) { @@ -291,7 +294,7 @@ void SelectQueryExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr auto interpreter_subquery = interpretSubquery(subquery_or_table_name, context, {}, query_options); BlockIO res = interpreter_subquery->execute(); - SetPtr set = std::make_shared(settings.size_limits_for_set, true); + SetPtr set = std::make_shared(settings.size_limits_for_set, true, context.getSettingsRef().transform_null_in); set->setHeader(res.in->getHeader()); res.in->readPrefix(); @@ -502,25 +505,11 @@ bool SelectQueryExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, b return true; } -static JoinPtr tryGetStorageJoin(const ASTTablesInSelectQueryElement & join_element, std::shared_ptr analyzed_join, - const Context & context) +static JoinPtr tryGetStorageJoin(std::shared_ptr analyzed_join) { - const auto & table_to_join = join_element.table_expression->as(); - - /// TODO This syntax does not support specifying a database name. - if (table_to_join.database_and_table_name) - { - auto table_id = context.resolveStorageID(table_to_join.database_and_table_name); - StoragePtr table = DatabaseCatalog::instance().tryGetTable(table_id); - - if (table) - { - auto * storage_join = dynamic_cast(table.get()); - if (storage_join) - return storage_join->getJoin(analyzed_join); - } - } - + if (auto * table = analyzed_join->joined_storage.get()) + if (auto * storage_join = dynamic_cast(table)) + return storage_join->getJoin(analyzed_join); return {}; } @@ -531,10 +520,44 @@ static ExpressionActionsPtr createJoinedBlockActions(const Context & context, co return ExpressionAnalyzer(expression_list, syntax_result, context).getActions(true, false); } -static std::shared_ptr makeJoin(std::shared_ptr analyzed_join, const Block & sample_block) +static bool allowDictJoin(StoragePtr joined_storage, const Context & context, String & dict_name, String & key_name) +{ + const auto * dict = dynamic_cast(joined_storage.get()); + if (!dict) + return false; + + dict_name = dict->dictionaryName(); + auto dictionary = context.getExternalDictionariesLoader().getDictionary(dict_name); + if (!dictionary) + return false; + + const DictionaryStructure & structure = dictionary->getStructure(); + if (structure.id) + { + key_name = structure.id->name; + return true; + } + return false; +} + +static std::shared_ptr makeJoin(std::shared_ptr analyzed_join, const Block & sample_block, const Context & context) { bool allow_merge_join = analyzed_join->allowMergeJoin(); + /// HashJoin with Dictionary optimisation + String dict_name; + String key_name; + if (analyzed_join->joined_storage && allowDictJoin(analyzed_join->joined_storage, context, dict_name, key_name)) + { + Names original_names; + NamesAndTypesList result_columns; + if (analyzed_join->allowDictJoin(key_name, sample_block, original_names, result_columns)) + { + analyzed_join->dictionary_reader = std::make_shared(dict_name, original_names, result_columns, context); + return std::make_shared(analyzed_join, sample_block); + } + } + if (analyzed_join->forceHashJoin() || (analyzed_join->preferMergeJoin() && !allow_merge_join)) return std::make_shared(analyzed_join, sample_block); else if (analyzed_join->forceMergeJoin() || (analyzed_join->preferMergeJoin() && allow_merge_join)) @@ -550,48 +573,49 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin(const ASTTablesInSelectQuer SubqueryForSet & subquery_for_join = subqueries_for_sets[join_subquery_id]; - /// Special case - if table name is specified on the right of JOIN, then the table has the type Join (the previously prepared mapping). + /// Use StorageJoin if any. if (!subquery_for_join.join) - subquery_for_join.join = tryGetStorageJoin(join_element, syntax->analyzed_join, context); + subquery_for_join.join = tryGetStorageJoin(syntax->analyzed_join); if (!subquery_for_join.join) { /// Actions which need to be calculated on joined block. ExpressionActionsPtr joined_block_actions = createJoinedBlockActions(context, analyzedJoin()); + Names original_right_columns; if (!subquery_for_join.source) { - NamesWithAliases required_columns_with_aliases = - analyzedJoin().getRequiredColumns(joined_block_actions->getSampleBlock(), joined_block_actions->getRequiredColumns()); - makeSubqueryForJoin(join_element, std::move(required_columns_with_aliases), subquery_for_join); + NamesWithAliases required_columns_with_aliases = analyzedJoin().getRequiredColumns( + joined_block_actions->getSampleBlock(), joined_block_actions->getRequiredColumns()); + for (auto & pr : required_columns_with_aliases) + original_right_columns.push_back(pr.first); + + /** For GLOBAL JOINs (in the case, for example, of the push method for executing GLOBAL subqueries), the following occurs + * - in the addExternalStorage function, the JOIN (SELECT ...) subquery is replaced with JOIN _data1, + * in the subquery_for_set object this subquery is exposed as source and the temporary table _data1 as the `table`. + * - this function shows the expression JOIN _data1. + */ + auto interpreter = interpretSubquery(join_element.table_expression, context, original_right_columns, query_options); + + subquery_for_join.makeSource(interpreter, std::move(required_columns_with_aliases)); } /// TODO You do not need to set this up when JOIN is only needed on remote servers. subquery_for_join.setJoinActions(joined_block_actions); /// changes subquery_for_join.sample_block inside - subquery_for_join.join = makeJoin(syntax->analyzed_join, subquery_for_join.sample_block); + subquery_for_join.join = makeJoin(syntax->analyzed_join, subquery_for_join.sample_block, context); + + /// Do not make subquery for join over dictionary. + if (syntax->analyzed_join->dictionary_reader) + { + JoinPtr join = subquery_for_join.join; + subqueries_for_sets.erase(join_subquery_id); + return join; + } } return subquery_for_join.join; } -void SelectQueryExpressionAnalyzer::makeSubqueryForJoin(const ASTTablesInSelectQueryElement & join_element, - NamesWithAliases && required_columns_with_aliases, - SubqueryForSet & subquery_for_set) const -{ - /** For GLOBAL JOINs (in the case, for example, of the push method for executing GLOBAL subqueries), the following occurs - * - in the addExternalStorage function, the JOIN (SELECT ...) subquery is replaced with JOIN _data1, - * in the subquery_for_set object this subquery is exposed as source and the temporary table _data1 as the `table`. - * - this function shows the expression JOIN _data1. - */ - Names original_columns; - for (auto & pr : required_columns_with_aliases) - original_columns.push_back(pr.first); - - auto interpreter = interpretSubquery(join_element.table_expression, context, original_columns, query_options); - - subquery_for_set.makeSource(interpreter, std::move(required_columns_with_aliases)); -} - bool SelectQueryExpressionAnalyzer::appendPrewhere( ExpressionActionsChain & chain, bool only_types, const Names & additional_required_columns) { @@ -854,7 +878,34 @@ void SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActionsChain & String result_name = ast->getAliasOrColumnName(); if (required_result_columns.empty() || required_result_columns.count(result_name)) { - result_columns.emplace_back(ast->getColumnName(), result_name); + std::string source_name = ast->getColumnName(); + + /* + * For temporary columns created by ExpressionAnalyzer for literals, + * use the correct source column. Using the default display name + * returned by getColumnName is not enough, and we have to use the + * column id set by EA. In principle, this logic applies to all kinds + * of columns, not only literals. Literals are especially problematic + * for two reasons: + * 1) confusing different literal columns leads to weird side + * effects (see 01101_literal_columns_clash); + * 2) the disambiguation mechanism in SyntaxAnalyzer, that, among + * other things, creates unique aliases for columns with same + * names from different tables, is applied before these temporary + * columns are created by ExpressionAnalyzer. + * Similar problems should also manifest for function columns, which + * are likewise created at a later stage by EA. + * In general, we need to have explicit separation between display + * names and identifiers for columns. This code is a workaround for + * a particular subclass of problems, and not a proper solution. + */ + if (const auto * as_literal = ast->as()) + { + source_name = as_literal->unique_column_name; + assert(!source_name.empty()); + } + + result_columns.emplace_back(source_name, result_name); step.required_output.push_back(result_columns.back().second); } } @@ -963,13 +1014,15 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( auto finalize_chain = [&](ExpressionActionsChain & chain) { + chain.finalize(); + if (!finalized) { - chain.finalize(); finalize(chain, context, where_step_num); - chain.clear(); + finalized = true; } - finalized = true; + + chain.clear(); }; { @@ -1133,7 +1186,7 @@ void ExpressionAnalysisResult::finalize(const ExpressionActionsChain & chain, co remove_where_filter = chain.steps.at(where_step_num).can_remove_required_output.at(0); } -void ExpressionAnalysisResult::removeExtraColumns() +void ExpressionAnalysisResult::removeExtraColumns() const { if (hasFilter()) filter_info->actions->prependProjectInput(); @@ -1143,7 +1196,7 @@ void ExpressionAnalysisResult::removeExtraColumns() before_having->prependProjectInput(); } -void ExpressionAnalysisResult::checkActions() +void ExpressionAnalysisResult::checkActions() const { /// Check that PREWHERE doesn't contain unusual actions. Unusual actions are that can change number of rows. if (hasPrewhere()) diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 4322a897378..72a4ead6a14 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -213,8 +213,8 @@ struct ExpressionAnalysisResult bool hasHaving() const { return before_having.get(); } bool hasLimitBy() const { return before_limit_by.get(); } - void removeExtraColumns(); - void checkActions(); + void removeExtraColumns() const; + void checkActions() const; void finalize(const ExpressionActionsChain & chain, const Context & context, size_t where_step_num); }; @@ -276,8 +276,6 @@ private: SetPtr isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name); JoinPtr makeTableJoin(const ASTTablesInSelectQueryElement & join_element); - void makeSubqueryForJoin(const ASTTablesInSelectQueryElement & join_element, NamesWithAliases && required_columns_with_aliases, - SubqueryForSet & subquery_for_set) const; const ASTSelectQuery * getAggregatingQuery() const; diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 1f6ac0a9926..dbbad2e8344 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -94,7 +94,7 @@ static ColumnData getColumnData(const IColumn * column) const bool is_const = isColumnConst(*column); if (is_const) column = &reinterpret_cast(column)->getDataColumn(); - if (auto * nullable = typeid_cast(column)) + if (const auto * nullable = typeid_cast(column)) { result.null = nullable->getNullMapColumn().getRawData().data; column = &nullable->getNestedColumn(); @@ -117,7 +117,7 @@ static llvm::TargetMachine * getNativeMachine() std::string error; auto cpu = llvm::sys::getHostCPUName(); auto triple = llvm::sys::getProcessTriple(); - auto target = llvm::TargetRegistry::lookupTarget(triple, error); + const auto * target = llvm::TargetRegistry::lookupTarget(triple, error); if (!target) throw Exception("Could not initialize native target: " + error, ErrorCodes::CANNOT_COMPILE_CODE); llvm::SubtargetFeatures features; @@ -290,7 +290,7 @@ public: std::vector columns(arguments.size() + 1); for (size_t i = 0; i < arguments.size(); ++i) { - auto * column = block.getByPosition(arguments[i]).column.get(); + const auto * column = block.getByPosition(arguments[i]).column.get(); if (!column) throw Exception("Column " + block.getByPosition(arguments[i]).name + " is missing", ErrorCodes::LOGICAL_ERROR); columns[i] = getColumnData(column); @@ -307,13 +307,13 @@ static void compileFunctionToLLVMByteCode(LLVMContext & context, const IFunction { ProfileEvents::increment(ProfileEvents::CompileFunction); - auto & arg_types = f.getArgumentTypes(); + const auto & arg_types = f.getArgumentTypes(); auto & b = context.builder; auto * size_type = b.getIntNTy(sizeof(size_t) * 8); auto * data_type = llvm::StructType::get(b.getInt8PtrTy(), b.getInt8PtrTy(), size_type); auto * func_type = llvm::FunctionType::get(b.getVoidTy(), { size_type, data_type->getPointerTo() }, /*isVarArg=*/false); auto * func = llvm::Function::Create(func_type, llvm::Function::ExternalLinkage, f.getName(), context.module.get()); - auto args = func->args().begin(); + auto * args = func->args().begin(); llvm::Value * counter_arg = &*args++; llvm::Value * columns_arg = &*args++; @@ -322,7 +322,7 @@ static void compileFunctionToLLVMByteCode(LLVMContext & context, const IFunction std::vector columns(arg_types.size() + 1); for (size_t i = 0; i <= arg_types.size(); ++i) { - auto & type = i == arg_types.size() ? f.getReturnType() : arg_types[i]; + const auto & type = i == arg_types.size() ? f.getReturnType() : arg_types[i]; auto * data = b.CreateLoad(b.CreateConstInBoundsGEP1_32(data_type, columns_arg, i)); columns[i].data_init = b.CreatePointerCast(b.CreateExtractValue(data, {0}), toNativeType(b, removeNullable(type))->getPointerTo()); columns[i].null_init = type->isNullable() ? b.CreateExtractValue(data, {1}) : nullptr; @@ -389,9 +389,9 @@ static llvm::Constant * getNativeValue(llvm::Type * type, const IColumn & column { if (!type || column.size() <= i) return nullptr; - if (auto * constant = typeid_cast(&column)) + if (const auto * constant = typeid_cast(&column)) return getNativeValue(type, constant->getDataColumn(), 0); - if (auto * nullable = typeid_cast(&column)) + if (const auto * nullable = typeid_cast(&column)) { auto * value = getNativeValue(type->getContainedType(0), nullable->getNestedColumn(), i); auto * is_null = llvm::ConstantInt::get(type->getContainedType(1), nullable->isNullAt(i)); @@ -510,7 +510,7 @@ bool LLVMFunction::isSuitableForConstantFolding() const return true; } -bool LLVMFunction::isInjective(const Block & sample_block) +bool LLVMFunction::isInjective(const Block & sample_block) const { for (const auto & f : originals) if (!f->isInjective(sample_block)) diff --git a/src/Interpreters/ExpressionJIT.h b/src/Interpreters/ExpressionJIT.h index 995fb35e52c..2053bbd222a 100644 --- a/src/Interpreters/ExpressionJIT.h +++ b/src/Interpreters/ExpressionJIT.h @@ -1,13 +1,15 @@ #pragma once -#include "config_core.h" -#if USE_EMBEDDED_COMPILER +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif -#include -#include -#include -#include -#include +#if USE_EMBEDDED_COMPILER +# include +# include +# include +# include +# include namespace DB @@ -51,7 +53,7 @@ public: bool isSuitableForConstantFolding() const override; - bool isInjective(const Block & sample_block) override; + bool isInjective(const Block & sample_block) const override; bool hasInformationAboutMonotonicity() const override; diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 02388028c73..4e958a8c12b 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -1,6 +1,10 @@ #include #include -#include "config_core.h" +#include + +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif #if USE_MYSQL # include @@ -30,11 +34,24 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( return DictionaryFactory::instance().create(name, config, key_in_config, context, dictionary_from_database); } + +DictionaryStructure +ExternalDictionariesLoader::getDictionaryStructure(const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config) +{ + return {config, key_in_config + ".structure"}; +} + +DictionaryStructure ExternalDictionariesLoader::getDictionaryStructure(const ObjectConfig & config) +{ + return getDictionaryStructure(*config.config, config.key_in_config); +} + + void ExternalDictionariesLoader::resetAll() { - #if USE_MYSQL - mysqlxx::PoolFactory::instance().reset(); - #endif +#if USE_MYSQL + mysqlxx::PoolFactory::instance().reset(); +#endif } } diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index 68913ffa166..e69046706a3 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -28,6 +28,9 @@ public: return std::static_pointer_cast(tryLoad(name)); } + static DictionaryStructure getDictionaryStructure(const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config = "dictionary"); + static DictionaryStructure getDictionaryStructure(const ObjectConfig & config); + static void resetAll(); protected: diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 893d9aa61f9..ccea5262213 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -94,15 +94,6 @@ namespace }; } -struct ExternalLoader::ObjectConfig -{ - Poco::AutoPtr config; - String key_in_config; - String repository_name; - bool from_temp_repository = false; - String path; -}; - /** Reads configurations from configuration repository and parses it. */ @@ -141,7 +132,7 @@ public: settings = settings_; } - using ObjectConfigsPtr = std::shared_ptr>; + using ObjectConfigsPtr = std::shared_ptr>>; /// Reads all repositories. ObjectConfigsPtr read() @@ -176,8 +167,9 @@ private: struct FileInfo { Poco::Timestamp last_update_time = 0; - std::vector> objects; // Parsed contents of the file. bool in_use = true; // Whether the `FileInfo` should be destroyed because the correspondent file is deleted. + Poco::AutoPtr file_contents; // Parsed contents of the file. + std::unordered_map objects; }; struct RepositoryInfo @@ -280,14 +272,15 @@ private: } LOG_TRACE(log, "Loading config file '" << path << "'."); - auto file_contents = repository.load(path); + file_info.file_contents = repository.load(path); + auto & file_contents = *file_info.file_contents; /// get all objects' definitions Poco::Util::AbstractConfiguration::Keys keys; - file_contents->keys(keys); + file_contents.keys(keys); /// for each object defined in repositories - std::vector> object_configs_from_file; + std::unordered_map objects; for (const auto & key : keys) { if (!startsWith(key, settings.external_config)) @@ -297,7 +290,7 @@ private: continue; } - String object_name = file_contents->getString(key + "." + settings.external_name); + String object_name = file_contents.getString(key + "." + settings.external_name); if (object_name.empty()) { LOG_WARNING(log, path << ": node '" << key << "' defines " << type_name << " with an empty name. It's not allowed"); @@ -306,14 +299,14 @@ private: String database; if (!settings.external_database.empty()) - database = file_contents->getString(key + "." + settings.external_database, ""); + database = file_contents.getString(key + "." + settings.external_database, ""); if (!database.empty()) object_name = database + "." + object_name; - object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}, {}}); + objects.emplace(object_name, key); } - file_info.objects = std::move(object_configs_from_file); + file_info.objects = std::move(objects); file_info.last_update_time = update_time_from_repository; file_info.in_use = true; return true; @@ -333,33 +326,36 @@ private: need_collect_object_configs = false; // Generate new result. - auto new_configs = std::make_shared>(); + auto new_configs = std::make_shared>>(); for (const auto & [repository, repository_info] : repositories) { for (const auto & [path, file_info] : repository_info.files) { - for (const auto & [object_name, object_config] : file_info.objects) + for (const auto & [object_name, key_in_config] : file_info.objects) { auto already_added_it = new_configs->find(object_name); if (already_added_it == new_configs->end()) { - auto & new_config = new_configs->emplace(object_name, object_config).first->second; - new_config.from_temp_repository = repository->isTemporary(); - new_config.repository_name = repository->getName(); - new_config.path = path; + auto new_config = std::make_shared(); + new_config->config = file_info.file_contents; + new_config->key_in_config = key_in_config; + new_config->repository_name = repository->getName(); + new_config->from_temp_repository = repository->isTemporary(); + new_config->path = path; + new_configs->emplace(object_name, std::move(new_config)); } else { const auto & already_added = already_added_it->second; - if (!already_added.from_temp_repository && !repository->isTemporary()) + if (!already_added->from_temp_repository && !repository->isTemporary()) { LOG_WARNING( log, type_name << " '" << object_name << "' is found " - << (((path == already_added.path) && (repository->getName() == already_added.repository_name)) + << (((path == already_added->path) && (repository->getName() == already_added->repository_name)) ? ("twice in the same file '" + path + "'") - : ("both in file '" + already_added.path + "' and '" + path + "'"))); + : ("both in file '" + already_added->path + "' and '" + path + "'"))); } } } @@ -440,13 +436,10 @@ public: else { const auto & new_config = new_config_it->second; - bool config_is_same = isSameConfiguration(*info.object_config.config, info.object_config.key_in_config, *new_config.config, new_config.key_in_config); - info.object_config = new_config; + bool config_is_same = isSameConfiguration(*info.config->config, info.config->key_in_config, *new_config->config, new_config->key_in_config); + info.config = new_config; if (!config_is_same) { - /// Configuration has been changed. - info.object_config = new_config; - if (info.triedToLoad()) { /// The object has been tried to load before, so it is currently in use or was in use @@ -531,7 +524,7 @@ public: /// Returns the load result of the object. template - ReturnType getCurrentLoadResult(const String & name) const + ReturnType getLoadResult(const String & name) const { std::lock_guard lock{mutex}; const Info * info = getInfo(name); @@ -543,13 +536,13 @@ public: /// Returns all the load results as a map. /// The function doesn't load anything, it just returns the current load results as is. template - ReturnType getCurrentLoadResults(const FilterByNameFunction & filter) const + ReturnType getLoadResults(const FilterByNameFunction & filter) const { std::lock_guard lock{mutex}; return collectLoadResults(filter); } - size_t getNumberOfCurrentlyLoadedObjects() const + size_t getNumberOfLoadedObjects() const { std::lock_guard lock{mutex}; size_t count = 0; @@ -562,10 +555,10 @@ public: return count; } - bool hasCurrentlyLoadedObjects() const + bool hasLoadedObjects() const { std::lock_guard lock{mutex}; - for (auto & name_info : infos) + for (const auto & name_info : infos) if (name_info.second.loaded()) return true; return false; @@ -575,12 +568,18 @@ public: { std::lock_guard lock{mutex}; Strings names; - for (auto & [name, info] : infos) + for (const auto & [name, info] : infos) if (info.triedToLoad()) names.push_back(name); return names; } + size_t getNumberOfObjects() const + { + std::lock_guard lock{mutex}; + return infos.size(); + } + /// Tries to load a specified object during the timeout. template ReturnType tryLoad(const String & name, Duration timeout) @@ -698,7 +697,7 @@ public: private: struct Info { - Info(const String & name_, const ObjectConfig & object_config_) : name(name_), object_config(object_config_) {} + Info(const String & name_, const std::shared_ptr & config_) : name(name_), config(config_) {} bool loaded() const { return object != nullptr; } bool failed() const { return !object && exception; } @@ -737,8 +736,7 @@ private: result.loading_start_time = loading_start_time; result.last_successful_update_time = last_successful_update_time; result.loading_duration = loadingDuration(); - result.origin = object_config.path; - result.repository_name = object_config.repository_name; + result.config = config; return result; } else @@ -750,7 +748,7 @@ private: String name; LoadablePtr object; - ObjectConfig object_config; + std::shared_ptr config; TimePoint loading_start_time; TimePoint loading_end_time; TimePoint last_successful_update_time; @@ -784,7 +782,7 @@ private: results.reserve(infos.size()); for (const auto & [name, info] : infos) { - if (filter(name)) + if (!filter || filter(name)) { auto result = info.template getLoadResult(); if constexpr (std::is_same_v) @@ -838,7 +836,7 @@ private: bool all_ready = true; for (auto & [name, info] : infos) { - if (!filter(name)) + if (filter && !filter(name)) continue; if (info.state_id >= min_id) @@ -955,7 +953,7 @@ private: previous_version_as_base_for_loading = nullptr; /// Need complete reloading, cannot use the previous version. /// Loading. - auto [new_object, new_exception] = loadSingleObject(name, info->object_config, previous_version_as_base_for_loading); + auto [new_object, new_exception] = loadSingleObject(name, *info->config, previous_version_as_base_for_loading); if (!new_object && !new_exception) throw Exception("No object created and no exception raised for " + type_name, ErrorCodes::LOGICAL_ERROR); @@ -1251,7 +1249,7 @@ private: ExternalLoader::ExternalLoader(const String & type_name_, Logger * log_) : config_files_reader(std::make_unique(type_name_, log_)) , loading_dispatcher(std::make_unique( - std::bind(&ExternalLoader::createObject, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3), + [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, type_name_, log_)) , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher)) @@ -1296,9 +1294,9 @@ void ExternalLoader::enablePeriodicUpdates(bool enable_) periodic_updater->enable(enable_); } -bool ExternalLoader::hasCurrentlyLoadedObjects() const +bool ExternalLoader::hasLoadedObjects() const { - return loading_dispatcher->hasCurrentlyLoadedObjects(); + return loading_dispatcher->hasLoadedObjects(); } ExternalLoader::Status ExternalLoader::getCurrentStatus(const String & name) const @@ -1307,30 +1305,35 @@ ExternalLoader::Status ExternalLoader::getCurrentStatus(const String & name) con } template -ReturnType ExternalLoader::getCurrentLoadResult(const String & name) const +ReturnType ExternalLoader::getLoadResult(const String & name) const { - return loading_dispatcher->getCurrentLoadResult(name); + return loading_dispatcher->getLoadResult(name); } template -ReturnType ExternalLoader::getCurrentLoadResults(const FilterByNameFunction & filter) const +ReturnType ExternalLoader::getLoadResults(const FilterByNameFunction & filter) const { - return loading_dispatcher->getCurrentLoadResults(filter); + return loading_dispatcher->getLoadResults(filter); } -ExternalLoader::Loadables ExternalLoader::getCurrentlyLoadedObjects() const +ExternalLoader::Loadables ExternalLoader::getLoadedObjects() const { - return getCurrentLoadResults(); + return getLoadResults(); } -ExternalLoader::Loadables ExternalLoader::getCurrentlyLoadedObjects(const FilterByNameFunction & filter) const +ExternalLoader::Loadables ExternalLoader::getLoadedObjects(const FilterByNameFunction & filter) const { - return getCurrentLoadResults(filter); + return getLoadResults(filter); } -size_t ExternalLoader::getNumberOfCurrentlyLoadedObjects() const +size_t ExternalLoader::getNumberOfLoadedObjects() const { - return loading_dispatcher->getNumberOfCurrentlyLoadedObjects(); + return loading_dispatcher->getNumberOfLoadedObjects(); +} + +size_t ExternalLoader::getNumberOfObjects() const +{ + return loading_dispatcher->getNumberOfObjects(); } template @@ -1456,10 +1459,10 @@ ExternalLoader::LoadablePtr ExternalLoader::createObject( return create(name, *config.config, config.key_in_config, config.repository_name); } -template ExternalLoader::LoadablePtr ExternalLoader::getCurrentLoadResult(const String &) const; -template ExternalLoader::LoadResult ExternalLoader::getCurrentLoadResult(const String &) const; -template ExternalLoader::Loadables ExternalLoader::getCurrentLoadResults(const FilterByNameFunction &) const; -template ExternalLoader::LoadResults ExternalLoader::getCurrentLoadResults(const FilterByNameFunction &) const; +template ExternalLoader::LoadablePtr ExternalLoader::getLoadResult(const String &) const; +template ExternalLoader::LoadResult ExternalLoader::getLoadResult(const String &) const; +template ExternalLoader::Loadables ExternalLoader::getLoadResults(const FilterByNameFunction &) const; +template ExternalLoader::LoadResults ExternalLoader::getLoadResults(const FilterByNameFunction &) const; template ExternalLoader::LoadablePtr ExternalLoader::tryLoad(const String &, Duration) const; template ExternalLoader::LoadResult ExternalLoader::tryLoad(const String &, Duration) const; diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index a9a94ca615e..bcf01eb6625 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -53,17 +53,25 @@ public: using Duration = std::chrono::milliseconds; using TimePoint = std::chrono::system_clock::time_point; + struct ObjectConfig + { + Poco::AutoPtr config; + String key_in_config; + String repository_name; + bool from_temp_repository = false; + String path; + }; + struct LoadResult { Status status = Status::NOT_EXIST; String name; LoadablePtr object; - String origin; TimePoint loading_start_time; TimePoint last_successful_update_time; Duration loading_duration; std::exception_ptr exception; - std::string repository_name; + std::shared_ptr config; }; using LoadResults = std::vector; @@ -99,26 +107,32 @@ public: /// Returns the result of loading the object. /// The function doesn't load anything, it just returns the current load result as is. template , void>> - ReturnType getCurrentLoadResult(const String & name) const; + ReturnType getLoadResult(const String & name) const; using FilterByNameFunction = std::function; /// Returns all the load results as a map. /// The function doesn't load anything, it just returns the current load results as is. template , void>> - ReturnType getCurrentLoadResults() const { return getCurrentLoadResults(alwaysTrue); } + ReturnType getLoadResults() const { return getLoadResults(FilterByNameFunction{}); } template , void>> - ReturnType getCurrentLoadResults(const FilterByNameFunction & filter) const; + ReturnType getLoadResults(const FilterByNameFunction & filter) const; /// Returns all loaded objects as a map. /// The function doesn't load anything, it just returns the current load results as is. - Loadables getCurrentlyLoadedObjects() const; - Loadables getCurrentlyLoadedObjects(const FilterByNameFunction & filter) const; + Loadables getLoadedObjects() const; + Loadables getLoadedObjects(const FilterByNameFunction & filter) const; /// Returns true if any object was loaded. - bool hasCurrentlyLoadedObjects() const; - size_t getNumberOfCurrentlyLoadedObjects() const; + bool hasLoadedObjects() const; + size_t getNumberOfLoadedObjects() const; + + /// Returns true if there is no object. + bool hasObjects() const { return getNumberOfObjects() == 0; } + + /// Returns number of objects. + size_t getNumberOfObjects() const; static constexpr Duration NO_WAIT = Duration::zero(); static constexpr Duration WAIT = Duration::max(); @@ -139,7 +153,7 @@ public: /// The function does nothing for already loaded objects, it just returns them. /// The function doesn't throw an exception if it's failed to load something. template , void>> - ReturnType tryLoadAll(Duration timeout = WAIT) const { return tryLoad(alwaysTrue, timeout); } + ReturnType tryLoadAll(Duration timeout = WAIT) const { return tryLoad(FilterByNameFunction{}, timeout); } /// Loads a specified object. /// The function does nothing if it's already loaded. @@ -157,7 +171,7 @@ public: /// The function does nothing for already loaded objects, it just returns them. /// The function throws an exception if it's failed to load something. template , void>> - ReturnType loadAll() const { return load(alwaysTrue); } + ReturnType loadAll() const { return load(FilterByNameFunction{}); } /// Loads or reloads a specified object. /// The function reloads the object if it's already loaded. @@ -174,7 +188,7 @@ public: /// Load or reloads all objects. Not recommended to use. /// The function throws an exception if it's failed to load or reload something. template , void>> - ReturnType loadOrReloadAll() const { return loadOrReload(alwaysTrue); } + ReturnType loadOrReloadAll() const { return loadOrReload(FilterByNameFunction{}); } /// Reloads objects by filter which were tried to load before (successfully or not). /// The function throws an exception if it's failed to load or reload something. @@ -197,10 +211,8 @@ private: void checkLoaded(const LoadResult & result, bool check_no_errors) const; void checkLoaded(const LoadResults & results, bool check_no_errors) const; - static bool alwaysTrue(const String &) { return true; } Strings getAllTriedToLoadNames() const; - struct ObjectConfig; LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; class LoadablesConfigReader; diff --git a/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp b/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp index 10f99262da7..7b3d57b192a 100644 --- a/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp @@ -1,30 +1,30 @@ #include -#include -#include #include + namespace DB { - namespace ErrorCodes { extern const int UNKNOWN_DICTIONARY; } + namespace { -String trimDatabaseName(const std::string & loadable_definition_name, const IDatabase & database) -{ - const auto & dbname = database.getDatabaseName(); - if (!startsWith(loadable_definition_name, dbname)) - throw Exception( - "Loadable '" + loadable_definition_name + "' is not from database '" + database.getDatabaseName(), ErrorCodes::UNKNOWN_DICTIONARY); - /// dbname.loadable_name - ///--> remove <--- - return loadable_definition_name.substr(dbname.length() + 1); -} + String trimDatabaseName(const std::string & loadable_definition_name, const IDatabase & database) + { + const auto & dbname = database.getDatabaseName(); + if (!startsWith(loadable_definition_name, dbname)) + throw Exception( + "Loadable '" + loadable_definition_name + "' is not from database '" + database.getDatabaseName(), ErrorCodes::UNKNOWN_DICTIONARY); + /// dbname.loadable_name + ///--> remove <--- + return loadable_definition_name.substr(dbname.length() + 1); + } } + ExternalLoaderDatabaseConfigRepository::ExternalLoaderDatabaseConfigRepository(IDatabase & database_, const Context & context_) : name(database_.getDatabaseName()) , database(database_) @@ -34,8 +34,7 @@ ExternalLoaderDatabaseConfigRepository::ExternalLoaderDatabaseConfigRepository(I LoadablesConfigurationPtr ExternalLoaderDatabaseConfigRepository::load(const std::string & loadable_definition_name) { - String dictname = trimDatabaseName(loadable_definition_name, database); - return getDictionaryConfigurationFromAST(database.getCreateDictionaryQuery(context, dictname)->as()); + return database.getDictionaryConfiguration(trimDatabaseName(loadable_definition_name, database)); } bool ExternalLoaderDatabaseConfigRepository::exists(const std::string & loadable_definition_name) diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index e577219629c..78d98805814 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -166,7 +167,19 @@ private: { if (func.name == "globalIn" || func.name == "globalNotIn") { - data.addExternalStorage(func.arguments->children[1]); + ASTPtr & ast = func.arguments->children[1]; + + /// Literal can use regular IN + if (ast->as()) + { + if (func.name == "globalIn") + func.name = "in"; + else + func.name = "notIn"; + return; + } + + data.addExternalStorage(ast); data.has_global_subqueries = true; } } diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index a3432ebebba..e5b3f9e55f3 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -1,19 +1,25 @@ #include +#include #include #include #include +#include #include #include #include +#include #include #include #include #include #include +#include + +#include #include #include @@ -21,8 +27,6 @@ #include #include #include -#include - namespace DB { @@ -88,7 +92,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, { /// We have to replace values masked by NULLs with defaults. if (column.column) - if (auto * nullable_column = checkAndGetColumn(*column.column)) + if (const auto * nullable_column = checkAndGetColumn(*column.column)) column.column = filterWithBlanks(column.column, nullable_column->getNullMapColumn().getData(), true); JoinCommon::removeColumnNullability(column); @@ -118,7 +122,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, static void changeNullability(MutableColumnPtr & mutable_column) { ColumnPtr column = std::move(mutable_column); - if (auto * nullable = checkAndGetColumn(*column)) + if (const auto * nullable = checkAndGetColumn(*column)) column = nullable->getNestedColumnPtr(); else column = makeNullable(column); @@ -158,7 +162,7 @@ static void changeColumnRepresentation(const ColumnPtr & src_column, ColumnPtr & if (nullable_src && !nullable_dst) { - auto * nullable = checkAndGetColumn(*src_column); + const auto * nullable = checkAndGetColumn(*src_column); if (change_lowcard) dst_column = changeLowCardinality(nullable->getNestedColumnPtr(), dst_column); else @@ -175,7 +179,7 @@ static void changeColumnRepresentation(const ColumnPtr & src_column, ColumnPtr & { if (change_lowcard) { - if (auto * nullable = checkAndGetColumn(*src_column)) + if (const auto * nullable = checkAndGetColumn(*src_column)) { dst_column = makeNullable(changeLowCardinality(nullable->getNestedColumnPtr(), dst_not_null)); assert_cast(*dst_column->assumeMutable()).applyNullMap(nullable->getNullMapColumn()); @@ -282,6 +286,42 @@ static KeyGetter createKeyGetter(const ColumnRawPtrs & key_columns, const Sizes return KeyGetter(key_columns, key_sizes, nullptr); } +class KeyGetterForDict +{ +public: + using Mapped = JoinStuff::MappedOne; + using FindResult = ColumnsHashing::columns_hashing_impl::FindResultImpl; + + KeyGetterForDict(const ColumnRawPtrs & key_columns_, const Sizes &, void *) + : key_columns(key_columns_) + {} + + FindResult findKey(const TableJoin & table_join, size_t row, const Arena &) + { + const DictionaryReader & reader = *table_join.dictionary_reader; + if (!read_result) + { + reader.readKeys(*key_columns[0], read_result, found, positions); + result.block = &read_result; + + if (table_join.forceNullableRight()) + for (auto & column : read_result) + if (table_join.rightBecomeNullable(column.type)) + JoinCommon::convertColumnToNullable(column); + } + + result.row_num = positions[row]; + return FindResult(&result, found[row]); + } + +private: + const ColumnRawPtrs & key_columns; + Block read_result; + Mapped result; + ColumnVector::Container found; + std::vector positions; +}; + template struct KeyGetterForTypeImpl; @@ -351,7 +391,7 @@ size_t HashJoin::getTotalRowCount() const for (const auto & block : data->blocks) res += block.rows(); } - else + else if (data->type != Type::DICT) { joinDispatch(kind, strictness, data->maps, [&](auto, auto, auto & map) { res += map.getTotalRowCount(data->type); }); } @@ -368,7 +408,7 @@ size_t HashJoin::getTotalByteCount() const for (const auto & block : data->blocks) res += block.bytes(); } - else + else if (data->type != Type::DICT) { joinDispatch(kind, strictness, data->maps, [&](auto, auto, auto & map) { res += map.getTotalByteCountImpl(data->type); }); res += data->pool.size(); @@ -400,7 +440,13 @@ void HashJoin::setSampleBlock(const Block & block) if (nullable_right_side) JoinCommon::convertColumnsToNullable(sample_block_with_columns_to_add); - if (strictness == ASTTableJoin::Strictness::Asof) + if (table_join->dictionary_reader) + { + data->type = Type::DICT; + std::get(data->maps).create(Type::DICT); + chooseMethod(key_columns, key_sizes); /// init key_sizes + } + else if (strictness == ASTTableJoin::Strictness::Asof) { if (kind != ASTTableJoin::Kind::Left and kind != ASTTableJoin::Kind::Inner) throw Exception("ASOF only supports LEFT and INNER as base joins", ErrorCodes::NOT_IMPLEMENTED); @@ -526,7 +572,8 @@ namespace switch (type) { case HashJoin::Type::EMPTY: break; - case HashJoin::Type::CROSS: break; /// Do nothing. We have already saved block, and it is enough. + case HashJoin::Type::CROSS: break; /// Do nothing. We have already saved block, and it is enough. + case HashJoin::Type::DICT: break; /// Noone should call it with Type::DICT. #define M(TYPE) \ case HashJoin::Type::TYPE: \ @@ -583,7 +630,7 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample) Block HashJoin::structureRightBlock(const Block & block) const { Block structured_block; - for (auto & sample_column : savedBlockSample().getColumnsWithTypeAndName()) + for (const auto & sample_column : savedBlockSample().getColumnsWithTypeAndName()) { ColumnWithTypeAndName column = block.getByName(sample_column.name); if (sample_column.column->isNullable()) @@ -598,6 +645,13 @@ bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) { if (empty()) throw Exception("Logical error: HashJoin was not initialized", ErrorCodes::LOGICAL_ERROR); + if (overDictionary()) + throw Exception("Logical error: insert into hash-map in HashJoin over dictionary", ErrorCodes::LOGICAL_ERROR); + + /// RowRef::SizeT is uint32_t (not size_t) for hash table Cell memory efficiency. + /// It's possible to split bigger blocks and insert them by parts here. But it would be a dead code. + if (unlikely(source_block.rows() > std::numeric_limits::max())) + throw Exception("Too many rows in right table block for HashJoin: " + toString(source_block.rows()), ErrorCodes::NOT_IMPLEMENTED); /// There's no optimization for right side const columns. Remove constness if any. Block block = materializeBlock(source_block); @@ -681,16 +735,14 @@ public: type_name.reserve(num_columns_to_add); right_indexes.reserve(num_columns_to_add); - for (size_t i = 0; i < num_columns_to_add; ++i) + for (const auto & src_column : block_with_columns_to_add) { - const ColumnWithTypeAndName & src_column = sample_block_with_columns_to_add.safeGetByPosition(i); - - /// Don't insert column if it's in left block or not explicitly required. - if (!block.has(src_column.name) && block_with_columns_to_add.has(src_column.name)) + /// Don't insert column if it's in left block + if (!block.has(src_column.name)) addColumn(src_column); } - for (auto & extra : extras) + for (const auto & extra : extras) addColumn(extra); for (auto & tn : type_name) @@ -932,8 +984,7 @@ IColumn::Filter switchJoinRightColumns(const Maps & maps_, AddedColumns & added_ case HashJoin::Type::TYPE: \ return joinRightColumnsSwitchNullability>::Type>(\ - *maps_.TYPE, added_columns, null_map);\ - break; + *maps_.TYPE, added_columns, null_map); APPLY_FOR_JOIN_VARIANTS(M) #undef M @@ -942,6 +993,20 @@ IColumn::Filter switchJoinRightColumns(const Maps & maps_, AddedColumns & added_ } } +template +IColumn::Filter dictionaryJoinRightColumns(const TableJoin & table_join, AddedColumns & added_columns, const ConstNullMapPtr & null_map) +{ + if constexpr (KIND == ASTTableJoin::Kind::Left && + (STRICTNESS == ASTTableJoin::Strictness::Any || + STRICTNESS == ASTTableJoin::Strictness::Semi || + STRICTNESS == ASTTableJoin::Strictness::Anti)) + { + return joinRightColumnsSwitchNullability(table_join, added_columns, null_map); + } + + throw Exception("Logical error: wrong JOIN combination", ErrorCodes::LOGICAL_ERROR); +} + } /// nameless @@ -1002,7 +1067,9 @@ void HashJoin::joinBlockImpl( bool has_required_right_keys = (required_right_keys.columns() != 0); added_columns.need_filter = need_filter || has_required_right_keys; - IColumn::Filter row_filter = switchJoinRightColumns(maps_, added_columns, data->type, null_map); + IColumn::Filter row_filter = overDictionary() ? + dictionaryJoinRightColumns(*table_join, added_columns, null_map) : + switchJoinRightColumns(maps_, added_columns, data->type, null_map); for (size_t i = 0; i < added_columns.size(); ++i) block.insert(added_columns.moveColumn(i)); @@ -1145,8 +1212,8 @@ void HashJoin::joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed) static void checkTypeOfKey(const Block & block_left, const Block & block_right) { - auto & [c1, left_type_origin, left_name] = block_left.safeGetByPosition(0); - auto & [c2, right_type_origin, right_name] = block_right.safeGetByPosition(0); + const auto & [c1, left_type_origin, left_name] = block_left.safeGetByPosition(0); + const auto & [c2, right_type_origin, right_name] = block_right.safeGetByPosition(0); auto left_type = removeNullable(left_type_origin); auto right_type = removeNullable(right_type_origin); @@ -1158,28 +1225,31 @@ static void checkTypeOfKey(const Block & block_left, const Block & block_right) } -DataTypePtr HashJoin::joinGetReturnType(const String & column_name) const +DataTypePtr HashJoin::joinGetReturnType(const String & column_name, bool or_null) const { std::shared_lock lock(data->rwlock); if (!sample_block_with_columns_to_add.has(column_name)) throw Exception("StorageJoin doesn't contain column " + column_name, ErrorCodes::LOGICAL_ERROR); - return sample_block_with_columns_to_add.getByName(column_name).type; + auto elem = sample_block_with_columns_to_add.getByName(column_name); + if (or_null) + elem.type = makeNullable(elem.type); + return elem.type; } template -void HashJoin::joinGetImpl(Block & block, const String & column_name, const Maps & maps_) const +void HashJoin::joinGetImpl(Block & block, const Block & block_with_columns_to_add, const Maps & maps_) const { joinBlockImpl( - block, {block.getByPosition(0).name}, {sample_block_with_columns_to_add.getByName(column_name)}, maps_); + block, {block.getByPosition(0).name}, block_with_columns_to_add, maps_); } // TODO: support composite key // TODO: return multiple columns as named tuple // TODO: return array of values when strictness == ASTTableJoin::Strictness::All -void HashJoin::joinGet(Block & block, const String & column_name) const +void HashJoin::joinGet(Block & block, const String & column_name, bool or_null) const { std::shared_lock lock(data->rwlock); @@ -1188,10 +1258,15 @@ void HashJoin::joinGet(Block & block, const String & column_name) const checkTypeOfKey(block, right_table_keys); + auto elem = sample_block_with_columns_to_add.getByName(column_name); + if (or_null) + elem.type = makeNullable(elem.type); + elem.column = elem.type->createColumn(); + if ((strictness == ASTTableJoin::Strictness::Any || strictness == ASTTableJoin::Strictness::RightAny) && kind == ASTTableJoin::Kind::Left) { - joinGetImpl(block, column_name, std::get(data->maps)); + joinGetImpl(block, {elem}, std::get(data->maps)); } else throw Exception("joinGet only supports StorageJoin of type Left Any", ErrorCodes::LOGICAL_ERROR); @@ -1205,7 +1280,36 @@ void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) const Names & key_names_left = table_join->keyNamesLeft(); JoinCommon::checkTypesOfKeys(block, key_names_left, right_table_keys, key_names_right); - if (joinDispatch(kind, strictness, data->maps, [&](auto kind_, auto strictness_, auto & map) + if (overDictionary()) + { + using Kind = ASTTableJoin::Kind; + using Strictness = ASTTableJoin::Strictness; + + auto & map = std::get(data->maps); + if (kind == Kind::Left) + { + switch (strictness) + { + case Strictness::Any: + case Strictness::All: + joinBlockImpl(block, key_names_left, sample_block_with_columns_to_add, map); + break; + case Strictness::Semi: + joinBlockImpl(block, key_names_left, sample_block_with_columns_to_add, map); + break; + case Strictness::Anti: + joinBlockImpl(block, key_names_left, sample_block_with_columns_to_add, map); + break; + default: + throw Exception("Logical error: wrong JOIN combination", ErrorCodes::LOGICAL_ERROR); + } + } + else if (kind == Kind::Inner && strictness == Strictness::All) + joinBlockImpl(block, key_names_left, sample_block_with_columns_to_add, map); + else + throw Exception("Logical error: wrong JOIN combination", ErrorCodes::LOGICAL_ERROR); + } + else if (joinDispatch(kind, strictness, data->maps, [&](auto kind_, auto strictness_, auto & map) { joinBlockImpl(block, key_names_left, sample_block_with_columns_to_add, map); })) diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 24ad2b871c9..9d4e0907f66 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -27,6 +27,7 @@ namespace DB { class TableJoin; +class DictionaryReader; namespace JoinStuff { @@ -148,7 +149,8 @@ class HashJoin : public IJoin public: HashJoin(std::shared_ptr table_join_, const Block & right_sample_block, bool any_take_last_row_ = false); - bool empty() { return data->type == Type::EMPTY; } + bool empty() const { return data->type == Type::EMPTY; } + bool overDictionary() const { return data->type == Type::DICT; } /** Add block of data from right hand of JOIN to the map. * Returns false, if some limit was exceeded and you should not insert more data. @@ -161,10 +163,10 @@ public: void joinBlock(Block & block, ExtraBlockPtr & not_processed) override; /// Infer the return type for joinGet function - DataTypePtr joinGetReturnType(const String & column_name) const; + DataTypePtr joinGetReturnType(const String & column_name, bool or_null) const; /// Used by joinGet function that turns StorageJoin into a dictionary - void joinGet(Block & block, const String & column_name) const; + void joinGet(Block & block, const String & column_name, bool or_null) const; /** Keep "totals" (separate part of dataset, see WITH TOTALS) to use later. */ @@ -186,7 +188,7 @@ public: /// Sum size in bytes of all buffers, used for JOIN maps and for all memory pools. size_t getTotalByteCount() const final; - bool alwaysReturnsEmptySet() const final { return isInnerOrRight(getKind()) && data->empty; } + bool alwaysReturnsEmptySet() const final { return isInnerOrRight(getKind()) && data->empty && !overDictionary(); } ASTTableJoin::Kind getKind() const { return kind; } ASTTableJoin::Strictness getStrictness() const { return strictness; } @@ -220,12 +222,12 @@ public: { EMPTY, CROSS, + DICT, #define M(NAME) NAME, APPLY_FOR_JOIN_VARIANTS(M) #undef M }; - /** Different data structures, that are used to perform JOIN. */ template @@ -247,6 +249,7 @@ public: { case Type::EMPTY: break; case Type::CROSS: break; + case Type::DICT: break; #define M(NAME) \ case Type::NAME: NAME = std::make_unique(); break; @@ -261,6 +264,7 @@ public: { case Type::EMPTY: return 0; case Type::CROSS: return 0; + case Type::DICT: return 0; #define M(NAME) \ case Type::NAME: return NAME ? NAME->size() : 0; @@ -277,6 +281,7 @@ public: { case Type::EMPTY: return 0; case Type::CROSS: return 0; + case Type::DICT: return 0; #define M(NAME) \ case Type::NAME: return NAME ? NAME->getBufferSizeInBytes() : 0; @@ -382,7 +387,7 @@ private: void joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed) const; template - void joinGetImpl(Block & block, const String & column_name, const Maps & maps) const; + void joinGetImpl(Block & block, const Block & block_with_columns_to_add, const Maps & maps_) const; static Type chooseMethod(const ColumnRawPtrs & key_columns, Sizes & key_sizes); }; diff --git a/src/Interpreters/InDepthNodeVisitor.h b/src/Interpreters/InDepthNodeVisitor.h index 7bb4f5e4d54..3e0a8e16185 100644 --- a/src/Interpreters/InDepthNodeVisitor.h +++ b/src/Interpreters/InDepthNodeVisitor.h @@ -51,21 +51,23 @@ private: template using ConstInDepthNodeVisitor = InDepthNodeVisitor; -/// Simple matcher for one node type without complex traversal logic. -template +struct NeedChild +{ + using Condition = bool (*)(const ASTPtr & node, const ASTPtr & child); + + static bool all(const ASTPtr &, const ASTPtr &) { return true; } + static bool none(const ASTPtr &, const ASTPtr &) { return false; } +}; + +/// Simple matcher for one node type. Use need_child function for complex traversal logic. +template class OneTypeMatcher { public: using Data = Data_; using TypeToVisit = typename Data::TypeToVisit; - static bool needChildVisit(const ASTPtr & node, const ASTPtr &) - { - if (node && node->as()) - return visit_children; - - return true; - } + static bool needChildVisit(const ASTPtr & node, const ASTPtr & child) { return need_child(node, child); } static void visit(T & ast, Data & data) { @@ -74,7 +76,7 @@ public: } }; -template -using ConstOneTypeMatcher = OneTypeMatcher; +template +using ConstOneTypeMatcher = OneTypeMatcher; } diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 7c6b9678325..c8517defdb7 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -82,7 +82,9 @@ BlockIO InterpreterAlterQuery::execute() if (!mutation_commands.empty()) { - auto table_lock_holder = table->lockStructureForShare(false /* because mutation is executed asyncronously */, context.getCurrentQueryId()); + auto table_lock_holder = table->lockStructureForShare( + false /* because mutation is executed asyncronously */, + context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); MutationsInterpreter(table, mutation_commands, context, false).validate(table_lock_holder); table->mutate(mutation_commands, context); } @@ -109,7 +111,8 @@ BlockIO InterpreterAlterQuery::execute() if (!alter_commands.empty()) { - auto table_lock_holder = table->lockAlterIntention(context.getCurrentQueryId()); + auto table_lock_holder = table->lockAlterIntention( + context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); alter_commands.validate(metadata, context); alter_commands.prepare(metadata); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index b605ce85bc2..2aaf7ec63b9 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include #include @@ -43,6 +45,8 @@ #include #include +#include + #include #include @@ -179,9 +183,9 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) ParserIdentifierWithOptionalParameters storage_p; String type_name = column.type->getName(); - auto pos = type_name.data(); - const auto end = pos + type_name.size(); - column_declaration->type = parseQuery(storage_p, pos, end, "data type", 0); + const char * pos = type_name.data(); + const char * end = pos + type_name.size(); + column_declaration->type = parseQuery(storage_p, pos, end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); columns_list->children.emplace_back(column_declaration); } @@ -205,9 +209,9 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns) ParserIdentifierWithOptionalParameters storage_p; String type_name = column.type->getName(); - auto type_name_pos = type_name.data(); - const auto type_name_end = type_name_pos + type_name.size(); - column_declaration->type = parseQuery(storage_p, type_name_pos, type_name_end, "data type", 0); + const char * type_name_pos = type_name.data(); + const char * type_name_end = type_name_pos + type_name.size(); + column_declaration->type = parseQuery(storage_p, type_name_pos, type_name_end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (column.default_desc.expression) { @@ -224,10 +228,10 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns) { String codec_desc = column.codec->getCodecDesc(); codec_desc = "CODEC(" + codec_desc + ")"; - auto codec_desc_pos = codec_desc.data(); - const auto codec_desc_end = codec_desc_pos + codec_desc.size(); + const char * codec_desc_pos = codec_desc.data(); + const char * codec_desc_end = codec_desc_pos + codec_desc.size(); ParserIdentifierWithParameters codec_p; - column_declaration->codec = parseQuery(codec_p, codec_desc_pos, codec_desc_end, "column codec", 0); + column_declaration->codec = parseQuery(codec_p, codec_desc_pos, codec_desc_end, "column codec", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); } if (column.ttl) @@ -295,7 +299,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres { const auto & final_column_name = col_decl.name; const auto tmp_column_name = final_column_name + "_tmp"; - const auto data_type_ptr = column_names_and_types.back().type.get(); + const auto * data_type_ptr = column_names_and_types.back().type.get(); default_expr_list->children.emplace_back( @@ -403,7 +407,8 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table}); /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table. - as_storage_lock = as_storage->lockStructureForShare(false, context.getCurrentQueryId()); + as_storage_lock = as_storage->lockStructureForShare( + false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); properties.columns = as_storage->getColumns(); /// Secondary indices make sense only for MergeTree family of storage engines. @@ -700,7 +705,11 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) } if (create.attach) - database->attachDictionary(dictionary_name, context); + { + auto config = getDictionaryConfigurationFromAST(create); + auto modification_time = database->getObjectMetadataModificationTime(dictionary_name); + database->attachDictionary(dictionary_name, DictionaryAttachInfo{query_ptr, config, modification_time}); + } else database->createDictionary(context, dictionary_name, query_ptr); diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/InterpreterCreateQuotaQuery.cpp index 13e772965ff..80987993c96 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuotaQuery.cpp @@ -34,7 +34,7 @@ void updateQuotaFromQueryImpl(Quota & quota, const ASTCreateQuotaQuery & query, auto duration = query_limits.duration; auto it = boost::range::find_if(quota_all_limits, [&](const Quota::Limits & x) { return x.duration == duration; }); - if (query_limits.unset_tracking) + if (query_limits.drop) { if (it != quota_all_limits.end()) quota_all_limits.erase(it); @@ -59,6 +59,8 @@ void updateQuotaFromQueryImpl(Quota & quota, const ASTCreateQuotaQuery & query, { if (query_limits.max[resource_type]) quota_limits.max[resource_type] = *query_limits.max[resource_type]; + else + quota_limits.max[resource_type] = Quota::UNLIMITED; } } diff --git a/src/Interpreters/InterpreterCreateUserQuery.cpp b/src/Interpreters/InterpreterCreateUserQuery.cpp index 78c7cc222ae..e4900e5c518 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/InterpreterCreateUserQuery.cpp @@ -68,7 +68,7 @@ namespace BlockIO InterpreterCreateUserQuery::execute() { - auto & query = query_ptr->as(); + const auto & query = query_ptr->as(); auto & access_control = context.getAccessControlManager(); auto access = context.getAccess(); access->checkAccess(query.alter ? AccessType::ALTER_USER : AccessType::CREATE_USER); diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 1353c01ebf6..f9c769a523e 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -89,7 +89,8 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() table = DatabaseCatalog::instance().getTable(table_id); } - auto table_lock = table->lockStructureForShare(false, context.getInitialQueryId()); + auto table_lock = table->lockStructureForShare( + false, context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout); columns = table->getColumns(); } diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index e3f5d467f38..71730c23788 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -93,7 +93,7 @@ BlockIO InterpreterDropQuery::executeToTable( context.checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockExclusively(context.getCurrentQueryId()); + auto table_lock = table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); /// Drop table from memory, don't touch data and metadata database->detachTable(table_name); } @@ -103,7 +103,7 @@ BlockIO InterpreterDropQuery::executeToTable( table->checkTableCanBeDropped(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockExclusively(context.getCurrentQueryId()); + auto table_lock = table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); /// Drop table data, don't touch metadata table->truncate(query_ptr, context, table_lock); } @@ -115,7 +115,7 @@ BlockIO InterpreterDropQuery::executeToTable( table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockExclusively(context.getCurrentQueryId()); + auto table_lock = table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); const std::string metadata_file_without_extension = database->getMetadataPath() + escapeForFileName(table_id.table_name); const auto prev_metadata_name = metadata_file_without_extension + ".sql"; @@ -188,7 +188,7 @@ BlockIO InterpreterDropQuery::executeToDictionary( { /// Drop dictionary from memory, don't touch data and metadata context.checkAccess(AccessType::DROP_DICTIONARY, database_name, dictionary_name); - database->detachDictionary(dictionary_name, context); + database->detachDictionary(dictionary_name); } else if (kind == ASTDropQuery::Kind::Truncate) { @@ -216,7 +216,8 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, if (kind == ASTDropQuery::Kind::Truncate) { /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockExclusively(context.getCurrentQueryId()); + auto table_lock = + table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); /// Drop table data, don't touch metadata table->truncate(query_ptr, context, table_lock); } @@ -225,7 +226,8 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, context_handle.removeExternalTable(table_name); table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockExclusively(context.getCurrentQueryId()); + auto table_lock = + table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); /// Delete table data table->drop(table_lock); table->is_dropped = true; @@ -252,21 +254,26 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS bool drop = kind == ASTDropQuery::Kind::Drop; context.checkAccess(AccessType::DROP_DATABASE, database_name); - /// DETACH or DROP all tables and dictionaries inside database - for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) + if (database->shouldBeEmptyOnDetach()) { - String current_table_name = iterator->name(); - executeToTable(database_name, current_table_name, kind, false, false, false); - } + /// DETACH or DROP all tables and dictionaries inside database. + /// First we should DETACH or DROP dictionaries because StorageDictionary + /// must be detached only by detaching corresponding dictionary. + for (auto iterator = database->getDictionariesIterator(context); iterator->isValid(); iterator->next()) + { + String current_dictionary = iterator->name(); + executeToDictionary(database_name, current_dictionary, kind, false, false, false); + } - for (auto iterator = database->getDictionariesIterator(context); iterator->isValid(); iterator->next()) - { - String current_dictionary = iterator->name(); - executeToDictionary(database_name, current_dictionary, kind, false, false, false); + for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) + { + String current_table_name = iterator->name(); + executeToTable(database_name, current_table_name, kind, false, false, false); + } } /// DETACH or DROP database itself - DatabaseCatalog::instance().detachDatabase(database_name, drop); + DatabaseCatalog::instance().detachDatabase(database_name, drop, database->shouldBeEmptyOnDetach()); } } diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 39b99b10c0d..d1a7568581b 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -67,7 +67,7 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) return DatabaseCatalog::instance().getTable(query.table_id); } -Block InterpreterInsertQuery::getSampleBlock(const ASTInsertQuery & query, const StoragePtr & table) +Block InterpreterInsertQuery::getSampleBlock(const ASTInsertQuery & query, const StoragePtr & table) const { Block table_sample_non_materialized = table->getSampleBlockNonMaterialized(); /// If the query does not include information about columns @@ -109,7 +109,8 @@ BlockIO InterpreterInsertQuery::execute() BlockIO res; StoragePtr table = getTable(query); - auto table_lock = table->lockStructureForShare(true, context.getInitialQueryId()); + auto table_lock = table->lockStructureForShare( + true, context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout); auto query_sample_block = getSampleBlock(query, table); if (!query.table_function) @@ -251,7 +252,7 @@ BlockIO InterpreterInsertQuery::execute() for (auto & in_stream : in_streams) { in_stream = std::make_shared( - context, in_stream, out_streams.at(0)->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); + in_stream, out_streams.at(0)->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); } Block in_header = in_streams.at(0)->getHeader(); diff --git a/src/Interpreters/InterpreterInsertQuery.h b/src/Interpreters/InterpreterInsertQuery.h index cead826c48a..476e86898d7 100644 --- a/src/Interpreters/InterpreterInsertQuery.h +++ b/src/Interpreters/InterpreterInsertQuery.h @@ -33,7 +33,7 @@ public: private: StoragePtr getTable(ASTInsertQuery & query); - Block getSampleBlock(const ASTInsertQuery & query, const StoragePtr & table); + Block getSampleBlock(const ASTInsertQuery & query, const StoragePtr & table) const; ASTPtr query_ptr; const Context & context; diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index b23d88524e1..23f39ab3fc5 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -267,7 +267,7 @@ BlockIO InterpreterKillQueryQuery::execute() else { ParserAlterCommand parser; - auto command_ast = parseQuery(parser, command_col.getDataAt(i).toString(), 0); + auto command_ast = parseQuery(parser, command_col.getDataAt(i).toString(), 0, context.getSettingsRef().max_parser_depth); required_access_rights = InterpreterAlterQuery::getRequiredAccessForCommand(command_ast->as(), table_id.database_name, table_id.table_name); if (!access->isGranted(&Poco::Logger::get("InterpreterKillQueryQuery"), required_access_rights)) { diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 4f54f759510..9a4f4b1b197 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -79,7 +79,8 @@ BlockIO InterpreterRenameQuery::execute() { database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name)); auto from_table = database_catalog.getTable({elem.from_database_name, elem.from_table_name}); - auto from_table_lock = from_table->lockExclusively(context.getCurrentQueryId()); + auto from_table_lock = + from_table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); database_catalog.getDatabase(elem.from_database_name)->renameTable( context, diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c58b0eab71b..2f98addd975 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -72,6 +72,7 @@ #include #include +#include #include #include #include @@ -84,10 +85,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -138,7 +137,7 @@ String InterpreterSelectQuery::generateFilterActions(ExpressionActionsPtr & acti for (const auto & column_str : prerequisite_columns) { ParserExpression expr_parser; - expr_list->children.push_back(parseQuery(expr_parser, column_str, 0)); + expr_list->children.push_back(parseQuery(expr_parser, column_str, 0, context->getSettingsRef().max_parser_depth)); } select_ast->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared()); @@ -255,7 +254,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) { - table_lock = storage->lockStructureForShare(false, context->getInitialQueryId()); + table_lock = storage->lockStructureForShare( + false, context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); table_id = storage->getStorageID(); } @@ -305,12 +305,13 @@ InterpreterSelectQuery::InterpreterSelectQuery( max_streams = settings.max_threads; ASTSelectQuery & query = getSelectQuery(); + std::shared_ptr table_join = joined_tables.makeTableJoin(query); auto analyze = [&] (bool try_move_to_prewhere = true) { syntax_analyzer_result = SyntaxAnalyzer(*context).analyzeSelect( query_ptr, SyntaxAnalyzerResult(source_header.getNamesAndTypesList(), storage), - options, joined_tables.tablesWithColumns(), required_result_column_names); + options, joined_tables.tablesWithColumns(), required_result_column_names, table_join); /// Save scalar sub queries's results in the query context if (context->hasQueryContext()) @@ -551,10 +552,10 @@ Block InterpreterSelectQuery::getSampleBlockImpl(bool try_move_to_prewhere) Block res; - for (auto & key : query_analyzer->aggregationKeys()) + for (const auto & key : query_analyzer->aggregationKeys()) res.insert({nullptr, header.getByName(key.name).type, key.name}); - for (auto & aggregate : query_analyzer->aggregates()) + for (const auto & aggregate : query_analyzer->aggregates()) { size_t arguments_size = aggregate.argument_names.size(); DataTypes argument_types(arguments_size); @@ -686,8 +687,8 @@ static std::pair getLimitLengthAndOffset(const ASTSelectQuery & static UInt64 getLimitForSorting(const ASTSelectQuery & query, const Context & context) { - /// Partial sort can be done if there is LIMIT but no DISTINCT or LIMIT BY. - if (!query.distinct && !query.limitBy() && !query.limit_with_ties) + /// Partial sort can be done if there is LIMIT but no DISTINCT or LIMIT BY, neither ARRAY JOIN. + if (!query.distinct && !query.limitBy() && !query.limit_with_ties && !query.arrayJoinExpressionList()) { auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); return limit_length + limit_offset; @@ -715,7 +716,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS auto & query = getSelectQuery(); const Settings & settings = context->getSettingsRef(); auto & expressions = analysis_result; - auto & subqueries_for_sets = query_analyzer->getSubqueriesForSets(); + const auto & subqueries_for_sets = query_analyzer->getSubqueriesForSets(); bool intermediate_stage = false; if (options.only_analyze) @@ -1550,7 +1551,7 @@ void InterpreterSelectQuery::executeFetchColumns( auto header = stream->getHeader(); auto mode = ConvertingBlockInputStream::MatchColumnsMode::Name; if (!blocksHaveEqualStructure(first_header, header)) - stream = std::make_shared(*context, stream, first_header, mode); + stream = std::make_shared(stream, first_header, mode); } } @@ -1712,7 +1713,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const auto transform_params = std::make_shared(params, final); - pipeline.dropTotalsIfHas(); + /// Forget about current totals and extremes. They will be calculated again after aggregation if needed. + pipeline.dropTotalsAndExtremes(); /// If there are several sources, then we perform parallel aggregation if (pipeline.getNumStreams() > 1) @@ -2541,8 +2543,7 @@ void InterpreterSelectQuery::executeExtremes(QueryPipeline & pipeline) if (!context->getSettingsRef().extremes) return; - auto transform = std::make_shared(pipeline.getHeader()); - pipeline.addExtremesTransform(std::move(transform)); + pipeline.addExtremesTransform(); } @@ -2592,7 +2593,7 @@ void InterpreterSelectQuery::unifyStreams(Pipeline & pipeline, Block header) auto mode = ConvertingBlockInputStream::MatchColumnsMode::Name; if (!blocksHaveEqualStructure(header, stream_header)) - stream = std::make_shared(*context, stream, header, mode); + stream = std::make_shared(stream, header, mode); } } diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 0208af2431f..c50f4a2f7b7 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -212,7 +212,7 @@ private: String generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const; /// Add ConvertingBlockInputStream to specified header. - void unifyStreams(Pipeline & pipeline, Block header); + static void unifyStreams(Pipeline & pipeline, Block header); enum class Modificator { diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index feec18a1af3..9cdb19b1934 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -193,7 +193,7 @@ BlockInputStreams InterpreterSelectWithUnionQuery::executeWithMultipleStreams(Qu if (nested_interpreters.size() > 1) { for (auto & stream : nested_streams) - stream = std::make_shared(*context, stream, result_header,ConvertingBlockInputStream::MatchColumnsMode::Position); + stream = std::make_shared(stream, result_header,ConvertingBlockInputStream::MatchColumnsMode::Position); parent_pipeline.addInterpreterContext(context); } @@ -269,7 +269,7 @@ QueryPipeline InterpreterSelectWithUnionQuery::executeWithProcessors() if (!pipelines.empty()) { auto common_header = getCommonHeaderForUnion(headers); - main_pipeline.unitePipelines(std::move(pipelines), common_header, *context); + main_pipeline.unitePipelines(std::move(pipelines), common_header); } main_pipeline.addInterpreterContext(context); diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp index 2a6f2317a9c..8f085d66c4c 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/InterpreterSetRoleQuery.cpp @@ -50,7 +50,7 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user->granted_roles.contains(id)) + if (!user->granted_roles.count(id)) throw Exception("Role should be granted to set current", ErrorCodes::SET_NON_GRANTED_ROLE); new_current_roles.push_back(id); } @@ -85,7 +85,7 @@ void InterpreterSetRoleQuery::updateUserSetDefaultRoles(User & user, const Exten { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user.granted_roles.contains(id)) + if (!user.granted_roles.count(id)) throw Exception("Role should be granted to set default", ErrorCodes::SET_NON_GRANTED_ROLE); } } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index d2f435106a8..0d3b88facce 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include #include +#include #include #include @@ -36,7 +38,7 @@ namespace ASTPtr getCreateQueryImpl( const User & user, const AccessControlManager * manager /* not used if attach_mode == true */, - bool attach_mode = false) + bool attach_mode) { auto query = std::make_shared(); query->name = user.getName(); @@ -71,7 +73,7 @@ namespace } - ASTPtr getCreateQueryImpl(const Role & role, const AccessControlManager * manager, bool attach_mode = false) + ASTPtr getCreateQueryImpl(const Role & role, const AccessControlManager * manager, bool attach_mode) { auto query = std::make_shared(); query->name = role.getName(); @@ -89,7 +91,7 @@ namespace } - ASTPtr getCreateQueryImpl(const SettingsProfile & profile, const AccessControlManager * manager, bool attach_mode = false) + ASTPtr getCreateQueryImpl(const SettingsProfile & profile, const AccessControlManager * manager, bool attach_mode) { auto query = std::make_shared(); query->name = profile.getName(); @@ -101,6 +103,8 @@ namespace query->settings = profile.elements.toAST(); else query->settings = profile.elements.toASTWithNames(*manager); + if (query->settings) + query->settings->setUseInheritKeyword(true); } if (!profile.to_roles.empty()) @@ -118,7 +122,7 @@ namespace ASTPtr getCreateQueryImpl( const Quota & quota, const AccessControlManager * manager /* not used if attach_mode == true */, - bool attach_mode = false) + bool attach_mode) { auto query = std::make_shared(); query->name = quota.getName(); @@ -133,7 +137,7 @@ namespace create_query_limits.duration = limits.duration; create_query_limits.randomize_interval = limits.randomize_interval; for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - if (limits.max[resource_type]) + if (limits.max[resource_type] != Quota::UNLIMITED) create_query_limits.max[resource_type] = limits.max[resource_type]; query->all_limits.push_back(create_query_limits); } @@ -153,7 +157,7 @@ namespace ASTPtr getCreateQueryImpl( const RowPolicy & policy, const AccessControlManager * manager /* not used if attach_mode == true */, - bool attach_mode = false) + bool attach_mode) { auto query = std::make_shared(); query->name_parts = RowPolicy::FullNameParts{policy.getDatabase(), policy.getTableName(), policy.getName()}; @@ -168,7 +172,7 @@ namespace if (!condition.empty()) { ParserExpression parser; - ASTPtr expr = parseQuery(parser, condition, 0); + ASTPtr expr = parseQuery(parser, condition, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); query->conditions.push_back(std::pair{index, expr}); } } @@ -187,7 +191,7 @@ namespace ASTPtr getCreateQueryImpl( const IAccessEntity & entity, const AccessControlManager * manager /* not used if attach_mode == true */, - bool attach_mode = false) + bool attach_mode) { if (const User * user = typeid_cast(&entity)) return getCreateQueryImpl(*user, manager, attach_mode); @@ -261,24 +265,24 @@ ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(const ASTShowCreat if (show_query.current_user) { auto user = context.getUser(); - return getCreateQueryImpl(*user, &access_control); + return getCreateQueryImpl(*user, &access_control, false); } if (show_query.current_quota) { auto quota = access_control.read(context.getQuota()->getUsageInfo().quota_id); - return getCreateQueryImpl(*quota, &access_control); + return getCreateQueryImpl(*quota, &access_control, false); } auto type = getType(show_query.kind); if (show_query.kind == Kind::ROW_POLICY) { RowPolicyPtr policy = access_control.read(show_query.row_policy_name.getFullName(context)); - return getCreateQueryImpl(*policy, &access_control); + return getCreateQueryImpl(*policy, &access_control, false); } auto entity = access_control.read(access_control.getID(type, show_query.name)); - return getCreateQueryImpl(*entity, &access_control); + return getCreateQueryImpl(*entity, &access_control, false); } diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 8bee0b88fe8..4161b3500bd 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -73,7 +73,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() throw Exception("Unable to show the create query of " + show_query->table + ". Maybe it was created by the system.", ErrorCodes::THERE_IS_NO_QUERY); std::stringstream stream; - formatAST(*create_query, stream, false, true); + formatAST(*create_query, stream, false, false); String res = stream.str(); MutableColumnPtr column = ColumnString::create(); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 056959d372c..bedd6c0f9cc 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -1,7 +1,6 @@ #include #include #include -#include "config_core.h" #include #include #include @@ -32,6 +31,10 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + namespace DB { @@ -326,7 +329,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, table->shutdown(); { /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockExclusively(context.getCurrentQueryId()); + auto table_lock = table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); create_ast = database->getCreateTableQuery(system_context, replica.table_name); database->detachTable(replica.table_name); @@ -390,7 +393,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &) context.checkAccess(AccessType::SYSTEM_SYNC_REPLICA, table_id); StoragePtr table = DatabaseCatalog::instance().getTable(table_id); - if (auto storage_replicated = dynamic_cast(table.get())) + if (auto * storage_replicated = dynamic_cast(table.get())) { LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); if (!storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.totalMilliseconds())) @@ -410,7 +413,7 @@ void InterpreterSystemQuery::flushDistributed(ASTSystemQuery &) { context.checkAccess(AccessType::SYSTEM_FLUSH_DISTRIBUTED, table_id); - if (auto storage_distributed = dynamic_cast(DatabaseCatalog::instance().getTable(table_id).get())) + if (auto * storage_distributed = dynamic_cast(DatabaseCatalog::instance().getTable(table_id).get())) storage_distributed->flushClusterNodesAllData(); else throw Exception("Table " + table_id.getNameForLogs() + " is not distributed", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Interpreters/InterserverIOHandler.h b/src/Interpreters/InterserverIOHandler.h index 4651c8cb978..952c99ae46d 100644 --- a/src/Interpreters/InterserverIOHandler.h +++ b/src/Interpreters/InterserverIOHandler.h @@ -32,7 +32,7 @@ class InterserverIOEndpoint public: virtual std::string getId(const std::string & path) const = 0; virtual void processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & body, WriteBuffer & out, Poco::Net::HTTPServerResponse & response) = 0; - virtual ~InterserverIOEndpoint() {} + virtual ~InterserverIOEndpoint() = default; /// You need to stop the data transfer if blocker is activated. ActionBlocker blocker; diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index ca21a53b5b0..7ed1bb9d1bb 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -9,10 +9,12 @@ #include #include #include +#include #include #include #include #include +#include namespace DB @@ -34,7 +36,7 @@ namespace ASTPtr makeSubqueryTemplate() { ParserTablesInSelectQueryElement parser(true); - ASTPtr subquery_template = parseQuery(parser, "(select * from _t) as `--.s`", 0); + ASTPtr subquery_template = parseQuery(parser, "(select * from _t) as `--.s`", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (!subquery_template) throw Exception("Cannot parse subquery template", ErrorCodes::LOGICAL_ERROR); return subquery_template; @@ -96,7 +98,7 @@ private: data.new_select_expression_list = std::make_shared(); data.new_select_expression_list->children.reserve(node.children.size()); - for (auto & child : node.children) + for (const auto & child : node.children) { if (child->as()) { @@ -129,6 +131,8 @@ private: /// Make aliases maps (alias -> column_name, column_name -> alias) struct ColumnAliasesMatcher { + using Visitor = ConstInDepthNodeVisitor; + struct Data { const std::vector tables; @@ -137,6 +141,7 @@ struct ColumnAliasesMatcher std::unordered_map aliases; /// alias -> long_name std::vector> compound_identifiers; std::set allowed_long_names; /// original names allowed as aliases '--t.x as t.x' (select expressions only). + bool inside_function = false; explicit Data(const std::vector && tables_) : tables(tables_) @@ -192,6 +197,10 @@ struct ColumnAliasesMatcher static bool needChildVisit(const ASTPtr & node, const ASTPtr &) { + /// Do not go into subqueries. Function visits children itself. + if (node->as() || + node->as()) + return false; return !node->as(); } @@ -199,11 +208,24 @@ struct ColumnAliasesMatcher { if (auto * t = ast->as()) visit(*t, ast, data); + else if (auto * f = ast->as()) + visit(*f, ast, data); - if (ast->as() || ast->as()) + /// Do not allow asterisks but ignore them inside functions. I.e. allow 'count(*)'. + if (!data.inside_function && (ast->as() || ast->as())) throw Exception("Multiple JOIN do not support asterisks for complex queries yet", ErrorCodes::NOT_IMPLEMENTED); } + static void visit(const ASTFunction &, const ASTPtr & ast, Data & data) + { + /// Grandchild case: Function -> (ExpressionList) -> Asterisk + data.inside_function = true; + Visitor visitor(data); + for (auto & child : ast->children) + visitor.visit(child); + data.inside_function = false; + } + static void visit(const ASTIdentifier & const_node, const ASTPtr &, Data & data) { ASTIdentifier & node = const_cast(const_node); /// we know it's not const @@ -215,7 +237,7 @@ struct ColumnAliasesMatcher if (auto table_pos = IdentifierSemantic::chooseTable(node, data.tables)) { - auto & table = data.tables[*table_pos]; + const auto & table = data.tables[*table_pos]; IdentifierSemantic::setColumnLongName(node, table); /// table_name.column_name -> table_alias.column_name long_name = node.name; if (&table == &data.tables.back()) @@ -348,7 +370,7 @@ bool needRewrite(ASTSelectQuery & select, std::vector; using RewriteVisitor = InDepthNodeVisitor; using ExtractAsterisksVisitor = ConstInDepthNodeVisitor; -using ColumnAliasesVisitor = ConstInDepthNodeVisitor; +using ColumnAliasesVisitor = ColumnAliasesMatcher::Visitor; using AppendSemanticMatcher = OneTypeMatcher; using AppendSemanticVisitor = InDepthNodeVisitor; @@ -461,13 +483,13 @@ struct TableNeededColumns String table_name = table.getQualifiedNamePrefix(false); - for (auto & column : no_clashes) + for (const auto & column : no_clashes) addShortName(column, expression_list); - for (auto & column : alias_clashes) + for (const auto & column : alias_clashes) addShortName(column, expression_list); - for (auto & [column, alias] : column_clashes) + for (const auto & [column, alias] : column_clashes) addAliasedName(table_name, column, alias, expression_list); } @@ -516,7 +538,7 @@ private: size_t countTablesWithColumn(const std::vector & tables, const String & short_name) { size_t count = 0; - for (auto & table : tables) + for (const auto & table : tables) if (table.hasColumn(short_name)) ++count; return count; @@ -557,7 +579,7 @@ std::vector normalizeColumnNamesExtractNeeded( NameSet restored_names; std::vector needed_columns; needed_columns.reserve(tables.size()); - for (auto & table : tables) + for (const auto & table : tables) needed_columns.push_back(TableNeededColumns{table.table}); for (ASTIdentifier * ident : identifiers) @@ -580,7 +602,7 @@ std::vector normalizeColumnNamesExtractNeeded( if (count > 1 || aliases.count(short_name)) { - auto & table = tables[*table_pos]; + const auto & table = tables[*table_pos]; IdentifierSemantic::setColumnLongName(*ident, table.table); /// table.column -> table_alias.column auto & unique_long_name = ident->name; @@ -630,7 +652,7 @@ std::shared_ptr subqueryExpressionList( /// Add needed right table columns needed_columns[table_pos].fillExpressionList(*expression_list); - for (auto & expr : alias_pushdown[table_pos]) + for (const auto & expr : alias_pushdown[table_pos]) expression_list->children.emplace_back(std::move(expr)); return expression_list; diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index cedf95bea06..8f96c19ae17 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -1,18 +1,26 @@ #include +#include #include #include #include #include #include + #include #include #include +#include +#include + #include +#include #include #include #include #include #include +#include +#include namespace DB { @@ -26,10 +34,38 @@ namespace ErrorCodes namespace { +void replaceJoinedTable(const ASTSelectQuery & select_query) +{ + const ASTTablesInSelectQueryElement * join = select_query.join(); + if (!join || !join->table_expression) + return; + + /// TODO: Push down for CROSS JOIN is not OK [disabled] + const auto & table_join = join->table_join->as(); + if (table_join.kind == ASTTableJoin::Kind::Cross) + return; + + auto & table_expr = join->table_expression->as(); + if (table_expr.database_and_table_name) + { + const auto & table_id = table_expr.database_and_table_name->as(); + String expr = "(select * from " + table_id.name + ") as " + table_id.shortName(); + + // FIXME: since the expression "a as b" exposes both "a" and "b" names, which is not equivalent to "(select * from a) as b", + // we can't replace aliased tables. + // FIXME: long table names include database name, which we can't save within alias. + if (table_id.alias.empty() && table_id.isShort()) + { + ParserTableExpression parser; + table_expr = parseQuery(parser, expr, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH)->as(); + } + } +} + template void checkTablesWithColumns(const std::vector & tables_with_columns, const Context & context) { - auto & settings = context.getSettingsRef(); + const auto & settings = context.getSettingsRef(); if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) { for (auto & t : tables_with_columns) @@ -68,7 +104,7 @@ private: return; bool rewritten = false; - for (auto & table : data) + for (const auto & table : data) { /// Table has an alias. We do not need to rewrite qualified names with table alias (match == ColumnMatch::TableName). auto match = IdentifierSemantic::canReferColumnToTable(identifier, table); @@ -89,7 +125,7 @@ private: { ASTIdentifier & identifier = *node.children[0]->as(); bool rewritten = false; - for (auto & table : data) + for (const auto & table : data) { if (identifier.name == table.table) { @@ -149,7 +185,7 @@ StoragePtr JoinedTables::getLeftTableStorage() if (auto view_source = context.getViewSource()) { - auto & storage_values = static_cast(*view_source); + const auto & storage_values = static_cast(*view_source); auto tmp_table_id = storage_values.getStorageID(); if (tmp_table_id.database_name == table_id.database_name && tmp_table_id.table_name == table_id.table_name) { @@ -209,4 +245,35 @@ void JoinedTables::rewriteDistributedInAndJoins(ASTPtr & query) } } +std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & select_query) +{ + if (tables_with_columns.size() < 2) + return {}; + + auto settings = context.getSettingsRef(); + auto table_join = std::make_shared(settings, context.getTemporaryVolume()); + + const ASTTablesInSelectQueryElement * ast_join = select_query.join(); + const auto & table_to_join = ast_join->table_expression->as(); + + /// TODO This syntax does not support specifying a database name. + if (table_to_join.database_and_table_name) + { + auto joined_table_id = context.resolveStorageID(table_to_join.database_and_table_name); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(joined_table_id); + if (table) + { + if (dynamic_cast(table.get()) || + dynamic_cast(table.get())) + table_join->joined_storage = table; + } + } + + if (!table_join->joined_storage && + settings.enable_optimize_predicate_expression) + replaceJoinedTable(select_query); + + return table_join; +} + } diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h index 66b3c8de609..399acdc0768 100644 --- a/src/Interpreters/JoinedTables.h +++ b/src/Interpreters/JoinedTables.h @@ -10,6 +10,7 @@ namespace DB class ASTSelectQuery; class Context; +class TableJoin; struct SelectQueryOptions; /// Joined tables' columns resolver. @@ -27,7 +28,10 @@ public: StoragePtr getLeftTableStorage(); bool resolveTables(); + + /// Make fake tables_with_columns[0] in case we have predefined input in InterpreterSelectQuery void makeFakeTable(StoragePtr storage, const Block & source_header); + std::shared_ptr makeTableJoin(const ASTSelectQuery & select_query); const std::vector & tablesWithColumns() const { return tables_with_columns; } diff --git a/src/Interpreters/LogicalExpressionsOptimizer.cpp b/src/Interpreters/LogicalExpressionsOptimizer.cpp index ca8fbb1c8f4..c0d5a16fa65 100644 --- a/src/Interpreters/LogicalExpressionsOptimizer.cpp +++ b/src/Interpreters/LogicalExpressionsOptimizer.cpp @@ -104,8 +104,8 @@ void LogicalExpressionsOptimizer::collectDisjunctiveEqualityChains() while (!to_visit.empty()) { auto edge = to_visit.back(); - auto from_node = edge.first; - auto to_node = edge.second; + auto * from_node = edge.first; + auto * to_node = edge.second; to_visit.pop_back(); @@ -118,7 +118,7 @@ void LogicalExpressionsOptimizer::collectDisjunctiveEqualityChains() if (expression_list) { /// The chain of elements of the OR expression. - for (auto & child : expression_list->children) + for (const auto & child : expression_list->children) { auto * equals = child->as(); if (equals && equals->name == "equals" && equals->children.size() == 1) @@ -227,7 +227,7 @@ void LogicalExpressionsOptimizer::addInExpression(const DisjunctiveEqualityChain /// Construct a list of literals `x1, ..., xN` from the string `expr = x1 OR ... OR expr = xN` ASTPtr value_list = std::make_shared(); - for (const auto function : equality_functions) + for (const auto * function : equality_functions) { const auto & operands = getFunctionOperands(function); value_list->children.push_back(operands[1]); @@ -245,7 +245,7 @@ void LogicalExpressionsOptimizer::addInExpression(const DisjunctiveEqualityChain /// Get the expression `expr` from the chain `expr = x1 OR ... OR expr = xN` ASTPtr equals_expr_lhs; { - auto function = equality_functions[0]; + auto * function = equality_functions[0]; const auto & operands = getFunctionOperands(function); equals_expr_lhs = operands[0]; } @@ -316,7 +316,7 @@ void LogicalExpressionsOptimizer::cleanupOrExpressions() /// Delete garbage. for (const auto & entry : garbage_map) { - auto function = entry.first; + const auto * function = entry.first; auto first_erased = entry.second; auto & operands = getFunctionOperands(function); @@ -333,7 +333,7 @@ void LogicalExpressionsOptimizer::fixBrokenOrExpressions() continue; const auto & or_with_expression = chain.first; - auto or_function = or_with_expression.or_function; + const auto * or_function = or_with_expression.or_function; auto & operands = getFunctionOperands(or_with_expression.or_function); if (operands.size() == 1) diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 1a3a84004dd..859a341846f 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -36,8 +36,8 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, if constexpr (has_nulls) { - auto * left_nullable = checkAndGetColumn(left_column); - auto * right_nullable = checkAndGetColumn(right_column); + const auto * left_nullable = checkAndGetColumn(left_column); + const auto * right_nullable = checkAndGetColumn(right_column); if (left_nullable && right_nullable) { @@ -79,7 +79,7 @@ Block extractMinMax(const Block & block, const Block & keys) for (size_t i = 0; i < columns.size(); ++i) { - auto & src_column = block.getByName(keys.getByPosition(i).name); + const auto & src_column = block.getByName(keys.getByPosition(i).name); columns[i]->insertFrom(*src_column.column, 0); columns[i]->insertFrom(*src_column.column, block.rows() - 1); @@ -151,8 +151,8 @@ public: for (size_t i = 0; i < impl.sort_columns.size(); ++i) { - auto & left_column = *impl.sort_columns[i]; - auto & right_column = *min_max.getByName(key_names[i]).column; /// cannot get by position cause of possible duplicates + const auto & left_column = *impl.sort_columns[i]; + const auto & right_column = *min_max.getByName(key_names[i]).column; /// cannot get by position cause of possible duplicates if (!first_vs_max) first_vs_max = nullableCompareAt(left_column, right_column, position(), 1); @@ -200,8 +200,8 @@ private: int res = 0; for (size_t i = 0; i < impl.sort_columns_size; ++i) { - auto * left_column = impl.sort_columns[i]; - auto * right_column = rhs.impl.sort_columns[i]; + const auto * left_column = impl.sort_columns[i]; + const auto * right_column = rhs.impl.sort_columns[i]; res = nullableCompareAt(*left_column, *right_column, lhs_pos, rhs_pos); if (res) @@ -252,7 +252,7 @@ MutableColumns makeMutableColumns(const Block & block, size_t rows_to_reserve = void makeSortAndMerge(const Names & keys, SortDescription & sort, SortDescription & merge) { NameSet unique_keys; - for (auto & key_name : keys) + for (const auto & key_name : keys) { merge.emplace_back(SortColumnDescription(key_name, 1, 1)); @@ -413,7 +413,7 @@ void MiniLSM::insert(const BlocksList & blocks) { BlockInputStreams inputs; inputs.reserve(blocks.size()); - for (auto & block : blocks) + for (const auto & block : blocks) inputs.push_back(std::make_shared(block)); MergingSortedBlockInputStream sorted_input(inputs, sort_description, rows_in_block); @@ -851,7 +851,7 @@ bool MergeJoin::semiLeftJoin(MergeJoinCursor & left_cursor, const Block & left_b return true; } -void MergeJoin::changeLeftColumns(Block & block, MutableColumns && columns) +void MergeJoin::changeLeftColumns(Block & block, MutableColumns && columns) const { if (is_left && is_any_join) return; diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index d62083df38e..b04720ca168 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -105,7 +105,7 @@ private: const size_t max_joined_block_rows; const size_t max_rows_in_right_block; - void changeLeftColumns(Block & block, MutableColumns && columns); + void changeLeftColumns(Block & block, MutableColumns && columns) const; void addRightColumns(Block & block, MutableColumns && columns); template diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index 5622e0c65b0..bd898170705 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -70,6 +70,13 @@ void MetricLog::stopCollectMetric() } +void MetricLog::shutdown() +{ + stopCollectMetric(); + stopFlushThread(); +} + + inline UInt64 time_in_milliseconds(std::chrono::time_point timepoint) { return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); diff --git a/src/Interpreters/MetricLog.h b/src/Interpreters/MetricLog.h index c55bad2c12f..a90ce923494 100644 --- a/src/Interpreters/MetricLog.h +++ b/src/Interpreters/MetricLog.h @@ -34,6 +34,8 @@ class MetricLog : public SystemLog using SystemLog::SystemLog; public: + void shutdown() override; + /// Launches a background thread to collect metrics with interval void startCollectMetric(size_t collect_interval_milliseconds_); diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 669b72c6317..5ba6424653c 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -36,34 +36,45 @@ namespace ErrorCodes namespace { -struct FirstNonDeterministicFuncData +/// Helps to detect situations, where non-deterministic functions may be used in mutations of Replicated*MergeTree. +class FirstNonDeterministicFuncMatcher { - using TypeToVisit = ASTFunction; - - explicit FirstNonDeterministicFuncData(const Context & context_) - : context{context_} - {} - - const Context & context; - std::optional nondeterministic_function_name; - - void visit(ASTFunction & function, ASTPtr &) +public: + struct Data { - if (nondeterministic_function_name) + const Context & context; + std::optional nondeterministic_function_name; + }; + + static bool needChildVisit(const ASTPtr & /*node*/, const ASTPtr & child) + { + return child != nullptr; + } + + static void visit(const ASTPtr & node, Data & data) + { + if (data.nondeterministic_function_name) return; - const auto func = FunctionFactory::instance().get(function.name, context); - if (!func->isDeterministic()) - nondeterministic_function_name = func->getName(); + if (const auto * function = typeid_cast(node.get())) + { + /// Property of being deterministic for lambda expression is completely determined + /// by the contents of its definition, so we just proceed to it. + if (function->name != "lambda") + { + const auto func = FunctionFactory::instance().get(function->name, data.context); + if (!func->isDeterministic()) + data.nondeterministic_function_name = func->getName(); + } + } } }; -using FirstNonDeterministicFuncFinder = - InDepthNodeVisitor, true>; +using FirstNonDeterministicFuncFinder = InDepthNodeVisitor; std::optional findFirstNonDeterministicFuncName(const MutationCommand & command, const Context & context) { - FirstNonDeterministicFuncData finder_data(context); + FirstNonDeterministicFuncMatcher::Data finder_data{context, std::nullopt}; switch (command.type) { @@ -661,9 +672,11 @@ BlockInputStreamPtr MutationsInterpreter::addStreamsForLaterStages(const std::ve void MutationsInterpreter::validate(TableStructureReadLockHolder &) { + const Settings & settings = context.getSettingsRef(); + /// For Replicated* storages mutations cannot employ non-deterministic functions /// because that produces inconsistencies between replicas - if (startsWith(storage->getName(), "Replicated")) + if (startsWith(storage->getName(), "Replicated") && !settings.allow_nondeterministic_mutations) { for (const auto & command : commands) { diff --git a/src/Interpreters/NullableUtils.cpp b/src/Interpreters/NullableUtils.cpp index fe2801f5d11..aea69f8ea6d 100644 --- a/src/Interpreters/NullableUtils.cpp +++ b/src/Interpreters/NullableUtils.cpp @@ -5,14 +5,14 @@ namespace DB { -ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullMapPtr & null_map) +ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullMapPtr & null_map, bool exact_null) { ColumnPtr null_map_holder; if (key_columns.size() == 1) { auto & column = key_columns[0]; - if (auto * column_nullable = checkAndGetColumn(*column)) + if (const auto * column_nullable = checkAndGetColumn(*column)) { null_map_holder = column_nullable->getNullMapColumnPtr(); null_map = &column_nullable->getNullMapData(); @@ -23,7 +23,7 @@ ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullM { for (auto & column : key_columns) { - if (auto * column_nullable = checkAndGetColumn(*column)) + if (const auto * column_nullable = checkAndGetColumn(*column)) { column = &column_nullable->getNestedColumn(); @@ -38,7 +38,12 @@ ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullM PaddedPODArray & mutable_null_map = assert_cast(*mutable_null_map_holder).getData(); const PaddedPODArray & other_null_map = column_nullable->getNullMapData(); for (size_t i = 0, size = mutable_null_map.size(); i < size; ++i) - mutable_null_map[i] |= other_null_map[i]; + { + if (exact_null) + mutable_null_map[i] &= other_null_map[i]; + else + mutable_null_map[i] |= other_null_map[i]; + } null_map_holder = std::move(mutable_null_map_holder); } diff --git a/src/Interpreters/NullableUtils.h b/src/Interpreters/NullableUtils.h index ee3193919cd..054835f8bef 100644 --- a/src/Interpreters/NullableUtils.h +++ b/src/Interpreters/NullableUtils.h @@ -8,6 +8,6 @@ namespace DB * In 'null_map' return a map of positions where at least one column was NULL. * @returns ownership column of null_map. */ -ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullMapPtr & null_map); +ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullMapPtr & null_map, bool exact_null = false); } diff --git a/src/Interpreters/PredicateRewriteVisitor.h b/src/Interpreters/PredicateRewriteVisitor.h index e07df922c15..cc1b6472a4c 100644 --- a/src/Interpreters/PredicateRewriteVisitor.h +++ b/src/Interpreters/PredicateRewriteVisitor.h @@ -16,6 +16,14 @@ public: void visit(ASTSelectWithUnionQuery & union_select_query, ASTPtr &); + static bool needChild(const ASTPtr & node, const ASTPtr &) + { + if (node && node->as()) + return false; + + return true; + } + PredicateRewriteVisitorData(const Context & context_, const ASTs & predicates_, const Names & column_names_, bool optimize_final_); private: @@ -31,6 +39,6 @@ private: bool rewriteSubquery(ASTSelectQuery & subquery, const Names & outer_columns, const Names & inner_columns); }; -using PredicateRewriteMatcher = OneTypeMatcher; +using PredicateRewriteMatcher = OneTypeMatcher; using PredicateRewriteVisitor = InDepthNodeVisitor; } diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index cf8001ab2c4..5ec04d0e11c 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -14,12 +14,6 @@ #include -namespace CurrentMetrics -{ - extern const Metric MemoryTracking; -} - - namespace DB { @@ -68,7 +62,6 @@ static bool isUnlimitedQuery(const IAST * ast) ProcessList::ProcessList(size_t max_size_) : max_size(max_size_) { - total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); } @@ -171,19 +164,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as process_it->setUserProcessList(&user_process_list); - /// Limits are only raised (to be more relaxed) or set to something instead of zero, - /// because settings for different queries will interfere each other: - /// setting from one query effectively sets values for all other queries. - - /// Track memory usage for all simultaneously running queries. - /// You should specify this value in configuration for default profile, - /// not for specific users, sessions or queries, - /// because this setting is effectively global. - total_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_all_queries); - total_memory_tracker.setDescription("(total)"); - /// Track memory usage for all simultaneously running queries from single user. - user_process_list.user_memory_tracker.setParent(&total_memory_tracker); user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); user_process_list.user_memory_tracker.setDescription("(for user)"); @@ -280,14 +261,9 @@ ProcessListEntry::~ProcessListEntry() if (user_process_list.queries.empty()) user_process_list.resetTrackers(); - /// This removes memory_tracker for all requests. At this time, no other memory_trackers live. + /// Reset throttler, similarly (see above). if (parent.processes.empty()) - { - /// Reset MemoryTracker, similarly (see above). - parent.total_memory_tracker.logPeakMemoryUsage(); - parent.total_memory_tracker.reset(); parent.total_network_throttler.reset(); - } } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index da6799c5eca..15487b8ec0e 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -295,9 +295,6 @@ protected: /// Stores info about queries grouped by their priority QueryPriorities priorities; - /// Limit and counter for memory of all simultaneously running queries. - MemoryTracker total_memory_tracker{VariableContext::Global}; - /// Limit network bandwidth for all users ThrottlerPtr total_network_throttler; diff --git a/src/Interpreters/ProfileEventsExt.cpp b/src/Interpreters/ProfileEventsExt.cpp index e5c0a3013d5..bca845c4248 100644 --- a/src/Interpreters/ProfileEventsExt.cpp +++ b/src/Interpreters/ProfileEventsExt.cpp @@ -14,8 +14,8 @@ namespace ProfileEvents void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_, DB::IColumn * column_values_, bool nonzero_only) { /// Convert ptr and make simple check - auto column_names = (column_names_) ? &typeid_cast(*column_names_) : nullptr; - auto column_values = (column_values_) ? &typeid_cast(*column_values_) : nullptr; + auto * column_names = (column_names_) ? &typeid_cast(*column_names_) : nullptr; + auto * column_values = (column_values_) ? &typeid_cast(*column_values_) : nullptr; size_t size = 0; diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 638ac68daa5..1b104fe1a6c 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -125,8 +125,8 @@ void QueryLogElement::appendToBlock(Block & block) const if (profile_counters) { - auto column_names = columns[i++].get(); - auto column_values = columns[i++].get(); + auto * column_names = columns[i++].get(); + auto * column_values = columns[i++].get(); ProfileEvents::dumpToArrayColumns(*profile_counters, column_names, column_values, true); } else @@ -137,8 +137,8 @@ void QueryLogElement::appendToBlock(Block & block) const if (query_settings) { - auto column_names = columns[i++].get(); - auto column_values = columns[i++].get(); + auto * column_names = columns[i++].get(); + auto * column_values = columns[i++].get(); query_settings->dumpToArrayColumns(column_names, column_values, true); } else diff --git a/src/Interpreters/QueryLog.h b/src/Interpreters/QueryLog.h index 836b37095e9..ec14f5e97fb 100644 --- a/src/Interpreters/QueryLog.h +++ b/src/Interpreters/QueryLog.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace ProfileEvents @@ -22,13 +23,7 @@ namespace DB /// A struct which will be inserted as row into query_log table struct QueryLogElement { - enum Type : int8_t // Make it signed for compatibility with DataTypeEnum8 - { - QUERY_START = 1, - QUERY_FINISH = 2, - EXCEPTION_BEFORE_START = 3, - EXCEPTION_WHILE_PROCESSING = 4, - }; + using Type = QueryLogElementType; Type type = QUERY_START; diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index 86fbd108f51..324c401eb8a 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -80,7 +80,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) /// We are alias for other column (node.name), but we are alias by /// ourselves to some other column - auto & alias_node = it_alias->second; + const auto & alias_node = it_alias->second; String our_alias_or_name = alias_node->getAliasOrColumnName(); std::optional our_name = IdentifierSemantic::getColumnName(alias_node); diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index 225e768eb3a..1a2975bd43b 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -98,8 +98,8 @@ void QueryThreadLogElement::appendToBlock(Block & block) const if (profile_counters) { - auto column_names = columns[i++].get(); - auto column_values = columns[i++].get(); + auto * column_names = columns[i++].get(); + auto * column_values = columns[i++].get(); dumpToArrayColumns(*profile_counters, column_names, column_values, true); } else diff --git a/src/Interpreters/RenameColumnVisitor.cpp b/src/Interpreters/RenameColumnVisitor.cpp index a22fa78e2cc..f94353cf646 100644 --- a/src/Interpreters/RenameColumnVisitor.cpp +++ b/src/Interpreters/RenameColumnVisitor.cpp @@ -3,7 +3,7 @@ namespace DB { -void RenameColumnData::visit(ASTIdentifier & identifier, ASTPtr &) +void RenameColumnData::visit(ASTIdentifier & identifier, ASTPtr &) const { std::optional identifier_column_name = IdentifierSemantic::getColumnName(identifier); if (identifier_column_name && identifier_column_name == column_name) diff --git a/src/Interpreters/RenameColumnVisitor.h b/src/Interpreters/RenameColumnVisitor.h index 318cde84f56..653cda4f51f 100644 --- a/src/Interpreters/RenameColumnVisitor.h +++ b/src/Interpreters/RenameColumnVisitor.h @@ -14,7 +14,7 @@ struct RenameColumnData String column_name; String rename_to; - void visit(ASTIdentifier & identifier, ASTPtr & ast); + void visit(ASTIdentifier & identifier, ASTPtr & ast) const; }; using RenameColumnMatcher = OneTypeMatcher; diff --git a/src/Interpreters/ColumnNamesContext.cpp b/src/Interpreters/RequiredSourceColumnsData.cpp similarity index 57% rename from src/Interpreters/ColumnNamesContext.cpp rename to src/Interpreters/RequiredSourceColumnsData.cpp index 4d23c6f0e8b..ec9f2ca4817 100644 --- a/src/Interpreters/ColumnNamesContext.cpp +++ b/src/Interpreters/RequiredSourceColumnsData.cpp @@ -1,21 +1,13 @@ -#include +#include +#include #include #include +#include namespace DB { -bool ColumnNamesContext::addTableAliasIfAny(const IAST & ast) -{ - String alias = ast.tryGetAlias(); - if (alias.empty()) - return false; - - table_aliases.insert(alias); - return true; -} - -bool ColumnNamesContext::addColumnAliasIfAny(const IAST & ast) +bool RequiredSourceColumnsData::addColumnAliasIfAny(const IAST & ast) { String alias = ast.tryGetAlias(); if (alias.empty()) @@ -28,7 +20,7 @@ bool ColumnNamesContext::addColumnAliasIfAny(const IAST & ast) return true; } -void ColumnNamesContext::addColumnIdentifier(const ASTIdentifier & node) +void RequiredSourceColumnsData::addColumnIdentifier(const ASTIdentifier & node) { if (!IdentifierSemantic::getColumnName(node)) return; @@ -38,7 +30,7 @@ void ColumnNamesContext::addColumnIdentifier(const ASTIdentifier & node) required_names[node.name].addInclusion(alias); } -bool ColumnNamesContext::addArrayJoinAliasIfAny(const IAST & ast) +bool RequiredSourceColumnsData::addArrayJoinAliasIfAny(const IAST & ast) { String alias = ast.tryGetAlias(); if (alias.empty()) @@ -48,12 +40,12 @@ bool ColumnNamesContext::addArrayJoinAliasIfAny(const IAST & ast) return true; } -void ColumnNamesContext::addArrayJoinIdentifier(const ASTIdentifier & node) +void RequiredSourceColumnsData::addArrayJoinIdentifier(const ASTIdentifier & node) { array_join_columns.insert(node.name); } -size_t ColumnNamesContext::nameInclusion(const String & name) const +size_t RequiredSourceColumnsData::nameInclusion(const String & name) const { auto it = required_names.find(name); if (it != required_names.end()) @@ -61,7 +53,7 @@ size_t ColumnNamesContext::nameInclusion(const String & name) const return 0; } -NameSet ColumnNamesContext::requiredColumns() const +NameSet RequiredSourceColumnsData::requiredColumns() const { NameSet required; for (const auto & pr : required_names) @@ -79,39 +71,25 @@ NameSet ColumnNamesContext::requiredColumns() const return required; } -std::ostream & operator << (std::ostream & os, const ColumnNamesContext & cols) +std::ostream & operator << (std::ostream & os, const RequiredSourceColumnsData & cols) { os << "required_names: "; for (const auto & pr : cols.required_names) { os << "'" << pr.first << "'"; - for (auto & alias : pr.second.aliases) + for (const auto & alias : pr.second.aliases) os << "/'" << alias << "'"; + os << ", "; } - os << " source_tables: "; - for (const auto & x : cols.tables) - { - auto alias = x.alias(); - auto name = x.name(); - if (alias && name) - os << "'" << *alias << "'/'" << *name << "' "; - else if (alias) - os << "'" << *alias << "' "; - else if (name) - os << "'" << *name << "' "; - } - os << "table_aliases: "; - for (const auto & x : cols.table_aliases) - os << "'" << x << "' "; os << "complex_aliases: "; for (const auto & x : cols.complex_aliases) - os << "'" << x << "' "; + os << "'" << x << "', "; os << "masked_columns: "; for (const auto & x : cols.masked_columns) - os << "'" << x << "' "; + os << "'" << x << "', "; os << "array_join_columns: "; for (const auto & x : cols.array_join_columns) - os << "'" << x << "' "; + os << "'" << x << "', "; return os; } diff --git a/src/Interpreters/RequiredSourceColumnsData.h b/src/Interpreters/RequiredSourceColumnsData.h new file mode 100644 index 00000000000..de1f3bc2721 --- /dev/null +++ b/src/Interpreters/RequiredSourceColumnsData.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace DB +{ + +class ASTIdentifier; + +/// Information about table and column names extracted from ASTSelectQuery block. Do not include info from subselects. +struct RequiredSourceColumnsData +{ + struct NameInfo + { + std::set aliases; + size_t appears = 0; + + void addInclusion(const String & alias) + { + if (!alias.empty()) + aliases.insert(alias); + ++appears; + } + }; + + std::unordered_map required_names; + NameSet private_aliases; /// lambda aliases that should not be interpreted as required columns + NameSet complex_aliases; /// aliases to functions results: they are not required cause calculated by query itself + NameSet masked_columns; /// columns names masked by function aliases: we still need them in required columns + NameSet array_join_columns; /// Tech debt: we exclude ArrayJoin columns from general logic cause they have own logic outside + + bool has_table_join = false; + bool has_array_join = false; + + bool addColumnAliasIfAny(const IAST & ast); + void addColumnIdentifier(const ASTIdentifier & node); + bool addArrayJoinAliasIfAny(const IAST & ast); + void addArrayJoinIdentifier(const ASTIdentifier & node); + + NameSet requiredColumns() const; + size_t nameInclusion(const String & name) const; +}; + +std::ostream & operator << (std::ostream & os, const RequiredSourceColumnsData & cols); + +} diff --git a/src/Interpreters/RequiredSourceColumnsVisitor.cpp b/src/Interpreters/RequiredSourceColumnsVisitor.cpp index 5a740805560..4eb7f477b05 100644 --- a/src/Interpreters/RequiredSourceColumnsVisitor.cpp +++ b/src/Interpreters/RequiredSourceColumnsVisitor.cpp @@ -88,15 +88,15 @@ void RequiredSourceColumnsMatcher::visit(const ASTPtr & ast, Data & data) visit(*t, ast, data); return; } + if (auto * t = ast->as()) { - data.addTableAliasIfAny(*ast); visit(*t, ast, data); return; } + if (ast->as()) { - data.addTableAliasIfAny(*ast); return; } @@ -122,7 +122,7 @@ void RequiredSourceColumnsMatcher::visit(const ASTSelectQuery & select, const AS } std::vector out; - for (auto & node : select.children) + for (const auto & node : select.children) if (node != select.select()) Visitor(data).visit(node); @@ -159,33 +159,14 @@ void RequiredSourceColumnsMatcher::visit(const ASTFunction & node, const ASTPtr void RequiredSourceColumnsMatcher::visit(const ASTTablesInSelectQueryElement & node, const ASTPtr &, Data & data) { - ASTTableExpression * expr = nullptr; - ASTTableJoin * join = nullptr; - - for (auto & child : node.children) - { - if (auto * e = child->as()) - expr = e; - if (auto * j = child->as()) - join = j; - } - - if (join) - data.has_table_join = true; - data.tables.emplace_back(ColumnNamesContext::JoinedTable{expr, join}); + for (const auto & child : node.children) + if (child->as()) + data.has_table_join = true; } /// ASTIdentifiers here are tables. Do not visit them as generic ones. -void RequiredSourceColumnsMatcher::visit(const ASTTableExpression & node, const ASTPtr &, Data & data) +void RequiredSourceColumnsMatcher::visit(const ASTTableExpression &, const ASTPtr &, Data &) { - if (node.database_and_table_name) - data.addTableAliasIfAny(*node.database_and_table_name); - - if (node.table_function) - data.addTableAliasIfAny(*node.table_function); - - if (node.subquery) - data.addTableAliasIfAny(*node.subquery); } void RequiredSourceColumnsMatcher::visit(const ASTArrayJoin & node, const ASTPtr &, Data & data) diff --git a/src/Interpreters/RequiredSourceColumnsVisitor.h b/src/Interpreters/RequiredSourceColumnsVisitor.h index 1f6ff482e3a..53decb3a849 100644 --- a/src/Interpreters/RequiredSourceColumnsVisitor.h +++ b/src/Interpreters/RequiredSourceColumnsVisitor.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace DB @@ -21,7 +21,7 @@ class RequiredSourceColumnsMatcher { public: using Visitor = ConstInDepthNodeVisitor; - using Data = ColumnNamesContext; + using Data = RequiredSourceColumnsData; static bool needChildVisit(const ASTPtr & node, const ASTPtr & child); static void visit(const ASTPtr & ast, Data & data); @@ -35,7 +35,7 @@ private: static void visit(const ASTSelectQuery & select, const ASTPtr &, Data & data); }; -/// Extracts all the information about columns and tables from ASTSelectQuery block into ColumnNamesContext object. +/// Extracts all the information about columns and tables from ASTSelectQuery block into Data object. /// It doesn't use anything but AST. It visits nodes from bottom to top except ASTFunction content to get aliases in right manner. /// @note There's some ambiguousness with nested columns names that can't be solved without schema. using RequiredSourceColumnsVisitor = RequiredSourceColumnsMatcher::Visitor; diff --git a/src/Interpreters/RowRefs.h b/src/Interpreters/RowRefs.h index 1da9acda685..65ae16e2b49 100644 --- a/src/Interpreters/RowRefs.h +++ b/src/Interpreters/RowRefs.h @@ -18,8 +18,10 @@ class Block; /// Reference to the row in block. struct RowRef { + using SizeT = uint32_t; /// Do not use size_t cause of memory economy + const Block * block = nullptr; - size_t row_num = 0; + SizeT row_num = 0; RowRef() {} RowRef(const Block * block_, size_t row_num_) : block(block_), row_num(row_num_) {} @@ -33,7 +35,7 @@ struct RowRefList : RowRef { static constexpr size_t MAX_SIZE = 7; /// Adequate values are 3, 7, 15, 31. - size_t size = 0; + SizeT size = 0; /// It's smaller than size_t but keeps align in Arena. Batch * next; RowRef row_refs[MAX_SIZE]; diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 3c79ea5174d..afc95e11fd8 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -87,6 +87,8 @@ void NO_INLINE Set::insertFromBlockImplCase( { if ((*null_map)[i]) { + has_null = true; + if constexpr (build_filter) { (*out_filter)[i] = false; @@ -128,7 +130,7 @@ void Set::setHeader(const Block & header) set_elements_types.emplace_back(header.safeGetByPosition(i).type); /// Convert low cardinality column to full. - if (auto * low_cardinality_type = typeid_cast(data_types.back().get())) + if (const auto * low_cardinality_type = typeid_cast(data_types.back().get())) { data_types.back() = low_cardinality_type->getDictionaryType(); materialized_columns.emplace_back(key_columns.back()->convertToFullColumnIfLowCardinality()); @@ -138,7 +140,7 @@ void Set::setHeader(const Block & header) /// We will insert to the Set only keys, where all components are not NULL. ConstNullMapPtr null_map{}; - ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map); + ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map, transform_null_in); if (fill_set_elements) { @@ -178,7 +180,7 @@ bool Set::insertFromBlock(const Block & block) /// We will insert to the Set only keys, where all components are not NULL. ConstNullMapPtr null_map{}; - ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map); + ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map, transform_null_in); /// Filter to extract distinct values from the block. ColumnUInt8::MutablePtr filter; @@ -243,13 +245,13 @@ void Set::createFromAST(const DataTypes & types, ASTPtr node, const Context & co DataTypePtr tuple_type; Row tuple_values; const auto & list = node->as(); - for (auto & elem : list.children) + for (const auto & elem : list.children) { if (num_columns == 1) { Field value = extractValueFromNode(elem, *types[0], context); - if (!value.isNull()) + if (!value.isNull() || context.getSettingsRef().transform_null_in) columns[0]->insert(value); } else if (const auto * func = elem->as()) @@ -284,7 +286,7 @@ void Set::createFromAST(const DataTypes & types, ASTPtr node, const Context & co : extractValueFromNode(func->arguments->children[i], *types[i], context); /// If at least one of the elements of the tuple has an impossible (outside the range of the type) value, then the entire tuple too. - if (value.isNull()) + if (value.isNull() && !context.getSettings().transform_null_in) break; tuple_values[i] = value; @@ -348,7 +350,8 @@ ColumnPtr Set::execute(const Block & block, bool negative) const /// We will check existence in Set only for keys, where all components are not NULL. ConstNullMapPtr null_map{}; - ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map); + + ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map, transform_null_in); executeOrdinary(key_columns, vec_res, negative, null_map); @@ -390,7 +393,12 @@ void NO_INLINE Set::executeImplCase( for (size_t i = 0; i < rows; ++i) { if (has_null_map && (*null_map)[i]) - vec_res[i] = negative; + { + if (transform_null_in && has_null) + vec_res[i] = !negative; + else + vec_res[i] = negative; + } else { auto find_result = state.findKey(method.data, i, pool); @@ -588,6 +596,14 @@ BoolMask MergeTreeSetIndex::checkInRange(const std::vector & key_ranges, }; } +bool MergeTreeSetIndex::hasMonotonicFunctionsChain() const +{ + for (const auto & mapping : indexes_mapping) + if (!mapping.functions.empty()) + return true; + return false; +} + void ValueWithInfinity::update(const Field & x) { /// Keep at most one element in column. @@ -599,8 +615,11 @@ void ValueWithInfinity::update(const Field & x) const IColumn & ValueWithInfinity::getColumnIfFinite() const { +#ifndef NDEBUG if (type != NORMAL) throw Exception("Trying to get column of infinite type", ErrorCodes::LOGICAL_ERROR); +#endif + return *column; } diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index c9605d4e11e..0f84c2f5da8 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -30,9 +30,9 @@ public: /// (that is useful only for checking that some value is in the set and may not store the original values), /// store all set elements in explicit form. /// This is needed for subsequent use for index. - Set(const SizeLimits & limits_, bool fill_set_elements_) + Set(const SizeLimits & limits_, bool fill_set_elements_, bool transform_null_in_) : log(&Logger::get("Set")), - limits(limits_), fill_set_elements(fill_set_elements_) + limits(limits_), fill_set_elements(fill_set_elements_), transform_null_in(transform_null_in_) { } @@ -113,6 +113,10 @@ private: /// Do we need to additionally store all elements of the set in explicit form for subsequent use for index. bool fill_set_elements; + bool transform_null_in; + + bool has_null = false; + /// Check if set contains all the data. bool is_created = false; @@ -227,6 +231,8 @@ public: size_t size() const { return ordered_set.at(0)->size(); } + bool hasMonotonicFunctionsChain() const; + BoolMask checkInRange(const std::vector & key_ranges, const DataTypes & data_types); private: diff --git a/src/Interpreters/SetVariants.cpp b/src/Interpreters/SetVariants.cpp index 52f54d2442a..739e6fffccd 100644 --- a/src/Interpreters/SetVariants.cpp +++ b/src/Interpreters/SetVariants.cpp @@ -74,7 +74,7 @@ typename SetVariantsTemplate::Type SetVariantsTemplate::choose for (const auto & col : key_columns) { - if (auto * nullable = checkAndGetColumn(*col)) + if (const auto * nullable = checkAndGetColumn(*col)) { nested_key_columns.push_back(&nullable->getNestedColumn()); has_nullable_key = true; diff --git a/src/Interpreters/SetVariants.h b/src/Interpreters/SetVariants.h index 454de98d35b..118cf0115ab 100644 --- a/src/Interpreters/SetVariants.h +++ b/src/Interpreters/SetVariants.h @@ -190,24 +190,24 @@ struct NonClearableSet * As in Aggregator, using consecutive keys cache doesn't improve performance * for FixedHashTables. */ - std::unique_ptr, false /* use_cache */>> key8; - std::unique_ptr, false /* use_cache */>> key16; + std::unique_ptr, false /* use_cache */>> key8; + std::unique_ptr, false /* use_cache */>> key16; /** Also for the experiment was tested the ability to use SmallSet, * as long as the number of elements in the set is small (and, if necessary, converted to a full-fledged HashSet). * But this experiment showed that there is an advantage only in rare cases. */ - std::unique_ptr>>> key32; - std::unique_ptr>>> key64; - std::unique_ptr>> key_string; - std::unique_ptr>> key_fixed_string; - std::unique_ptr>> keys128; - std::unique_ptr>> keys256; - std::unique_ptr>> hashed; + std::unique_ptr>>> key32; + std::unique_ptr>>> key64; + std::unique_ptr>> key_string; + std::unique_ptr>> key_fixed_string; + std::unique_ptr>> keys128; + std::unique_ptr>> keys256; + std::unique_ptr>> hashed; /// Support for nullable keys (for DISTINCT implementation). - std::unique_ptr, true>> nullable_keys128; - std::unique_ptr, true>> nullable_keys256; + std::unique_ptr, true>> nullable_keys128; + std::unique_ptr, true>> nullable_keys256; /** Unlike Aggregator, `concat` method is not used here. * This is done because `hashed` method, although slower, but in this case, uses less RAM. * since when you use it, the key values themselves are not stored. @@ -216,20 +216,20 @@ struct NonClearableSet struct ClearableSet { - std::unique_ptr, false /* use_cache */>> key8; - std::unique_ptr, false /*use_cache */>> key16; + std::unique_ptr, false /* use_cache */>> key8; + std::unique_ptr, false /*use_cache */>> key16; - std::unique_ptr>>> key32; - std::unique_ptr>>> key64; - std::unique_ptr>> key_string; - std::unique_ptr>> key_fixed_string; - std::unique_ptr>> keys128; - std::unique_ptr>> keys256; - std::unique_ptr>> hashed; + std::unique_ptr>>> key32; + std::unique_ptr>>> key64; + std::unique_ptr>> key_string; + std::unique_ptr>> key_fixed_string; + std::unique_ptr>> keys128; + std::unique_ptr>> keys256; + std::unique_ptr>> hashed; /// Support for nullable keys (for DISTINCT implementation). - std::unique_ptr, true>> nullable_keys128; - std::unique_ptr, true>> nullable_keys256; + std::unique_ptr, true>> nullable_keys128; + std::unique_ptr, true>> nullable_keys256; /** Unlike Aggregator, `concat` method is not used here. * This is done because `hashed` method, although slower, but in this case, uses less RAM. * since when you use it, the key values themselves are not stored. diff --git a/src/Interpreters/StorageID.cpp b/src/Interpreters/StorageID.cpp index ef4e4257ce7..9bf9047a2dd 100644 --- a/src/Interpreters/StorageID.cpp +++ b/src/Interpreters/StorageID.cpp @@ -33,9 +33,9 @@ StorageID::StorageID(const ASTIdentifier & table_identifier_node) StorageID::StorageID(const ASTPtr & node) { - if (auto identifier = dynamic_cast(node.get())) + if (const auto * identifier = dynamic_cast(node.get())) *this = StorageID(*identifier); - else if (auto simple_query = dynamic_cast(node.get())) + else if (const auto * simple_query = dynamic_cast(node.get())) *this = StorageID(*simple_query); else throw Exception("Unexpected AST", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp index 8a9a63206ba..7011d240feb 100644 --- a/src/Interpreters/SyntaxAnalyzer.cpp +++ b/src/Interpreters/SyntaxAnalyzer.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -28,10 +29,10 @@ #include #include #include -#include -#include #include +#include + #include #include @@ -60,25 +61,40 @@ namespace using LogAST = DebugASTLog; /// set to true to enable logs -/// Select implementation of countDistinct based on settings. +/// Select implementation of a function based on settings. /// Important that it is done as query rewrite. It means rewritten query /// will be sent to remote servers during distributed query execution, /// and on all remote servers, function implementation will be same. +template struct CustomizeFunctionsData { using TypeToVisit = ASTFunction; - const String & count_distinct; + const String & customized_func_name; void visit(ASTFunction & func, ASTPtr &) { - if (Poco::toLower(func.name) == "countdistinct") - func.name = count_distinct; + if (Poco::toLower(func.name) == func_name) + { + func.name = customized_func_name; + } } }; -using CustomizeFunctionsMatcher = OneTypeMatcher; -using CustomizeFunctionsVisitor = InDepthNodeVisitor; +char countdistinct[] = "countdistinct"; +using CustomizeFunctionsVisitor = InDepthNodeVisitor>, true>; + +char in[] = "in"; +using CustomizeInVisitor = InDepthNodeVisitor>, true>; + +char notIn[] = "notin"; +using CustomizeNotInVisitor = InDepthNodeVisitor>, true>; + +char globalIn[] = "globalin"; +using CustomizeGlobalInVisitor = InDepthNodeVisitor>, true>; + +char globalNotIn[] = "globalnotin"; +using CustomizeGlobalNotInVisitor = InDepthNodeVisitor>, true>; /// Translate qualified names such as db.table.column, table.column, table_alias.column to names' normal form. @@ -200,28 +216,6 @@ void executeScalarSubqueries(ASTPtr & query, const Context & context, size_t sub ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query); } -/** Calls to these functions in the GROUP BY statement would be - * replaced by their immediate argument. - */ -const std::unordered_set injective_function_names -{ - "negate", - "bitNot", - "reverse", - "reverseUTF8", - "toString", - "toFixedString", - "IPv4NumToString", - "IPv4StringToNum", - "hex", - "unhex", - "bitmaskToList", - "bitmaskToArray", - "tuple", - "regionToName", - "concatAssumeInjective", -}; - const std::unordered_set possibly_injective_function_names { "dictGetString", @@ -262,6 +256,8 @@ void appendUnusedGroupByColumn(ASTSelectQuery * select_query, const NameSet & so /// Eliminates injective function calls and constant expressions from group by statement. void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_columns, const Context & context) { + const FunctionFactory & function_factory = FunctionFactory::instance(); + if (!select_query->groupBy()) { // If there is a HAVING clause without GROUP BY, make sure we have some aggregation happen. @@ -311,7 +307,7 @@ void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_colum continue; } } - else if (!injective_function_names.count(function->name)) + else if (!function_factory.get(function->name, context)->isInjective(Block{})) { ++i; continue; @@ -549,34 +545,6 @@ void collectJoinedColumns(TableJoin & analyzed_join, const ASTSelectQuery & sele } } -void replaceJoinedTable(const ASTSelectQuery & select_query) -{ - const ASTTablesInSelectQueryElement * join = select_query.join(); - if (!join || !join->table_expression) - return; - - /// TODO: Push down for CROSS JOIN is not OK [disabled] - const auto & table_join = join->table_join->as(); - if (table_join.kind == ASTTableJoin::Kind::Cross) - return; - - auto & table_expr = join->table_expression->as(); - if (table_expr.database_and_table_name) - { - const auto & table_id = table_expr.database_and_table_name->as(); - String expr = "(select * from " + table_id.name + ") as " + table_id.shortName(); - - // FIXME: since the expression "a as b" exposes both "a" and "b" names, which is not equivalent to "(select * from a) as b", - // we can't replace aliased tables. - // FIXME: long table names include database name, which we can't save within alias. - if (table_id.alias.empty() && table_id.isShort()) - { - ParserTableExpression parser; - table_expr = parseQuery(parser, expr, 0)->as(); - } - } -} - std::vector getAggregates(ASTPtr & query, const ASTSelectQuery & select_query) { /// There can not be aggregate functions inside the WHERE and PREWHERE. @@ -641,7 +609,7 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query) /// Add columns obtained by JOIN (if needed). for (const auto & joined_column : analyzed_join->columnsFromJoinedTable()) { - auto & name = joined_column.name; + const auto & name = joined_column.name; if (available_columns.count(name)) continue; @@ -783,7 +751,8 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( SyntaxAnalyzerResult && result, const SelectQueryOptions & select_options, const std::vector & tables_with_columns, - const Names & required_result_columns) const + const Names & required_result_columns, + std::shared_ptr table_join) const { auto * select_query = query->as(); if (!select_query) @@ -795,14 +764,13 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( const auto & settings = context.getSettingsRef(); const NameSet & source_columns_set = result.source_columns_set; - result.analyzed_join = std::make_shared(settings, context.getTemporaryVolume()); + result.analyzed_join = table_join; + if (!result.analyzed_join) /// ExpressionAnalyzer expects some not empty object here + result.analyzed_join = std::make_shared(); if (remove_duplicates) renameDuplicatedColumns(select_query); - if (settings.enable_optimize_predicate_expression) - replaceJoinedTable(*select_query); - /// TODO: Remove unneeded conversion std::vector tables_with_column_names; tables_with_column_names.reserve(tables_with_columns.size()); @@ -889,6 +857,21 @@ void SyntaxAnalyzer::normalize(ASTPtr & query, Aliases & aliases, const Settings CustomizeFunctionsVisitor::Data data{settings.count_distinct_implementation}; CustomizeFunctionsVisitor(data).visit(query); + if (settings.transform_null_in) + { + CustomizeInVisitor::Data data_null_in{"nullIn"}; + CustomizeInVisitor(data_null_in).visit(query); + + CustomizeNotInVisitor::Data data_not_null_in{"notNullIn"}; + CustomizeNotInVisitor(data_not_null_in).visit(query); + + CustomizeGlobalInVisitor::Data data_global_null_in{"globalNullIn"}; + CustomizeGlobalInVisitor(data_global_null_in).visit(query); + + CustomizeGlobalNotInVisitor::Data data_global_not_null_in{"globalNotNullIn"}; + CustomizeGlobalNotInVisitor(data_global_not_null_in).visit(query); + } + /// Creates a dictionary `aliases`: alias -> ASTPtr QueryAliasesVisitor(aliases).visit(query); diff --git a/src/Interpreters/SyntaxAnalyzer.h b/src/Interpreters/SyntaxAnalyzer.h index 23e8a4b79aa..08afd14b83c 100644 --- a/src/Interpreters/SyntaxAnalyzer.h +++ b/src/Interpreters/SyntaxAnalyzer.h @@ -94,7 +94,8 @@ public: SyntaxAnalyzerResult && result, const SelectQueryOptions & select_options = {}, const std::vector & tables_with_columns = {}, - const Names & required_result_columns = {}) const; + const Names & required_result_columns = {}, + std::shared_ptr table_join = {}) const; private: const Context & context; diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index a78342f8b17..cfb31ab2a41 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -83,6 +83,27 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds"); metric_log->startCollectMetric(collect_interval_milliseconds); } + + if (query_log) + logs.emplace_back(query_log.get()); + if (query_thread_log) + logs.emplace_back(query_thread_log.get()); + if (part_log) + logs.emplace_back(part_log.get()); + if (trace_log) + logs.emplace_back(trace_log.get()); + if (text_log) + logs.emplace_back(text_log.get()); + if (metric_log) + logs.emplace_back(metric_log.get()); + + bool lazy_load = config.getBool("system_tables_lazy_load", true); + for (auto & log : logs) + { + if (!lazy_load) + log->prepareTable(); + log->startup(); + } } @@ -93,21 +114,8 @@ SystemLogs::~SystemLogs() void SystemLogs::shutdown() { - if (query_log) - query_log->shutdown(); - if (query_thread_log) - query_thread_log->shutdown(); - if (part_log) - part_log->shutdown(); - if (trace_log) - trace_log->shutdown(); - if (text_log) - text_log->shutdown(); - if (metric_log) - { - metric_log->stopCollectMetric(); - metric_log->shutdown(); - } + for (auto & log : logs) + log->shutdown(); } } diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 87da342ae1f..066956ed53a 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -2,10 +2,14 @@ #include #include +#include +#include + #include #include #include #include +#include #include #include #include @@ -59,13 +63,22 @@ namespace ErrorCodes #define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 + class Context; -class QueryLog; -class QueryThreadLog; -class PartLog; -class TextLog; -class TraceLog; -class MetricLog; + + +class ISystemLog +{ +public: + virtual String getName() = 0; + virtual ASTPtr getCreateTableQuery() = 0; + virtual void flush() = 0; + virtual void prepareTable() = 0; + virtual void startup() = 0; + virtual void shutdown() = 0; + virtual ~ISystemLog() = default; +}; + /// System logs should be destroyed in destructor of the last Context and before tables, /// because SystemLog destruction makes insert query while flushing data into underlying tables @@ -82,11 +95,13 @@ struct SystemLogs std::shared_ptr trace_log; /// Used to log traces from query profiler std::shared_ptr text_log; /// Used to log all text messages. std::shared_ptr metric_log; /// Used to log all metrics. + + std::vector logs; }; template -class SystemLog : private boost::noncopyable +class SystemLog : public ISystemLog, private boost::noncopyable { public: using Self = SystemLog; @@ -106,18 +121,31 @@ public: const String & storage_def_, size_t flush_interval_milliseconds_); - ~SystemLog(); - /** Append a record into log. * Writing to table will be done asynchronously and in case of failure, record could be lost. */ void add(const LogElement & element); + void stopFlushThread(); + /// Flush data in the buffer to disk - void flush(); + void flush() override; + + /// Start the background thread. + void startup() override; /// Stop the background flush thread before destructor. No more data will be written. - void shutdown(); + void shutdown() override + { + stopFlushThread(); + } + + String getName() override + { + return LogElement::name(); + } + + ASTPtr getCreateTableQuery() override; protected: Logger * log; @@ -154,7 +182,7 @@ private: * Renames old table if its structure is not suitable. * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created. */ - void prepareTable(); + void prepareTable() override; /// flushImpl can be executed only in saving_thread. void flushImpl(const std::vector & to_flush, uint64_t to_flush_end); @@ -174,7 +202,12 @@ SystemLog::SystemLog(Context & context_, { assert(database_name_ == DatabaseCatalog::SYSTEM_DATABASE); log = &Logger::get("SystemLog (" + database_name_ + "." + table_name_ + ")"); +} + +template +void SystemLog::startup() +{ saving_thread = ThreadFromGlobalPool([this] { savingThreadFunction(); }); } @@ -250,7 +283,7 @@ void SystemLog::flush() template -void SystemLog::shutdown() +void SystemLog::stopFlushThread() { { std::unique_lock lock(mutex); @@ -270,13 +303,6 @@ void SystemLog::shutdown() } -template -SystemLog::~SystemLog() -{ - shutdown(); -} - - template void SystemLog::savingThreadFunction() { @@ -399,7 +425,7 @@ void SystemLog::prepareTable() rename->elements.emplace_back(elem); LOG_DEBUG(log, "Existing table " << description << " for system log has obsolete or different structure." - " Renaming it to " << backQuoteIfNeed(to.table)); + " Renaming it to " << backQuoteIfNeed(to.table)); InterpreterRenameQuery(rename, context).execute(); @@ -415,22 +441,7 @@ void SystemLog::prepareTable() /// Create the table. LOG_DEBUG(log, "Creating new table " << description << " for " + LogElement::name()); - auto create = std::make_shared(); - - create->database = table_id.database_name; - create->table = table_id.table_name; - - Block sample = LogElement::createBlock(); - - auto new_columns_list = std::make_shared(); - new_columns_list->set(new_columns_list->columns, InterpreterCreateQuery::formatColumns(sample.getNamesAndTypesList())); - create->set(create->columns_list, new_columns_list); - - ParserStorage storage_parser; - ASTPtr storage_ast = parseQuery( - storage_parser, storage_def.data(), storage_def.data() + storage_def.size(), - "Storage to create table for " + LogElement::name(), 0); - create->set(create->storage, storage_ast); + auto create = getCreateTableQuery(); InterpreterCreateQuery interpreter(create, context); interpreter.setInternal(true); @@ -442,4 +453,28 @@ void SystemLog::prepareTable() is_prepared = true; } + +template +ASTPtr SystemLog::getCreateTableQuery() +{ + auto create = std::make_shared(); + + create->database = table_id.database_name; + create->table = table_id.table_name; + + Block sample = LogElement::createBlock(); + + auto new_columns_list = std::make_shared(); + new_columns_list->set(new_columns_list->columns, InterpreterCreateQuery::formatColumns(sample.getNamesAndTypesList())); + create->set(create->columns_list, new_columns_list); + + ParserStorage storage_parser; + ASTPtr storage_ast = parseQuery( + storage_parser, storage_def.data(), storage_def.data() + storage_def.size(), + "Storage to create table for " + LogElement::name(), 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + create->set(create->storage, storage_ast); + + return create; +} + } diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 30b5e8e4483..c06dd50803a 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include @@ -79,7 +81,9 @@ void TableJoin::deduplicateAndQualifyColumnNames(const NameSet & left_table_colu dedup_columns.push_back(column); auto & inserted = dedup_columns.back(); - if (left_table_columns.count(column.name)) + /// Also qualify unusual column names - that does not look like identifiers. + + if (left_table_columns.count(column.name) || !isValidIdentifierBegin(column.name.at(0))) inserted.name = right_table_prefix + column.name; original_names[inserted.name] = column.name; @@ -155,22 +159,26 @@ NamesWithAliases TableJoin::getRequiredColumns(const Block & sample, const Names return getNamesWithAliases(required_columns); } +bool TableJoin::leftBecomeNullable(const DataTypePtr & column_type) const +{ + return forceNullableLeft() && column_type->canBeInsideNullable(); +} + +bool TableJoin::rightBecomeNullable(const DataTypePtr & column_type) const +{ + return forceNullableRight() && column_type->canBeInsideNullable(); +} + void TableJoin::addJoinedColumn(const NameAndTypePair & joined_column) { - if (join_use_nulls && isLeftOrFull(table_join.kind)) - { - auto type = joined_column.type->canBeInsideNullable() ? makeNullable(joined_column.type) : joined_column.type; - columns_added_by_join.emplace_back(NameAndTypePair(joined_column.name, std::move(type))); - } + if (rightBecomeNullable(joined_column.type)) + columns_added_by_join.emplace_back(NameAndTypePair(joined_column.name, makeNullable(joined_column.type))); else columns_added_by_join.push_back(joined_column); } void TableJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) const { - bool right_or_full_join = isRightOrFull(table_join.kind); - bool left_or_full_join = isLeftOrFull(table_join.kind); - for (auto & col : sample_block) { /// Materialize column. @@ -179,9 +187,7 @@ void TableJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) cons if (col.column) col.column = nullptr; - bool make_nullable = join_use_nulls && right_or_full_join; - - if (make_nullable && col.type->canBeInsideNullable()) + if (leftBecomeNullable(col.type)) col.type = makeNullable(col.type); } @@ -189,9 +195,7 @@ void TableJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) cons { auto res_type = col.type; - bool make_nullable = join_use_nulls && left_or_full_join; - - if (make_nullable && res_type->canBeInsideNullable()) + if (rightBecomeNullable(res_type)) res_type = makeNullable(res_type); sample_block.insert(ColumnWithTypeAndName(nullptr, res_type, col.name)); @@ -238,4 +242,31 @@ bool TableJoin::allowMergeJoin() const return allow_merge_join; } +bool TableJoin::allowDictJoin(const String & dict_key, const Block & sample_block, Names & names, NamesAndTypesList & result_columns) const +{ + /// Support ALL INNER, [ANY | ALL | SEMI | ANTI] LEFT + if (!isLeft(kind()) && !(isInner(kind()) && strictness() == ASTTableJoin::Strictness::All)) + return false; + + const Names & right_keys = keyNamesRight(); + if (right_keys.size() != 1) + return false; + + for (const auto & col : sample_block) + { + String original = original_names.find(col.name)->second; + if (col.name == right_keys[0]) + { + if (original != dict_key) + return false; /// JOIN key != Dictionary key + continue; /// do not extract key column + } + + names.push_back(original); + result_columns.push_back({col.name, col.type}); + } + + return true; +} + } diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 0b5ed82411a..2047f935966 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ class Context; class ASTSelectQuery; struct DatabaseAndTableWithAlias; class Block; +class DictionaryReader; struct Settings; @@ -42,10 +44,10 @@ class TableJoin friend class SyntaxAnalyzer; const SizeLimits size_limits; - const size_t default_max_bytes; - const bool join_use_nulls; + const size_t default_max_bytes = 0; + const bool join_use_nulls = false; const size_t max_joined_block_rows = 0; - JoinAlgorithm join_algorithm; + JoinAlgorithm join_algorithm = JoinAlgorithm::AUTO; const bool partial_merge_join_optimizations = false; const size_t partial_merge_join_rows_in_right_blocks = 0; @@ -69,6 +71,7 @@ class TableJoin VolumePtr tmp_volume; public: + TableJoin() = default; TableJoin(const Settings &, VolumePtr tmp_volume); /// for StorageJoin @@ -84,12 +87,16 @@ public: table_join.strictness = strictness; } + StoragePtr joined_storage; + std::shared_ptr dictionary_reader; + ASTTableJoin::Kind kind() const { return table_join.kind; } ASTTableJoin::Strictness strictness() const { return table_join.strictness; } bool sameStrictnessAndKind(ASTTableJoin::Strictness, ASTTableJoin::Kind) const; const SizeLimits & sizeLimits() const { return size_limits; } VolumePtr getTemporaryVolume() { return tmp_volume; } bool allowMergeJoin() const; + bool allowDictJoin(const String & dict_key, const Block & sample_block, Names &, NamesAndTypesList &) const; bool preferMergeJoin() const { return join_algorithm == JoinAlgorithm::PREFER_PARTIAL_MERGE; } bool forceMergeJoin() const { return join_algorithm == JoinAlgorithm::PARTIAL_MERGE; } bool forceHashJoin() const { return join_algorithm == JoinAlgorithm::HASH; } @@ -115,6 +122,8 @@ public: size_t rightKeyInclusion(const String & name) const; NameSet requiredRightKeys() const; + bool leftBecomeNullable(const DataTypePtr & column_type) const; + bool rightBecomeNullable(const DataTypePtr & column_type) const; void addJoinedColumn(const NameAndTypePair & joined_column); void addJoinedColumnsAndCorrectNullability(Block & sample_block) const; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 42b0721859f..c518eef7b0a 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -140,7 +140,7 @@ void ThreadStatus::finalizePerformanceCounters() { if (global_context && query_context) { - auto & settings = query_context->getSettingsRef(); + const auto & settings = query_context->getSettingsRef(); if (settings.log_queries && settings.log_query_threads) if (auto thread_log = global_context->getQueryThreadLog()) logToQueryThreadLog(*thread_log); diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 17b1bc004f8..b03bb4b0de3 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -36,7 +37,7 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const const String & short_name = identifier.shortName(); const Names & column_names = tables[table_pos].columns; - for (auto & known_name : column_names) + for (const auto & known_name : column_names) { if (short_name == known_name) return false; @@ -47,7 +48,7 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const } const Names & hidden_names = tables[table_pos].hidden_columns; - for (auto & known_name : hidden_names) + for (const auto & known_name : hidden_names) { if (short_name == known_name) return false; @@ -107,8 +108,9 @@ void TranslateQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &, IdentifierSemantic::setMembership(identifier, table_pos); /// In case if column from the joined table are in source columns, change it's name to qualified. - auto & table = data.tables[table_pos].table; - if (table_pos && data.hasColumn(short_name)) + /// Also always leave unusual identifiers qualified. + const auto & table = data.tables[table_pos].table; + if (table_pos && (data.hasColumn(short_name) || !isValidIdentifierBegin(short_name.at(0)))) IdentifierSemantic::setColumnLongName(identifier, table); else IdentifierSemantic::setColumnShortName(identifier, table); @@ -128,7 +130,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTFunction & node, const ASTPtr &, D func_arguments->children.clear(); } -void TranslateQualifiedNamesMatcher::visit(const ASTQualifiedAsterisk & , const ASTPtr & ast, Data & data) +void TranslateQualifiedNamesMatcher::visit(const ASTQualifiedAsterisk &, const ASTPtr & ast, Data & data) { if (ast->children.size() != 1) throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR); @@ -155,7 +157,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTTableJoin & join, const ASTPtr & , void TranslateQualifiedNamesMatcher::visit(ASTSelectQuery & select, const ASTPtr & , Data & data) { - if (auto join = select.join()) + if (const auto * join = select.join()) extractJoinUsingColumns(join->table_join, data); /// If the WHERE clause or HAVING consists of a single qualified column, the reference must be translated not only in children, diff --git a/src/Interpreters/castColumn.cpp b/src/Interpreters/castColumn.cpp index dd281540b51..2e6604f7df5 100644 --- a/src/Interpreters/castColumn.cpp +++ b/src/Interpreters/castColumn.cpp @@ -38,9 +38,4 @@ ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type return temporary_block.getByPosition(2).column; } -ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context &) -{ - return castColumn(arg, type); -} - } diff --git a/src/Interpreters/castColumn.h b/src/Interpreters/castColumn.h index 28914f34977..44c7f071c26 100644 --- a/src/Interpreters/castColumn.h +++ b/src/Interpreters/castColumn.h @@ -1,12 +1,9 @@ #pragma once #include -#include namespace DB { ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type); -ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context & context); - } diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index e5d468c88a6..9fc4c3f9d04 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -196,9 +196,9 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID if (which_type.isInt64()) return convertNumericType(src, type); if (which_type.isFloat32()) return convertNumericType(src, type); if (which_type.isFloat64()) return convertNumericType(src, type); - if (auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); - if (auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); - if (auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); + if (const auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); + if (const auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); + if (const auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); if (!which_type.isDateOrDateTime() && !which_type.isUUID() && !which_type.isEnum()) throw Exception{"Logical error: unknown numeric type " + type.getName(), ErrorCodes::LOGICAL_ERROR}; @@ -230,7 +230,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID } else if (which_type.isDateTime64()) { - const auto date_time64 = typeid_cast(&type); + const auto * date_time64 = typeid_cast(&type); /// Convert 'YYYY-MM-DD hh:mm:ss.NNNNNNNNN' Strings to DateTime return stringToDateTime64(src.get(), date_time64->getScale()); } @@ -257,7 +257,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID const Array & src_arr = src.get(); size_t src_arr_size = src_arr.size(); - auto & element_type = *(type_array->getNestedType()); + const auto & element_type = *(type_array->getNestedType()); bool have_unconvertible_element = false; Array res(src_arr_size); for (size_t i = 0; i < src_arr_size; ++i) @@ -289,7 +289,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID bool have_unconvertible_element = false; for (size_t i = 0; i < dst_tuple_size; ++i) { - auto & element_type = *(type_tuple->getElements()[i]); + const auto & element_type = *(type_tuple->getElements()[i]); res[i] = convertFieldToType(src_tuple[i], element_type); if (!res[i].isNull() || element_type.isNullable()) continue; @@ -321,7 +321,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID throw Exception(String("Cannot convert ") + src.getTypeName() + " to " + agg_func_type->getName(), ErrorCodes::TYPE_MISMATCH); - auto & name = src.get().name; + const auto & name = src.get().name; if (agg_func_type->getName() != name) throw Exception("Cannot convert " + name + " to " + agg_func_type->getName(), ErrorCodes::TYPE_MISMATCH); @@ -353,9 +353,9 @@ Field convertFieldToType(const Field & from_value, const IDataType & to_type, co if (from_type_hint && from_type_hint->equals(to_type)) return from_value; - if (auto * low_cardinality_type = typeid_cast(&to_type)) + if (const auto * low_cardinality_type = typeid_cast(&to_type)) return convertFieldToType(from_value, *low_cardinality_type->getDictionaryType(), from_type_hint); - else if (auto * nullable_type = typeid_cast(&to_type)) + else if (const auto * nullable_type = typeid_cast(&to_type)) { const IDataType & nested_type = *nullable_type->getNestedType(); if (from_type_hint && from_type_hint->equals(nested_type)) diff --git a/src/Interpreters/createBlockSelector.cpp b/src/Interpreters/createBlockSelector.cpp index 2b08ca0845c..0759b9d9601 100644 --- a/src/Interpreters/createBlockSelector.cpp +++ b/src/Interpreters/createBlockSelector.cpp @@ -5,8 +5,8 @@ #include -#ifdef __SSE2__ - #define LIBDIVIDE_USE_SSE2 1 +#if defined(__SSE2__) +# define LIBDIVIDE_SSE2 1 #endif #include diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c9c66832f08..288e1e398f1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -80,7 +80,7 @@ static String prepareQueryForLogging(const String & query, Context & context) // wiping sensitive data before cropping query by log_queries_cut_to_length, // otherwise something like credit card without last digit can go to log - if (auto masker = SensitiveDataMasker::getInstance()) + if (auto * masker = SensitiveDataMasker::getInstance()) { auto matches = masker->wipeSensitiveData(res); if (matches > 0) @@ -157,7 +157,7 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c /// Log the start of query execution into the table if necessary. QueryLogElement elem; - elem.type = QueryLogElement::EXCEPTION_BEFORE_START; + elem.type = QueryLogElementType::EXCEPTION_BEFORE_START; elem.event_time = current_time; elem.query_start_time = current_time; @@ -175,7 +175,7 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c /// Update performance counters before logging to query_log CurrentThread::finalizePerformanceCounters(); - if (settings.log_queries) + if (settings.log_queries && elem.type >= settings.log_queries_min_type) if (auto query_log = context.getQueryLog()) query_log->add(elem); } @@ -332,7 +332,7 @@ static std::tuple executeQueryImpl( else res = interpreter->execute(); - if (auto * insert_interpreter = typeid_cast(&*interpreter)) + if (const auto * insert_interpreter = typeid_cast(&*interpreter)) { /// Save insertion table (not table function). TODO: support remote() table function. auto table_id = insert_interpreter->getDatabaseTable(); @@ -389,7 +389,7 @@ static std::tuple executeQueryImpl( if (res.out) { - if (auto stream = dynamic_cast(res.out.get())) + if (auto * stream = dynamic_cast(res.out.get())) { stream->setProcessListElement(context.getProcessListElement()); } @@ -400,7 +400,7 @@ static std::tuple executeQueryImpl( { QueryLogElement elem; - elem.type = QueryLogElement::QUERY_START; + elem.type = QueryLogElementType::QUERY_START; elem.event_time = current_time; elem.query_start_time = current_time; @@ -412,7 +412,7 @@ static std::tuple executeQueryImpl( bool log_queries = settings.log_queries && !internal; /// Log into system table start of query execution, if need. - if (log_queries) + if (log_queries && elem.type >= settings.log_queries_min_type) { if (settings.log_query_settings) elem.query_settings = std::make_shared(context.getSettingsRef()); @@ -422,7 +422,7 @@ static std::tuple executeQueryImpl( } /// Also make possible for caller to log successful query finish and exception during execution. - auto finish_callback = [elem, &context, log_queries] (IBlockInputStream * stream_in, IBlockOutputStream * stream_out) mutable + auto finish_callback = [elem, &context, log_queries, log_queries_min_type = settings.log_queries_min_type] (IBlockInputStream * stream_in, IBlockOutputStream * stream_out) mutable { QueryStatus * process_list_elem = context.getProcessListElement(); @@ -436,7 +436,7 @@ static std::tuple executeQueryImpl( double elapsed_seconds = info.elapsed_seconds; - elem.type = QueryLogElement::QUERY_FINISH; + elem.type = QueryLogElementType::QUERY_FINISH; elem.event_time = time(nullptr); elem.query_duration_ms = elapsed_seconds * 1000; @@ -464,7 +464,7 @@ static std::tuple executeQueryImpl( } else if (stream_out) /// will be used only for ordinary INSERT queries { - if (auto counting_stream = dynamic_cast(stream_out)) + if (const auto * counting_stream = dynamic_cast(stream_out)) { /// NOTE: Redundancy. The same values could be extracted from process_list_elem->progress_out.query_settings = process_list_elem->progress_in elem.result_rows = counting_stream->getProgress().read_rows; @@ -484,19 +484,19 @@ static std::tuple executeQueryImpl( elem.thread_ids = std::move(info.thread_ids); elem.profile_counters = std::move(info.profile_counters); - if (log_queries) + if (log_queries && elem.type >= log_queries_min_type) { if (auto query_log = context.getQueryLog()) query_log->add(elem); } }; - auto exception_callback = [elem, &context, log_queries, quota(quota)] () mutable + auto exception_callback = [elem, &context, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota)] () mutable { if (quota) quota->used(Quota::ERRORS, 1, /* check_exceeded = */ false); - elem.type = QueryLogElement::EXCEPTION_WHILE_PROCESSING; + elem.type = QueryLogElementType::EXCEPTION_WHILE_PROCESSING; elem.event_time = time(nullptr); elem.query_duration_ms = 1000 * (elem.event_time - elem.query_start_time); @@ -529,7 +529,7 @@ static std::tuple executeQueryImpl( logException(context, elem); /// In case of exception we log internal queries also - if (log_queries) + if (log_queries && elem.type >= log_queries_min_type) { if (auto query_log = context.getQueryLog()) query_log->add(elem); diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 7625de70776..92fa4305824 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -84,9 +84,9 @@ static NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression else if (table_expression.table_function) { const auto table_function = table_expression.table_function; - auto query_context = const_cast(&context.getQueryContext()); + auto * query_context = const_cast(&context.getQueryContext()); const auto & function_storage = query_context->executeTableFunction(table_function); - auto & columns = function_storage->getColumns(); + const auto & columns = function_storage->getColumns(); names_and_type_list = columns.getOrdinary(); materialized = columns.getMaterialized(); aliases = columns.getAliases(); @@ -96,7 +96,7 @@ static NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression { auto table_id = context.resolveStorageID(table_expression.database_and_table_name); const auto & table = DatabaseCatalog::instance().getTable(table_id); - auto & columns = table->getColumns(); + const auto & columns = table->getColumns(); names_and_type_list = columns.getOrdinary(); materialized = columns.getMaterialized(); aliases = columns.getAliases(); diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp index cd875234243..0521ab7fd8b 100644 --- a/src/Interpreters/interpretSubquery.cpp +++ b/src/Interpreters/interpretSubquery.cpp @@ -87,7 +87,7 @@ std::shared_ptr interpretSubquery( /// get columns list for target table if (function) { - auto query_context = const_cast(&context.getQueryContext()); + auto * query_context = const_cast(&context.getQueryContext()); const auto & storage = query_context->executeTableFunction(table_expression); columns = storage->getColumns().getOrdinary(); select_query->addTableFunction(*const_cast(&table_expression)); // XXX: const_cast should be avoided! diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 7ef8d66d752..701c520c9f8 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -41,7 +41,7 @@ void removeColumnNullability(ColumnWithTypeAndName & column) column.type = static_cast(*column.type).getNestedType(); if (column.column) { - auto * nullable_column = checkAndGetColumn(*column.column); + const auto * nullable_column = checkAndGetColumn(*column.column); ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); MutableColumnPtr mutable_column = (*std::move(nested_column)).mutate(); column.column = std::move(mutable_column); @@ -53,7 +53,7 @@ ColumnRawPtrs materializeColumnsInplace(Block & block, const Names & names) ColumnRawPtrs ptrs; ptrs.reserve(names.size()); - for (auto & column_name : names) + for (const auto & column_name : names) { auto & column = block.getByName(column_name).column; column = recursiveRemoveLowCardinality(column->convertToFullColumnIfConst()); @@ -68,7 +68,7 @@ Columns materializeColumns(const Block & block, const Names & names) Columns materialized; materialized.reserve(names.size()); - for (auto & column_name : names) + for (const auto & column_name : names) { const auto & src_column = block.getByName(column_name).column; materialized.emplace_back(recursiveRemoveLowCardinality(src_column->convertToFullColumnIfConst())); @@ -82,7 +82,7 @@ ColumnRawPtrs getRawPointers(const Columns & columns) ColumnRawPtrs ptrs; ptrs.reserve(columns.size()); - for (auto & column : columns) + for (const auto & column : columns) ptrs.push_back(column.get()); return ptrs; @@ -125,7 +125,7 @@ ColumnRawPtrs extractKeysForJoin(const Block & block_keys, const Names & key_nam key_columns[i] = block_keys.getByName(column_name).column.get(); /// We will join only keys, where all components are not NULL. - if (auto * nullable = checkAndGetColumn(*key_columns[i])) + if (const auto * nullable = checkAndGetColumn(*key_columns[i])) key_columns[i] = &nullable->getNestedColumn(); } diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 53954faa2c0..42bef47a501 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -36,7 +36,7 @@ static void executeCreateQuery( bool has_force_restore_data_flag) { ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "in file " + file_name, 0); + ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "in file " + file_name, 0, context.getSettingsRef().max_parser_depth); auto & ast_create_query = ast->as(); ast_create_query.attach = true; diff --git a/src/Interpreters/misc.h b/src/Interpreters/misc.h index e2f34375dc0..0fd0e12a4bb 100644 --- a/src/Interpreters/misc.h +++ b/src/Interpreters/misc.h @@ -5,12 +5,12 @@ namespace DB inline bool functionIsInOperator(const std::string & name) { - return name == "in" || name == "notIn"; + return name == "in" || name == "notIn" || name == "nullIn" || name == "notNullIn"; } inline bool functionIsInOrGlobalInOperator(const std::string & name) { - return functionIsInOperator(name) || name == "globalIn" || name == "globalNotIn"; + return functionIsInOperator(name) || name == "globalIn" || name == "globalNotIn" || name == "globalNullIn" || name == "globalNotNullIn"; } inline bool functionIsLikeOperator(const std::string & name) diff --git a/src/Interpreters/tests/CMakeLists.txt b/src/Interpreters/tests/CMakeLists.txt index da45c1a5153..26ebf007e6c 100644 --- a/src/Interpreters/tests/CMakeLists.txt +++ b/src/Interpreters/tests/CMakeLists.txt @@ -47,9 +47,6 @@ add_executable (two_level_hash_map two_level_hash_map.cpp) target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (two_level_hash_map PRIVATE dbms) -add_executable (logical_expressions_optimizer logical_expressions_optimizer.cpp) -target_link_libraries (logical_expressions_optimizer PRIVATE dbms clickhouse_parsers) - add_executable (in_join_subqueries_preprocessor in_join_subqueries_preprocessor.cpp) target_link_libraries (in_join_subqueries_preprocessor PRIVATE dbms clickhouse_parsers) add_check(in_join_subqueries_preprocessor) diff --git a/src/Interpreters/tests/create_query.cpp b/src/Interpreters/tests/create_query.cpp index 20a0bfcb062..82bb8db5d78 100644 --- a/src/Interpreters/tests/create_query.cpp +++ b/src/Interpreters/tests/create_query.cpp @@ -76,9 +76,10 @@ try ") ENGINE = Log"; ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); context.setPath("./"); diff --git a/src/Interpreters/tests/expression.cpp b/src/Interpreters/tests/expression.cpp index 8d64b4f64ce..8327514b3d3 100644 --- a/src/Interpreters/tests/expression.cpp +++ b/src/Interpreters/tests/expression.cpp @@ -41,12 +41,13 @@ int main(int argc, char ** argv) "s1 < s2 AND x % 3 < x % 5"; ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); formatAST(*ast, std::cerr); std::cerr << std::endl; - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); NamesAndTypesList columns { diff --git a/src/Interpreters/tests/expression_analyzer.cpp b/src/Interpreters/tests/expression_analyzer.cpp index 2f8b6b2aef2..d4cbb1e2630 100644 --- a/src/Interpreters/tests/expression_analyzer.cpp +++ b/src/Interpreters/tests/expression_analyzer.cpp @@ -71,8 +71,8 @@ private: { ParserSelectQuery parser; std::string message; - auto text = query.data(); - if (ASTPtr ast = tryParseQuery(parser, text, text + query.size(), message, false, "", false, 0)) + const auto * text = query.data(); + if (ASTPtr ast = tryParseQuery(parser, text, text + query.size(), message, false, "", false, 0, 0)) return ast; throw Exception(message, ErrorCodes::SYNTAX_ERROR); } @@ -95,7 +95,8 @@ int main() } }; - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); auto system_database = std::make_shared("system"); diff --git a/src/Interpreters/tests/gtest_cycle_aliases.cpp b/src/Interpreters/tests/gtest_cycle_aliases.cpp index c8037b23d84..593db93de3e 100644 --- a/src/Interpreters/tests/gtest_cycle_aliases.cpp +++ b/src/Interpreters/tests/gtest_cycle_aliases.cpp @@ -12,11 +12,11 @@ TEST(QueryNormalizer, SimpleCycleAlias) { String query = "a as b, b as a"; ParserExpressionList parser(false); - ASTPtr ast = parseQuery(parser, query, 0); + ASTPtr ast = parseQuery(parser, query, 0, 0); Aliases aliases; - aliases["a"] = parseQuery(parser, "b as a", 0)->children[0]; - aliases["b"] = parseQuery(parser, "a as b", 0)->children[0]; + aliases["a"] = parseQuery(parser, "b as a", 0, 0)->children[0]; + aliases["b"] = parseQuery(parser, "a as b", 0, 0)->children[0]; Settings settings; QueryNormalizer::Data normalizer_data(aliases, settings); diff --git a/src/Interpreters/tests/gtest_merge_tree_set_index.cpp b/src/Interpreters/tests/gtest_merge_tree_set_index.cpp index d7c0e46d1ca..4bdbc9c9fc5 100644 --- a/src/Interpreters/tests/gtest_merge_tree_set_index.cpp +++ b/src/Interpreters/tests/gtest_merge_tree_set_index.cpp @@ -63,7 +63,7 @@ TEST(MergeTreeSetIndex, checkInRangeTuple) { auto values = {1, 1, 3, 3, 3, 10}; auto mut = types[0]->createColumn(); - for (auto & val : values) + for (const auto & val : values) mut->insert(val); columns.push_back(std::move(mut)); } @@ -71,7 +71,7 @@ TEST(MergeTreeSetIndex, checkInRangeTuple) { auto values = {"a", "b", "a", "a", "b", "c"}; auto mut = types[1]->createColumn(); - for (auto & val : values) + for (const auto & val : values) mut->insert(val); columns.push_back(std::move(mut)); } diff --git a/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index 9a17f03f32a..2b53277d02f 100644 --- a/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -1156,7 +1156,8 @@ static bool run() TestResult check(const TestEntry & entry) { - static DB::Context context = DB::Context::createGlobal(); + static DB::SharedContextHolder shared_context = DB::Context::createShared(); + static DB::Context context = DB::Context::createGlobal(shared_context.get()); context.makeGlobalContext(); try @@ -1222,9 +1223,9 @@ bool parse(DB::ASTPtr & ast, const std::string & query) { DB::ParserSelectQuery parser; std::string message; - auto begin = query.data(); - auto end = begin + query.size(); - ast = DB::tryParseQuery(parser, begin, end, message, false, "", false, 0); + const auto * begin = query.data(); + const auto * end = begin + query.size(); + ast = DB::tryParseQuery(parser, begin, end, message, false, "", false, 0, 0); return ast != nullptr; } diff --git a/src/Interpreters/tests/logical_expressions_optimizer.cpp b/src/Interpreters/tests/logical_expressions_optimizer.cpp deleted file mode 100644 index c21c4dda299..00000000000 --- a/src/Interpreters/tests/logical_expressions_optimizer.cpp +++ /dev/null @@ -1,295 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - - -namespace -{ - -struct TestEntry -{ - std::string input; - std::string expected_output; - UInt64 limit; -}; - -using TestEntries = std::vector; -using TestResult = std::pair; - -void run(); -void performTests(const TestEntries & entries); -TestResult check(const TestEntry & entry); -bool parse(DB::ASTPtr & ast, const std::string & query); -bool equals(const DB::ASTPtr & lhs, const DB::ASTPtr & rhs); -void reorder(DB::IAST * ast); - - -void run() -{ - /// NOTE: Queries are not always realistic, but we are only interested in the syntax. - TestEntries entries = - { - { - "SELECT 1", - "SELECT 1", - 3 - }, - - // WHERE - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - 4 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol')", - 3 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE (value = 1000) OR name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol') OR (value = 2000)", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol') OR value IN (1000, 2000)", - 2 - }, - - { - "SELECT value FROM report WHERE ((value + 1) = 1000) OR ((2 * value) = 2000) OR ((2 * value) = 4000) OR ((value + 1) = 3000)", - "SELECT value FROM report WHERE ((value + 1) IN (1000, 3000)) OR ((2 * value) IN (2000, 4000))", - 2 - }, - - { - "SELECT name, value FROM report WHERE ((name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')) AND ((value = 1000) OR (value = 2000))", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol') AND ((value = 1000) OR (value = 2000))", - 3 - }, - - // PREWHERE - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - 4 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE name IN ('Alice', 'Bob', 'Carol')", - 3 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE (value = 1000) OR name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol') OR (value = 2000)", - "SELECT name, value FROM report PREWHERE name IN ('Alice', 'Bob', 'Carol') OR value IN (1000, 2000)", - 2 - }, - - { - "SELECT value FROM report PREWHERE ((value + 1) = 1000) OR ((2 * value) = 2000) OR ((2 * value) = 4000) OR ((value + 1) = 3000)", - "SELECT value FROM report PREWHERE (value + 1) IN (1000, 3000) OR (2 * value) IN (2000, 4000)", - 2 - }, - - // HAVING - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING number = 1", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING number = 1", - 2 - }, - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING number IN (1, 2)", - 2 - }, - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING (number = 1) OR (number = 2)", - 3 - }, - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING ((number + 1) = 1) OR ((number + 1) = 2) OR ((number + 3) = 7)", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING ((number + 3) = 7) OR (number + 1) IN (1, 2)", - 2 - }, - - // PREWHERE + WHERE + HAVING - - { - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE (U = 1) OR (U = 2) " - "WHERE (T = 1) OR (T = 2) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE U IN (1, 2) " - "WHERE T IN (1, 2) GROUP BY number HAVING number IN (1, 2)", - 2 - }, - - { - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE (U = 1) OR (U = 2) OR (U = 3) " - "WHERE (T = 1) OR (T = 2) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE U IN (1, 2, 3) " - "WHERE (T = 1) OR (T = 2) GROUP BY number HAVING (number = 1) OR (number = 2)", - 3 - }, - - { - "SELECT x = 1 OR x=2 OR (x = 3 AS x3) AS y, 4 AS x", - "SELECT x IN (1, 2, 3) AS y, 4 AS x", - 2 - } - }; - - performTests(entries); -} - -void performTests(const TestEntries & entries) -{ - unsigned int count = 0; - unsigned int i = 1; - - for (const auto & entry : entries) - { - auto res = check(entry); - if (res.first) - { - ++count; - std::cout << "Test " << i << " passed.\n"; - } - else - std::cout << "Test " << i << " failed. Expected: " << entry.expected_output << ". Received: " << res.second << "\n"; - - ++i; - } - std::cout << count << " out of " << entries.size() << " test(s) passed.\n"; -} - -TestResult check(const TestEntry & entry) -{ - try - { - /// Parse and optimize the incoming query. - DB::ASTPtr ast_input; - if (!parse(ast_input, entry.input)) - return TestResult(false, "parse error"); - - auto select_query = typeid_cast(&*ast_input); - - DB::LogicalExpressionsOptimizer optimizer(select_query, entry.limit); - optimizer.perform(); - - /// Parse the expected result. - DB::ASTPtr ast_expected; - if (!parse(ast_expected, entry.expected_output)) - return TestResult(false, "parse error"); - - /// Compare the optimized query and the expected result. - bool res = equals(ast_input, ast_expected); - std::string output = DB::queryToString(ast_input); - - return TestResult(res, output); - } - catch (DB::Exception & e) - { - return TestResult(false, e.displayText()); - } -} - -bool parse(DB::ASTPtr & ast, const std::string & query) -{ - DB::ParserSelectQuery parser; - std::string message; - auto begin = query.data(); - auto end = begin + query.size(); - ast = DB::tryParseQuery(parser, begin, end, message, false, "", false, 0); - return ast != nullptr; -} - -bool equals(const DB::ASTPtr & lhs, const DB::ASTPtr & rhs) -{ - DB::ASTPtr lhs_reordered = lhs->clone(); - reorder(&*lhs_reordered); - - DB::ASTPtr rhs_reordered = rhs->clone(); - reorder(&*rhs_reordered); - - return lhs_reordered->getTreeHash() == rhs_reordered->getTreeHash(); -} - -void reorderImpl(DB::IAST * ast) -{ - if (ast == nullptr) - return; - - auto & children = ast->children; - if (children.empty()) - return; - - for (auto & child : children) - reorderImpl(&*child); - - std::sort(children.begin(), children.end(), [](const DB::ASTPtr & lhs, const DB::ASTPtr & rhs) - { - return lhs->getTreeHash() < rhs->getTreeHash(); - }); -} - -void reorder(DB::IAST * ast) -{ - if (ast == nullptr) - return; - - auto select_query = typeid_cast(ast); - if (select_query == nullptr) - return; - - reorderImpl(select_query->where().get()); - reorderImpl(select_query->prewhere().get()); - reorderImpl(select_query->having().get()); -} - -} - -int main() -{ - run(); - return 0; -} diff --git a/src/Interpreters/tests/select_query.cpp b/src/Interpreters/tests/select_query.cpp index 3775e5a3532..8ce65ede05f 100644 --- a/src/Interpreters/tests/select_query.cpp +++ b/src/Interpreters/tests/select_query.cpp @@ -30,7 +30,8 @@ try /// Pre-initialize the `DateLUT` so that the first initialization does not affect the measured execution speed. DateLUT::instance(); - Context context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); context.setPath("./"); diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make new file mode 100644 index 00000000000..51b39c02f49 --- /dev/null +++ b/src/Interpreters/ya.make @@ -0,0 +1,139 @@ +LIBRARY() + +ADDINCL( + contrib/libs/libdivide + contrib/libs/pdqsort +) + +PEERDIR( + clickhouse/src/Core + contrib/libs/libdivide + contrib/libs/pdqsort +) + +NO_COMPILER_WARNINGS() + +SRCS( + ActionLocksManager.cpp + ActionsVisitor.cpp + addMissingDefaults.cpp + addTypeConversionToAST.cpp + Aggregator.cpp + ArrayJoinAction.cpp + AsynchronousMetrics.cpp + BloomFilter.cpp + castColumn.cpp + CatBoostModel.cpp + ClientInfo.cpp + Cluster.cpp + ClusterProxy/executeQuery.cpp + ClusterProxy/SelectStreamFactory.cpp + CollectJoinOnKeysVisitor.cpp + Context.cpp + convertFieldToType.cpp + createBlockSelector.cpp + CrossToInnerJoinVisitor.cpp + DatabaseAndTableWithAlias.cpp + DatabaseCatalog.cpp + DDLWorker.cpp + DictionaryReader.cpp + DNSCacheUpdater.cpp + EmbeddedDictionaries.cpp + evaluateConstantExpression.cpp + executeQuery.cpp + ExecuteScalarSubqueriesVisitor.cpp + ExpressionActions.cpp + ExpressionAnalyzer.cpp + ExternalDictionariesLoader.cpp + ExternalLoader.cpp + ExternalLoaderDatabaseConfigRepository.cpp + ExternalLoaderTempConfigRepository.cpp + ExternalLoaderXMLConfigRepository.cpp + ExternalModelsLoader.cpp + ExtractExpressionInfoVisitor.cpp + FillingRow.cpp + getClusterName.cpp + getTableExpressions.cpp + HashJoin.cpp + IdentifierSemantic.cpp + IExternalLoadable.cpp + InJoinSubqueriesPreprocessor.cpp + inplaceBlockConversions.cpp + InternalTextLogsQueue.cpp + InterpreterAlterQuery.cpp + InterpreterCheckQuery.cpp + InterpreterCreateQuery.cpp + InterpreterCreateQuotaQuery.cpp + InterpreterCreateRoleQuery.cpp + InterpreterCreateRowPolicyQuery.cpp + InterpreterCreateSettingsProfileQuery.cpp + InterpreterCreateUserQuery.cpp + InterpreterDescribeQuery.cpp + InterpreterDropAccessEntityQuery.cpp + InterpreterDropQuery.cpp + InterpreterExistsQuery.cpp + InterpreterExplainQuery.cpp + InterpreterFactory.cpp + InterpreterGrantQuery.cpp + InterpreterInsertQuery.cpp + InterpreterKillQueryQuery.cpp + InterpreterOptimizeQuery.cpp + InterpreterRenameQuery.cpp + InterpreterSelectQuery.cpp + InterpreterSelectWithUnionQuery.cpp + InterpreterSetQuery.cpp + InterpreterSetRoleQuery.cpp + InterpreterShowCreateAccessEntityQuery.cpp + InterpreterShowCreateQuery.cpp + InterpreterShowGrantsQuery.cpp + InterpreterShowProcesslistQuery.cpp + InterpreterShowQuotasQuery.cpp + InterpreterShowRowPoliciesQuery.cpp + InterpreterShowTablesQuery.cpp + InterpreterSystemQuery.cpp + InterpreterUseQuery.cpp + InterpreterWatchQuery.cpp + interpretSubquery.cpp + join_common.cpp + JoinedTables.cpp + JoinSwitcher.cpp + JoinToSubqueryTransformVisitor.cpp + loadMetadata.cpp + LogicalExpressionsOptimizer.cpp + MarkTableIdentifiersVisitor.cpp + MergeJoin.cpp + MetricLog.cpp + MutationsInterpreter.cpp + NullableUtils.cpp + OptimizeIfChains.cpp + OptimizeIfWithConstantConditionVisitor.cpp + PartLog.cpp + PredicateExpressionsOptimizer.cpp + PredicateRewriteVisitor.cpp + ProcessList.cpp + ProfileEventsExt.cpp + QueryAliasesVisitor.cpp + QueryLog.cpp + QueryNormalizer.cpp + QueryThreadLog.cpp + RenameColumnVisitor.cpp + ReplaceQueryParameterVisitor.cpp + RequiredSourceColumnsData.cpp + RequiredSourceColumnsVisitor.cpp + RowRefs.cpp + Set.cpp + SetVariants.cpp + sortBlock.cpp + StorageID.cpp + SubqueryForSet.cpp + SyntaxAnalyzer.cpp + SystemLog.cpp + TableJoin.cpp + TablesStatus.cpp + TextLog.cpp + ThreadStatusExt.cpp + TraceLog.cpp + TranslateQualifiedNamesVisitor.cpp +) + +END() diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index 8fa0dbb0d31..edb510d1a2e 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -28,16 +28,17 @@ namespace } - void formatLimit(ResourceType resource_type, ResourceAmount max, const IAST::FormatSettings & settings) + void formatLimit(ResourceType resource_type, ResourceAmount max, bool first, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX " << Quota::resourceTypeToKeyword(resource_type) - << (settings.hilite ? IAST::hilite_none : ""); + if (first) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX" << (settings.hilite ? IAST::hilite_none : ""); + else + settings.ostr << ","; - settings.ostr << (settings.hilite ? IAST::hilite_operator : "") << " = " << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << " " << (settings.hilite ? IAST::hilite_keyword : "") << Quota::resourceTypeToKeyword(resource_type) + << (settings.hilite ? IAST::hilite_none : "") << " "; - if (max == Quota::UNLIMITED) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ANY" << (settings.hilite ? IAST::hilite_none : ""); - else if (resource_type == Quota::EXECUTION_TIME) + if (resource_type == Quota::EXECUTION_TIME) settings.ostr << Quota::executionTimeToSeconds(max); else settings.ostr << max; @@ -59,9 +60,9 @@ namespace << interval_kind.toKeyword() << (settings.hilite ? IAST::hilite_none : ""); - if (limits.unset_tracking) + if (limits.drop) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " UNSET TRACKING" << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NO LIMITS" << (settings.hilite ? IAST::hilite_none : ""); } else { @@ -70,21 +71,19 @@ namespace { if (limits.max[resource_type]) { - if (limit_found) - settings.ostr << ","; + formatLimit(resource_type, *limits.max[resource_type], !limit_found, settings); limit_found = true; - formatLimit(resource_type, *limits.max[resource_type], settings); } } if (!limit_found) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TRACKING" << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TRACKING ONLY" << (settings.hilite ? IAST::hilite_none : ""); } } void formatAllLimits(const std::vector & all_limits, const IAST::FormatSettings & settings) { bool need_comma = false; - for (auto & limits : all_limits) + for (const auto & limits : all_limits) { if (need_comma) settings.ostr << ","; @@ -150,7 +149,7 @@ void ASTCreateQuotaQuery::formatImpl(const FormatSettings & settings, FormatStat } -void ASTCreateQuotaQuery::replaceCurrentUserTagWithName(const String & current_user_name) +void ASTCreateQuotaQuery::replaceCurrentUserTagWithName(const String & current_user_name) const { if (roles) roles->replaceCurrentUserTagWithName(current_user_name); diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index 09ceaea9825..b001ec44a0c 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -13,17 +13,16 @@ class ASTExtendedRoleSet; /** CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | - * [SET] TRACKING} [,...]] + * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} [,...] | + * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] * * ALTER QUOTA [IF EXISTS] name * [RENAME TO new_name] * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | - * [SET] TRACKING | - * UNSET TRACKING} [,...]] + * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} [,...] | + * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ class ASTCreateQuotaQuery : public IAST, public ASTQueryWithOnCluster @@ -48,7 +47,7 @@ public: struct Limits { std::optional max[MAX_RESOURCE_TYPE]; - bool unset_tracking = false; + bool drop = false; std::chrono::seconds duration = std::chrono::seconds::zero(); bool randomize_interval = false; }; @@ -59,7 +58,7 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - void replaceCurrentUserTagWithName(const String & current_user_name); + void replaceCurrentUserTagWithName(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index 9b36f5500c1..fc6af51da6f 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -172,7 +172,7 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format } -void ASTCreateRowPolicyQuery::replaceCurrentUserTagWithName(const String & current_user_name) +void ASTCreateRowPolicyQuery::replaceCurrentUserTagWithName(const String & current_user_name) const { if (roles) roles->replaceCurrentUserTagWithName(current_user_name); diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index 85ba674eeb0..2ad64255c04 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -48,7 +48,7 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - void replaceCurrentUserTagWithName(const String & current_user_name); + void replaceCurrentUserTagWithName(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/ASTCreateSettingsProfileQuery.cpp index 8db82b0e1cb..601425d3446 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ASTCreateSettingsProfileQuery.cpp @@ -74,7 +74,7 @@ void ASTCreateSettingsProfileQuery::formatImpl(const FormatSettings & format, Fo } -void ASTCreateSettingsProfileQuery::replaceCurrentUserTagWithName(const String & current_user_name) +void ASTCreateSettingsProfileQuery::replaceCurrentUserTagWithName(const String & current_user_name) const { if (to_roles) to_roles->replaceCurrentUserTagWithName(current_user_name); diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.h b/src/Parsers/ASTCreateSettingsProfileQuery.h index cc133397db4..cd470283410 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/ASTCreateSettingsProfileQuery.h @@ -12,10 +12,12 @@ class ASTExtendedRoleSet; /** CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ class ASTCreateSettingsProfileQuery : public IAST, public ASTQueryWithOnCluster { @@ -37,7 +39,7 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; - void replaceCurrentUserTagWithName(const String & current_user_name); + void replaceCurrentUserTagWithName(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index d901ed8f5a1..c8e2a76dfa2 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -109,7 +109,7 @@ namespace { if (std::exchange(need_comma, true)) settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NAME REGEXP " << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "REGEXP " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma2 = false; for (const auto & host_regexp : name_regexps) { diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index 5a5cc0d9550..54dc51d783b 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -13,14 +13,14 @@ class ASTSettingsProfileElements; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name * [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] - * [HOST {LOCAL | NAME 'name' | NAME REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] * [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] - * [[ADD|DROP] HOST {LOCAL | NAME 'name' | NAME REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] */ diff --git a/src/Parsers/ASTExtendedRoleSet.cpp b/src/Parsers/ASTExtendedRoleSet.cpp index 9eb06a6a101..1803af11ab3 100644 --- a/src/Parsers/ASTExtendedRoleSet.cpp +++ b/src/Parsers/ASTExtendedRoleSet.cpp @@ -37,7 +37,7 @@ void ASTExtendedRoleSet::formatImpl(const FormatSettings & settings, FormatState } else { - for (auto & role : names) + for (const auto & role : names) { if (std::exchange(need_comma, true)) settings.ostr << ", "; @@ -57,7 +57,7 @@ void ASTExtendedRoleSet::formatImpl(const FormatSettings & settings, FormatState settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (settings.hilite ? IAST::hilite_none : ""); need_comma = false; - for (auto & except_role : except_names) + for (const auto & except_role : except_names) { if (std::exchange(need_comma, true)) settings.ostr << ", "; diff --git a/src/Parsers/ASTGrantQuery.cpp b/src/Parsers/ASTGrantQuery.cpp index f91a5416011..e6764fc067a 100644 --- a/src/Parsers/ASTGrantQuery.cpp +++ b/src/Parsers/ASTGrantQuery.cpp @@ -155,7 +155,7 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F } -void ASTGrantQuery::replaceCurrentUserTagWithName(const String & current_user_name) +void ASTGrantQuery::replaceCurrentUserTagWithName(const String & current_user_name) const { if (to_roles) to_roles->replaceCurrentUserTagWithName(current_user_name); diff --git a/src/Parsers/ASTGrantQuery.h b/src/Parsers/ASTGrantQuery.h index e1ad8dc5dc5..7e3321799fb 100644 --- a/src/Parsers/ASTGrantQuery.h +++ b/src/Parsers/ASTGrantQuery.h @@ -35,7 +35,7 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - void replaceCurrentUserTagWithName(const String & current_user_name); + void replaceCurrentUserTagWithName(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 9e28a1461ca..c13c2c3f977 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -40,6 +40,8 @@ public: bool isShort() const { return name_parts.empty() || name == name_parts.back(); } void setShortName(const String & new_name); + + /// Restore name field from name_parts in case it was cropped by analyzer but we need a full form for future (re)analyze. void restoreCompoundName(); const String & shortName() const diff --git a/src/Parsers/ASTLiteral.h b/src/Parsers/ASTLiteral.h index 552f5da04a2..18f440a81a4 100644 --- a/src/Parsers/ASTLiteral.h +++ b/src/Parsers/ASTLiteral.h @@ -21,6 +21,17 @@ public: std::optional begin; std::optional end; + /* + * The name of the column corresponding to this literal. Only used to + * disambiguate the literal columns with the same display name that are + * created at the expression analyzer stage. In the future, we might want to + * have a full separation between display names and column identifiers. For + * now, this field is effectively just some private EA data. + */ + String unique_column_name; + + +public: ASTLiteral(Field && value_) : value(value_) {} ASTLiteral(const Field & value_) : value(value_) {} diff --git a/src/Parsers/ASTSettingsProfileElement.cpp b/src/Parsers/ASTSettingsProfileElement.cpp index b3f4032d14c..24f1aa60813 100644 --- a/src/Parsers/ASTSettingsProfileElement.cpp +++ b/src/Parsers/ASTSettingsProfileElement.cpp @@ -25,7 +25,8 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form { if (!parent_profile.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "PROFILE " << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (use_inherit_keyword ? "INHERIT" : "PROFILE") << " " + << (settings.hilite ? IAST::hilite_none : ""); formatProfileNameOrID(parent_profile, id_mode, settings); return; } @@ -85,4 +86,11 @@ void ASTSettingsProfileElements::formatImpl(const FormatSettings & settings, For } } + +void ASTSettingsProfileElements::setUseInheritKeyword(bool use_inherit_keyword_) +{ + for (auto & element : elements) + element->use_inherit_keyword = use_inherit_keyword_; +} + } diff --git a/src/Parsers/ASTSettingsProfileElement.h b/src/Parsers/ASTSettingsProfileElement.h index 0470b51cf85..ee1ee28c383 100644 --- a/src/Parsers/ASTSettingsProfileElement.h +++ b/src/Parsers/ASTSettingsProfileElement.h @@ -19,6 +19,7 @@ public: Field max_value; std::optional readonly; bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. + bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. bool empty() const { return parent_profile.empty() && name.empty(); } @@ -41,5 +42,7 @@ public: String getID(char) const override { return "SettingsProfileElements"; } ASTPtr clone() const override { return std::make_shared(*this); } void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + + void setUseInheritKeyword(bool use_inherit_keyword_); }; } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index f5b9afde4b3..eac96e50cb1 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -1,9 +1,12 @@ #pragma once -#include "config_core.h" #include #include +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + namespace DB { diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 30fa4a2e9fb..e0164f50df6 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1000,6 +1000,7 @@ bool ParserCollectionOfLiterals::parseImpl(Pos & pos, ASTPtr & node, Collection arr; ParserLiteral literal_p; + ParserCollectionOfLiterals collection_p(opening_bracket, closing_bracket); ++pos; while (pos.isValid()) @@ -1032,7 +1033,7 @@ bool ParserCollectionOfLiterals::parseImpl(Pos & pos, ASTPtr & node, } ASTPtr literal_node; - if (!literal_p.parse(pos, literal_node, expected)) + if (!literal_p.parse(pos, literal_node, expected) && !collection_p.parse(pos, literal_node, expected)) return false; arr.push_back(literal_node->as().value); diff --git a/src/Parsers/IParser.h b/src/Parsers/IParser.h index 925140bd25e..69c199c201e 100644 --- a/src/Parsers/IParser.h +++ b/src/Parsers/IParser.h @@ -56,8 +56,6 @@ public: /// Token iterator augmented with depth information. This allows to control recursion depth. struct Pos : TokenIterator { - using TokenIterator::TokenIterator; - uint32_t depth = 0; uint32_t max_depth = 0; @@ -126,7 +124,7 @@ public: return parse(pos, node, expected); } - virtual ~IParser() {} + virtual ~IParser() = default; }; using ParserPtr = std::unique_ptr; diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index 66e72ee4968..6007d6206ec 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -63,12 +63,22 @@ namespace }); } - bool parseLimit(IParserBase::Pos & pos, Expected & expected, ResourceType & resource_type, ResourceAmount & max) + bool parseLimit(IParserBase::Pos & pos, Expected & expected, bool first, ResourceType & resource_type, ResourceAmount & max) { return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserKeyword{"MAX"}.ignore(pos, expected)) - return false; + if (first) + { + if (!ParserKeyword{"MAX"}.ignore(pos, expected)) + return false; + } + else + { + if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) + return false; + + ParserKeyword{"MAX"}.ignore(pos, expected); + } bool resource_type_set = false; for (auto rt : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) @@ -83,9 +93,6 @@ namespace if (!resource_type_set) return false; - if (!ParserToken{TokenType::Equals}.ignore(pos, expected)) - return false; - ASTPtr max_ast; if (ParserNumber{}.parse(pos, max_ast, expected)) { @@ -95,10 +102,6 @@ namespace else max = applyVisitor(FieldVisitorConvertToNumber(), max_field); } - else if (ParserKeyword{"ANY"}.ignore(pos, expected)) - { - max = Quota::UNLIMITED; - } else return false; @@ -106,18 +109,7 @@ namespace }); } - bool parseCommaAndLimit(IParserBase::Pos & pos, Expected & expected, ResourceType & resource_type, ResourceAmount & max) - { - return IParserBase::wrapParseImpl(pos, [&] - { - if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) - return false; - - return parseLimit(pos, expected, resource_type, max); - }); - } - - bool parseLimits(IParserBase::Pos & pos, Expected & expected, bool alter, ASTCreateQuotaQuery::Limits & limits) + bool parseLimits(IParserBase::Pos & pos, Expected & expected, ASTCreateQuotaQuery::Limits & limits) { return IParserBase::wrapParseImpl(pos, [&] { @@ -142,23 +134,22 @@ namespace new_limits.duration = std::chrono::seconds(static_cast(num_intervals * interval_kind.toAvgSeconds())); - if (alter && ParserKeyword{"UNSET TRACKING"}.ignore(pos, expected)) + if (ParserKeyword{"NO LIMITS"}.ignore(pos, expected)) { - new_limits.unset_tracking = true; + new_limits.drop = true; } - else if (ParserKeyword{"SET TRACKING"}.ignore(pos, expected) || ParserKeyword{"TRACKING"}.ignore(pos, expected)) + else if (ParserKeyword{"TRACKING ONLY"}.ignore(pos, expected)) { } else { - ParserKeyword{"SET"}.ignore(pos, expected); ResourceType resource_type; ResourceAmount max; - if (!parseLimit(pos, expected, resource_type, max)) + if (!parseLimit(pos, expected, true, resource_type, max)) return false; new_limits.max[resource_type] = max; - while (parseCommaAndLimit(pos, expected, resource_type, max)) + while (parseLimit(pos, expected, false, resource_type, max)) new_limits.max[resource_type] = max; } @@ -167,7 +158,7 @@ namespace }); } - bool parseAllLimits(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector & all_limits) + bool parseAllLimits(IParserBase::Pos & pos, Expected & expected, std::vector & all_limits) { return IParserBase::wrapParseImpl(pos, [&] { @@ -175,7 +166,7 @@ namespace do { ASTCreateQuotaQuery::Limits limits; - if (!parseLimits(pos, expected, alter, limits)) + if (!parseLimits(pos, expected, limits)) { all_limits.resize(old_size); return false; @@ -199,6 +190,14 @@ namespace return true; }); } + + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); + }); + } } @@ -238,16 +237,10 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (!parseIdentifierOrStringLiteral(pos, expected, name)) return false; - String cluster; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) - return false; - } - String new_name; std::optional key_type; std::vector all_limits; + String cluster; while (true) { @@ -257,7 +250,10 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (!key_type && parseKeyType(pos, expected, key_type)) continue; - if (parseAllLimits(pos, expected, alter, all_limits)) + if (parseAllLimits(pos, expected, all_limits)) + continue; + + if (cluster.empty() && parseOnCluster(pos, expected, cluster)) continue; break; @@ -266,6 +262,9 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe std::shared_ptr roles; parseToRoles(pos, expected, attach_mode, roles); + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); + auto query = std::make_shared(); node = query; diff --git a/src/Parsers/ParserCreateQuotaQuery.h b/src/Parsers/ParserCreateQuotaQuery.h index 18e6ef6f9f7..786c8292b15 100644 --- a/src/Parsers/ParserCreateQuotaQuery.h +++ b/src/Parsers/ParserCreateQuotaQuery.h @@ -9,17 +9,16 @@ namespace DB * CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | - * [SET] TRACKING} [,...]] + * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} [,...] | + * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] * * ALTER QUOTA [IF EXISTS] name * [RENAME TO new_name] * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | - * [SET] TRACKING | - * UNSET TRACKING} [,...]] + * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} } [,...] | + * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ class ParserCreateQuotaQuery : public IParserBase diff --git a/src/Parsers/ParserCreateRoleQuery.cpp b/src/Parsers/ParserCreateRoleQuery.cpp index 05143108480..2a6f2dd2c90 100644 --- a/src/Parsers/ParserCreateRoleQuery.cpp +++ b/src/Parsers/ParserCreateRoleQuery.cpp @@ -41,6 +41,14 @@ namespace return true; }); } + + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); + }); + } } @@ -80,15 +88,10 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!parseRoleName(pos, expected, name)) return false; - String cluster; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) - return false; - } - String new_name; std::shared_ptr settings; + String cluster; + while (true) { if (alter && parseRenameTo(pos, expected, new_name)) @@ -97,6 +100,9 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (parseSettings(pos, expected, attach_mode, settings)) continue; + if (cluster.empty() && parseOnCluster(pos, expected, cluster)) + continue; + break; } diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index 8bfe54b87b2..b6840f0ed6a 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -83,14 +83,13 @@ namespace static constexpr char delete_op[] = "DELETE"; std::vector ops; - bool keyword_for = false; if (ParserKeyword{"FOR"}.ignore(pos, expected)) { - keyword_for = true; do { if (ParserKeyword{"SELECT"}.ignore(pos, expected)) ops.push_back(select_op); +#if 0 /// INSERT, UPDATE, DELETE are not supported yet else if (ParserKeyword{"INSERT"}.ignore(pos, expected)) ops.push_back(insert_op); else if (ParserKeyword{"UPDATE"}.ignore(pos, expected)) @@ -100,6 +99,7 @@ namespace else if (ParserKeyword{"ALL"}.ignore(pos, expected)) { } +#endif else return false; } @@ -109,9 +109,11 @@ namespace if (ops.empty()) { ops.push_back(select_op); +#if 0 /// INSERT, UPDATE, DELETE are not supported yet ops.push_back(insert_op); ops.push_back(update_op); ops.push_back(delete_op); +#endif } std::optional filter; @@ -123,14 +125,15 @@ namespace if (!parseConditionalExpression(pos, expected, filter)) return false; } +#if 0 /// INSERT, UPDATE, DELETE are not supported yet if (ParserKeyword{"WITH CHECK"}.ignore(pos, expected)) { keyword_with_check = true; if (!parseConditionalExpression(pos, expected, check)) return false; } - - if (!keyword_for && !keyword_using && !keyword_with_check) +#endif + if (!keyword_using && !keyword_with_check) return false; if (filter && !check && !alter) @@ -200,6 +203,14 @@ namespace return true; }); } + + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); + }); + } } @@ -243,16 +254,10 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; - String cluster; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) - return false; - } - String new_policy_name; std::optional is_restrictive; std::vector> conditions; + String cluster; while (true) { @@ -265,12 +270,18 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if (parseMultipleConditions(pos, expected, alter, conditions)) continue; + if (cluster.empty() && parseOnCluster(pos, expected, cluster)) + continue; + break; } std::shared_ptr roles; parseToRoles(pos, expected, attach_mode, roles); + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); + auto query = std::make_shared(); node = query; diff --git a/src/Parsers/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/ParserCreateSettingsProfileQuery.cpp index 4d3ed2f6e63..83d0f0c1d91 100644 --- a/src/Parsers/ParserCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ParserCreateSettingsProfileQuery.cpp @@ -33,7 +33,7 @@ namespace return false; ASTPtr new_settings_ast; - if (!ParserSettingsProfileElements{}.useIDMode(id_mode).parse(pos, new_settings_ast, expected)) + if (!ParserSettingsProfileElements{}.useIDMode(id_mode).enableInheritKeyword(true).parse(pos, new_settings_ast, expected)) return false; if (!settings) @@ -57,6 +57,14 @@ namespace return true; }); } + + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); + }); + } } @@ -96,15 +104,10 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec if (!parseIdentifierOrStringLiteral(pos, expected, name)) return false; - String cluster; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) - return false; - } - String new_name; std::shared_ptr settings; + String cluster; + while (true) { if (alter && parseRenameTo(pos, expected, new_name)) @@ -113,12 +116,18 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec if (parseSettings(pos, expected, attach_mode, settings)) continue; + if (cluster.empty() && parseOnCluster(pos, expected, cluster)) + continue; + break; } std::shared_ptr to_roles; parseToRoles(pos, expected, attach_mode, to_roles); + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); + auto query = std::make_shared(); node = query; diff --git a/src/Parsers/ParserCreateSettingsProfileQuery.h b/src/Parsers/ParserCreateSettingsProfileQuery.h index 6797fc884fa..073a8ca75ae 100644 --- a/src/Parsers/ParserCreateSettingsProfileQuery.h +++ b/src/Parsers/ParserCreateSettingsProfileQuery.h @@ -7,11 +7,11 @@ namespace DB { /** Parses queries like * CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] */ class ParserCreateSettingsProfileQuery : public IParserBase { diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index 3968c26d42e..76a06a0282f 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -166,7 +166,7 @@ namespace { new_hosts.addLocalHost(); } - else if (ParserKeyword{"NAME REGEXP"}.ignore(pos, expected)) + else if (ParserKeyword{"REGEXP"}.ignore(pos, expected)) { ASTPtr ast; if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) @@ -250,6 +250,14 @@ namespace return true; }); } + + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); + }); + } } @@ -290,13 +298,6 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!parseUserName(pos, expected, name, host_pattern)) return false; - String cluster; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) - return false; - } - String new_name; std::optional new_host_pattern; std::optional authentication; @@ -305,6 +306,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec std::optional remove_hosts; std::shared_ptr default_roles; std::shared_ptr settings; + String cluster; while (true) { @@ -320,6 +322,9 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!default_roles && parseDefaultRoles(pos, expected, attach_mode, default_roles)) continue; + if (cluster.empty() && parseOnCluster(pos, expected, cluster)) + continue; + if (alter) { if (new_name.empty() && parseRenameTo(pos, expected, new_name, new_host_pattern)) diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h index 4b2af34c003..d609894a7ec 100644 --- a/src/Parsers/ParserCreateUserQuery.h +++ b/src/Parsers/ParserCreateUserQuery.h @@ -8,13 +8,13 @@ namespace DB /** Parses queries like * CREATE USER [IF NOT EXISTS | OR REPLACE] name * [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] - * [HOST {LOCAL | NAME 'name' | NAME REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] * [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] - * [[ADD|DROP] HOST {LOCAL | NAME 'name' | NAME REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] */ class ParserCreateUserQuery : public IParserBase diff --git a/src/Parsers/ParserGrantQuery.cpp b/src/Parsers/ParserGrantQuery.cpp index f6eecbe5dba..64dde8f6524 100644 --- a/src/Parsers/ParserGrantQuery.cpp +++ b/src/Parsers/ParserGrantQuery.cpp @@ -237,6 +237,14 @@ namespace return true; }); } + + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); + }); + } } @@ -260,11 +268,8 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; String cluster; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) - return false; - } + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); bool grant_option = false; bool admin_option = false; @@ -281,10 +286,16 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!parseAccessRightsElements(pos, expected, elements) && !parseRoles(pos, expected, attach, roles)) return false; + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); + std::shared_ptr to_roles; if (!parseToRoles(pos, expected, kind, to_roles)) return false; + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); + if (kind == Kind::GRANT) { if (ParserKeyword{"WITH GRANT OPTION"}.ignore(pos, expected)) @@ -293,6 +304,9 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) admin_option = true; } + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); + if (grant_option && roles) throw Exception("GRANT OPTION should be specified for access types", ErrorCodes::SYNTAX_ERROR); if (admin_option && !elements.empty()) diff --git a/src/Parsers/ParserPartition.cpp b/src/Parsers/ParserPartition.cpp index d5ba5739ab1..a3ec4943e1c 100644 --- a/src/Parsers/ParserPartition.cpp +++ b/src/Parsers/ParserPartition.cpp @@ -48,12 +48,12 @@ bool ParserPartition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) else fields_count = 0; } - else if (auto literal = value->as()) + else if (const auto * literal = value->as()) { if (literal->value.getType() == Field::Types::Tuple) { surrounded_by_parens = true; - fields_count = literal->value.get().size(); + fields_count = literal->value.get().size(); } else { diff --git a/src/Parsers/ParserSettingsProfileElement.cpp b/src/Parsers/ParserSettingsProfileElement.cpp index 06fa58fde4e..31bc339f544 100644 --- a/src/Parsers/ParserSettingsProfileElement.cpp +++ b/src/Parsers/ParserSettingsProfileElement.cpp @@ -108,7 +108,8 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected Field max_value; std::optional readonly; - if (ParserKeyword{"PROFILE"}.ignore(pos, expected)) + if (ParserKeyword{"PROFILE"}.ignore(pos, expected) || + (enable_inherit_keyword && ParserKeyword{"INHERIT"}.ignore(pos, expected))) { if (!parseProfileNameOrID(pos, expected, id_mode, parent_profile)) return false; @@ -120,9 +121,15 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected return false; name = getIdentifierName(name_ast); + bool has_value_or_constraint = false; while (parseValue(pos, expected, value) || parseMinMaxValue(pos, expected, min_value, max_value) || parseReadonlyOrWritableKeyword(pos, expected, readonly)) - ; + { + has_value_or_constraint = true; + } + + if (!has_value_or_constraint) + return false; } auto result = std::make_shared(); @@ -133,6 +140,7 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected result->max_value = std::move(max_value); result->readonly = readonly; result->id_mode = id_mode; + result->use_inherit_keyword = enable_inherit_keyword; node = result; return true; } @@ -142,12 +150,15 @@ bool ParserSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Expected { std::vector> elements; - if (!ParserKeyword{"NONE"}.ignore(pos, expected)) + if (ParserKeyword{"NONE"}.ignore(pos, expected)) + { + } + else { do { ASTPtr ast; - if (!ParserSettingsProfileElement{}.useIDMode(id_mode).parse(pos, ast, expected)) + if (!ParserSettingsProfileElement{}.useIDMode(id_mode).enableInheritKeyword(enable_inherit_keyword).parse(pos, ast, expected)) return false; auto element = typeid_cast>(ast); elements.push_back(std::move(element)); diff --git a/src/Parsers/ParserSettingsProfileElement.h b/src/Parsers/ParserSettingsProfileElement.h index ec8e1abb5b5..309c797e645 100644 --- a/src/Parsers/ParserSettingsProfileElement.h +++ b/src/Parsers/ParserSettingsProfileElement.h @@ -12,6 +12,7 @@ class ParserSettingsProfileElement : public IParserBase { public: ParserSettingsProfileElement & useIDMode(bool enable_) { id_mode = enable_; return *this; } + ParserSettingsProfileElement & enableInheritKeyword(bool enable_) { enable_inherit_keyword = enable_; return *this; } protected: const char * getName() const override { return "SettingsProfileElement"; } @@ -19,6 +20,7 @@ protected: private: bool id_mode = false; + bool enable_inherit_keyword = false; }; @@ -26,6 +28,7 @@ class ParserSettingsProfileElements : public IParserBase { public: ParserSettingsProfileElements & useIDMode(bool enable_) { id_mode = enable_; return *this; } + ParserSettingsProfileElements & enableInheritKeyword(bool enable_) { enable_inherit_keyword = enable_; return *this; } protected: const char * getName() const override { return "SettingsProfileElements"; } @@ -33,4 +36,7 @@ protected: private: bool id_mode = false; -};} + bool enable_inherit_keyword = false; +}; + +} diff --git a/src/Parsers/makeASTForLogicalFunction.cpp b/src/Parsers/makeASTForLogicalFunction.cpp new file mode 100644 index 00000000000..eaae38740aa --- /dev/null +++ b/src/Parsers/makeASTForLogicalFunction.cpp @@ -0,0 +1,103 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +ASTPtr makeASTForLogicalNot(ASTPtr argument) +{ + bool b; + if (tryGetLiteralBool(argument.get(), b)) + return std::make_shared(Field{UInt8(!b)}); + + auto function = std::make_shared(); + auto exp_list = std::make_shared(); + function->name = "not"; + function->arguments = exp_list; + function->children.push_back(exp_list); + exp_list->children.push_back(argument); + return function; +} + + +ASTPtr makeASTForLogicalAnd(ASTs && arguments) +{ + bool partial_result = true; + boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool + { + bool b; + if (!tryGetLiteralBool(argument.get(), b)) + return false; + partial_result &= b; + return true; + }); + + if (!partial_result) + return std::make_shared(Field{UInt8(0)}); + if (arguments.empty()) + return std::make_shared(Field{UInt8(1)}); + if (arguments.size() == 1) + return arguments[0]; + + auto function = std::make_shared(); + auto exp_list = std::make_shared(); + function->name = "and"; + function->arguments = exp_list; + function->children.push_back(exp_list); + exp_list->children = std::move(arguments); + return function; +} + + +ASTPtr makeASTForLogicalOr(ASTs && arguments) +{ + bool partial_result = false; + boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool + { + bool b; + if (!tryGetLiteralBool(argument.get(), b)) + return false; + partial_result |= b; + return true; + }); + + if (partial_result) + return std::make_shared(Field{UInt8(1)}); + if (arguments.empty()) + return std::make_shared(Field{UInt8(0)}); + if (arguments.size() == 1) + return arguments[0]; + + auto function = std::make_shared(); + auto exp_list = std::make_shared(); + function->name = "or"; + function->arguments = exp_list; + function->children.push_back(exp_list); + exp_list->children = std::move(arguments); + return function; +} + + +bool tryGetLiteralBool(const IAST * ast, bool & value) +{ + if (!ast) + return false; + + try + { + if (const ASTLiteral * literal = ast->as()) + { + value = !literal->value.isNull() && applyVisitor(FieldVisitorConvertToNumber(), literal->value); + return true; + } + return false; + } + catch (...) + { + return false; + } +} +} diff --git a/src/Parsers/makeASTForLogicalFunction.h b/src/Parsers/makeASTForLogicalFunction.h new file mode 100644 index 00000000000..5c1096cab6e --- /dev/null +++ b/src/Parsers/makeASTForLogicalFunction.h @@ -0,0 +1,19 @@ +#pragma once + +#include + + +namespace DB +{ +/// Makes an AST calculating NOT argument. +ASTPtr makeASTForLogicalNot(ASTPtr argument); + +/// Makes an AST calculating argument1 AND argument2 AND ... AND argumentN. +ASTPtr makeASTForLogicalAnd(ASTs && arguments); + +/// Makes an AST calculating argument1 OR argument2 OR ... OR argumentN. +ASTPtr makeASTForLogicalOr(ASTs && arguments); + +/// Tries to extract a literal bool from AST. +bool tryGetLiteralBool(const IAST * ast, bool & value); +} diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp index b8ab940f2dd..dcc587c83ef 100644 --- a/src/Parsers/parseQuery.cpp +++ b/src/Parsers/parseQuery.cpp @@ -319,7 +319,7 @@ ASTPtr parseQuery( size_t max_query_size, size_t max_parser_depth) { - auto pos = begin; + const char * pos = begin; return parseQueryAndMovePosition(parser, pos, end, query_description, false, max_query_size, max_parser_depth); } @@ -328,19 +328,28 @@ ASTPtr parseQuery( IParser & parser, const std::string & query, const std::string & query_description, - size_t max_query_size) + size_t max_query_size, + size_t max_parser_depth) { - return parseQuery(parser, query.data(), query.data() + query.size(), query_description, max_query_size); + return parseQuery(parser, query.data(), query.data() + query.size(), query_description, max_query_size, max_parser_depth); } -ASTPtr parseQuery(IParser & parser, const std::string & query, size_t max_query_size) +ASTPtr parseQuery( + IParser & parser, + const std::string & query, + size_t max_query_size, + size_t max_parser_depth) { - return parseQuery(parser, query.data(), query.data() + query.size(), parser.getName(), max_query_size); + return parseQuery(parser, query.data(), query.data() + query.size(), parser.getName(), max_query_size, max_parser_depth); } -std::pair splitMultipartQuery(const std::string & queries, std::vector & queries_list) +std::pair splitMultipartQuery( + const std::string & queries, + std::vector & queries_list, + size_t max_query_size, + size_t max_parser_depth) { ASTPtr ast; @@ -356,7 +365,7 @@ std::pair splitMultipartQuery(const std::string & queries, s { begin = pos; - ast = parseQueryAndMovePosition(parser, pos, end, "", true, 0); + ast = parseQueryAndMovePosition(parser, pos, end, "", true, max_query_size, max_parser_depth); auto * insert = ast->as(); diff --git a/src/Parsers/parseQuery.h b/src/Parsers/parseQuery.h index feea204181e..14a9a85b22c 100644 --- a/src/Parsers/parseQuery.h +++ b/src/Parsers/parseQuery.h @@ -17,7 +17,7 @@ ASTPtr tryParseQuery( bool allow_multi_statements, /// If false, check for non-space characters after semicolon and set error message if any. size_t max_query_size, /// If (end - pos) > max_query_size and query is longer than max_query_size then throws "Max query size exceeded". /// Disabled if zero. Is used in order to check query size if buffer can contains data for INSERT query. - size_t max_parser_depth = 0); + size_t max_parser_depth); /// Parse query or throw an exception with error message. @@ -27,8 +27,8 @@ ASTPtr parseQueryAndMovePosition( const char * end, const std::string & description, bool allow_multi_statements, - size_t max_query_size = 0, - size_t max_parser_depth = 0); + size_t max_query_size, + size_t max_parser_depth); ASTPtr parseQuery( IParser & parser, @@ -36,24 +36,30 @@ ASTPtr parseQuery( const char * end, const std::string & description, size_t max_query_size, - size_t max_parser_depth = 0); + size_t max_parser_depth); ASTPtr parseQuery( IParser & parser, const std::string & query, const std::string & query_description, - size_t max_query_size); + size_t max_query_size, + size_t max_parser_depth); ASTPtr parseQuery( IParser & parser, const std::string & query, - size_t max_query_size); + size_t max_query_size, + size_t max_parser_depth); /** Split queries separated by ; on to list of single queries * Returns pointer to the end of last successfully parsed query (first), and true if all queries are successfully parsed (second) * NOTE: INSERT's data should be placed in single line. */ -std::pair splitMultipartQuery(const std::string & queries, std::vector & queries_list); +std::pair splitMultipartQuery( + const std::string & queries, + std::vector & queries_list, + size_t max_query_size, + size_t max_parser_depth); } diff --git a/src/Parsers/tests/create_parser.cpp b/src/Parsers/tests/create_parser.cpp index 6137d4d18da..fbdc967fa2a 100644 --- a/src/Parsers/tests/create_parser.cpp +++ b/src/Parsers/tests/create_parser.cpp @@ -12,7 +12,7 @@ int main(int, char **) std::string input = "CREATE TABLE hits (URL String, UserAgentMinor2 FixedString(2), EventTime DateTime) ENGINE = Log"; ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); formatAST(*ast, std::cerr); std::cerr << std::endl; diff --git a/src/Parsers/tests/gtest_dictionary_parser.cpp b/src/Parsers/tests/gtest_dictionary_parser.cpp index 25dcbb326ad..31b6d6cdd81 100644 --- a/src/Parsers/tests/gtest_dictionary_parser.cpp +++ b/src/Parsers/tests/gtest_dictionary_parser.cpp @@ -43,7 +43,7 @@ TEST(ParserDictionaryDDL, SimpleDictionary) " RANGE(MIN second_column MAX third_column)"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); EXPECT_EQ(create->table, "dict1"); EXPECT_EQ(create->database, "test"); @@ -76,24 +76,24 @@ TEST(ParserDictionaryDDL, SimpleDictionary) EXPECT_EQ(children[5]->as()->second->as()->value.get(), "table_for_dict"); /// layout test - auto layout = create->dictionary->layout; + auto * layout = create->dictionary->layout; EXPECT_EQ(layout->layout_type, "flat"); EXPECT_EQ(layout->children.size(), 0); /// lifetime test - auto lifetime = create->dictionary->lifetime; + auto * lifetime = create->dictionary->lifetime; EXPECT_EQ(lifetime->min_sec, 1); EXPECT_EQ(lifetime->max_sec, 10); /// primary key test - auto primary_key = create->dictionary->primary_key; + auto * primary_key = create->dictionary->primary_key; EXPECT_EQ(primary_key->children.size(), 1); EXPECT_EQ(primary_key->children[0]->as()->name, "key_column"); /// range test - auto range = create->dictionary->range; + auto * range = create->dictionary->range; EXPECT_EQ(range->min_attr_name, "second_column"); EXPECT_EQ(range->max_attr_name, "third_column"); @@ -139,7 +139,7 @@ TEST(ParserDictionaryDDL, AttributesWithMultipleProperties) " SOURCE(CLICKHOUSE(HOST 'localhost'))"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); EXPECT_EQ(create->table, "dict2"); EXPECT_EQ(create->database, ""); @@ -186,7 +186,7 @@ TEST(ParserDictionaryDDL, CustomAttributePropertiesOrder) " LIFETIME(300)"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); /// test attributes @@ -219,7 +219,7 @@ TEST(ParserDictionaryDDL, CustomAttributePropertiesOrder) EXPECT_EQ(attributes_children[2]->as()->is_object_id, false); /// lifetime test - auto lifetime = create->dictionary->lifetime; + auto * lifetime = create->dictionary->lifetime; EXPECT_EQ(lifetime->min_sec, 0); EXPECT_EQ(lifetime->max_sec, 300); @@ -241,7 +241,7 @@ TEST(ParserDictionaryDDL, NestedSource) " RANGE(MIN second_column MAX third_column)"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); EXPECT_EQ(create->table, "dict4"); EXPECT_EQ(create->database, ""); @@ -289,7 +289,7 @@ TEST(ParserDictionaryDDL, Formatting) " RANGE(MIN second_column MAX third_column)"; ParserCreateDictionaryQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); auto str = serializeAST(*create, true); EXPECT_EQ(str, "CREATE DICTIONARY test.dict5 (`key_column1` UInt64 DEFAULT 1 HIERARCHICAL INJECTIVE, `key_column2` String DEFAULT '', `second_column` UInt8 EXPRESSION intDiv(50, rand() % 1000), `third_column` UInt8) PRIMARY KEY key_column1, key_column2 SOURCE(MYSQL(HOST 'localhost' PORT 9000 USER 'default' REPLICA (HOST '127.0.0.1' PRIORITY 1) PASSWORD '')) LIFETIME(MIN 1 MAX 10) LAYOUT(CACHE(SIZE_IN_CELLS 50)) RANGE(MIN second_column MAX third_column)"); @@ -300,7 +300,7 @@ TEST(ParserDictionaryDDL, ParseDropQuery) String input1 = "DROP DICTIONARY test.dict1"; ParserDropQuery parser; - ASTPtr ast1 = parseQuery(parser, input1.data(), input1.data() + input1.size(), "", 0); + ASTPtr ast1 = parseQuery(parser, input1.data(), input1.data() + input1.size(), "", 0, 0); ASTDropQuery * drop1 = ast1->as(); EXPECT_TRUE(drop1->is_dictionary); @@ -311,7 +311,7 @@ TEST(ParserDictionaryDDL, ParseDropQuery) String input2 = "DROP DICTIONARY IF EXISTS dict2"; - ASTPtr ast2 = parseQuery(parser, input2.data(), input2.data() + input2.size(), "", 0); + ASTPtr ast2 = parseQuery(parser, input2.data(), input2.data() + input2.size(), "", 0, 0); ASTDropQuery * drop2 = ast2->as(); EXPECT_TRUE(drop2->is_dictionary); @@ -326,7 +326,7 @@ TEST(ParserDictionaryDDL, ParsePropertiesQueries) String input1 = "SHOW CREATE DICTIONARY test.dict1"; ParserTablePropertiesQuery parser; - ASTPtr ast1 = parseQuery(parser, input1.data(), input1.data() + input1.size(), "", 0); + ASTPtr ast1 = parseQuery(parser, input1.data(), input1.data() + input1.size(), "", 0, 0); ASTShowCreateDictionaryQuery * show1 = ast1->as(); EXPECT_EQ(show1->table, "dict1"); @@ -335,7 +335,7 @@ TEST(ParserDictionaryDDL, ParsePropertiesQueries) String input2 = "EXISTS DICTIONARY dict2"; - ASTPtr ast2 = parseQuery(parser, input2.data(), input2.data() + input2.size(), "", 0); + ASTPtr ast2 = parseQuery(parser, input2.data(), input2.data() + input2.size(), "", 0, 0); ASTExistsDictionaryQuery * show2 = ast2->as(); EXPECT_EQ(show2->table, "dict2"); diff --git a/src/Parsers/tests/select_parser.cpp b/src/Parsers/tests/select_parser.cpp index f5d94746aa1..7711f0d2527 100644 --- a/src/Parsers/tests/select_parser.cpp +++ b/src/Parsers/tests/select_parser.cpp @@ -22,7 +22,7 @@ try " FORMAT TabSeparated"; ParserQueryWithOutput parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); std::cout << "Success." << std::endl; formatAST(*ast, std::cerr); diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make new file mode 100644 index 00000000000..942124c2c7a --- /dev/null +++ b/src/Parsers/ya.make @@ -0,0 +1,113 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + ASTAlterQuery.cpp + ASTAsterisk.cpp + ASTColumnDeclaration.cpp + ASTColumnsMatcher.cpp + ASTConstraintDeclaration.cpp + ASTCreateQuery.cpp + ASTCreateQuotaQuery.cpp + ASTCreateRoleQuery.cpp + ASTCreateRowPolicyQuery.cpp + ASTCreateSettingsProfileQuery.cpp + ASTCreateUserQuery.cpp + ASTDictionary.cpp + ASTDictionaryAttributeDeclaration.cpp + ASTDropAccessEntityQuery.cpp + ASTDropQuery.cpp + ASTExpressionList.cpp + ASTExtendedRoleSet.cpp + ASTFunction.cpp + ASTFunctionWithKeyValueArguments.cpp + ASTGrantQuery.cpp + ASTIdentifier.cpp + ASTInsertQuery.cpp + ASTKillQueryQuery.cpp + ASTLiteral.cpp + ASTOptimizeQuery.cpp + ASTOrderByElement.cpp + ASTPartition.cpp + ASTQualifiedAsterisk.cpp + ASTQueryParameter.cpp + ASTQueryWithOnCluster.cpp + ASTQueryWithOutput.cpp + ASTQueryWithTableAndOutput.cpp + ASTSampleRatio.cpp + ASTSelectQuery.cpp + ASTSelectWithUnionQuery.cpp + ASTSetRoleQuery.cpp + ASTSettingsProfileElement.cpp + ASTShowCreateAccessEntityQuery.cpp + ASTShowGrantsQuery.cpp + ASTShowQuotasQuery.cpp + ASTShowRowPoliciesQuery.cpp + ASTShowTablesQuery.cpp + ASTSubquery.cpp + ASTSystemQuery.cpp + ASTTablesInSelectQuery.cpp + ASTTTLElement.cpp + ASTWithAlias.cpp + CommonParsers.cpp + ExpressionElementParsers.cpp + ExpressionListParsers.cpp + formatAST.cpp + IAST.cpp + iostream_debug_helpers.cpp + IParserBase.cpp + Lexer.cpp + makeASTForLogicalFunction.cpp + parseDatabaseAndTableName.cpp + parseIdentifierOrStringLiteral.cpp + parseIntervalKind.cpp + parseQuery.cpp + ParserAlterQuery.cpp + ParserCase.cpp + ParserCheckQuery.cpp + ParserCreateQuery.cpp + ParserCreateQuotaQuery.cpp + ParserCreateRoleQuery.cpp + ParserCreateRowPolicyQuery.cpp + ParserCreateSettingsProfileQuery.cpp + ParserCreateUserQuery.cpp + ParserDescribeTableQuery.cpp + ParserDictionary.cpp + ParserDictionaryAttributeDeclaration.cpp + ParserDropAccessEntityQuery.cpp + ParserDropQuery.cpp + ParserExtendedRoleSet.cpp + ParserGrantQuery.cpp + ParserInsertQuery.cpp + ParserKillQueryQuery.cpp + ParserOptimizeQuery.cpp + ParserPartition.cpp + ParserQuery.cpp + ParserQueryWithOutput.cpp + ParserRenameQuery.cpp + ParserSampleRatio.cpp + ParserSelectQuery.cpp + ParserSelectWithUnionQuery.cpp + ParserSetQuery.cpp + ParserSetRoleQuery.cpp + ParserSettingsProfileElement.cpp + ParserShowCreateAccessEntityQuery.cpp + ParserShowGrantsQuery.cpp + ParserShowQuotasQuery.cpp + ParserShowRowPoliciesQuery.cpp + ParserShowTablesQuery.cpp + ParserSystemQuery.cpp + ParserTablePropertiesQuery.cpp + ParserTablesInSelectQuery.cpp + ParserUnionQueryElement.cpp + ParserUseQuery.cpp + ParserWatchQuery.cpp + parseUserName.cpp + queryToString.cpp + TokenIterator.cpp +) + +END() diff --git a/src/Processors/Chunk.cpp b/src/Processors/Chunk.cpp index 554ccab2af3..d68c2bea5ae 100644 --- a/src/Processors/Chunk.cpp +++ b/src/Processors/Chunk.cpp @@ -141,7 +141,7 @@ UInt64 Chunk::allocatedBytes() const std::string Chunk::dumpStructure() const { WriteBufferFromOwnString out; - for (auto & column : columns) + for (const auto & column : columns) out << ' ' << column->dumpStructure(); return out.str(); diff --git a/src/Processors/ConcatProcessor.cpp b/src/Processors/ConcatProcessor.cpp index d3333c320c0..27338c7c879 100644 --- a/src/Processors/ConcatProcessor.cpp +++ b/src/Processors/ConcatProcessor.cpp @@ -31,17 +31,13 @@ ConcatProcessor::Status ConcatProcessor::prepare() /// Check can input. - if (current_input == inputs.end()) - return Status::Finished; - - if (current_input->isFinished()) - { + while (current_input != inputs.end() && current_input->isFinished()) ++current_input; - if (current_input == inputs.end()) - { - output.finish(); - return Status::Finished; - } + + if (current_input == inputs.end()) + { + output.finish(); + return Status::Finished; } auto & input = *current_input; diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index 672f2645c16..5a6297413e1 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -19,7 +19,7 @@ DelayedPortsProcessor::DelayedPortsProcessor(const Block & header, size_t num_po ++output_it; } - for (auto & delayed : delayed_ports) + for (const auto & delayed : delayed_ports) port_pairs[delayed].is_delayed = true; } @@ -63,13 +63,13 @@ IProcessor::Status DelayedPortsProcessor::prepare(const PortNumbers & updated_in bool skip_delayed = (num_finished + num_delayed) < port_pairs.size(); bool need_data = false; - for (auto & output_number : updated_outputs) + for (const auto & output_number : updated_outputs) { if (!skip_delayed || !port_pairs[output_number].is_delayed) need_data = processPair(port_pairs[output_number]) || need_data; } - for (auto & input_number : updated_inputs) + for (const auto & input_number : updated_inputs) { if (!skip_delayed || !port_pairs[input_number].is_delayed) need_data = processPair(port_pairs[input_number]) || need_data; diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index f2d2477991e..687736eef61 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -263,6 +263,8 @@ bool PipelineExecutor::tryAddProcessorToStackIfUpdated(Edge & edge, Queue & queu node.status = ExecStatus::Preparing; return prepareProcessor(edge.to, thread_number, queue, std::move(lock)); } + else + graph[edge.to].processor->onUpdatePorts(); return true; } @@ -342,14 +344,14 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue { for (auto & edge_id : *node.post_updated_input_ports) { - auto edge = static_cast(edge_id); + auto * edge = static_cast(edge_id); updated_back_edges.emplace_back(edge); edge->update_info.trigger(); } for (auto & edge_id : *node.post_updated_output_ports) { - auto edge = static_cast(edge_id); + auto * edge = static_cast(edge_id); updated_direct_edges.emplace_back(edge); edge->update_info.trigger(); } @@ -618,7 +620,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads Queue queue; ++num_processing_executors; - while (auto task = expand_pipeline_task.load()) + while (auto * task = expand_pipeline_task.load()) doExpandPipeline(task, true); /// Execute again if can. @@ -659,7 +661,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads } --num_processing_executors; - while (auto task = expand_pipeline_task.load()) + while (auto * task = expand_pipeline_task.load()) doExpandPipeline(task, false); } @@ -778,7 +780,7 @@ void PipelineExecutor::executeImpl(size_t num_threads) String PipelineExecutor::dumpPipeline() const { - for (auto & node : graph) + for (const auto & node : graph) { if (node.execution_state) { @@ -800,7 +802,7 @@ String PipelineExecutor::dumpPipeline() const statuses.reserve(graph.size()); proc_list.reserve(graph.size()); - for (auto & proc : graph) + for (const auto & proc : graph) { proc_list.emplace_back(proc.processor); statuses.emplace_back(proc.last_processor_status); diff --git a/src/Processors/Executors/TreeExecutorBlockInputStream.cpp b/src/Processors/Executors/TreeExecutorBlockInputStream.cpp index ee5b254ccf9..333dcc84b39 100644 --- a/src/Processors/Executors/TreeExecutorBlockInputStream.cpp +++ b/src/Processors/Executors/TreeExecutorBlockInputStream.cpp @@ -30,11 +30,14 @@ static void checkProcessorHasSingleOutput(IProcessor * processor) /// Check tree invariants (described in TreeExecutor.h). /// Collect sources with progress. -static void validateTree(const Processors & processors, IProcessor * root, IProcessor * totals_root, std::vector & sources) +static void validateTree( + const Processors & processors, + IProcessor * root, IProcessor * totals_root, IProcessor * extremes_root, + std::vector & sources) { std::unordered_map index; - for (auto & processor : processors) + for (const auto & processor : processors) { bool is_inserted = index.try_emplace(processor.get(), index.size()).second; @@ -49,6 +52,8 @@ static void validateTree(const Processors & processors, IProcessor * root, IProc stack.push(root); if (totals_root) stack.push(totals_root); + if (extremes_root) + stack.push(extremes_root); while (!stack.empty()) { @@ -104,11 +109,15 @@ void TreeExecutorBlockInputStream::init() root = &output_port.getProcessor(); IProcessor * totals_root = nullptr; + IProcessor * extremes_root = nullptr; if (totals_port) totals_root = &totals_port->getProcessor(); - validateTree(processors, root, totals_root, sources_with_progress); + if (extremes_port) + extremes_root = &extremes_port->getProcessor(); + + validateTree(processors, root, totals_root, extremes_root, sources_with_progress); input_port = std::make_unique(getHeader(), root); connect(output_port, *input_port); @@ -121,15 +130,24 @@ void TreeExecutorBlockInputStream::init() input_totals_port->setNeeded(); } + if (extremes_port) + { + input_extremes_port = std::make_unique(extremes_port->getHeader(), root); + connect(*extremes_port, *input_extremes_port); + input_extremes_port->setNeeded(); + } + initRowsBeforeLimit(); } -void TreeExecutorBlockInputStream::execute(bool on_totals) +void TreeExecutorBlockInputStream::execute(bool on_totals, bool on_extremes) { std::stack stack; if (on_totals) stack.push(&totals_port->getProcessor()); + else if (on_extremes) + stack.push(&extremes_port->getProcessor()); else stack.push(root); @@ -146,7 +164,7 @@ void TreeExecutorBlockInputStream::execute(bool on_totals) } }; - while (!stack.empty()) + while (!stack.empty() && !is_cancelled) { IProcessor * node = stack.top(); @@ -224,7 +242,7 @@ void TreeExecutorBlockInputStream::initRowsBeforeLimit() while (!stack.empty()) { - auto processor = stack.top().processor; + auto * processor = stack.top().processor; bool visited_limit = stack.top().visited_limit; stack.pop(); @@ -277,17 +295,24 @@ void TreeExecutorBlockInputStream::initRowsBeforeLimit() Block TreeExecutorBlockInputStream::readImpl() { - while (true) + while (!is_cancelled) { if (input_port->isFinished()) { if (totals_port && !input_totals_port->isFinished()) { - execute(true); + execute(true, false); if (input_totals_port->hasData()) totals = getHeader().cloneWithColumns(input_totals_port->pull().detachColumns()); } + if (extremes_port && !input_extremes_port->isFinished()) + { + execute(false, true); + if (input_extremes_port->hasData()) + extremes = getHeader().cloneWithColumns(input_extremes_port->pull().detachColumns()); + } + if (rows_before_limit_at_least && rows_before_limit_at_least->hasAppliedLimit()) info.setRowsBeforeLimit(rows_before_limit_at_least->get()); @@ -299,9 +324,9 @@ Block TreeExecutorBlockInputStream::readImpl() auto chunk = input_port->pull(); Block block = getHeader().cloneWithColumns(chunk.detachColumns()); - if (auto & chunk_info = chunk.getChunkInfo()) + if (const auto & chunk_info = chunk.getChunkInfo()) { - if (auto * agg_info = typeid_cast(chunk_info.get())) + if (const auto * agg_info = typeid_cast(chunk_info.get())) { block.info.bucket_num = agg_info->bucket_num; block.info.is_overflows = agg_info->is_overflows; @@ -311,8 +336,10 @@ Block TreeExecutorBlockInputStream::readImpl() return block; } - execute(false); + execute(false, false); } + + return {}; } void TreeExecutorBlockInputStream::setProgressCallback(const ProgressCallback & callback) @@ -348,4 +375,12 @@ void TreeExecutorBlockInputStream::addTotalRowsApprox(size_t value) sources_with_progress.front()->addTotalRowsApprox(value); } +void TreeExecutorBlockInputStream::cancel(bool kill) +{ + IBlockInputStream::cancel(kill); + + for (auto & processor : processors) + processor->cancel(); +} + } diff --git a/src/Processors/Executors/TreeExecutorBlockInputStream.h b/src/Processors/Executors/TreeExecutorBlockInputStream.h index 24cab387eb8..d96492b3fb8 100644 --- a/src/Processors/Executors/TreeExecutorBlockInputStream.h +++ b/src/Processors/Executors/TreeExecutorBlockInputStream.h @@ -31,6 +31,7 @@ public: interpreter_context.emplace_back(context); totals_port = pipe.getTotalsPort(); + extremes_port = pipe.getExtremesPort(); processors = std::move(pipe).detachProcessors(); init(); } @@ -38,6 +39,8 @@ public: String getName() const override { return "TreeExecutor"; } Block getHeader() const override { return root->getOutputs().front().getHeader(); } + void cancel(bool kill) override; + /// This methods does not affect TreeExecutor as IBlockInputStream itself. /// They just passed to all SourceWithProgress processors. void setProgressCallback(const ProgressCallback & callback) final; @@ -52,10 +55,12 @@ protected: private: OutputPort & output_port; OutputPort * totals_port = nullptr; + OutputPort * extremes_port = nullptr; Processors processors; IProcessor * root = nullptr; std::unique_ptr input_port; std::unique_ptr input_totals_port; + std::unique_ptr input_extremes_port; RowsBeforeLimitCounterPtr rows_before_limit_at_least; /// Remember sources that support progress. @@ -65,7 +70,7 @@ private: void init(); /// Execute tree step-by-step until root returns next chunk or execution is finished. - void execute(bool on_totals); + void execute(bool on_totals, bool on_extremes); void initRowsBeforeLimit(); diff --git a/src/Processors/Formats/IRowOutputFormat.cpp b/src/Processors/Formats/IRowOutputFormat.cpp index ac898b693c4..bb74f1ce59e 100644 --- a/src/Processors/Formats/IRowOutputFormat.cpp +++ b/src/Processors/Formats/IRowOutputFormat.cpp @@ -15,7 +15,7 @@ void IRowOutputFormat::consume(DB::Chunk chunk) writePrefixIfNot(); auto num_rows = chunk.getNumRows(); - auto & columns = chunk.getColumns(); + const auto & columns = chunk.getColumns(); for (size_t row = 0; row < num_rows; ++row) { @@ -39,7 +39,7 @@ void IRowOutputFormat::consumeTotals(DB::Chunk chunk) if (num_rows != 1) throw Exception("Got " + toString(num_rows) + " in totals chunk, expected 1", ErrorCodes::LOGICAL_ERROR); - auto & columns = chunk.getColumns(); + const auto & columns = chunk.getColumns(); writeBeforeTotals(); writeTotals(columns, 0); @@ -52,7 +52,7 @@ void IRowOutputFormat::consumeExtremes(DB::Chunk chunk) writeSuffixIfNot(); auto num_rows = chunk.getNumRows(); - auto & columns = chunk.getColumns(); + const auto & columns = chunk.getColumns(); if (num_rows != 2) throw Exception("Got " + toString(num_rows) + " in extremes chunk, expected 2", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 0fa1de9da65..24e8571b119 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -292,11 +292,11 @@ namespace DB if (arrow_type == arrow::Type::DECIMAL) { - const auto decimal_type = static_cast(arrow_column->type().get()); + const auto * decimal_type = static_cast(arrow_column->type().get()); internal_nested_type = std::make_shared>(decimal_type->precision(), decimal_type->scale()); } - else if (auto internal_type_it = std::find_if(arrow_type_to_internal_type.begin(), arrow_type_to_internal_type.end(), + else if (const auto * internal_type_it = std::find_if(arrow_type_to_internal_type.begin(), arrow_type_to_internal_type.end(), [=](auto && elem) { return elem.first == arrow_type; }); internal_type_it != arrow_type_to_internal_type.end()) { diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 52ceaf063b7..4bd3d5c2ada 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -102,10 +102,6 @@ private: ReadBuffer & in; }; -static void deserializeNoop(IColumn &, avro::Decoder &) -{ -} - /// Insert value with conversion to the column of target type. template static void insertNumber(IColumn & column, WhichDataType type, T value) @@ -420,7 +416,7 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) } return [field_skip_fns](avro::Decoder & decoder) { - for (auto & skip_fn : field_skip_fns) + for (const auto & skip_fn : field_skip_fns) skip_fn(decoder); }; } @@ -441,8 +437,43 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) } } +void AvroDeserializer::createActions(const Block & header, const avro::NodePtr& node, std::string current_path) +{ + if (node->type() == avro::AVRO_RECORD) + { + for (size_t i = 0; i < node->leaves(); ++i) + { + const auto & field_node = node->leafAt(i); + const auto & field_name = node->nameAt(i); + auto field_path = current_path.empty() ? field_name : current_path + "." + field_name; + createActions(header, field_node, field_path); + } + } + else + { + if (header.has(current_path)) + { + auto target_column_idx = header.getPositionByName(current_path); + const auto & column = header.getByPosition(target_column_idx); + try + { + actions.emplace_back(target_column_idx, createDeserializeFn(node, column.type)); + } + catch (Exception & e) + { + e.addMessage("column " + column.name); + throw; + } + column_found[target_column_idx] = true; + } + else + { + actions.emplace_back(createSkipFn(node)); + } + } +} -AvroDeserializer::AvroDeserializer(const ColumnsWithTypeAndName & columns, avro::ValidSchema schema) +AvroDeserializer::AvroDeserializer(const Block & header, avro::ValidSchema schema) { const auto & schema_root = schema.root(); if (schema_root->type() != avro::AVRO_RECORD) @@ -450,48 +481,23 @@ AvroDeserializer::AvroDeserializer(const ColumnsWithTypeAndName & columns, avro: throw Exception("Root schema must be a record", ErrorCodes::TYPE_MISMATCH); } - field_mapping.resize(schema_root->leaves(), -1); + column_found.resize(header.columns()); + createActions(header, schema_root); - for (size_t i = 0; i < schema_root->leaves(); ++i) + for (size_t i = 0; i < header.columns(); ++i) { - skip_fns.push_back(createSkipFn(schema_root->leafAt(i))); - deserialize_fns.push_back(&deserializeNoop); - } - - for (size_t i = 0; i < columns.size(); ++i) - { - const auto & column = columns[i]; - size_t field_index = 0; - if (!schema_root->nameIndex(column.name, field_index)) + if (!column_found[i]) { - throw Exception("Field " + column.name + " not found in Avro schema", ErrorCodes::THERE_IS_NO_COLUMN); + throw Exception("Field " + header.getByPosition(i).name + " not found in Avro schema", ErrorCodes::THERE_IS_NO_COLUMN); } - auto field_schema = schema_root->leafAt(field_index); - try - { - deserialize_fns[field_index] = createDeserializeFn(field_schema, column.type); - } - catch (Exception & e) - { - e.addMessage("column " + column.name); - throw; - } - field_mapping[field_index] = i; } } void AvroDeserializer::deserializeRow(MutableColumns & columns, avro::Decoder & decoder) const { - for (size_t i = 0; i < field_mapping.size(); i++) + for (const auto& action : actions) { - if (field_mapping[i] >= 0) - { - deserialize_fns[i](*columns[field_mapping[i]], decoder); - } - else - { - skip_fns[i](decoder); - } + action.execute(columns, decoder); } } @@ -499,7 +505,7 @@ void AvroDeserializer::deserializeRow(MutableColumns & columns, avro::Decoder & AvroRowInputFormat::AvroRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) : IRowInputFormat(header_, in_, params_) , file_reader(std::make_unique(in_)) - , deserializer(header_.getColumnsWithTypeAndName(), file_reader.dataSchema()) + , deserializer(output.getHeader(), file_reader.dataSchema()) { file_reader.init(); } @@ -555,7 +561,7 @@ private: session->sendRequest(request); Poco::Net::HTTPResponse response; - auto response_body = receiveResponse(*session, request, response, false); + auto * response_body = receiveResponse(*session, request, response, false); Poco::JSON::Parser parser; auto json_body = parser.parse(*response_body).extract(); @@ -584,7 +590,6 @@ private: } } -private: Poco::URI base_url; LRUCache schema_cache; }; @@ -626,8 +631,7 @@ static uint32_t readConfluentSchemaId(ReadBuffer & in) AvroConfluentRowInputFormat::AvroConfluentRowInputFormat( const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) - : IRowInputFormat(header_.cloneEmpty(), in_, params_) - , header_columns(header_.getColumnsWithTypeAndName()) + : IRowInputFormat(header_, in_, params_) , schema_registry(getConfluentSchemaRegistry(format_settings_)) , input_stream(std::make_unique(in)) , decoder(avro::binaryDecoder()) @@ -643,7 +647,7 @@ bool AvroConfluentRowInputFormat::readRow(MutableColumns & columns, RowReadExten return false; } SchemaId schema_id = readConfluentSchemaId(in); - auto & deserializer = getOrCreateDeserializer(schema_id); + const auto & deserializer = getOrCreateDeserializer(schema_id); deserializer.deserializeRow(columns, *decoder); decoder->drain(); return true; @@ -655,7 +659,7 @@ const AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(Sc if (it == deserializer_cache.end()) { auto schema = schema_registry->getSchema(schema_id); - AvroDeserializer deserializer(header_columns, schema); + AvroDeserializer deserializer(output.getHeader(), schema); it = deserializer_cache.emplace(schema_id, deserializer).first; } return it->second; diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.h b/src/Processors/Formats/Impl/AvroRowInputFormat.h index b54c8ecede5..6245d704e74 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -22,7 +22,7 @@ namespace DB class AvroDeserializer { public: - AvroDeserializer(const ColumnsWithTypeAndName & columns, avro::ValidSchema schema); + AvroDeserializer(const Block & header, avro::ValidSchema schema); void deserializeRow(MutableColumns & columns, avro::Decoder & decoder) const; private: @@ -31,15 +31,46 @@ private: static DeserializeFn createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type); SkipFn createSkipFn(avro::NodePtr root_node); - /// Map from field index in Avro schema to column number in block header. Or -1 if there is no corresponding column. - std::vector field_mapping; + struct Action + { + enum Type { Deserialize, Skip }; + Type type; + /// Deserialize + int target_column_idx; + DeserializeFn deserialize_fn; + /// Skip + SkipFn skip_fn; - /// How to skip the corresponding field in Avro schema. - std::vector skip_fns; + Action(int target_column_idx_, DeserializeFn deserialize_fn_) + : type(Deserialize) + , target_column_idx(target_column_idx_) + , deserialize_fn(deserialize_fn_) {} - /// How to deserialize the corresponding field in Avro schema. - std::vector deserialize_fns; + Action(SkipFn skip_fn_) + : type(Skip) + , skip_fn(skip_fn_) {} + void execute(MutableColumns & columns, avro::Decoder & decoder) const + { + switch (type) + { + case Deserialize: + deserialize_fn(*columns[target_column_idx], decoder); + break; + case Skip: + skip_fn(decoder); + break; + } + } + }; + + /// Populate actions by recursively traversing root schema + void createActions(const Block & header, const avro::NodePtr& node, std::string current_path = ""); + + /// Bitmap of columns found in Avro schema + std::vector column_found; + /// Deserialize/Skip actions for a row + std::vector actions; /// Map from name of named Avro type (record, enum, fixed) to SkipFn. /// This is to avoid infinite recursion when Avro schema contains self-references. e.g. LinkedList std::map symbolic_skip_fn_map; @@ -73,7 +104,6 @@ public: class SchemaRegistry; private: - const ColumnsWithTypeAndName header_columns; std::shared_ptr schema_registry; using SchemaId = uint32_t; std::unordered_map deserializer_cache; diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 97106d5e297..620eafa2fd7 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -286,7 +286,7 @@ AvroSerializer::AvroSerializer(const ColumnsWithTypeAndName & columns) avro::RecordSchema record_schema("row"); size_t type_name_increment = 0; - for (auto & column : columns) + for (const auto & column : columns) { try { diff --git a/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp index 726a9ca45e8..f228044f13d 100644 --- a/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp @@ -16,7 +16,7 @@ BinaryRowOutputFormat::BinaryRowOutputFormat(WriteBuffer & out_, const Block & h void BinaryRowOutputFormat::writePrefix() { - auto & header = getPort(PortKind::Main).getHeader(); + const auto & header = getPort(PortKind::Main).getHeader(); size_t columns = header.columns(); if (with_names || with_types) diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index bcb3ca44e5d..bd16de69f75 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -30,7 +30,7 @@ CSVRowInputFormat::CSVRowInputFormat(const Block & header_, ReadBuffer & in_, co throw Exception(String("CSV format may not work correctly with delimiter '") + format_settings.csv.delimiter + "'. Try use CustomSeparated format instead.", ErrorCodes::BAD_ARGUMENTS); - auto & sample = getPort().getHeader(); + const auto & sample = getPort().getHeader(); size_t num_columns = sample.columns(); data_types.resize(num_columns); @@ -152,7 +152,7 @@ void CSVRowInputFormat::readPrefix() skipBOMIfExists(in); size_t num_columns = data_types.size(); - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); if (with_names) { @@ -275,7 +275,7 @@ bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, if (column_indexes_for_input_fields[file_column].has_value()) { - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); size_t col_idx = column_indexes_for_input_fields[file_column].value(); if (!deserializeFieldAndPrintDiagnosticInfo(header.getByPosition(col_idx).name, data_types[col_idx], *columns[col_idx], out, file_column)) diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp index e5ad8b007c1..27d022250f2 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp @@ -11,7 +11,7 @@ namespace DB CSVRowOutputFormat::CSVRowOutputFormat(WriteBuffer & out_, const Block & header_, bool with_names_, FormatFactory::WriteCallback callback, const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_, callback), with_names(with_names_), format_settings(format_settings_) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); data_types.resize(columns); for (size_t i = 0; i < columns; ++i) @@ -21,7 +21,7 @@ CSVRowOutputFormat::CSVRowOutputFormat(WriteBuffer & out_, const Block & header_ void CSVRowOutputFormat::writePrefix() { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); if (with_names) diff --git a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp index 5c758624d84..331747bca11 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp @@ -192,7 +192,7 @@ CapnProtoRowInputFormat::CapnProtoRowInputFormat(ReadBuffer & in_, Block header, * Here we gather the list of fields and sort them in a way so that fields in the same structure are adjacent, * and the nesting level doesn't decrease to make traversal easier. */ - auto & sample = getPort().getHeader(); + const auto & sample = getPort().getHeader(); NestedFieldList list; size_t num_columns = sample.columns(); for (size_t i = 0; i < num_columns; ++i) diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index 99369ffa26b..7254cf91245 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -77,7 +77,7 @@ static void fillLiteralInfo(DataTypes & nested_types, LiteralInfo & info) { /// It can be Array(Nullable(nested_type)) or Tuple(..., Nullable(nested_type), ...) bool is_nullable = false; - if (auto nullable = dynamic_cast(nested_type.get())) + if (const auto * nullable = dynamic_cast(nested_type.get())) { nested_type = nullable->getNestedType(); is_nullable = true; @@ -105,6 +105,14 @@ static void fillLiteralInfo(DataTypes & nested_types, LiteralInfo & info) { field_type = Field::Types::String; } + else if (type_info.isArray()) + { + field_type = Field::Types::Array; + } + else if (type_info.isTuple()) + { + field_type = Field::Types::Tuple; + } else throw Exception("Unexpected literal type inside Array: " + nested_type->getName() + ". It's a bug", ErrorCodes::LOGICAL_ERROR); @@ -130,7 +138,7 @@ public: { if (visitIfLiteral(ast, force_nullable)) return; - if (auto function = ast->as()) + if (auto * function = ast->as()) visit(*function, force_nullable); else if (ast->as()) return; @@ -464,7 +472,7 @@ bool ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, co { const auto & [nested_field_type, is_nullable] = type_info.nested_types[i]; if (is_nullable) - if (auto nullable = dynamic_cast(nested_types[i].get())) + if (const auto * nullable = dynamic_cast(nested_types[i].get())) nested_types[i] = nullable->getNestedType(); WhichDataType nested_type_info(nested_types[i]); @@ -560,10 +568,10 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll(BlockMissingValues & nulls, si return res; /// Extract column with evaluated expression and mask for NULLs - auto & tuple = assert_cast(*res); + const auto & tuple = assert_cast(*res); if (tuple.tupleSize() != 2) throw Exception("Invalid tuple size, it'a a bug", ErrorCodes::LOGICAL_ERROR); - auto & is_null = assert_cast(tuple.getColumn(1)); + const auto & is_null = assert_cast(tuple.getColumn(1)); for (size_t i = 0; i < is_null.size(); ++i) if (is_null.getUInt(i)) diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp index ec4d5143ba6..2895abc29f1 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp @@ -24,7 +24,7 @@ JSONCompactEachRowRowInputFormat::JSONCompactEachRowRowInputFormat(ReadBuffer & { /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. skipBOMIfExists(in); - auto & sample = getPort().getHeader(); + const auto & sample = getPort().getHeader(); size_t num_columns = sample.columns(); data_types.resize(num_columns); diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp index 02b7cc220a0..e155dcb4247 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp @@ -15,7 +15,7 @@ JSONCompactEachRowRowOutputFormat::JSONCompactEachRowRowOutputFormat(WriteBuffer bool with_names_) : IRowOutputFormat(header_, out_, callback), settings(settings_), with_names(with_names_) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); NamesAndTypesList columns(sample.getNamesAndTypesList()); fields.assign(columns.begin(), columns.end()); } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index d9cd8a21769..421a54df061 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -71,7 +71,7 @@ inline size_t JSONEachRowRowInputFormat::columnIndex(const StringRef & name, siz } else { - const auto it = name_map.find(name); + auto * it = name_map.find(name); if (it) { @@ -256,7 +256,7 @@ bool JSONEachRowRowInputFormat::readRow(MutableColumns & columns, RowReadExtensi nested_prefix_length = 0; readJSONObject(columns); - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); /// Fill non-visited columns with the default values. for (size_t i = 0; i < num_columns; ++i) if (!seen_columns[i]) diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp index f862a07173b..910a9710de3 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -11,7 +11,7 @@ namespace DB JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_) : IRowOutputFormat(header_, out_, callback), settings(settings_) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); fields.resize(columns); diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index c23f3812c0b..b3255f2894e 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -10,7 +10,7 @@ namespace DB JSONRowOutputFormat::JSONRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_) : IRowOutputFormat(header, out_, callback), settings(settings_) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); NamesAndTypesList columns(sample.getNamesAndTypesList()); fields.assign(columns.begin(), columns.end()); diff --git a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp new file mode 100644 index 00000000000..a48454a5d02 --- /dev/null +++ b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp @@ -0,0 +1,62 @@ +#include +#include + +namespace DB +{ + +MarkdownRowOutputFormat::MarkdownRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & format_settings_) + : IRowOutputFormat(header_, out_, callback), format_settings(format_settings_) {} + +void MarkdownRowOutputFormat::writePrefix() +{ + const auto & header = getPort(PortKind::Main).getHeader(); + size_t columns = header.columns(); + + writeChar('|', out); + for (size_t i = 0; i < columns; ++i) + { + writeEscapedString(header.safeGetByPosition(i).name, out); + writeChar('|', out); + } + writeCString("\n|", out); + String format = ":-:|"; + for (size_t i = 0; i < columns; ++i) + { + writeString(format, out); + } + writeChar('\n', out); +} + +void MarkdownRowOutputFormat::writeRowStartDelimiter() +{ + writeChar('|', out); +} + +void MarkdownRowOutputFormat::writeFieldDelimiter() +{ + writeChar('|', out); +} + +void MarkdownRowOutputFormat::writeRowEndDelimiter() +{ + writeCString("|\n", out); +} + +void MarkdownRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) +{ + type.serializeAsTextEscaped(column, row_num, out, format_settings); +} + +void registerOutputFormatProcessorMarkdown(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("Markdown", []( + WriteBuffer & buf, + const Block & sample, + FormatFactory::WriteCallback callback, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, callback, settings); + }); +} + +} diff --git a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h new file mode 100644 index 00000000000..2f39cce0181 --- /dev/null +++ b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class ReadBuffer; + +class MarkdownRowOutputFormat : public IRowOutputFormat +{ +public: + MarkdownRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & format_settings_); + + /// Write higher part of markdown table like this: + /// |columnName1|columnName2|...|columnNameN| + /// |:-:|:-:|...|:-:| + void writePrefix() override; + + /// Write '|' before each row + void writeRowStartDelimiter() override; + + /// Write '|' between values + void writeFieldDelimiter() override; + + /// Write '|\n' after each row + void writeRowEndDelimiter() override ; + + void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; + String getName() const override { return "MarkdownRowOutputFormat"; } + +protected: + const FormatSettings format_settings; +}; + + +} diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp new file mode 100644 index 00000000000..3e112fb1ce6 --- /dev/null +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -0,0 +1,199 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int INCORRECT_DATA; +} + +MsgPackRowInputFormat::MsgPackRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) + : IRowInputFormat(header_, in_, std::move(params_)), buf(in), parser(visitor), data_types(header_.getDataTypes()) {} + +void MsgPackVisitor::set_info(IColumn & column, DataTypePtr type) // NOLINT +{ + while (!info_stack.empty()) + { + info_stack.pop(); + } + info_stack.push(Info{column, type}); +} + +void MsgPackVisitor::insert_integer(UInt64 value) // NOLINT +{ + Info & info = info_stack.top(); + switch (info.type->getTypeId()) + { + case TypeIndex::UInt8: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::Date: [[fallthrough]]; + case TypeIndex::UInt16: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::DateTime: [[fallthrough]]; + case TypeIndex::UInt32: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::UInt64: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::Int8: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::Int16: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::Int32: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::Int64: + { + assert_cast(info.column).insertValue(value); + break; + } + case TypeIndex::DateTime64: + { + assert_cast(info.column).insertValue(value); + break; + } + default: + throw Exception("Type " + info.type->getName() + " is not supported for MsgPack input format", ErrorCodes::ILLEGAL_COLUMN); + } +} + +bool MsgPackVisitor::visit_positive_integer(UInt64 value) // NOLINT +{ + insert_integer(value); + return true; +} + +bool MsgPackVisitor::visit_negative_integer(Int64 value) // NOLINT +{ + insert_integer(value); + return true; +} + +bool MsgPackVisitor::visit_str(const char* value, size_t size) // NOLINT +{ + info_stack.top().column.insertData(value, size); + return true; +} + +bool MsgPackVisitor::visit_float32(Float32 value) // NOLINT +{ + assert_cast(info_stack.top().column).insertValue(value); + return true; +} + +bool MsgPackVisitor::visit_float64(Float64 value) // NOLINT +{ + assert_cast(info_stack.top().column).insertValue(value); + return true; +} + +bool MsgPackVisitor::start_array(size_t size) // NOLINT +{ + auto nested_type = assert_cast(*info_stack.top().type).getNestedType(); + ColumnArray & column_array = assert_cast(info_stack.top().column); + ColumnArray::Offsets & offsets = column_array.getOffsets(); + IColumn & nested_column = column_array.getData(); + offsets.push_back(offsets.back() + size); + info_stack.push(Info{nested_column, nested_type}); + return true; +} + +bool MsgPackVisitor::end_array() // NOLINT +{ + info_stack.pop(); + return true; +} + +void MsgPackVisitor::parse_error(size_t, size_t) // NOLINT +{ + throw Exception("Error occurred while parsing msgpack data.", ErrorCodes::INCORRECT_DATA); +} + +bool MsgPackRowInputFormat::readObject() +{ + if (buf.eof()) + return false; + + PeekableReadBufferCheckpoint checkpoint{buf}; + size_t offset = 0; + while (!parser.execute(buf.position(), buf.available(), offset)) + { + buf.position() = buf.buffer().end(); + if (buf.eof()) + throw Exception("Unexpected end of file while parsing msgpack object.", ErrorCodes::INCORRECT_DATA); + buf.position() = buf.buffer().end(); + buf.makeContinuousMemoryFromCheckpointToPos(); + buf.rollbackToCheckpoint(); + } + buf.position() += offset; + return true; +} + +bool MsgPackRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + size_t column_index = 0; + bool has_more_data = true; + for (; column_index != columns.size(); ++column_index) + { + visitor.set_info(*columns[column_index], data_types[column_index]); + has_more_data = readObject(); + if (!has_more_data) + break; + } + if (!has_more_data) + { + if (column_index != 0) + throw Exception("Not enough values to complete the row.", ErrorCodes::INCORRECT_DATA); + return false; + } + return true; +} + +void registerInputFormatProcessorMsgPack(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("MsgPack", []( + ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & params, + const FormatSettings &) + { + return std::make_shared(sample, buf, params); + }); +} + +} diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h new file mode 100644 index 00000000000..92e4f5d0bd7 --- /dev/null +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +class ReadBuffer; + +class MsgPackVisitor : public msgpack::null_visitor +{ +public: + struct Info + { + IColumn & column; + DataTypePtr type; + }; + + /// These functions are called when parser meets corresponding object in parsed data + bool visit_positive_integer(UInt64 value); + bool visit_negative_integer(Int64 value); + bool visit_float32(Float32 value); + bool visit_float64(Float64 value); + bool visit_str(const char* value, size_t size); + bool start_array(size_t size); + bool end_array(); + + /// This function will be called if error occurs in parsing + [[noreturn]] void parse_error(size_t parsed_offset, size_t error_offset); + + /// Update info_stack + void set_info(IColumn & column, DataTypePtr type); + + void insert_integer(UInt64 value); + +private: + /// Stack is needed to process nested arrays + std::stack info_stack; +}; + +class MsgPackRowInputFormat : public IRowInputFormat +{ +public: + MsgPackRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); + + bool readRow(MutableColumns & columns, RowReadExtension & ext) override; + String getName() const override { return "MagPackRowInputFormat"; } +private: + bool readObject(); + + PeekableReadBuffer buf; + MsgPackVisitor visitor; + msgpack::detail::parse_helper parser; + DataTypes data_types; +}; + +} diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp new file mode 100644 index 00000000000..cef7b001505 --- /dev/null +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp @@ -0,0 +1,159 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + +MsgPackRowOutputFormat::MsgPackRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback) + : IRowOutputFormat(header_, out_, callback), packer(out_) {} + +void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr data_type, size_t row_num) +{ + switch (data_type->getTypeId()) + { + case TypeIndex::UInt8: + { + packer.pack_uint8(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::Date: [[fallthrough]]; + case TypeIndex::UInt16: + { + packer.pack_uint16(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::DateTime: [[fallthrough]]; + case TypeIndex::UInt32: + { + packer.pack_uint32(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::UInt64: + { + packer.pack_uint64(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::Int8: + { + packer.pack_int8(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::Int16: + { + packer.pack_int16(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::Int32: + { + packer.pack_int32(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::Int64: + { + packer.pack_int64(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::Float32: + { + packer.pack_float(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::Float64: + { + packer.pack_double(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::DateTime64: + { + packer.pack_uint64(assert_cast(column).getElement(row_num)); + return; + } + case TypeIndex::String: + { + const StringRef & string = assert_cast(column).getDataAt(row_num); + packer.pack_str(string.size); + packer.pack_str_body(string.data, string.size); + return; + } + case TypeIndex::FixedString: + { + const StringRef & string = assert_cast(column).getDataAt(row_num); + packer.pack_str(string.size); + packer.pack_str_body(string.data, string.size); + return; + } + case TypeIndex::Array: + { + auto nested_type = assert_cast(*data_type).getNestedType(); + const ColumnArray & column_array = assert_cast(column); + const IColumn & nested_column = column_array.getData(); + const ColumnArray::Offsets & offsets = column_array.getOffsets(); + size_t offset = offsets[row_num - 1]; + size_t size = offsets[row_num] - offset; + packer.pack_array(size); + for (size_t i = 0; i < size; ++i) + { + serializeField(nested_column, nested_type, offset + i); + } + return; + } + case TypeIndex::Nullable: + { + auto nested_type = removeNullable(data_type); + const ColumnNullable & column_nullable = assert_cast(column); + if (!column_nullable.isNullAt(row_num)) + serializeField(column_nullable.getNestedColumn(), nested_type, row_num); + else + packer.pack_nil(); + return; + } + case TypeIndex::Nothing: + { + packer.pack_nil(); + return; + } + default: + break; + } + throw Exception("Type " + data_type->getName() + " is not supported for MsgPack output format", ErrorCodes::ILLEGAL_COLUMN); +} + +void MsgPackRowOutputFormat::write(const Columns & columns, size_t row_num) +{ + size_t num_columns = columns.size(); + for (size_t i = 0; i < num_columns; ++i) + { + serializeField(*columns[i], types[i], row_num); + } +} + +void registerOutputFormatProcessorMsgPack(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("MsgPack", []( + WriteBuffer & buf, + const Block & sample, + FormatFactory::WriteCallback callback, + const FormatSettings &) + { + return std::make_shared(buf, sample, callback); + }); +} + +} diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h new file mode 100644 index 00000000000..351920eb7c8 --- /dev/null +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ + +class MsgPackRowOutputFormat : public IRowOutputFormat +{ +public: + MsgPackRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback); + + String getName() const override { return "MsgPackRowOutputFormat"; } + + void write(const Columns & columns, size_t row_num) override; + void writeField(const IColumn &, const IDataType &, size_t) override {} + void serializeField(const IColumn & column, DataTypePtr data_type, size_t row_num); + +private: + msgpack::packer packer; +}; + +} diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index b948422a930..8f1967d3704 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -24,7 +24,7 @@ void MySQLOutputFormat::initialize() return; initialized = true; - auto & header = getPort(PortKind::Main).getHeader(); + const auto & header = getPort(PortKind::Main).getHeader(); data_types = header.getDataTypes(); if (header.columns()) diff --git a/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp b/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp index 5cad02cf2cf..b7018a1c1e8 100644 --- a/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp @@ -29,7 +29,7 @@ void ODBCDriver2BlockOutputFormat::writeRow(const Block & header, const Columns for (size_t column_idx = 0; column_idx < num_columns; ++column_idx) { buffer.clear(); - auto & column = columns[column_idx]; + const auto & column = columns[column_idx]; if (column->isNullAt(row_idx)) { @@ -49,8 +49,8 @@ void ODBCDriver2BlockOutputFormat::writeRow(const Block & header, const Columns void ODBCDriver2BlockOutputFormat::write(Chunk chunk, PortKind port_kind) { String text_value; - auto & header = getPort(port_kind).getHeader(); - auto & columns = chunk.getColumns(); + const auto & header = getPort(port_kind).getHeader(); + const auto & columns = chunk.getColumns(); const size_t rows = chunk.getNumRows(); for (size_t i = 0; i < rows; ++i) writeRow(header, columns, i, text_value); @@ -75,7 +75,7 @@ void ODBCDriver2BlockOutputFormat::finalize() void ODBCDriver2BlockOutputFormat::writePrefix() { - auto & header = getPort(PortKind::Main).getHeader(); + const auto & header = getPort(PortKind::Main).getHeader(); const size_t columns = header.columns(); /// Number of header rows. diff --git a/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp index eaccc47346d..741424da282 100644 --- a/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp @@ -19,8 +19,8 @@ void ODBCDriverBlockOutputFormat::consume(Chunk chunk) const size_t num_rows = chunk.getNumRows(); const size_t num_columns = chunk.getNumColumns(); - auto & columns = chunk.getColumns(); - auto & header = getPort(PortKind::Main).getHeader(); + const auto & columns = chunk.getColumns(); + const auto & header = getPort(PortKind::Main).getHeader(); String text_value; for (size_t i = 0; i < num_rows; ++i) @@ -28,8 +28,8 @@ void ODBCDriverBlockOutputFormat::consume(Chunk chunk) for (size_t j = 0; j < num_columns; ++j) { text_value.resize(0); - auto & column = columns[j]; - auto & type = header.getByPosition(j).type; + const auto & column = columns[j]; + const auto & type = header.getByPosition(j).type; { WriteBufferFromString text_out(text_value); @@ -43,7 +43,7 @@ void ODBCDriverBlockOutputFormat::consume(Chunk chunk) void ODBCDriverBlockOutputFormat::writePrefix() { - auto & header = getPort(PortKind::Main).getHeader(); + const auto & header = getPort(PortKind::Main).getHeader(); const size_t columns = header.columns(); /// Number of columns. diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index d03597b0f6e..2080dc479a1 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -27,7 +27,7 @@ Chunk ORCBlockInputFormat::generate() { Chunk res; - auto &header = getPort().getHeader(); + const auto & header = getPort().getHeader(); if (!in.eof()) { diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index dbc19944a56..221e1906a0a 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -84,7 +84,7 @@ private: static std::shared_ptr as_arrow_file(ReadBuffer & in) { - if (auto fd_in = dynamic_cast(&in)) + if (auto * fd_in = dynamic_cast(&in)) { struct stat stat; auto res = ::fstat(fd_in->getFD(), &stat); @@ -137,7 +137,7 @@ ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_ Chunk ParquetBlockInputFormat::generate() { Chunk res; - auto &header = getPort().getHeader(); + const auto & header = getPort().getHeader(); if (row_group_current >= row_group_total) return res; diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 4fab1235a0e..2681d862c25 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -131,7 +131,7 @@ static void fillArrowArrayWithDateColumnData( static void fillArrowArrayWithDateTimeColumnData( ColumnPtr write_column, std::shared_ptr & arrow_array, const PaddedPODArray * null_bytemap) { - auto & internal_data = assert_cast &>(*write_column).getData(); + const auto & internal_data = assert_cast &>(*write_column).getData(); //arrow::Date64Builder builder; arrow::UInt32Builder builder; arrow::Status status; @@ -291,7 +291,7 @@ private: void ParquetBlockOutputFormat::consume(Chunk chunk) { - auto & header = getPort(PortKind::Main).getHeader(); + const auto & header = getPort(PortKind::Main).getHeader(); const size_t columns_num = chunk.getNumColumns(); /// For arrow::Schema and arrow::Table creation diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp index a816cdd5318..96b1ff5a511 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp @@ -34,7 +34,7 @@ void PrettyBlockOutputFormat::calculateWidths( { size_t num_rows = chunk.getNumRows(); size_t num_columns = chunk.getNumColumns(); - auto & columns = chunk.getColumns(); + const auto & columns = chunk.getColumns(); widths.resize(num_columns); max_widths.resize_fill(num_columns); @@ -45,8 +45,8 @@ void PrettyBlockOutputFormat::calculateWidths( size_t prefix = 2; // Tab character adjustment for (size_t i = 0; i < num_columns; ++i) { - auto & elem = header.getByPosition(i); - auto & column = columns[i]; + const auto & elem = header.getByPosition(i); + const auto & column = columns[i]; widths[i].resize(num_rows); @@ -86,8 +86,8 @@ void PrettyBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind) auto num_rows = chunk.getNumRows(); auto num_columns = chunk.getNumColumns(); - auto & columns = chunk.getColumns(); - auto & header = getPort(port_kind).getHeader(); + const auto & columns = chunk.getColumns(); + const auto & header = getPort(port_kind).getHeader(); WidthsPerColumn widths; Widths max_widths; @@ -142,7 +142,7 @@ void PrettyBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind) if (i != 0) writeCString(" ┃ ", out); - auto & col = header.getByPosition(i); + const auto & col = header.getByPosition(i); if (format_settings.pretty.color) writeCString("\033[1m", out); @@ -181,7 +181,7 @@ void PrettyBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind) if (j != 0) writeCString(" │ ", out); - auto & type = *header.getByPosition(j).type; + const auto & type = *header.getByPosition(j).type; writeValueWithPadding(*columns[j], type, i, widths[j].empty() ? max_widths[j] : widths[j][i], max_widths[j]); } @@ -225,7 +225,7 @@ void PrettyBlockOutputFormat::consumeTotals(Chunk chunk) { total_rows = 0; writeSuffixIfNot(); - writeCString("\nExtremes:\n", out); + writeCString("\nTotals:\n", out); write(chunk, PortKind::Totals); } @@ -233,7 +233,7 @@ void PrettyBlockOutputFormat::consumeExtremes(Chunk chunk) { total_rows = 0; writeSuffixIfNot(); - writeCString("\nTotals:\n", out); + writeCString("\nExtremes:\n", out); write(chunk, PortKind::Extremes); } diff --git a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp index 554f0f00e39..4726332e554 100644 --- a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp @@ -90,8 +90,8 @@ void PrettyCompactBlockOutputFormat::writeRow( if (j != 0) writeCString(" │ ", out); - auto & type = *header.getByPosition(j).type; - auto & cur_widths = widths[j].empty() ? max_widths[j] : widths[j][row_num]; + const auto & type = *header.getByPosition(j).type; + const auto & cur_widths = widths[j].empty() ? max_widths[j] : widths[j][row_num]; writeValueWithPadding(*columns[j], type, row_num, cur_widths, max_widths[j]); } @@ -109,8 +109,8 @@ void PrettyCompactBlockOutputFormat::write(const Chunk & chunk, PortKind port_ki } size_t num_rows = chunk.getNumRows(); - auto & header = getPort(port_kind).getHeader(); - auto & columns = chunk.getColumns(); + const auto & header = getPort(port_kind).getHeader(); + const auto & columns = chunk.getColumns(); WidthsPerColumn widths; Widths max_widths; diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp index 8b852b3d194..ca9d1b674d4 100644 --- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp @@ -21,8 +21,8 @@ void PrettySpaceBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind size_t num_rows = chunk.getNumRows(); size_t num_columns = chunk.getNumColumns(); - auto & header = getPort(port_kind).getHeader(); - auto & columns = chunk.getColumns(); + const auto & header = getPort(port_kind).getHeader(); + const auto & columns = chunk.getColumns(); WidthsPerColumn widths; Widths max_widths; @@ -69,7 +69,7 @@ void PrettySpaceBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind if (column != 0) writeCString(" ", out); - auto & type = *header.getByPosition(column).type; + const auto & type = *header.getByPosition(column).type; auto & cur_width = widths[column].empty() ? max_widths[column] : widths[column][row]; writeValueWithPadding(*columns[column], type, row, cur_width, max_widths[column]); } diff --git a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h index 029b2c8329e..8dd36722efc 100644 --- a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h @@ -1,11 +1,13 @@ #pragma once -#include "config_formats.h" -#if USE_PROTOBUF +#if !defined(ARCADIA_BUILD) +# include "config_formats.h" +#endif -#include -#include -#include +#if USE_PROTOBUF +# include +# include +# include namespace DB { diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h index ae8ebbf28f4..f216af01d80 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h @@ -1,13 +1,15 @@ #pragma once -#include "config_formats.h" -#if USE_PROTOBUF +#if !defined(ARCADIA_BUILD) +# include "config_formats.h" +#endif -#include -#include -#include -#include -#include +#if USE_PROTOBUF +# include +# include +# include +# include +# include namespace google diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index 1e4fa511983..86e905344e1 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -94,7 +94,7 @@ bool TSKVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ex if (in.eof()) return false; - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); size_t num_columns = columns.size(); /// Set of columns for which the values were read. The rest will be filled with default values. @@ -119,7 +119,7 @@ bool TSKVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ex /// NOTE Optimization is possible by caching the order of fields (which is almost always the same) /// and quickly checking for the next expected field, instead of searching the hash table. - auto it = name_map.find(name_ref); + auto * it = name_map.find(name_ref); if (!it) { if (!format_settings.skip_unknown_fields) diff --git a/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp index 9d9cc132e2b..17677acfb86 100644 --- a/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp @@ -10,7 +10,7 @@ namespace DB TSKVRowOutputFormat::TSKVRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & format_settings_) : TabSeparatedRowOutputFormat(out_, header, false, false, callback, format_settings_) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); NamesAndTypesList columns(sample.getNamesAndTypesList()); fields.assign(columns.begin(), columns.end()); diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 706941a3dc6..014795cade1 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -46,7 +46,7 @@ TabSeparatedRowInputFormat::TabSeparatedRowInputFormat(const Block & header_, Re bool with_names_, bool with_types_, const FormatSettings & format_settings_) : RowInputFormatWithDiagnosticInfo(header_, in_, params_), with_names(with_names_), with_types(with_types_), format_settings(format_settings_) { - auto & sample = getPort().getHeader(); + const auto & sample = getPort().getHeader(); size_t num_columns = sample.columns(); data_types.resize(num_columns); @@ -67,7 +67,7 @@ TabSeparatedRowInputFormat::TabSeparatedRowInputFormat(const Block & header_, Re void TabSeparatedRowInputFormat::setupAllColumnsByTableSchema() { - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); read_columns.assign(header.columns(), true); column_indexes_for_input_fields.resize(header.columns()); @@ -126,7 +126,7 @@ void TabSeparatedRowInputFormat::fillUnreadColumnsWithDefaults(MutableColumns & void TabSeparatedRowInputFormat::readPrefix() { - if (with_names || with_types) + if (with_names || with_types || data_types.at(0)->textCanContainOnlyValidUTF8()) { /// In this format, we assume that column name or type cannot contain BOM, /// so, if format has header, @@ -238,7 +238,7 @@ bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & if (column_indexes_for_input_fields[file_column].has_value()) { - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); size_t col_idx = column_indexes_for_input_fields[file_column].value(); if (!deserializeFieldAndPrintDiagnosticInfo(header.getByPosition(col_idx).name, data_types[col_idx], *columns[col_idx], out, file_column)) @@ -350,7 +350,7 @@ void TabSeparatedRowInputFormat::resetParser() void registerInputFormatProcessorTabSeparated(FormatFactory & factory) { - for (auto name : {"TabSeparated", "TSV"}) + for (const auto * name : {"TabSeparated", "TSV"}) { factory.registerInputFormatProcessor(name, []( ReadBuffer & buf, @@ -362,7 +362,7 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory) }); } - for (auto name : {"TabSeparatedWithNames", "TSVWithNames"}) + for (const auto * name : {"TabSeparatedWithNames", "TSVWithNames"}) { factory.registerInputFormatProcessor(name, []( ReadBuffer & buf, @@ -374,7 +374,7 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory) }); } - for (auto name : {"TabSeparatedWithNamesAndTypes", "TSVWithNamesAndTypes"}) + for (const auto * name : {"TabSeparatedWithNamesAndTypes", "TSVWithNamesAndTypes"}) { factory.registerInputFormatProcessor(name, []( ReadBuffer & buf, @@ -421,7 +421,7 @@ static bool fileSegmentationEngineTabSeparatedImpl(ReadBuffer & in, DB::Memory<> void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) { // We can use the same segmentation engine for TSKV. - for (auto name : {"TabSeparated", "TSV", "TSKV"}) + for (const auto * name : {"TabSeparated", "TSV", "TSKV"}) { factory.registerFileSegmentationEngine(name, &fileSegmentationEngineTabSeparatedImpl); } diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp index 63dff316fc5..1c493cd1ab4 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp @@ -20,7 +20,7 @@ TabSeparatedRowOutputFormat::TabSeparatedRowOutputFormat( void TabSeparatedRowOutputFormat::writePrefix() { - auto & header = getPort(PortKind::Main).getHeader(); + const auto & header = getPort(PortKind::Main).getHeader(); size_t columns = header.columns(); if (with_names) @@ -75,7 +75,7 @@ void TabSeparatedRowOutputFormat::writeBeforeExtremes() void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) { - for (auto name : {"TabSeparated", "TSV"}) + for (const auto * name : {"TabSeparated", "TSV"}) { factory.registerOutputFormatProcessor(name, []( WriteBuffer & buf, @@ -87,7 +87,7 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) }); } - for (auto name : {"TabSeparatedRaw", "TSVRaw"}) + for (const auto * name : {"TabSeparatedRaw", "TSVRaw"}) { factory.registerOutputFormatProcessor(name, []( WriteBuffer & buf, @@ -99,7 +99,7 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) }); } - for (auto name : {"TabSeparatedWithNames", "TSVWithNames"}) + for (const auto * name : {"TabSeparatedWithNames", "TSVWithNames"}) { factory.registerOutputFormatProcessor(name, []( WriteBuffer & buf, @@ -111,7 +111,7 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory) }); } - for (auto name : {"TabSeparatedWithNamesAndTypes", "TSVWithNamesAndTypes"}) + for (const auto * name : {"TabSeparatedWithNamesAndTypes", "TSVWithNamesAndTypes"}) { factory.registerOutputFormatProcessor(name, []( WriteBuffer & buf, diff --git a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp index bd49b25934e..9adf631bfaa 100644 --- a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp @@ -19,7 +19,7 @@ TemplateBlockOutputFormat::TemplateBlockOutputFormat(const Block & header_, Writ : IOutputFormat(header_, out_), settings(settings_), format(std::move(format_)) , row_format(std::move(row_format_)), row_between_delimiter(std::move(row_between_delimiter_)) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); types.resize(columns); for (size_t i = 0; i < columns; ++i) diff --git a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp index 9b007d9f29b..2a0fe6d6ec6 100644 --- a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp @@ -364,7 +364,7 @@ bool TemplateRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & col skipSpaces(); if (row_format.format_idx_to_column_idx[i]) { - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); size_t col_idx = *row_format.format_idx_to_column_idx[i]; if (!deserializeFieldAndPrintDiagnosticInfo(header.getByPosition(col_idx).name, data_types[col_idx], *columns[col_idx], out, i)) diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp b/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp index ccef2a0898a..7c1f0e3b363 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp @@ -14,7 +14,7 @@ VerticalRowOutputFormat::VerticalRowOutputFormat( WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_, callback), format_settings(format_settings_) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); using Widths = std::vector; @@ -142,7 +142,7 @@ void VerticalRowOutputFormat::writeSpecialRow(const Columns & columns, size_t ro row_number = 0; field_number = 0; - auto & header = getPort(port_kind).getHeader(); + const auto & header = getPort(port_kind).getHeader(); size_t num_columns = columns.size(); writeCString(title, out); @@ -158,7 +158,7 @@ void VerticalRowOutputFormat::writeSpecialRow(const Columns & columns, size_t ro if (i != 0) writeFieldDelimiter(); - auto & col = header.getByPosition(i); + const auto & col = header.getByPosition(i); writeField(*columns[i], *col.type, row_num); } } diff --git a/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp b/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp index 9c2384691f6..dba07660503 100644 --- a/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp @@ -10,7 +10,7 @@ namespace DB XMLRowOutputFormat::XMLRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_, callback), format_settings(format_settings_) { - auto & sample = getPort(PortKind::Main).getHeader(); + const auto & sample = getPort(PortKind::Main).getHeader(); NamesAndTypesList columns(sample.getNamesAndTypesList()); fields.assign(columns.begin(), columns.end()); field_tag_names.resize(sample.columns()); @@ -124,7 +124,7 @@ void XMLRowOutputFormat::writeBeforeTotals() void XMLRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) { size_t totals_columns = columns.size(); - auto & header = getPort(PortKind::Totals).getHeader(); + const auto & header = getPort(PortKind::Totals).getHeader(); for (size_t i = 0; i < totals_columns; ++i) { const ColumnWithTypeAndName & column = header.safeGetByPosition(i); @@ -167,7 +167,7 @@ void XMLRowOutputFormat::writeAfterExtremes() void XMLRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) { - auto & header = getPort(PortKind::Extremes).getHeader(); + const auto & header = getPort(PortKind::Extremes).getHeader(); writeCString("\t\t<", *ostr); writeCString(title, *ostr); diff --git a/src/Processors/Formats/LazyOutputFormat.cpp b/src/Processors/Formats/LazyOutputFormat.cpp index e440743b0fb..920b34b3813 100644 --- a/src/Processors/Formats/LazyOutputFormat.cpp +++ b/src/Processors/Formats/LazyOutputFormat.cpp @@ -27,7 +27,7 @@ Block LazyOutputFormat::getBlock(UInt64 milliseconds) if (auto chunk_info = chunk.getChunkInfo()) { - if (auto * agg_info = typeid_cast(chunk_info.get())) + if (const auto * agg_info = typeid_cast(chunk_info.get())) { block.info.bucket_num = agg_info->bucket_num; block.info.is_overflows = agg_info->is_overflows; diff --git a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp index 369c0740264..ff40597073b 100644 --- a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp +++ b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp @@ -42,7 +42,7 @@ String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() WriteBufferFromOwnString out; - auto & header = getPort().getHeader(); + const auto & header = getPort().getHeader(); MutableColumns columns = header.cloneEmptyColumns(); /// It is possible to display detailed diagnostics only if the last and next to last rows are still in the read buffer. @@ -101,8 +101,8 @@ bool RowInputFormatWithDiagnosticInfo::deserializeFieldAndPrintDiagnosticInfo(co << "name: " << alignedName(col_name, max_length_of_column_name) << "type: " << alignedName(type->getName(), max_length_of_data_type_name); - auto prev_position = in.position(); - auto curr_position = in.position(); + auto * prev_position = in.position(); + auto * curr_position = in.position(); std::exception_ptr exception; try diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h index a613e8008d0..b7c230cb6de 100644 --- a/src/Processors/IProcessor.h +++ b/src/Processors/IProcessor.h @@ -233,6 +233,10 @@ public: onCancel(); } + /// Additional method which is called in case if ports were updated while work() method. + /// May be used to stop execution in rare cases. + virtual void onUpdatePorts() {} + virtual ~IProcessor() = default; auto & getInputs() { return inputs; } diff --git a/src/Processors/Merges/AggregatingSortedTransform.h b/src/Processors/Merges/AggregatingSortedTransform.h new file mode 100644 index 00000000000..a0425d4c376 --- /dev/null +++ b/src/Processors/Merges/AggregatingSortedTransform.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class ColumnAggregateFunction; + +/// Implementation of IMergingTransform via AggregatingSortedAlgorithm. +class AggregatingSortedTransform final : public IMergingTransform +{ +public: + AggregatingSortedTransform( + const Block & header, size_t num_inputs, + SortDescription description_, size_t max_block_size) + : IMergingTransform( + num_inputs, header, header, true, + header, + num_inputs, + std::move(description_), + max_block_size) + { + } + + String getName() const override { return "AggregatingSortedTransform"; } +}; + +} diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp new file mode 100644 index 00000000000..269c69f2747 --- /dev/null +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp @@ -0,0 +1,353 @@ +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +AggregatingSortedAlgorithm::ColumnsDefinition::ColumnsDefinition() = default; +AggregatingSortedAlgorithm::ColumnsDefinition::ColumnsDefinition(ColumnsDefinition &&) noexcept = default; +AggregatingSortedAlgorithm::ColumnsDefinition::~ColumnsDefinition() = default; + +/// Stores information for aggregation of AggregateFunction columns +struct AggregatingSortedAlgorithm::AggregateDescription +{ + ColumnAggregateFunction * column = nullptr; + const size_t column_number = 0; /// Position in header. + + AggregateDescription() = default; + explicit AggregateDescription(size_t col_number) : column_number(col_number) {} +}; + +/// Stores information for aggregation of SimpleAggregateFunction columns +struct AggregatingSortedAlgorithm::SimpleAggregateDescription +{ + /// An aggregate function 'anyLast', 'sum'... + AggregateFunctionPtr function; + IAggregateFunction::AddFunc add_function = nullptr; + + size_t column_number = 0; + IColumn * column = nullptr; + + /// For LowCardinality, convert is converted to nested type. nested_type is nullptr if no conversion needed. + const DataTypePtr nested_type; /// Nested type for LowCardinality, if it is. + const DataTypePtr real_type; /// Type in header. + + AlignedBuffer state; + bool created = false; + + SimpleAggregateDescription( + AggregateFunctionPtr function_, const size_t column_number_, + DataTypePtr nested_type_, DataTypePtr real_type_) + : function(std::move(function_)), column_number(column_number_) + , nested_type(std::move(nested_type_)), real_type(std::move(real_type_)) + { + add_function = function->getAddressOfAddFunction(); + state.reset(function->sizeOfData(), function->alignOfData()); + } + + void createState() + { + if (created) + return; + function->create(state.data()); + created = true; + } + + void destroyState() + { + if (!created) + return; + function->destroy(state.data()); + created = false; + } + + /// Explicitly destroy aggregation state if the stream is terminated + ~SimpleAggregateDescription() + { + destroyState(); + } + + SimpleAggregateDescription() = default; + SimpleAggregateDescription(SimpleAggregateDescription &&) = default; + SimpleAggregateDescription(const SimpleAggregateDescription &) = delete; +}; + +static AggregatingSortedAlgorithm::ColumnsDefinition defineColumns( + const Block & header, const SortDescription & description) +{ + AggregatingSortedAlgorithm::ColumnsDefinition def = {}; + size_t num_columns = header.columns(); + + /// Fill in the column numbers that need to be aggregated. + for (size_t i = 0; i < num_columns; ++i) + { + const ColumnWithTypeAndName & column = header.safeGetByPosition(i); + + /// We leave only states of aggregate functions. + if (!dynamic_cast(column.type.get()) + && !dynamic_cast(column.type->getCustomName())) + { + def.column_numbers_not_to_aggregate.push_back(i); + continue; + } + + /// Included into PK? + auto it = description.begin(); + for (; it != description.end(); ++it) + if (it->column_name == column.name || (it->column_name.empty() && it->column_number == i)) + break; + + if (it != description.end()) + { + def.column_numbers_not_to_aggregate.push_back(i); + continue; + } + + if (const auto * simple = dynamic_cast(column.type->getCustomName())) + { + auto type = recursiveRemoveLowCardinality(column.type); + if (type.get() == column.type.get()) + type = nullptr; + + // simple aggregate function + AggregatingSortedAlgorithm::SimpleAggregateDescription desc(simple->getFunction(), i, type, column.type); + if (desc.function->allocatesMemoryInArena()) + def.allocates_memory_in_arena = true; + + def.columns_to_simple_aggregate.emplace_back(std::move(desc)); + } + else + { + // standard aggregate function + def.columns_to_aggregate.emplace_back(i); + } + } + + return def; +} + +static MutableColumns getMergedColumns(const Block & header, const AggregatingSortedAlgorithm::ColumnsDefinition & def) +{ + MutableColumns columns; + columns.resize(header.columns()); + + for (const auto & desc : def.columns_to_simple_aggregate) + { + const auto & type = desc.nested_type ? desc.nested_type + : desc.real_type; + columns[desc.column_number] = type->createColumn(); + } + + for (size_t i = 0; i < columns.size(); ++i) + if (!columns[i]) + columns[i] = header.getByPosition(i).type->createColumn(); + + return columns; +} + +/// Remove constants and LowCardinality for SimpleAggregateFunction +static void preprocessChunk(Chunk & chunk, const AggregatingSortedAlgorithm::ColumnsDefinition & def) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + for (auto & column : columns) + column = column->convertToFullColumnIfConst(); + + for (const auto & desc : def.columns_to_simple_aggregate) + if (desc.nested_type) + columns[desc.column_number] = recursiveRemoveLowCardinality(columns[desc.column_number]); + + chunk.setColumns(std::move(columns), num_rows); +} + +/// Return back LowCardinality for SimpleAggregateFunction +static void postprocessChunk(Chunk & chunk, const AggregatingSortedAlgorithm::ColumnsDefinition & def) +{ + size_t num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + for (const auto & desc : def.columns_to_simple_aggregate) + { + if (desc.nested_type) + { + const auto & from_type = desc.nested_type; + const auto & to_type = desc.real_type; + columns[desc.column_number] = recursiveTypeConversion(columns[desc.column_number], from_type, to_type); + } + } + + chunk.setColumns(std::move(columns), num_rows); +} + + +AggregatingSortedAlgorithm::AggregatingMergedData::AggregatingMergedData( + MutableColumns columns_, UInt64 max_block_size_, ColumnsDefinition & def_) + : MergedData(std::move(columns_), false, max_block_size_), def(def_) +{ + initAggregateDescription(); +} + +void AggregatingSortedAlgorithm::AggregatingMergedData::startGroup(const ColumnRawPtrs & raw_columns, size_t row) +{ + /// We will write the data for the group. We copy the values of ordinary columns. + for (auto column_number : def.column_numbers_not_to_aggregate) + columns[column_number]->insertFrom(*raw_columns[column_number], row); + + /// Add the empty aggregation state to the aggregate columns. The state will be updated in the `addRow` function. + for (auto & column_to_aggregate : def.columns_to_aggregate) + column_to_aggregate.column->insertDefault(); + + /// Reset simple aggregation states for next row + for (auto & desc : def.columns_to_simple_aggregate) + desc.createState(); + + if (def.allocates_memory_in_arena) + arena = std::make_unique(); + + is_group_started = true; +} + +void AggregatingSortedAlgorithm::AggregatingMergedData::finishGroup() +{ + /// Write the simple aggregation result for the current group. + for (auto & desc : def.columns_to_simple_aggregate) + { + desc.function->insertResultInto(desc.state.data(), *desc.column); + desc.destroyState(); + } + + is_group_started = false; + ++total_merged_rows; + ++merged_rows; + /// TODO: sum_blocks_granularity += block_size; +} + +void AggregatingSortedAlgorithm::AggregatingMergedData::addRow(SortCursor & cursor) +{ + if (!is_group_started) + throw Exception("Can't add a row to the group because it was not started.", ErrorCodes::LOGICAL_ERROR); + + for (auto & desc : def.columns_to_aggregate) + desc.column->insertMergeFrom(*cursor->all_columns[desc.column_number], cursor->pos); + + for (auto & desc : def.columns_to_simple_aggregate) + { + auto & col = cursor->all_columns[desc.column_number]; + desc.add_function(desc.function.get(), desc.state.data(), &col, cursor->pos, arena.get()); + } +} + +Chunk AggregatingSortedAlgorithm::AggregatingMergedData::pull() +{ + if (is_group_started) + throw Exception("Can't pull chunk because group was not finished.", ErrorCodes::LOGICAL_ERROR); + + auto chunk = MergedData::pull(); + postprocessChunk(chunk, def); + + initAggregateDescription(); + + return chunk; +} + +void AggregatingSortedAlgorithm::AggregatingMergedData::initAggregateDescription() +{ + for (auto & desc : def.columns_to_simple_aggregate) + desc.column = columns[desc.column_number].get(); + + for (auto & desc : def.columns_to_aggregate) + desc.column = typeid_cast(columns[desc.column_number].get()); +} + + +AggregatingSortedAlgorithm::AggregatingSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, size_t max_block_size) + : IMergingAlgorithmWithDelayedChunk(num_inputs, std::move(description_)) + , columns_definition(defineColumns(header, description_)) + , merged_data(getMergedColumns(header, columns_definition), max_block_size, columns_definition) +{ +} + +void AggregatingSortedAlgorithm::initialize(Chunks chunks) +{ + for (auto & chunk : chunks) + if (chunk) + preprocessChunk(chunk, columns_definition); + + initializeQueue(std::move(chunks)); +} + +void AggregatingSortedAlgorithm::consume(Chunk chunk, size_t source_num) +{ + preprocessChunk(chunk, columns_definition); + updateCursor(std::move(chunk), source_num); +} + +IMergingAlgorithm::Status AggregatingSortedAlgorithm::merge() +{ + /// We take the rows in the correct order and put them in `merged_block`, while the rows are no more than `max_block_size` + while (queue.isValid()) + { + bool key_differs; + SortCursor current = queue.current(); + + { + detail::RowRef current_key; + current_key.set(current); + + key_differs = last_key.empty() || !last_key.hasEqualSortColumnsWith(current_key); + + last_key = current_key; + last_chunk_sort_columns.clear(); + } + + if (key_differs) + { + if (merged_data.isGroupStarted()) + merged_data.finishGroup(); + + /// if there are enough rows accumulated and the last one is calculated completely + if (merged_data.hasEnoughRows()) + { + last_key.reset(); + return Status(merged_data.pull()); + } + + merged_data.startGroup(current->all_columns, current->pos); + } + + merged_data.addRow(current); + + if (!current->isLast()) + { + queue.next(); + } + else + { + /// We get the next block from the corresponding source, if there is one. + queue.removeTop(); + return Status(current.impl->order); + } + } + + /// Write the simple aggregation result for the previous group. + if (merged_data.isGroupStarted()) + merged_data.finishGroup(); + + last_chunk_sort_columns.clear(); + return Status(merged_data.pull(), true); +} + +} diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h new file mode 100644 index 00000000000..b4819ad030c --- /dev/null +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h @@ -0,0 +1,88 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/** Merges several sorted inputs to one. + * During this for each group of consecutive identical values of the primary key (the columns by which the data is sorted), + * merges them into one row. When merging, the data is pre-aggregated - merge of states of aggregate functions, + * corresponding to a one value of the primary key. For columns that are not part of the primary key and which do not have the AggregateFunction type, + * when merged, the first value is selected. + */ +class AggregatingSortedAlgorithm final : public IMergingAlgorithmWithDelayedChunk +{ +public: + AggregatingSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, size_t max_block_size); + + void initialize(Chunks chunks) override; + void consume(Chunk chunk, size_t source_num) override; + Status merge() override; + + struct SimpleAggregateDescription; + struct AggregateDescription; + + /// This structure define columns into one of three types: + /// * columns which are not aggregate functions and not needed to be aggregated + /// * usual aggregate functions, which stores states into ColumnAggregateFunction + /// * simple aggregate functions, which store states into ordinary columns + struct ColumnsDefinition + { + ColumnsDefinition(); /// Is needed because destructor is defined. + ColumnsDefinition(ColumnsDefinition &&) noexcept; /// Is needed because destructor is defined. + ~ColumnsDefinition(); /// Is needed because otherwise std::vector's destructor uses incomplete types. + + /// Columns with which numbers should not be aggregated. + ColumnNumbers column_numbers_not_to_aggregate; + std::vector columns_to_aggregate; + std::vector columns_to_simple_aggregate; + + /// Does SimpleAggregateFunction allocates memory in arena? + bool allocates_memory_in_arena = false; + }; + +private: + /// Specialization for AggregatingSortedAlgorithm. + struct AggregatingMergedData : public MergedData + { + private: + using MergedData::pull; + using MergedData::insertRow; + + public: + AggregatingMergedData(MutableColumns columns_, UInt64 max_block_size_, ColumnsDefinition & def_); + + /// Group is a group of rows with the same sorting key. It represents single row in result. + /// Algorithm is: start group, add several rows, finish group. + /// Then pull chunk when enough groups were added. + void startGroup(const ColumnRawPtrs & raw_columns, size_t row); + void finishGroup(); + + bool isGroupStarted() const { return is_group_started; } + void addRow(SortCursor & cursor); /// Possible only when group was started. + + Chunk pull(); /// Possible only if group was finished. + + private: + ColumnsDefinition & def; + + /// Memory pool for SimpleAggregateFunction + /// (only when allocates_memory_in_arena == true). + std::unique_ptr arena; + + bool is_group_started = false; + + /// Initialize aggregate descriptions with columns. + void initAggregateDescription(); + }; + + /// Order between members is important because merged_data has reference to columns_definition. + ColumnsDefinition columns_definition; + AggregatingMergedData merged_data; +}; + +} diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp new file mode 100644 index 00000000000..cd3a193105d --- /dev/null +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp @@ -0,0 +1,193 @@ +#include + +#include +#include +#include +#include + +#include + +/// Maximum number of messages about incorrect data in the log. +#define MAX_ERROR_MESSAGES 10 + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} + +CollapsingSortedAlgorithm::CollapsingSortedAlgorithm( + const Block & header, + size_t num_inputs, + SortDescription description_, + const String & sign_column, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_, + bool use_average_block_sizes, + Logger * log_) + : IMergingAlgorithmWithSharedChunks(num_inputs, std::move(description_), out_row_sources_buf_, max_row_refs) + , merged_data(header.cloneEmptyColumns(), use_average_block_sizes, max_block_size) + , sign_column_number(header.getPositionByName(sign_column)) + , log(log_) +{ +} + +void CollapsingSortedAlgorithm::reportIncorrectData() +{ + if (!log) + return; + + std::stringstream s; + s << "Incorrect data: number of rows with sign = 1 (" << count_positive + << ") differs with number of rows with sign = -1 (" << count_negative + << ") by more than one (for key: "; + + auto & sort_columns = *last_row.sort_columns; + for (size_t i = 0, size = sort_columns.size(); i < size; ++i) + { + if (i != 0) + s << ", "; + s << applyVisitor(FieldVisitorToString(), (*sort_columns[i])[last_row.row_num]); + } + + s << ")."; + + /** Fow now we limit ourselves to just logging such situations, + * since the data is generated by external programs. + * With inconsistent data, this is an unavoidable error that can not be easily corrected by admins. Therefore Warning. + */ + LOG_WARNING(log, s.rdbuf()); +} + +void CollapsingSortedAlgorithm::insertRow(RowRef & row) +{ + merged_data.insertRow(*row.all_columns, row.row_num, row.owned_chunk->getNumRows()); +} + +void CollapsingSortedAlgorithm::insertRows() +{ + if (count_positive == 0 && count_negative == 0) + { + /// No input rows have been read. + return; + } + + if (last_is_positive || count_positive != count_negative) + { + if (count_positive <= count_negative) + { + insertRow(first_negative_row); + + if (out_row_sources_buf) + current_row_sources[first_negative_pos].setSkipFlag(false); + } + + if (count_positive >= count_negative) + { + insertRow(last_positive_row); + + if (out_row_sources_buf) + current_row_sources[last_positive_pos].setSkipFlag(false); + } + + if (!(count_positive == count_negative || count_positive + 1 == count_negative || count_positive == count_negative + 1)) + { + if (count_incorrect_data < MAX_ERROR_MESSAGES) + reportIncorrectData(); + ++count_incorrect_data; + } + } + + first_negative_row.clear(); + last_positive_row.clear(); + + if (out_row_sources_buf) + out_row_sources_buf->write( + reinterpret_cast(current_row_sources.data()), + current_row_sources.size() * sizeof(RowSourcePart)); +} + +IMergingAlgorithm::Status CollapsingSortedAlgorithm::merge() +{ + /// Take rows in required order and put them into `merged_data`, while the rows are no more than `max_block_size` + while (queue.isValid()) + { + auto current = queue.current(); + Int8 sign = assert_cast(*current->all_columns[sign_column_number]).getData()[current->pos]; + + RowRef current_row; + setRowRef(current_row, current); + + if (last_row.empty()) + setRowRef(last_row, current); + + bool key_differs = !last_row.hasEqualSortColumnsWith(current_row); + + /// if there are enough rows and the last one is calculated completely + if (key_differs && merged_data.hasEnoughRows()) + return Status(merged_data.pull()); + + if (key_differs) + { + /// We write data for the previous primary key. + insertRows(); + + current_row.swap(last_row); + + count_negative = 0; + count_positive = 0; + + current_pos = 0; + first_negative_pos = 0; + last_positive_pos = 0; + current_row_sources.resize(0); + } + + /// Initially, skip all rows. On insert, unskip "corner" rows. + if (out_row_sources_buf) + current_row_sources.emplace_back(current.impl->order, true); + + if (sign == 1) + { + ++count_positive; + last_is_positive = true; + + setRowRef(last_positive_row, current); + last_positive_pos = current_pos; + } + else if (sign == -1) + { + if (!count_negative) + { + setRowRef(first_negative_row, current); + first_negative_pos = current_pos; + } + + ++count_negative; + last_is_positive = false; + } + else + throw Exception("Incorrect data: Sign = " + toString(sign) + " (must be 1 or -1).", + ErrorCodes::INCORRECT_DATA); + + ++current_pos; + + if (!current->isLast()) + { + queue.next(); + } + else + { + /// We take next block from the corresponding source, if there is one. + queue.removeTop(); + return Status(current.impl->order); + } + } + + insertRows(); + return Status(merged_data.pull(), true); +} + +} diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.h new file mode 100644 index 00000000000..4158f55a7cd --- /dev/null +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.h @@ -0,0 +1,71 @@ +#pragma once +#include +#include +#include + +namespace Poco +{ + class Logger; +} + +namespace DB +{ + +/** Merges several sorted inputs to one. + * For each group of consecutive identical values of the primary key (the columns by which the data is sorted), + * keeps no more than one row with the value of the column `sign_column = -1` ("negative row") + * and no more than a row with the value of the column `sign_column = 1` ("positive row"). + * That is, it collapses the records from the change log. + * + * If the number of positive and negative rows is the same, and the last row is positive, then the first negative and last positive rows are written. + * If the number of positive and negative rows is the same, and the last line is negative, it writes nothing. + * If the positive by 1 is greater than the negative rows, then only the last positive row is written. + * If negative by 1 is greater than positive rows, then only the first negative row is written. + * Otherwise, a logical error. + */ +class CollapsingSortedAlgorithm final : public IMergingAlgorithmWithSharedChunks +{ +public: + CollapsingSortedAlgorithm( + const Block & header, + size_t num_inputs, + SortDescription description_, + const String & sign_column, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_, + bool use_average_block_sizes, + Logger * log_); + + Status merge() override; + +private: + MergedData merged_data; + + const size_t sign_column_number; + + static constexpr size_t max_row_refs = 4; /// first_negative, last_positive, last, current. + RowRef first_negative_row; + RowRef last_positive_row; + RowRef last_row; + + size_t count_positive = 0; /// The number of positive rows for the current primary key. + size_t count_negative = 0; /// The number of negative rows for the current primary key. + bool last_is_positive = false; /// true if the last row for the current primary key is positive. + + /// Fields specific for VERTICAL merge algorithm. + /// Row numbers are relative to the start of current primary key. + size_t current_pos = 0; /// Current row number + size_t first_negative_pos = 0; /// Row number of first_negative + size_t last_positive_pos = 0; /// Row number of last_positive + PODArray current_row_sources; /// Sources of rows with the current primary key + + size_t count_incorrect_data = 0; /// To prevent too many error messages from writing to the log. + Logger * log; + + void reportIncorrectData(); + void insertRow(RowRef & row); + void insertRows(); +}; + +} + diff --git a/src/DataStreams/VersionedCollapsingSortedBlockInputStream.h b/src/Processors/Merges/Algorithms/FixedSizeDequeWithGaps.h similarity index 68% rename from src/DataStreams/VersionedCollapsingSortedBlockInputStream.h rename to src/Processors/Merges/Algorithms/FixedSizeDequeWithGaps.h index c64972d9266..35cfded4214 100644 --- a/src/DataStreams/VersionedCollapsingSortedBlockInputStream.h +++ b/src/Processors/Merges/Algorithms/FixedSizeDequeWithGaps.h @@ -1,24 +1,13 @@ #pragma once -#include - -#include -#include - -#include - - namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; +extern const int LOGICAL_ERROR; } -static const size_t MAX_ROWS_IN_MULTIVERSION_QUEUE = 8192; - - /* Deque with fixed memory size. Allows pushing gaps. * frontGap() returns the number of gaps were inserted before front. * @@ -154,7 +143,7 @@ private: { if (size() + 1 == container.size()) throw Exception("Not enough space to insert into FixedSizeDequeWithGaps with capacity " - + toString(container.size() - 1), ErrorCodes::LOGICAL_ERROR); + + std::to_string(container.size() - 1), ErrorCodes::LOGICAL_ERROR); } void checkHasValuesToRemove() const @@ -170,46 +159,4 @@ private: } }; -class VersionedCollapsingSortedBlockInputStream : public MergingSortedBlockInputStream -{ -public: - /// Don't need version column. It's in primary key. - /// max_rows_in_queue should be about max_block_size_ if we won't store a lot of extra blocks (RowRef holds SharedBlockPtr). - VersionedCollapsingSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, - const String & sign_column_, size_t max_block_size_, - WriteBuffer * out_row_sources_buf_ = nullptr, bool average_block_sizes_ = false); - - String getName() const override { return "VersionedCollapsingSorted"; } - -protected: - /// Can return 1 more records than max_block_size. - Block readImpl() override; - -private: - size_t sign_column_number = 0; - - Logger * log = &Logger::get("VersionedCollapsingSortedBlockInputStream"); - - /// Read is finished. - bool finished = false; - - Int8 sign_in_queue = 0; - const size_t max_rows_in_queue; - /// Rows with the same primary key and sign. - FixedSizeDequeWithGaps current_keys; - - size_t blocks_written = 0; - - /// Sources of rows for VERTICAL merge algorithm. Size equals to (size + number of gaps) in current_keys. - std::queue current_row_sources; - - void merge(MutableColumns & merged_columns, SortingHeap & queue); - - /// Output to result row for the current primary key. - void insertRow(size_t skip_rows, const SharedBlockRowRef & row, MutableColumns & merged_columns); - - void insertGap(size_t gap_size); -}; - } diff --git a/src/Processors/Merges/Algorithms/Graphite.h b/src/Processors/Merges/Algorithms/Graphite.h new file mode 100644 index 00000000000..3ff297987de --- /dev/null +++ b/src/Processors/Merges/Algorithms/Graphite.h @@ -0,0 +1,133 @@ +#pragma once +#include + +namespace DB +{ + +class IAggregateFunction; +using AggregateFunctionPtr = std::shared_ptr; + +} + +/** Intended for implementation of "rollup" - aggregation (rounding) of older data + * for a table with Graphite data (Graphite is the system for time series monitoring). + * + * Table with graphite data has at least the following columns (accurate to the name): + * Path, Time, Value, Version + * + * Path - name of metric (sensor); + * Time - time of measurement; + * Value - value of measurement; + * Version - a number, that for equal pairs of Path and Time, need to leave only record with maximum version. + * + * Each row in a table correspond to one value of one sensor. + * + * Pattern should contain function, retention scheme, or both of them. The order of patterns does mean as well: + * * Aggregation OR retention patterns should be first + * * Then aggregation AND retention full patterns have to be placed + * * default pattern without regexp must be the last + * + * Rollup rules are specified in the following way: + * + * pattern + * regexp + * function + * pattern + * regexp + * age -> precision + * age -> precision + * ... + * pattern + * regexp + * function + * age -> precision + * age -> precision + * ... + * pattern + * ... + * default + * function + * age -> precision + * ... + * + * regexp - pattern for sensor name + * default - if no pattern has matched + * + * age - minimal data age (in seconds), to start rounding with specified precision. + * precision - rounding precision (in seconds) + * + * function - name of aggregate function to be applied for values, that time was rounded to same. + * + * Example: + * + * + * + * \.max$ + * max + * + * + * click_cost + * any + * + * 0 + * 5 + * + * + * 86400 + * 60 + * + * + * + * max + * + * 0 + * 60 + * + * + * 3600 + * 300 + * + * + * 86400 + * 3600 + * + * + * + */ +namespace DB::Graphite +{ + +struct Retention +{ + UInt32 age; + UInt32 precision; +}; + +using Retentions = std::vector; + +struct Pattern +{ + std::shared_ptr regexp; + std::string regexp_str; + AggregateFunctionPtr function; + Retentions retentions; /// Must be ordered by 'age' descending. + enum { TypeUndef, TypeRetention, TypeAggregation, TypeAll } type = TypeAll; /// The type of defined pattern, filled automatically +}; + +using Patterns = std::vector; +using RetentionPattern = Pattern; +using AggregationPattern = Pattern; + +struct Params +{ + String config_name; + String path_column_name; + String time_column_name; + String value_column_name; + String version_column_name; + Graphite::Patterns patterns; +}; + +using RollupRule = std::pair; + +} diff --git a/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp b/src/Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.cpp similarity index 59% rename from src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp rename to src/Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.cpp index 1b81881ab1a..f26fe96876f 100644 --- a/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp +++ b/src/Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.cpp @@ -1,21 +1,36 @@ -#include -#include - +#include +#include +#include +#include namespace DB { -namespace ErrorCodes +static GraphiteRollupSortedAlgorithm::ColumnsDefinition defineColumns( + const Block & header, const Graphite::Params & params) { - extern const int LOGICAL_ERROR; + GraphiteRollupSortedAlgorithm::ColumnsDefinition def; + + def.path_column_num = header.getPositionByName(params.path_column_name); + def.time_column_num = header.getPositionByName(params.time_column_name); + def.value_column_num = header.getPositionByName(params.value_column_name); + def.version_column_num = header.getPositionByName(params.version_column_name); + + size_t num_columns = header.columns(); + for (size_t i = 0; i < num_columns; ++i) + if (i != def.time_column_num && i != def.value_column_num && i != def.version_column_num) + def.unmodified_column_numbers.push_back(i); + + return def; } - -GraphiteRollupSortedBlockInputStream::GraphiteRollupSortedBlockInputStream( - const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_, - const Graphite::Params & params_, time_t time_of_merge_) - : MergingSortedBlockInputStream(inputs_, description_, max_block_size_), - params(params_), time_of_merge(time_of_merge_) +GraphiteRollupSortedAlgorithm::GraphiteRollupSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, size_t max_block_size, + Graphite::Params params_, time_t time_of_merge_) + : IMergingAlgorithmWithSharedChunks(num_inputs, std::move(description_), nullptr, max_row_refs) + , merged_data(header.cloneEmptyColumns(), false, max_block_size) + , params(std::move(params_)), time_of_merge(time_of_merge_) { size_t max_size_of_aggregate_state = 0; size_t max_alignment_of_aggregate_state = 1; @@ -29,21 +44,11 @@ GraphiteRollupSortedBlockInputStream::GraphiteRollupSortedBlockInputStream( } } - place_for_aggregate_state.reset(max_size_of_aggregate_state, max_alignment_of_aggregate_state); - - /// Memoize column numbers in block. - path_column_num = header.getPositionByName(params.path_column_name); - time_column_num = header.getPositionByName(params.time_column_name); - value_column_num = header.getPositionByName(params.value_column_name); - version_column_num = header.getPositionByName(params.version_column_name); - - for (size_t i = 0; i < num_columns; ++i) - if (i != time_column_num && i != value_column_num && i != version_column_num) - unmodified_column_numbers.push_back(i); + merged_data.allocMemForAggregates(max_size_of_aggregate_state, max_alignment_of_aggregate_state); + columns_definition = defineColumns(header, params); } - -Graphite::RollupRule GraphiteRollupSortedBlockInputStream::selectPatternForPath(StringRef path) const +Graphite::RollupRule GraphiteRollupSortedAlgorithm::selectPatternForPath(StringRef path) const { const Graphite::Pattern * first_match = &undef_pattern; @@ -74,8 +79,8 @@ Graphite::RollupRule GraphiteRollupSortedBlockInputStream::selectPatternForPath( /// General pattern with matched path if (pattern.type == pattern.TypeAll) { - /// Only for not default patterns with both function and retention parameters - return std::pair(&pattern, &pattern); + /// Only for not default patterns with both function and retention parameters + return std::pair(&pattern, &pattern); } if (first_match->type == first_match->TypeUndef) { @@ -99,8 +104,7 @@ Graphite::RollupRule GraphiteRollupSortedBlockInputStream::selectPatternForPath( return {nullptr, nullptr}; } - -UInt32 GraphiteRollupSortedBlockInputStream::selectPrecision(const Graphite::Retentions & retentions, time_t time) const +UInt32 GraphiteRollupSortedAlgorithm::selectPrecision(const Graphite::Retentions & retentions, time_t time) const { static_assert(is_signed_v, "time_t must be signed type"); @@ -114,7 +118,6 @@ UInt32 GraphiteRollupSortedBlockInputStream::selectPrecision(const Graphite::Ret return 1; } - /** Round the unix timestamp to seconds precision. * In this case, the date should not change. The date is calculated using the local time zone. * @@ -140,33 +143,11 @@ static time_t roundTimeToPrecision(const DateLUTImpl & date_lut, time_t time, UI } } - -Block GraphiteRollupSortedBlockInputStream::readImpl() -{ - if (finished) - return Block(); - - MutableColumns merged_columns; - init(merged_columns); - - if (has_collation) - throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR); - - if (merged_columns.empty()) - return Block(); - - merge(merged_columns, queue_without_collation); - return header.cloneWithColumns(std::move(merged_columns)); -} - - -void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap & queue) +IMergingAlgorithm::Status GraphiteRollupSortedAlgorithm::merge() { const DateLUTImpl & date_lut = DateLUT::instance(); - size_t started_rows = 0; /// Number of times startNextRow() has been called. - - /// Take rows in needed order and put them into `merged_columns` until we get `max_block_size` rows. + /// Take rows in needed order and put them into `merged_data` until we get `max_block_size` rows. /// /// Variables starting with current_* refer to the rows previously popped from the queue that will /// contribute towards current output row. @@ -174,24 +155,24 @@ void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns while (queue.isValid()) { - SortCursor next_cursor = queue.current(); + SortCursor current = queue.current(); - StringRef next_path = next_cursor->all_columns[path_column_num]->getDataAt(next_cursor->pos); + StringRef next_path = current->all_columns[columns_definition.path_column_num]->getDataAt(current->pos); bool new_path = is_first || next_path != current_group_path; is_first = false; - time_t next_row_time = next_cursor->all_columns[time_column_num]->getUInt(next_cursor->pos); + time_t next_row_time = current->all_columns[columns_definition.time_column_num]->getUInt(current->pos); /// Is new key before rounding. bool is_new_key = new_path || next_row_time != current_time; if (is_new_key) { /// Accumulate the row that has maximum version in the previous group of rows with the same key: - if (started_rows) + if (merged_data.wasGroupStarted()) accumulateRow(current_subgroup_newest_row); - Graphite::RollupRule next_rule = current_rule; + Graphite::RollupRule next_rule = merged_data.currentRule(); if (new_path) next_rule = selectPatternForPath(next_path); @@ -213,22 +194,21 @@ void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns if (will_be_new_key) { - if (started_rows) + if (merged_data.wasGroupStarted()) { - finishCurrentGroup(merged_columns); + finishCurrentGroup(); /// We have enough rows - return, but don't advance the loop. At the beginning of the /// next call to merge() the same next_cursor will be processed once more and /// the next output row will be created from it. - if (started_rows >= max_block_size) - return; + if (merged_data.hasEnoughRows()) + return Status(merged_data.pull()); } /// At this point previous row has been fully processed, so we can advance the loop /// (substitute current_* values for next_*, advance the cursor). - startNextGroup(merged_columns, next_cursor, next_rule); - ++started_rows; + startNextGroup(current, next_rule); current_time_rounded = next_time_rounded; } @@ -239,12 +219,12 @@ void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns /// Within all rows with same key, we should leave only one row with maximum version; /// and for rows with same maximum version - only last row. if (is_new_key - || next_cursor->all_columns[version_column_num]->compareAt( - next_cursor->pos, current_subgroup_newest_row.row_num, - *(*current_subgroup_newest_row.columns)[version_column_num], + || current->all_columns[columns_definition.version_column_num]->compareAt( + current->pos, current_subgroup_newest_row.row_num, + *(*current_subgroup_newest_row.all_columns)[columns_definition.version_column_num], /* nan_direction_hint = */ 1) >= 0) { - setRowRef(current_subgroup_newest_row, next_cursor); + current_subgroup_newest_row.set(current, source_chunks[current.impl->order]); /// Small hack: group and subgroups have the same path, so we can set current_group_path here instead of startNextGroup /// But since we keep in memory current_subgroup_newest_row's block, we could use StringRef for current_group_path and don't @@ -252,37 +232,52 @@ void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns current_group_path = next_path; } - if (!next_cursor->isLast()) + if (!current->isLast()) { queue.next(); } else { /// We get the next block from the appropriate source, if there is one. - fetchNextBlock(next_cursor, queue); + queue.removeTop(); + return Status(current.impl->order); } } /// Write result row for the last group. - if (started_rows) + if (merged_data.wasGroupStarted()) { accumulateRow(current_subgroup_newest_row); - finishCurrentGroup(merged_columns); + finishCurrentGroup(); } - finished = true; + return Status(merged_data.pull(), true); } +void GraphiteRollupSortedAlgorithm::startNextGroup(SortCursor & cursor, Graphite::RollupRule next_rule) +{ + merged_data.startNextGroup(cursor->all_columns, cursor->pos, next_rule, columns_definition); +} -template -void GraphiteRollupSortedBlockInputStream::startNextGroup(MutableColumns & merged_columns, TSortCursor & cursor, - Graphite::RollupRule next_rule) +void GraphiteRollupSortedAlgorithm::finishCurrentGroup() +{ + merged_data.insertRow(current_time_rounded, current_subgroup_newest_row, columns_definition); +} + +void GraphiteRollupSortedAlgorithm::accumulateRow(RowRef & row) +{ + merged_data.accumulateRow(row, columns_definition); +} + +void GraphiteRollupSortedAlgorithm::GraphiteRollupMergedData::startNextGroup( + const ColumnRawPtrs & raw_columns, size_t row, + Graphite::RollupRule next_rule, ColumnsDefinition & def) { const Graphite::AggregationPattern * aggregation_pattern = std::get<1>(next_rule); /// Copy unmodified column values (including path column). - for (size_t j : unmodified_column_numbers) - merged_columns[j]->insertFrom(*cursor->all_columns[j], cursor->pos); + for (size_t j : def.unmodified_column_numbers) + columns[j]->insertFrom(*raw_columns[j], row); if (aggregation_pattern) { @@ -291,34 +286,49 @@ void GraphiteRollupSortedBlockInputStream::startNextGroup(MutableColumns & merge } current_rule = next_rule; + was_group_started = true; } - -void GraphiteRollupSortedBlockInputStream::finishCurrentGroup(MutableColumns & merged_columns) +void GraphiteRollupSortedAlgorithm::GraphiteRollupMergedData::insertRow( + time_t time, RowRef & row, ColumnsDefinition & def) { /// Insert calculated values of the columns `time`, `value`, `version`. - merged_columns[time_column_num]->insert(current_time_rounded); - merged_columns[version_column_num]->insertFrom( - *(*current_subgroup_newest_row.columns)[version_column_num], current_subgroup_newest_row.row_num); + columns[def.time_column_num]->insert(time); + auto & row_ref_version_column = (*row.all_columns)[def.version_column_num]; + columns[def.version_column_num]->insertFrom(*row_ref_version_column, row.row_num); + auto & value_column = columns[def.value_column_num]; const Graphite::AggregationPattern * aggregation_pattern = std::get<1>(current_rule); if (aggregate_state_created) { - aggregation_pattern->function->insertResultInto(place_for_aggregate_state.data(), *merged_columns[value_column_num]); + aggregation_pattern->function->insertResultInto(place_for_aggregate_state.data(), *value_column); aggregation_pattern->function->destroy(place_for_aggregate_state.data()); aggregate_state_created = false; } else - merged_columns[value_column_num]->insertFrom( - *(*current_subgroup_newest_row.columns)[value_column_num], current_subgroup_newest_row.row_num); + value_column->insertFrom(*(*row.all_columns)[def.value_column_num], row.row_num); + + ++total_merged_rows; + ++merged_rows; + /// TODO: sum_blocks_granularity += block_size; + + was_group_started = false; } - -void GraphiteRollupSortedBlockInputStream::accumulateRow(SharedBlockRowRef & row) +void GraphiteRollupSortedAlgorithm::GraphiteRollupMergedData::accumulateRow(RowRef & row, ColumnsDefinition & def) { const Graphite::AggregationPattern * aggregation_pattern = std::get<1>(current_rule); if (aggregate_state_created) - aggregation_pattern->function->add(place_for_aggregate_state.data(), &(*row.columns)[value_column_num], row.row_num, nullptr); + { + auto & column = (*row.all_columns)[def.value_column_num]; + aggregation_pattern->function->add(place_for_aggregate_state.data(), &column, row.row_num, nullptr); + } +} + +GraphiteRollupSortedAlgorithm::GraphiteRollupMergedData::~GraphiteRollupMergedData() +{ + if (aggregate_state_created) + std::get<1>(current_rule)->function->destroy(place_for_aggregate_state.data()); } } diff --git a/src/Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.h b/src/Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.h new file mode 100644 index 00000000000..a0e8f1662aa --- /dev/null +++ b/src/Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.h @@ -0,0 +1,125 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ + +/** Merges several sorted inputs into one. + * + * For each group of consecutive identical values of the `path` column, + * and the same `time` values, rounded to some precision + * (where rounding accuracy depends on the template set for `path` + * and the amount of time elapsed from `time` to the specified time), + * keeps one line, + * performing the rounding of time, + * merge `value` values using the specified aggregate functions, + * as well as keeping the maximum value of the `version` column. + */ +class GraphiteRollupSortedAlgorithm final : public IMergingAlgorithmWithSharedChunks +{ +public: + GraphiteRollupSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, size_t max_block_size, + Graphite::Params params_, time_t time_of_merge_); + + Status merge() override; + + struct ColumnsDefinition + { + size_t path_column_num; + size_t time_column_num; + size_t value_column_num; + size_t version_column_num; + + /// All columns other than 'time', 'value', 'version'. They are unmodified during rollup. + ColumnNumbers unmodified_column_numbers; + }; + + using RowRef = detail::RowRefWithOwnedChunk; + + /// Specialization for SummingSortedTransform. + class GraphiteRollupMergedData : public MergedData + { + public: + using MergedData::MergedData; + ~GraphiteRollupMergedData(); + + void startNextGroup(const ColumnRawPtrs & raw_columns, size_t row, + Graphite::RollupRule next_rule, ColumnsDefinition & def); + void insertRow(time_t time, RowRef & row, ColumnsDefinition & def); + void accumulateRow(RowRef & row, ColumnsDefinition & def); + bool wasGroupStarted() const { return was_group_started; } + + const Graphite::RollupRule & currentRule() const { return current_rule; } + void allocMemForAggregates(size_t size, size_t alignment) { place_for_aggregate_state.reset(size, alignment); } + + private: + Graphite::RollupRule current_rule = {nullptr, nullptr}; + AlignedBuffer place_for_aggregate_state; + bool aggregate_state_created = false; /// Invariant: if true then current_rule is not NULL. + bool was_group_started = false; + }; + +private: + GraphiteRollupMergedData merged_data; + + const Graphite::Params params; + ColumnsDefinition columns_definition; + + time_t time_of_merge; + + /// No data has been read. + bool is_first = true; + + /* | path | time | rounded_time | version | value | unmodified | + * ----------------------------------------------------------------------------------- + * | A | 11 | 10 | 1 | 1 | a | | + * | A | 11 | 10 | 3 | 2 | b |> subgroup(A, 11) | + * | A | 11 | 10 | 2 | 3 | c | |> group(A, 10) + * ----------------------------------------------------------------------------------|> + * | A | 12 | 10 | 0 | 4 | d | |> Outputs (A, 10, avg(2, 5), a) + * | A | 12 | 10 | 1 | 5 | e |> subgroup(A, 12) | + * ----------------------------------------------------------------------------------- + * | A | 21 | 20 | 1 | 6 | f | + * | B | 11 | 10 | 1 | 7 | g | + * ... + */ + + /// Path name of current bucket + StringRef current_group_path; + + static constexpr size_t max_row_refs = 2; /// current_subgroup_newest_row, current_row. + /// Last row with maximum version for current primary key (time bucket). + RowRef current_subgroup_newest_row; + + /// Time of last read row + time_t current_time = 0; + time_t current_time_rounded = 0; + + const Graphite::Pattern undef_pattern = + { /// temporary empty pattern for selectPatternForPath + .regexp = nullptr, + .regexp_str = "", + .function = nullptr, + .retentions = DB::Graphite::Retentions(), + .type = undef_pattern.TypeUndef, + }; + + Graphite::RollupRule selectPatternForPath(StringRef path) const; + UInt32 selectPrecision(const Graphite::Retentions & retentions, time_t time) const; + + /// Insert the values into the resulting columns, which will not be changed in the future. + void startNextGroup(SortCursor & cursor, Graphite::RollupRule next_rule); + + /// Insert the calculated `time`, `value`, `version` values into the resulting columns by the last group of rows. + void finishCurrentGroup(); + + /// Update the state of the aggregate function with the new `value`. + void accumulateRow(RowRef & row); +}; + +} diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithm.h b/src/Processors/Merges/Algorithms/IMergingAlgorithm.h new file mode 100644 index 00000000000..263acee4c2d --- /dev/null +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithm.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class IMergingAlgorithm +{ +public: + struct Status + { + Chunk chunk; + bool is_finished = false; + ssize_t required_source = -1; + + explicit Status(Chunk chunk_) : chunk(std::move(chunk_)) {} + explicit Status(Chunk chunk_, bool is_finished_) : chunk(std::move(chunk_)), is_finished(is_finished_) {} + explicit Status(size_t source) : required_source(source) {} + }; + + virtual void initialize(Chunks chunks) = 0; + virtual void consume(Chunk chunk, size_t source_num) = 0; + virtual Status merge() = 0; + + IMergingAlgorithm() = default; + virtual ~IMergingAlgorithm() = default; +}; + +// TODO: use when compile with clang which could support it +// template +// concept MergingAlgorithm = std::is_base_of::value; + +} diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp new file mode 100644 index 00000000000..6777109982e --- /dev/null +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp @@ -0,0 +1,45 @@ +#include + + +namespace DB +{ + +IMergingAlgorithmWithDelayedChunk::IMergingAlgorithmWithDelayedChunk( + size_t num_inputs, + SortDescription description_) + : description(std::move(description_)) + , source_chunks(num_inputs) + , cursors(num_inputs) +{ +} + +void IMergingAlgorithmWithDelayedChunk::initializeQueue(Chunks chunks) +{ + source_chunks = std::move(chunks); + + for (size_t source_num = 0; source_num < source_chunks.size(); ++source_num) + { + if (!source_chunks[source_num]) + continue; + + cursors[source_num] = SortCursorImpl(source_chunks[source_num].getColumns(), description, source_num); + } + + queue = SortingHeap(cursors); +} + +void IMergingAlgorithmWithDelayedChunk::updateCursor(Chunk chunk, size_t source_num) +{ + auto & source_chunk = source_chunks[source_num]; + + /// Extend lifetime of last chunk. + last_chunk = std::move(source_chunk); + last_chunk_sort_columns = std::move(cursors[source_num].sort_columns); + + source_chunk = std::move(chunk); + cursors[source_num].reset(source_chunk.getColumns(), {}); + + queue.push(cursors[source_num]); +} + +} diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h new file mode 100644 index 00000000000..d02b9dfcb7a --- /dev/null +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class IMergingAlgorithmWithDelayedChunk : public IMergingAlgorithm +{ +public: + IMergingAlgorithmWithDelayedChunk( + size_t num_inputs, + SortDescription description_); + +protected: + SortingHeap queue; + SortDescription description; + + /// Previous row. May refer to last_chunk_sort_columns or row from source_chunks. + detail::RowRef last_key; + + ColumnRawPtrs last_chunk_sort_columns; /// Point to last_chunk if valid. + + void initializeQueue(Chunks chunks); + void updateCursor(Chunk chunk, size_t source_num); + +private: + /// Chunks currently being merged. + std::vector source_chunks; + SortCursorImpls cursors; + + /// In merging algorithm, we need to compare current sort key with the last one. + /// So, sorting columns for last row needed to be stored. + /// In order to do it, we extend lifetime of last chunk and it's sort columns (from corresponding sort cursor). + Chunk last_chunk; +}; + +} diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp new file mode 100644 index 00000000000..bcea74b5f38 --- /dev/null +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp @@ -0,0 +1,66 @@ +#include + +namespace DB +{ + +IMergingAlgorithmWithSharedChunks::IMergingAlgorithmWithSharedChunks( + size_t num_inputs, + SortDescription description_, + WriteBuffer * out_row_sources_buf_, + size_t max_row_refs) + : description(std::move(description_)) + , chunk_allocator(num_inputs + max_row_refs) + , cursors(num_inputs) + , source_chunks(num_inputs) + , out_row_sources_buf(out_row_sources_buf_) +{ +} + +static void prepareChunk(Chunk & chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->convertToFullColumnIfConst(); + + chunk.setColumns(std::move(columns), num_rows); +} + +void IMergingAlgorithmWithSharedChunks::initialize(Chunks chunks) +{ + source_chunks.resize(chunks.size()); + + for (size_t source_num = 0; source_num < source_chunks.size(); ++source_num) + { + if (!chunks[source_num]) + continue; + + prepareChunk(chunks[source_num]); + + auto & source_chunk = source_chunks[source_num]; + + source_chunk = chunk_allocator.alloc(std::move(chunks[source_num])); + cursors[source_num] = SortCursorImpl(source_chunk->getColumns(), description, source_num); + + source_chunk->all_columns = cursors[source_num].all_columns; + source_chunk->sort_columns = cursors[source_num].sort_columns; + } + + queue = SortingHeap(cursors); +} + +void IMergingAlgorithmWithSharedChunks::consume(Chunk chunk, size_t source_num) +{ + prepareChunk(chunk); + + auto & source_chunk = source_chunks[source_num]; + source_chunk = chunk_allocator.alloc(std::move(chunk)); + cursors[source_num].reset(source_chunk->getColumns(), {}); + + source_chunk->all_columns = cursors[source_num].all_columns; + source_chunk->sort_columns = cursors[source_num].sort_columns; + + queue.push(cursors[source_num]); +} + +} diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h new file mode 100644 index 00000000000..1ef7f540f96 --- /dev/null +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +class IMergingAlgorithmWithSharedChunks : public IMergingAlgorithm +{ +public: + IMergingAlgorithmWithSharedChunks( + size_t num_inputs, + SortDescription description_, + WriteBuffer * out_row_sources_buf_, + size_t max_row_refs); + + void initialize(Chunks chunks) override; + void consume(Chunk chunk, size_t source_num) override; + +private: + SortDescription description; + + /// Allocator must be destroyed after source_chunks. + detail::SharedChunkAllocator chunk_allocator; + + SortCursorImpls cursors; + +protected: + /// Chunks currently being merged. + using SourceChunks = std::vector; + SourceChunks source_chunks; + + SortingHeap queue; + + /// Used in Vertical merge algorithm to gather non-PK/non-index columns (on next step) + /// If it is not nullptr then it should be populated during execution + WriteBuffer * out_row_sources_buf = nullptr; + + using RowRef = detail::RowRefWithOwnedChunk; + void setRowRef(RowRef & row, SortCursor & cursor) { row.set(cursor, source_chunks[cursor.impl->order]); } +}; + +} diff --git a/src/Processors/Merges/Algorithms/MergedData.h b/src/Processors/Merges/Algorithms/MergedData.h new file mode 100644 index 00000000000..c96cc9ad6f4 --- /dev/null +++ b/src/Processors/Merges/Algorithms/MergedData.h @@ -0,0 +1,120 @@ +#pragma once + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +/// Class which represents current merging chunk of data. +/// Also it calculates the number of merged rows and other profile info. +class MergedData +{ +public: + explicit MergedData(MutableColumns columns_, bool use_average_block_size_, UInt64 max_block_size_) + : columns(std::move(columns_)), max_block_size(max_block_size_), use_average_block_size(use_average_block_size_) + { + } + + /// Pull will be called at next prepare call. + void flush() { need_flush = true; } + + void insertRow(const ColumnRawPtrs & raw_columns, size_t row, size_t block_size) + { + size_t num_columns = raw_columns.size(); + for (size_t i = 0; i < num_columns; ++i) + columns[i]->insertFrom(*raw_columns[i], row); + + ++total_merged_rows; + ++merged_rows; + sum_blocks_granularity += block_size; + } + + void insertFromChunk(Chunk && chunk, size_t limit_rows) + { + if (merged_rows) + throw Exception("Cannot insert to MergedData from Chunk because MergedData is not empty.", + ErrorCodes::LOGICAL_ERROR); + + auto num_rows = chunk.getNumRows(); + columns = chunk.mutateColumns(); + if (limit_rows && num_rows > limit_rows) + { + num_rows = limit_rows; + for (auto & column : columns) + column = (*column->cut(0, num_rows)).mutate(); + } + + need_flush = true; + total_merged_rows += num_rows; + merged_rows = num_rows; + + /// We don't cate about granularity here. Because, for fast-forward optimization, chunk will be moved as-is. + /// sum_blocks_granularity += block_size * num_rows; + } + + Chunk pull() + { + MutableColumns empty_columns; + empty_columns.reserve(columns.size()); + + for (const auto & column : columns) + empty_columns.emplace_back(column->cloneEmpty()); + + empty_columns.swap(columns); + Chunk chunk(std::move(empty_columns), merged_rows); + + merged_rows = 0; + sum_blocks_granularity = 0; + ++total_chunks; + total_allocated_bytes += chunk.allocatedBytes(); + need_flush = false; + + return chunk; + } + + bool hasEnoughRows() const + { + /// If full chunk was or is going to be inserted, then we must pull it. + /// It is needed for fast-forward optimization. + if (need_flush) + return true; + + /// Never return more then max_block_size. + if (merged_rows >= max_block_size) + return true; + + if (!use_average_block_size) + return false; + + /// Zero rows always not enough. + if (merged_rows == 0) + return false; + + size_t average = sum_blocks_granularity / merged_rows; + return merged_rows >= average; + } + + UInt64 mergedRows() const { return merged_rows; } + UInt64 totalMergedRows() const { return total_merged_rows; } + UInt64 totalChunks() const { return total_chunks; } + UInt64 totalAllocatedBytes() const { return total_allocated_bytes; } + +protected: + MutableColumns columns; + + UInt64 sum_blocks_granularity = 0; + UInt64 merged_rows = 0; + UInt64 total_merged_rows = 0; + UInt64 total_chunks = 0; + UInt64 total_allocated_bytes = 0; + + const UInt64 max_block_size; + const bool use_average_block_size; + + bool need_flush = false; +}; + +} diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp new file mode 100644 index 00000000000..d06e3c1179a --- /dev/null +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -0,0 +1,202 @@ +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +MergingSortedAlgorithm::MergingSortedAlgorithm( + const Block & header, + size_t num_inputs, + SortDescription description_, + size_t max_block_size, + UInt64 limit_, + WriteBuffer * out_row_sources_buf_, + bool use_average_block_sizes) + : merged_data(header.cloneEmptyColumns(), use_average_block_sizes, max_block_size) + , description(std::move(description_)) + , limit(limit_) + , out_row_sources_buf(out_row_sources_buf_) + , source_chunks(num_inputs) + , cursors(num_inputs) +{ + /// Replace column names in description to positions. + for (auto & column_description : description) + { + has_collation |= column_description.collator != nullptr; + if (!column_description.column_name.empty()) + { + column_description.column_number = header.getPositionByName(column_description.column_name); + column_description.column_name.clear(); + } + } +} + +void MergingSortedAlgorithm::addInput() +{ + source_chunks.emplace_back(); + cursors.emplace_back(); +} + +static void prepareChunk(Chunk & chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->convertToFullColumnIfConst(); + + chunk.setColumns(std::move(columns), num_rows); +} + +void MergingSortedAlgorithm::initialize(Chunks chunks) +{ + source_chunks = std::move(chunks); + + for (size_t source_num = 0; source_num < source_chunks.size(); ++source_num) + { + auto & chunk = source_chunks[source_num]; + + if (!chunk) + continue; + + prepareChunk(chunk); + cursors[source_num] = SortCursorImpl(chunk.getColumns(), description, source_num); + } + + if (has_collation) + queue_with_collation = SortingHeap(cursors); + else + queue_without_collation = SortingHeap(cursors); +} + +void MergingSortedAlgorithm::consume(Chunk chunk, size_t source_num) +{ + prepareChunk(chunk); + source_chunks[source_num] = std::move(chunk); + cursors[source_num].reset(source_chunks[source_num].getColumns(), {}); + + if (has_collation) + queue_with_collation.push(cursors[source_num]); + else + queue_without_collation.push(cursors[source_num]); +} + +IMergingAlgorithm::Status MergingSortedAlgorithm::merge() +{ + if (has_collation) + return mergeImpl(queue_with_collation); + else + return mergeImpl(queue_without_collation); +} + +template +IMergingAlgorithm::Status MergingSortedAlgorithm::mergeImpl(TSortingHeap & queue) +{ + /// Take rows in required order and put them into `merged_data`, while the rows are no more than `max_block_size` + while (queue.isValid()) + { + if (merged_data.hasEnoughRows()) + return Status(merged_data.pull()); + + auto current = queue.current(); + + /** And what if the block is totally less or equal than the rest for the current cursor? + * Or is there only one data source left in the queue? Then you can take the entire block on current cursor. + */ + if (current.impl->isFirst() + && (queue.size() == 1 + || (queue.size() >= 2 && current.totallyLessOrEquals(queue.nextChild())))) + { + //std::cerr << "current block is totally less or equals\n"; + + /// If there are already data in the current block, we first return it. + /// We'll get here again the next time we call the merge function. + if (merged_data.mergedRows() != 0) + { + //std::cerr << "merged rows is non-zero\n"; + // merged_data.flush(); + return Status(merged_data.pull()); + } + + /// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl) + size_t source_num = current.impl->order; + queue.removeTop(); + return insertFromChunk(source_num); + } + + //std::cerr << "total_merged_rows: " << total_merged_rows << ", merged_rows: " << merged_rows << "\n"; + //std::cerr << "Inserting row\n"; + merged_data.insertRow(current->all_columns, current->pos, current->rows); + + if (out_row_sources_buf) + { + /// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl) + RowSourcePart row_source(current.impl->order); + out_row_sources_buf->write(row_source.data); + } + + if (limit && merged_data.totalMergedRows() >= limit) + return Status(merged_data.pull(), true); + + if (!current->isLast()) + { + //std::cerr << "moving to next row\n"; + queue.next(); + } + else + { + /// We will get the next block from the corresponding source, if there is one. + queue.removeTop(); + //std::cerr << "It was last row, fetching next block\n"; + return Status(current.impl->order); + } + } + + return Status(merged_data.pull(), true); +} + +IMergingAlgorithm::Status MergingSortedAlgorithm::insertFromChunk(size_t source_num) +{ + if (source_num >= cursors.size()) + throw Exception("Logical error in MergingSortedTransform", ErrorCodes::LOGICAL_ERROR); + + //std::cerr << "copied columns\n"; + + auto num_rows = source_chunks[source_num].getNumRows(); + + UInt64 total_merged_rows_after_insertion = merged_data.mergedRows() + num_rows; + bool is_finished = limit && total_merged_rows_after_insertion >= limit; + + if (limit && total_merged_rows_after_insertion > limit) + { + num_rows = total_merged_rows_after_insertion - limit; + merged_data.insertFromChunk(std::move(source_chunks[source_num]), num_rows); + } + else + merged_data.insertFromChunk(std::move(source_chunks[source_num]), 0); + + source_chunks[source_num] = Chunk(); + + /// Write order of rows for other columns + /// this data will be used in gather stream + if (out_row_sources_buf) + { + RowSourcePart row_source(source_num); + for (size_t i = 0; i < num_rows; ++i) + out_row_sources_buf->write(row_source.data); + } + + auto status = Status(merged_data.pull(), is_finished); + + if (!is_finished) + status.required_source = source_num; + + return status; +} + +} diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h new file mode 100644 index 00000000000..6ff48b520bd --- /dev/null +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h @@ -0,0 +1,57 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ + +/// Merges several sorted inputs into one sorted output. +class MergingSortedAlgorithm final : public IMergingAlgorithm +{ +public: + MergingSortedAlgorithm( + const Block & header, + size_t num_inputs, + SortDescription description_, + size_t max_block_size, + UInt64 limit_, + WriteBuffer * out_row_sources_buf_, + bool use_average_block_sizes); + + void addInput(); + + void initialize(Chunks chunks) override; + void consume(Chunk chunk, size_t source_num) override; + Status merge() override; + + const MergedData & getMergedData() const { return merged_data; } + +private: + MergedData merged_data; + + /// Settings + SortDescription description; + UInt64 limit; + bool has_collation = false; + + /// Used in Vertical merge algorithm to gather non-PK/non-index columns (on next step) + /// If it is not nullptr then it should be populated during execution + WriteBuffer * out_row_sources_buf = nullptr; + + /// Chunks currently being merged. + std::vector source_chunks; + + SortCursorImpls cursors; + + SortingHeap queue_without_collation; + SortingHeap queue_with_collation; + + Status insertFromChunk(size_t source_num); + + template + Status mergeImpl(TSortingHeap & queue); +}; + +} diff --git a/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.cpp new file mode 100644 index 00000000000..4a0f7493637 --- /dev/null +++ b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.cpp @@ -0,0 +1,96 @@ +#include +#include + +namespace DB +{ + +ReplacingSortedAlgorithm::ReplacingSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, const String & version_column, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_, + bool use_average_block_sizes) + : IMergingAlgorithmWithSharedChunks(num_inputs, std::move(description_), out_row_sources_buf_, max_row_refs) + , merged_data(header.cloneEmptyColumns(), use_average_block_sizes, max_block_size) +{ + if (!version_column.empty()) + version_column_number = header.getPositionByName(version_column); +} + +void ReplacingSortedAlgorithm::insertRow() +{ + if (out_row_sources_buf) + { + /// true flag value means "skip row" + current_row_sources[max_pos].setSkipFlag(false); + + out_row_sources_buf->write(reinterpret_cast(current_row_sources.data()), + current_row_sources.size() * sizeof(RowSourcePart)); + current_row_sources.resize(0); + } + + merged_data.insertRow(*selected_row.all_columns, selected_row.row_num, selected_row.owned_chunk->getNumRows()); + selected_row.clear(); +} + +IMergingAlgorithm::Status ReplacingSortedAlgorithm::merge() +{ + /// Take the rows in needed order and put them into `merged_columns` until rows no more than `max_block_size` + while (queue.isValid()) + { + SortCursor current = queue.current(); + + RowRef current_row; + setRowRef(current_row, current); + + bool key_differs = selected_row.empty() || !current_row.hasEqualSortColumnsWith(selected_row); + if (key_differs) + { + /// if there are enough rows and the last one is calculated completely + if (merged_data.hasEnoughRows()) + return Status(merged_data.pull()); + + /// Write the data for the previous primary key. + if (!selected_row.empty()) + insertRow(); + + selected_row.clear(); + } + + /// Initially, skip all rows. Unskip last on insert. + size_t current_pos = current_row_sources.size(); + if (out_row_sources_buf) + current_row_sources.emplace_back(current.impl->order, true); + + /// A non-strict comparison, since we select the last row for the same version values. + if (version_column_number == -1 + || selected_row.empty() + || current->all_columns[version_column_number]->compareAt( + current->pos, selected_row.row_num, + *(*selected_row.all_columns)[version_column_number], + /* nan_direction_hint = */ 1) >= 0) + { + max_pos = current_pos; + setRowRef(selected_row, current); + } + + if (!current->isLast()) + { + queue.next(); + } + else + { + /// We get the next block from the corresponding source, if there is one. + queue.removeTop(); + return Status(current.impl->order); + } + } + + /// We will write the data for the last primary key. + if (!selected_row.empty()) + insertRow(); + + return Status(merged_data.pull(), true); +} + +} diff --git a/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.h new file mode 100644 index 00000000000..7a193470f89 --- /dev/null +++ b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.h @@ -0,0 +1,46 @@ +#pragma once +#include +#include +#include + +namespace Poco +{ +class Logger; +} + +namespace DB +{ + +/** Merges several sorted inputs into one. + * For each group of consecutive identical values of the primary key (the columns by which the data is sorted), + * keeps row with max `version` value. + */ +class ReplacingSortedAlgorithm final : public IMergingAlgorithmWithSharedChunks +{ +public: + ReplacingSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, const String & version_column, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_ = nullptr, + bool use_average_block_sizes = false); + + Status merge() override; + +private: + MergedData merged_data; + + ssize_t version_column_number = -1; + + using RowRef = detail::RowRefWithOwnedChunk; + static constexpr size_t max_row_refs = 2; /// last, current. + RowRef selected_row; /// Last row with maximum version for current primary key. + size_t max_pos = 0; /// The position (into current_row_sources) of the row with the highest version. + + /// Sources of rows with the current primary key. + PODArray current_row_sources; + + void insertRow(); +}; + +} diff --git a/src/Processors/Merges/Algorithms/RowRef.h b/src/Processors/Merges/Algorithms/RowRef.h new file mode 100644 index 00000000000..5aeae952067 --- /dev/null +++ b/src/Processors/Merges/Algorithms/RowRef.h @@ -0,0 +1,216 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace DB::detail +{ + +class SharedChunkAllocator; + +/// Allows you refer to the row in the block and hold the block ownership, +/// and thus avoid creating a temporary row object. +/// Do not use std::shared_ptr, since there is no need for a place for `weak_count` and `deleter`; +/// does not use Poco::SharedPtr, since you need to allocate a block and `refcount` in one piece; +/// does not use Poco::AutoPtr, since it does not have a `move` constructor and there are extra checks for nullptr; +/// The reference counter is not atomic, since it is used from one thread. +struct SharedChunk : Chunk +{ + ColumnRawPtrs all_columns; + ColumnRawPtrs sort_columns; + + using Chunk::Chunk; + using Chunk::operator=; + +private: + int refcount = 0; + size_t position = 0; + SharedChunkAllocator * allocator = nullptr; + + friend class SharedChunkAllocator; + friend void intrusive_ptr_add_ref(SharedChunk * ptr); + friend void intrusive_ptr_release(SharedChunk * ptr); +}; + +using SharedChunkPtr = boost::intrusive_ptr; + +/// Custom allocator for shared chunk. +/// It helps to avoid explicit new/delete calls if we know maximum required capacity. +/// Thanks to that, SharedChunk does not own any memory. +/// It improves leaks detection, because memory is allocated only once in constructor. +class SharedChunkAllocator +{ +public: + explicit SharedChunkAllocator(size_t max_chunks) + { + if (max_chunks == 0) + max_chunks = 1; + + chunks.resize(max_chunks); + free_chunks.reserve(max_chunks); + + for (size_t i = 0; i < max_chunks; ++i) + free_chunks.push_back(i); + } + + SharedChunkPtr alloc(Chunk && chunk) + { + if (free_chunks.empty()) + throw Exception("Not enough space in SharedChunkAllocator. " + "Chunks allocated: " + std::to_string(chunks.size()), ErrorCodes::LOGICAL_ERROR); + + auto pos = free_chunks.back(); + free_chunks.pop_back(); + + chunks[pos] = std::move(chunk); + chunks[pos].position = pos; + chunks[pos].allocator = this; + + return SharedChunkPtr(&chunks[pos]); + } + + ~SharedChunkAllocator() + { + if (free_chunks.size() != chunks.size()) + { + LOG_ERROR(&Logger::get("SharedChunkAllocator"), + "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " + << StackTrace().toString()); + + return; + } + } + +private: + std::vector chunks; + std::vector free_chunks; + + void release(SharedChunk * ptr) noexcept + { + if (chunks.empty()) + { + /// This may happen if allocator was removed before chunks. + /// Log message and exit, because we don't want to throw exception in destructor. + + LOG_ERROR(&Logger::get("SharedChunkAllocator"), + "SharedChunkAllocator was destroyed before RowRef was released. StackTrace: " + << StackTrace().toString()); + + return; + } + + /// Release memory. It is not obligatory. + ptr->clear(); + ptr->all_columns.clear(); + ptr->sort_columns.clear(); + + free_chunks.push_back(ptr->position); + } + + friend void intrusive_ptr_release(SharedChunk * ptr); +}; + +inline void intrusive_ptr_add_ref(SharedChunk * ptr) +{ + ++ptr->refcount; +} + +inline void intrusive_ptr_release(SharedChunk * ptr) +{ + if (0 == --ptr->refcount) + ptr->allocator->release(ptr); +} + +/// This class represents a row in a chunk. +struct RowRef +{ + const IColumn ** sort_columns = nullptr; /// Point to sort_columns from SortCursor or last_chunk_sort_columns. + size_t num_columns = 0; + UInt64 row_num = 0; + + bool empty() const { return sort_columns == nullptr; } + void reset() { sort_columns = nullptr; } + + void set(SortCursor & cursor) + { + sort_columns = cursor.impl->sort_columns.data(); + num_columns = cursor.impl->sort_columns.size(); + row_num = cursor.impl->pos; + } + + static bool checkEquals(size_t size, const IColumn ** lhs, size_t lhs_row, const IColumn ** rhs, size_t rhs_row) + { + for (size_t col_number = 0; col_number < size; ++col_number) + { + auto & cur_column = lhs[col_number]; + auto & other_column = rhs[col_number]; + + if (0 != cur_column->compareAt(lhs_row, rhs_row, *other_column, 1)) + return false; + } + + return true; + } + + bool hasEqualSortColumnsWith(const RowRef & other) + { + return checkEquals(num_columns, sort_columns, row_num, other.sort_columns, other.row_num); + } +}; + +/// This class also represents a row in a chunk. +/// RowRefWithOwnedChunk hold shared pointer to this chunk, possibly extending its life time. +/// It is needed, for example, in CollapsingTransform, where we need to store first negative row for current sort key. +/// We do not copy data itself, because it may be potentially changed for each row. Performance for `set` is important. +struct RowRefWithOwnedChunk +{ + detail::SharedChunkPtr owned_chunk = nullptr; + + ColumnRawPtrs * all_columns = nullptr; + ColumnRawPtrs * sort_columns = nullptr; + UInt64 row_num = 0; + + void swap(RowRefWithOwnedChunk & other) + { + owned_chunk.swap(other.owned_chunk); + std::swap(all_columns, other.all_columns); + std::swap(sort_columns, other.sort_columns); + std::swap(row_num, other.row_num); + } + + bool empty() const { return owned_chunk == nullptr; } + + void clear() + { + owned_chunk.reset(); + all_columns = nullptr; + sort_columns = nullptr; + row_num = 0; + } + + void set(SortCursor & cursor, SharedChunkPtr chunk) + { + owned_chunk = std::move(chunk); + row_num = cursor.impl->pos; + all_columns = &owned_chunk->all_columns; + sort_columns = &owned_chunk->sort_columns; + } + + bool hasEqualSortColumnsWith(const RowRefWithOwnedChunk & other) + { + return RowRef::checkEquals(sort_columns->size(), sort_columns->data(), row_num, + other.sort_columns->data(), other.row_num); + } +}; + +} diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp new file mode 100644 index 00000000000..5bcbf778d64 --- /dev/null +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -0,0 +1,698 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int CORRUPTED_DATA; +} + +SummingSortedAlgorithm::ColumnsDefinition::ColumnsDefinition() = default; +SummingSortedAlgorithm::ColumnsDefinition::ColumnsDefinition(ColumnsDefinition &&) noexcept = default; +SummingSortedAlgorithm::ColumnsDefinition::~ColumnsDefinition() = default; + +/// Stores numbers of key-columns and value-columns. +struct SummingSortedAlgorithm::MapDescription +{ + std::vector key_col_nums; + std::vector val_col_nums; +}; + +/// Stores aggregation function, state, and columns to be used as function arguments. +struct SummingSortedAlgorithm::AggregateDescription +{ + /// An aggregate function 'sumWithOverflow' or 'sumMapWithOverflow' for summing. + AggregateFunctionPtr function; + IAggregateFunction::AddFunc add_function = nullptr; + std::vector column_numbers; + IColumn * merged_column = nullptr; + AlignedBuffer state; + bool created = false; + + /// In case when column has type AggregateFunction: + /// use the aggregate function from itself instead of 'function' above. + bool is_agg_func_type = false; + + void init(const char * function_name, const DataTypes & argument_types) + { + function = AggregateFunctionFactory::instance().get(function_name, argument_types); + add_function = function->getAddressOfAddFunction(); + state.reset(function->sizeOfData(), function->alignOfData()); + } + + void createState() + { + if (created) + return; + if (is_agg_func_type) + merged_column->insertDefault(); + else + function->create(state.data()); + created = true; + } + + void destroyState() + { + if (!created) + return; + if (!is_agg_func_type) + function->destroy(state.data()); + created = false; + } + + /// Explicitly destroy aggregation state if the stream is terminated + ~AggregateDescription() + { + destroyState(); + } + + AggregateDescription() = default; + AggregateDescription(AggregateDescription &&) = default; + AggregateDescription(const AggregateDescription &) = delete; +}; + + +static bool isInPrimaryKey(const SortDescription & description, const std::string & name, const size_t number) +{ + for (const auto & desc : description) + if (desc.column_name == name || (desc.column_name.empty() && desc.column_number == number)) + return true; + + return false; +} + +/// Returns true if merge result is not empty +static bool mergeMap(const SummingSortedAlgorithm::MapDescription & desc, + Row & row, const ColumnRawPtrs & raw_columns, size_t row_number) +{ + /// Strongly non-optimal. + + Row & left = row; + Row right(left.size()); + + for (size_t col_num : desc.key_col_nums) + right[col_num] = (*raw_columns[col_num])[row_number].template get(); + + for (size_t col_num : desc.val_col_nums) + right[col_num] = (*raw_columns[col_num])[row_number].template get(); + + auto at_ith_column_jth_row = [&](const Row & matrix, size_t i, size_t j) -> const Field & + { + return matrix[i].get()[j]; + }; + + auto tuple_of_nth_columns_at_jth_row = [&](const Row & matrix, const ColumnNumbers & col_nums, size_t j) -> Array + { + size_t size = col_nums.size(); + Array res(size); + for (size_t col_num_index = 0; col_num_index < size; ++col_num_index) + res[col_num_index] = at_ith_column_jth_row(matrix, col_nums[col_num_index], j); + return res; + }; + + std::map merged; + + auto accumulate = [](Array & dst, const Array & src) + { + bool has_non_zero = false; + size_t size = dst.size(); + for (size_t i = 0; i < size; ++i) + if (applyVisitor(FieldVisitorSum(src[i]), dst[i])) + has_non_zero = true; + return has_non_zero; + }; + + auto merge = [&](const Row & matrix) + { + size_t rows = matrix[desc.key_col_nums[0]].get().size(); + + for (size_t j = 0; j < rows; ++j) + { + Array key = tuple_of_nth_columns_at_jth_row(matrix, desc.key_col_nums, j); + Array value = tuple_of_nth_columns_at_jth_row(matrix, desc.val_col_nums, j); + + auto it = merged.find(key); + if (merged.end() == it) + merged.emplace(std::move(key), std::move(value)); + else + { + if (!accumulate(it->second, value)) + merged.erase(it); + } + } + }; + + merge(left); + merge(right); + + for (size_t col_num : desc.key_col_nums) + row[col_num] = Array(merged.size()); + for (size_t col_num : desc.val_col_nums) + row[col_num] = Array(merged.size()); + + size_t row_num = 0; + for (const auto & key_value : merged) + { + for (size_t col_num_index = 0, size = desc.key_col_nums.size(); col_num_index < size; ++col_num_index) + row[desc.key_col_nums[col_num_index]].get()[row_num] = key_value.first[col_num_index]; + + for (size_t col_num_index = 0, size = desc.val_col_nums.size(); col_num_index < size; ++col_num_index) + row[desc.val_col_nums[col_num_index]].get()[row_num] = key_value.second[col_num_index]; + + ++row_num; + } + + return row_num != 0; +} + +static SummingSortedAlgorithm::ColumnsDefinition defineColumns( + const Block & header, + const SortDescription & description, + const Names & column_names_to_sum) +{ + size_t num_columns = header.columns(); + SummingSortedAlgorithm::ColumnsDefinition def; + def.column_names = header.getNames(); + + /// name of nested structure -> the column numbers that refer to it. + std::unordered_map> discovered_maps; + + /** Fill in the column numbers, which must be summed. + * This can only be numeric columns that are not part of the sort key. + * If a non-empty column_names_to_sum is specified, then we only take these columns. + * Some columns from column_names_to_sum may not be found. This is ignored. + */ + for (size_t i = 0; i < num_columns; ++i) + { + const ColumnWithTypeAndName & column = header.safeGetByPosition(i); + + /// Discover nested Maps and find columns for summation + if (typeid_cast(column.type.get())) + { + const auto map_name = Nested::extractTableName(column.name); + /// if nested table name ends with `Map` it is a possible candidate for special handling + if (map_name == column.name || !endsWith(map_name, "Map")) + { + def.column_numbers_not_to_aggregate.push_back(i); + continue; + } + + discovered_maps[map_name].emplace_back(i); + } + else + { + bool is_agg_func = WhichDataType(column.type).isAggregateFunction(); + + /// There are special const columns for example after prewhere sections. + if ((!column.type->isSummable() && !is_agg_func) || isColumnConst(*column.column)) + { + def.column_numbers_not_to_aggregate.push_back(i); + continue; + } + + /// Are they inside the PK? + if (isInPrimaryKey(description, column.name, i)) + { + def.column_numbers_not_to_aggregate.push_back(i); + continue; + } + + if (column_names_to_sum.empty() + || column_names_to_sum.end() != + std::find(column_names_to_sum.begin(), column_names_to_sum.end(), column.name)) + { + // Create aggregator to sum this column + SummingSortedAlgorithm::AggregateDescription desc; + desc.is_agg_func_type = is_agg_func; + desc.column_numbers = {i}; + + if (!is_agg_func) + { + desc.init("sumWithOverflow", {column.type}); + } + + def.columns_to_aggregate.emplace_back(std::move(desc)); + } + else + { + // Column is not going to be summed, use last value + def.column_numbers_not_to_aggregate.push_back(i); + } + } + } + + /// select actual nested Maps from list of candidates + for (const auto & map : discovered_maps) + { + /// map should contain at least two elements (key -> value) + if (map.second.size() < 2) + { + for (auto col : map.second) + def.column_numbers_not_to_aggregate.push_back(col); + continue; + } + + /// no elements of map could be in primary key + auto column_num_it = map.second.begin(); + for (; column_num_it != map.second.end(); ++column_num_it) + if (isInPrimaryKey(description, header.safeGetByPosition(*column_num_it).name, *column_num_it)) + break; + if (column_num_it != map.second.end()) + { + for (auto col : map.second) + def.column_numbers_not_to_aggregate.push_back(col); + continue; + } + + DataTypes argument_types; + SummingSortedAlgorithm::AggregateDescription desc; + SummingSortedAlgorithm::MapDescription map_desc; + + column_num_it = map.second.begin(); + for (; column_num_it != map.second.end(); ++column_num_it) + { + const ColumnWithTypeAndName & key_col = header.safeGetByPosition(*column_num_it); + const String & name = key_col.name; + const IDataType & nested_type = *assert_cast(*key_col.type).getNestedType(); + + if (column_num_it == map.second.begin() + || endsWith(name, "ID") + || endsWith(name, "Key") + || endsWith(name, "Type")) + { + if (!nested_type.isValueRepresentedByInteger() && !isStringOrFixedString(nested_type)) + break; + + map_desc.key_col_nums.push_back(*column_num_it); + } + else + { + if (!nested_type.isSummable()) + break; + + map_desc.val_col_nums.push_back(*column_num_it); + } + + // Add column to function arguments + desc.column_numbers.push_back(*column_num_it); + argument_types.push_back(key_col.type); + } + + if (column_num_it != map.second.end()) + { + for (auto col : map.second) + def.column_numbers_not_to_aggregate.push_back(col); + continue; + } + + if (map_desc.key_col_nums.size() == 1) + { + // Create summation for all value columns in the map + desc.init("sumMapWithOverflow", argument_types); + def.columns_to_aggregate.emplace_back(std::move(desc)); + } + else + { + // Fall back to legacy mergeMaps for composite keys + for (auto col : map.second) + def.column_numbers_not_to_aggregate.push_back(col); + def.maps_to_sum.emplace_back(std::move(map_desc)); + } + } + + return def; +} + +static MutableColumns getMergedDataColumns( + const Block & header, + const SummingSortedAlgorithm::ColumnsDefinition & def) +{ + MutableColumns columns; + size_t num_columns = def.column_numbers_not_to_aggregate.size() + def.columns_to_aggregate.size(); + columns.reserve(num_columns); + + for (const auto & desc : def.columns_to_aggregate) + { + // Wrap aggregated columns in a tuple to match function signature + if (!desc.is_agg_func_type && isTuple(desc.function->getReturnType())) + { + size_t tuple_size = desc.column_numbers.size(); + MutableColumns tuple_columns(tuple_size); + for (size_t i = 0; i < tuple_size; ++i) + tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column->cloneEmpty(); + + columns.emplace_back(ColumnTuple::create(std::move(tuple_columns))); + } + else + columns.emplace_back(header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty()); + } + + for (const auto & column_number : def.column_numbers_not_to_aggregate) + columns.emplace_back(header.safeGetByPosition(column_number).type->createColumn()); + + return columns; +} + +static void preprocessChunk(Chunk & chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + for (auto & column : columns) + column = column->convertToFullColumnIfConst(); + + chunk.setColumns(std::move(columns), num_rows); +} + +static void postprocessChunk( + Chunk & chunk, size_t num_result_columns, + const SummingSortedAlgorithm::ColumnsDefinition & def) +{ + size_t num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + Columns res_columns(num_result_columns); + size_t next_column = 0; + + for (const auto & desc : def.columns_to_aggregate) + { + auto column = std::move(columns[next_column]); + ++next_column; + + if (!desc.is_agg_func_type && isTuple(desc.function->getReturnType())) + { + /// Unpack tuple into block. + size_t tuple_size = desc.column_numbers.size(); + for (size_t i = 0; i < tuple_size; ++i) + res_columns[desc.column_numbers[i]] = assert_cast(*column).getColumnPtr(i); + } + else + res_columns[desc.column_numbers[0]] = std::move(column); + } + + for (auto column_number : def.column_numbers_not_to_aggregate) + { + auto column = std::move(columns[next_column]); + ++next_column; + + res_columns[column_number] = std::move(column); + } + + chunk.setColumns(std::move(res_columns), num_rows); +} + +static void setRow(Row & row, const ColumnRawPtrs & raw_columns, size_t row_num, const Names & column_names) +{ + size_t num_columns = row.size(); + for (size_t i = 0; i < num_columns; ++i) + { + try + { + raw_columns[i]->get(row_num, row[i]); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + + /// Find out the name of the column and throw more informative exception. + + String column_name; + if (i < column_names.size()) + column_name = column_names[i]; + + throw Exception("MergingSortedBlockInputStream failed to read row " + toString(row_num) + + " of column " + toString(i) + (column_name.empty() ? "" : " (" + column_name + ")"), + ErrorCodes::CORRUPTED_DATA); + } + } +} + + +SummingSortedAlgorithm::SummingMergedData::SummingMergedData( + MutableColumns columns_, UInt64 max_block_size_, ColumnsDefinition & def_) + : MergedData(std::move(columns_), false, max_block_size_) + , def(def_) +{ + current_row.resize(def.column_names.size()); + initAggregateDescription(); +} + +void SummingSortedAlgorithm::SummingMergedData::startGroup(ColumnRawPtrs & raw_columns, size_t row) +{ + is_group_started = true; + + setRow(current_row, raw_columns, row, def.column_names); + + /// Reset aggregation states for next row + for (auto & desc : def.columns_to_aggregate) + desc.createState(); + + if (def.maps_to_sum.empty()) + { + /// We have only columns_to_aggregate. The status of current row will be determined + /// in 'insertCurrentRowIfNeeded' method on the values of aggregate functions. + current_row_is_zero = true; // NOLINT + } + else + { + /// We have complex maps that will be summed with 'mergeMap' method. + /// The single row is considered non zero, and the status after merging with other rows + /// will be determined in the branch below (when key_differs == false). + current_row_is_zero = false; // NOLINT + } + + addRowImpl(raw_columns, row); +} + +void SummingSortedAlgorithm::SummingMergedData::finishGroup() +{ + is_group_started = false; + + /// We have nothing to aggregate. It means that it could be non-zero, because we have columns_not_to_aggregate. + if (def.columns_to_aggregate.empty()) + current_row_is_zero = false; + + for (auto & desc : def.columns_to_aggregate) + { + // Do not insert if the aggregation state hasn't been created + if (desc.created) + { + if (desc.is_agg_func_type) + { + current_row_is_zero = false; + } + else + { + try + { + desc.function->insertResultInto(desc.state.data(), *desc.merged_column); + + /// Update zero status of current row + if (desc.column_numbers.size() == 1) + { + // Flag row as non-empty if at least one column number if non-zero + current_row_is_zero = current_row_is_zero + && desc.merged_column->isDefaultAt(desc.merged_column->size() - 1); + } + else + { + /// It is sumMapWithOverflow aggregate function. + /// Assume that the row isn't empty in this case + /// (just because it is compatible with previous version) + current_row_is_zero = false; + } + } + catch (...) + { + desc.destroyState(); + throw; + } + } + desc.destroyState(); + } + else + desc.merged_column->insertDefault(); + } + + /// If it is "zero" row, then rollback the insertion + /// (at this moment we need rollback only cols from columns_to_aggregate) + if (current_row_is_zero) + { + for (auto & desc : def.columns_to_aggregate) + desc.merged_column->popBack(1); + + return; + } + + size_t next_column = columns.size() - def.column_numbers_not_to_aggregate.size(); + for (auto column_number : def.column_numbers_not_to_aggregate) + { + columns[next_column]->insert(current_row[column_number]); + ++next_column; + } + + ++total_merged_rows; + ++merged_rows; + /// TODO: sum_blocks_granularity += block_size; +} + +void SummingSortedAlgorithm::SummingMergedData::addRow(ColumnRawPtrs & raw_columns, size_t row) +{ + // Merge maps only for same rows + for (const auto & desc : def.maps_to_sum) + if (mergeMap(desc, current_row, raw_columns, row)) + current_row_is_zero = false; + + addRowImpl(raw_columns, row); +} + +void SummingSortedAlgorithm::SummingMergedData::addRowImpl(ColumnRawPtrs & raw_columns, size_t row) +{ + for (auto & desc : def.columns_to_aggregate) + { + if (!desc.created) + throw Exception("Logical error in SummingSortedBlockInputStream, there are no description", + ErrorCodes::LOGICAL_ERROR); + + if (desc.is_agg_func_type) + { + // desc.state is not used for AggregateFunction types + auto & col = raw_columns[desc.column_numbers[0]]; + assert_cast(*desc.merged_column).insertMergeFrom(*col, row); + } + else + { + // Specialized case for unary functions + if (desc.column_numbers.size() == 1) + { + auto & col = raw_columns[desc.column_numbers[0]]; + desc.add_function(desc.function.get(), desc.state.data(), &col, row, nullptr); + } + else + { + // Gather all source columns into a vector + ColumnRawPtrs column_ptrs(desc.column_numbers.size()); + for (size_t i = 0; i < desc.column_numbers.size(); ++i) + column_ptrs[i] = raw_columns[desc.column_numbers[i]]; + + desc.add_function(desc.function.get(), desc.state.data(), column_ptrs.data(), row, nullptr); + } + } + } +} + +void SummingSortedAlgorithm::SummingMergedData::initAggregateDescription() +{ + size_t num_columns = def.columns_to_aggregate.size(); + for (size_t column_number = 0; column_number < num_columns; ++column_number) + def.columns_to_aggregate[column_number].merged_column = columns[column_number].get(); +} + + +Chunk SummingSortedAlgorithm::SummingMergedData::pull() +{ + auto chunk = MergedData::pull(); + postprocessChunk(chunk, def.column_names.size(), def); + + initAggregateDescription(); + + return chunk; +} + + +SummingSortedAlgorithm::SummingSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, + const Names & column_names_to_sum, + size_t max_block_size) + : IMergingAlgorithmWithDelayedChunk(num_inputs, std::move(description_)) + , columns_definition(defineColumns(header, description, column_names_to_sum)) + , merged_data(getMergedDataColumns(header, columns_definition), max_block_size, columns_definition) +{ +} + +void SummingSortedAlgorithm::initialize(Chunks chunks) +{ + for (auto & chunk : chunks) + if (chunk) + preprocessChunk(chunk); + + initializeQueue(std::move(chunks)); +} + +void SummingSortedAlgorithm::consume(Chunk chunk, size_t source_num) +{ + preprocessChunk(chunk); + updateCursor(std::move(chunk), source_num); +} + +IMergingAlgorithm::Status SummingSortedAlgorithm::merge() +{ + /// Take the rows in needed order and put them in `merged_columns` until rows no more than `max_block_size` + while (queue.isValid()) + { + bool key_differs; + + SortCursor current = queue.current(); + + { + detail::RowRef current_key; + current_key.set(current); + + key_differs = last_key.empty() || !last_key.hasEqualSortColumnsWith(current_key); + + last_key = current_key; + last_chunk_sort_columns.clear(); + } + + if (key_differs) + { + if (merged_data.isGroupStarted()) + /// Write the data for the previous group. + merged_data.finishGroup(); + + if (merged_data.hasEnoughRows()) + { + /// The block is now full and the last row is calculated completely. + last_key.reset(); + return Status(merged_data.pull()); + } + + merged_data.startGroup(current->all_columns, current->pos); + } + else + merged_data.addRow(current->all_columns, current->pos); + + if (!current->isLast()) + { + queue.next(); + } + else + { + /// We get the next block from the corresponding source, if there is one. + queue.removeTop(); + return Status(current.impl->order); + } + } + + /// We will write the data for the last group, if it is non-zero. + if (merged_data.isGroupStarted()) + merged_data.finishGroup(); + + last_chunk_sort_columns.clear(); + return Status(merged_data.pull(), true); +} + +} diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h new file mode 100644 index 00000000000..a38df215ccc --- /dev/null +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/** Merges several sorted inputs into one. + * For each group of consecutive identical values of the primary key (the columns by which the data is sorted), + * collapses them into one row, summing all the numeric columns except the primary key. + * If in all numeric columns, except for the primary key, the result is zero, it deletes the row. + */ +class SummingSortedAlgorithm final : public IMergingAlgorithmWithDelayedChunk +{ +public: + SummingSortedAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, + /// List of columns to be summed. If empty, all numeric columns that are not in the description are taken. + const Names & column_names_to_sum, + size_t max_block_size); + + void initialize(Chunks chunks) override; + void consume(Chunk chunk, size_t source_num) override; + Status merge() override; + + struct AggregateDescription; + struct MapDescription; + + /// This structure define columns into one of three types: + /// * columns which values not needed to be aggregated + /// * aggregate functions and columns which needed to be summed + /// * mapping for nested columns + struct ColumnsDefinition + { + ColumnsDefinition(); /// Is needed because destructor is defined. + ColumnsDefinition(ColumnsDefinition &&) noexcept; /// Is needed because destructor is defined. + ~ColumnsDefinition(); /// Is needed because otherwise std::vector's destructor uses incomplete types. + + /// Columns with which values should not be aggregated. + ColumnNumbers column_numbers_not_to_aggregate; + /// Columns which should be aggregated. + std::vector columns_to_aggregate; + /// Mapping for nested columns. + std::vector maps_to_sum; + + /// Names of columns from header. + Names column_names; + }; + + /// Specialization for SummingSortedTransform. Inserts only data for non-aggregated columns. + class SummingMergedData : public MergedData + { + private: + using MergedData::pull; + using MergedData::insertRow; + + public: + SummingMergedData(MutableColumns columns_, UInt64 max_block_size_, ColumnsDefinition & def_); + + void startGroup(ColumnRawPtrs & raw_columns, size_t row); + void finishGroup(); + + bool isGroupStarted() const { return is_group_started; } + void addRow(ColumnRawPtrs & raw_columns, size_t row); /// Possible only when group was started. + + Chunk pull(); + + private: + ColumnsDefinition & def; + + bool is_group_started = false; + + Row current_row; + bool current_row_is_zero = true; /// Are all summed columns zero (or empty)? It is updated incrementally. + + void addRowImpl(ColumnRawPtrs & raw_columns, size_t row); + + /// Initialize aggregate descriptions with columns. + void initAggregateDescription(); + }; + +private: + /// Order between members is important because merged_data has reference to columns_definition. + ColumnsDefinition columns_definition; + SummingMergedData merged_data; +}; + +} diff --git a/src/Processors/Merges/Algorithms/VersionedCollapsingAlgorithm.cpp b/src/Processors/Merges/Algorithms/VersionedCollapsingAlgorithm.cpp new file mode 100644 index 00000000000..e039c383995 --- /dev/null +++ b/src/Processors/Merges/Algorithms/VersionedCollapsingAlgorithm.cpp @@ -0,0 +1,150 @@ +#include +#include +#include + +namespace DB +{ + +static const size_t MAX_ROWS_IN_MULTIVERSION_QUEUE = 8192; + +VersionedCollapsingAlgorithm::VersionedCollapsingAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, const String & sign_column_, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_, + bool use_average_block_sizes) + : IMergingAlgorithmWithSharedChunks( + num_inputs, std::move(description_), out_row_sources_buf_, MAX_ROWS_IN_MULTIVERSION_QUEUE) + , merged_data(header.cloneEmptyColumns(), use_average_block_sizes, max_block_size) + /// -1 for +1 in FixedSizeDequeWithGaps's internal buffer. 3 is a reasonable minimum size to collapse anything. + , max_rows_in_queue(std::min(std::max(3, max_block_size), MAX_ROWS_IN_MULTIVERSION_QUEUE) - 1) + , current_keys(max_rows_in_queue) +{ + sign_column_number = header.getPositionByName(sign_column_); +} + +inline ALWAYS_INLINE static void writeRowSourcePart(WriteBuffer & buffer, RowSourcePart row_source) +{ + if constexpr (sizeof(RowSourcePart) == 1) + buffer.write(*reinterpret_cast(&row_source)); + else + buffer.write(reinterpret_cast(&row_source), sizeof(RowSourcePart)); +} + +void VersionedCollapsingAlgorithm::insertGap(size_t gap_size) +{ + if (out_row_sources_buf) + { + for (size_t i = 0; i < gap_size; ++i) + { + writeRowSourcePart(*out_row_sources_buf, current_row_sources.front()); + current_row_sources.pop(); + } + } +} + +void VersionedCollapsingAlgorithm::insertRow(size_t skip_rows, const RowRef & row) +{ + merged_data.insertRow(*row.all_columns, row.row_num, row.owned_chunk->getNumRows()); + + insertGap(skip_rows); + + if (out_row_sources_buf) + { + current_row_sources.front().setSkipFlag(false); + writeRowSourcePart(*out_row_sources_buf, current_row_sources.front()); + current_row_sources.pop(); + } +} + +IMergingAlgorithm::Status VersionedCollapsingAlgorithm::merge() +{ + /// Take rows in correct order and put them into `merged_columns` until the rows no more than `max_block_size` + while (queue.isValid()) + { + SortCursor current = queue.current(); + + RowRef current_row; + + Int8 sign = assert_cast(*current->all_columns[sign_column_number]).getData()[current->pos]; + + setRowRef(current_row, current); + + /// At first, let's decide the number of rows needed to insert right now. + size_t num_rows_to_insert = 0; + if (!current_keys.empty()) + { + auto key_differs = !current_row.hasEqualSortColumnsWith(current_keys.back()); + + if (key_differs) /// Flush whole queue + num_rows_to_insert = current_keys.size(); + else if (current_keys.size() >= max_rows_in_queue) /// Flush single row if queue is big + num_rows_to_insert = 1; + } + + /// Insert ready roes if any. + while (num_rows_to_insert) + { + const auto & row = current_keys.front(); + auto gap = current_keys.frontGap(); + + insertRow(gap, row); + + current_keys.popFront(); + + --num_rows_to_insert; + + /// It's ok to return here, because we didn't affect queue. + if (merged_data.hasEnoughRows()) + return Status(merged_data.pull()); + } + + if (current_keys.empty()) + { + sign_in_queue = sign; + current_keys.pushBack(current_row); + } + else /// If queue is not empty, then current_row has the same key as in current_keys queue + { + if (sign == sign_in_queue) + current_keys.pushBack(current_row); + else + { + current_keys.popBack(); + current_keys.pushGap(2); + } + } + + if (out_row_sources_buf) + current_row_sources.emplace(current->order, true); + + if (!current->isLast()) + { + queue.next(); + } + else + { + /// We take next block from the corresponding source, if there is one. + queue.removeTop(); + return Status(current.impl->order); + } + } + + while (!current_keys.empty()) + { + const auto & row = current_keys.front(); + auto gap = current_keys.frontGap(); + + insertRow(gap, row); + current_keys.popFront(); + + if (merged_data.hasEnoughRows()) + return Status(merged_data.pull()); + } + + /// Write information about last collapsed rows. + insertGap(current_keys.frontGap()); + return Status(merged_data.pull(), true); +} + +} diff --git a/src/Processors/Merges/Algorithms/VersionedCollapsingAlgorithm.h b/src/Processors/Merges/Algorithms/VersionedCollapsingAlgorithm.h new file mode 100644 index 00000000000..e5f861a268e --- /dev/null +++ b/src/Processors/Merges/Algorithms/VersionedCollapsingAlgorithm.h @@ -0,0 +1,45 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +/** Merges several sorted inputs to one. + * For each group of consecutive identical values of the sorting key + * (the columns by which the data is sorted, including specially specified version column), + * merges any pair of consecutive rows with opposite sign. + */ +class VersionedCollapsingAlgorithm final : public IMergingAlgorithmWithSharedChunks +{ +public: + /// Don't need version column. It's in primary key. + VersionedCollapsingAlgorithm( + const Block & header, size_t num_inputs, + SortDescription description_, const String & sign_column_, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_ = nullptr, + bool use_average_block_sizes = false); + + Status merge() override; + +private: + MergedData merged_data; + + size_t sign_column_number = 0; + + const size_t max_rows_in_queue; + + /// Rows with the same primary key and sign. + FixedSizeDequeWithGaps current_keys; + Int8 sign_in_queue = 0; + + std::queue current_row_sources; /// Sources of rows with the current primary key + + void insertGap(size_t gap_size); + void insertRow(size_t skip_rows, const RowRef & row); +}; + +} diff --git a/src/Processors/Merges/CollapsingSortedTransform.h b/src/Processors/Merges/CollapsingSortedTransform.h new file mode 100644 index 00000000000..d4f40c60938 --- /dev/null +++ b/src/Processors/Merges/CollapsingSortedTransform.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Implementation of IMergingTransform via CollapsingSortedAlgorithm. +class CollapsingSortedTransform final : public IMergingTransform +{ +public: + CollapsingSortedTransform( + const Block & header, + size_t num_inputs, + SortDescription description_, + const String & sign_column, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_ = nullptr, + bool use_average_block_sizes = false) + : IMergingTransform( + num_inputs, header, header, true, + header, + num_inputs, + std::move(description_), + sign_column, + max_block_size, + out_row_sources_buf_, + use_average_block_sizes, + &Logger::get("CollapsingSortedTransform")) + { + } + + String getName() const override { return "CollapsingSortedTransform"; } +}; + +} diff --git a/src/Processors/Merges/GraphiteRollupSortedTransform.h b/src/Processors/Merges/GraphiteRollupSortedTransform.h new file mode 100644 index 00000000000..5104801aa0d --- /dev/null +++ b/src/Processors/Merges/GraphiteRollupSortedTransform.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Implementation of IMergingTransform via GraphiteRollupSortedAlgorithm. +class GraphiteRollupSortedTransform final : public IMergingTransform +{ +public: + GraphiteRollupSortedTransform( + const Block & header, size_t num_inputs, + SortDescription description_, size_t max_block_size, + Graphite::Params params_, time_t time_of_merge_) + : IMergingTransform( + num_inputs, header, header, true, + header, + num_inputs, + std::move(description_), + max_block_size, + std::move(params_), + time_of_merge_) + { + } + + String getName() const override { return "GraphiteRollupSortedTransform"; } +}; + +} diff --git a/src/Processors/Merges/IMergingTransform.cpp b/src/Processors/Merges/IMergingTransform.cpp new file mode 100644 index 00000000000..0dc4cd41991 --- /dev/null +++ b/src/Processors/Merges/IMergingTransform.cpp @@ -0,0 +1,177 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} + +IMergingTransformBase::IMergingTransformBase( + size_t num_inputs, + const Block & input_header, + const Block & output_header, + bool have_all_inputs_) + : IProcessor(InputPorts(num_inputs, input_header), {output_header}) + , have_all_inputs(have_all_inputs_) +{ +} + +void IMergingTransformBase::onNewInput() +{ + throw Exception("onNewInput is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + +void IMergingTransformBase::addInput() +{ + if (have_all_inputs) + throw Exception("IMergingTransform already have all inputs.", ErrorCodes::LOGICAL_ERROR); + + inputs.emplace_back(outputs.front().getHeader(), this); + onNewInput(); +} + +void IMergingTransformBase::setHaveAllInputs() +{ + if (have_all_inputs) + throw Exception("IMergingTransform already have all inputs.", ErrorCodes::LOGICAL_ERROR); + + have_all_inputs = true; +} + +IProcessor::Status IMergingTransformBase::prepareInitializeInputs() +{ + /// Add information about inputs. + if (input_states.empty()) + { + input_states.reserve(inputs.size()); + for (auto & input : inputs) + input_states.emplace_back(input); + + state.init_chunks.resize(inputs.size()); + } + + /// Check for inputs we need. + bool all_inputs_has_data = true; + auto it = inputs.begin(); + for (size_t i = 0; it != inputs.end(); ++i, ++it) + { + auto & input = *it; + if (input.isFinished()) + continue; + + if (input_states[i].is_initialized) + { + // input.setNotNeeded(); + continue; + } + + input.setNeeded(); + + if (!input.hasData()) + { + all_inputs_has_data = false; + continue; + } + + auto chunk = input.pull(); + if (!chunk.hasRows()) + { + + if (!input.isFinished()) + all_inputs_has_data = false; + + continue; + } + + state.init_chunks[i] = std::move(chunk); + input_states[i].is_initialized = true; + } + + if (!all_inputs_has_data) + return Status::NeedData; + + is_initialized = true; + return Status::Ready; +} + +IProcessor::Status IMergingTransformBase::prepare() +{ + if (!have_all_inputs) + return Status::NeedData; + + auto & output = outputs.front(); + + /// Special case for no inputs. + if (inputs.empty()) + { + output.finish(); + onFinish(); + return Status::Finished; + } + + /// Check can output. + + if (output.isFinished()) + { + for (auto & in : inputs) + in.close(); + + onFinish(); + return Status::Finished; + } + + /// Do not disable inputs, so it will work in the same way as with AsynchronousBlockInputStream, like before. + bool is_port_full = !output.canPush(); + + /// Push if has data. + if (state.output_chunk && !is_port_full) + output.push(std::move(state.output_chunk)); + + if (!is_initialized) + return prepareInitializeInputs(); + + if (state.is_finished) + { + + if (is_port_full) + return Status::PortFull; + + for (auto & input : inputs) + input.close(); + + outputs.front().finish(); + + onFinish(); + return Status::Finished; + } + + if (state.need_data) + { + auto & input = input_states[state.next_input_to_read].port; + if (!input.isFinished()) + { + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + auto chunk = input.pull(); + if (!chunk.hasRows() && !input.isFinished()) + return Status::NeedData; + + state.input_chunk = std::move(chunk); + } + + state.need_data = false; + } + + if (is_port_full) + return Status::PortFull; + + return Status::Ready; +} + +} diff --git a/src/Processors/Merges/IMergingTransform.h b/src/Processors/Merges/IMergingTransform.h new file mode 100644 index 00000000000..f9c2dba8271 --- /dev/null +++ b/src/Processors/Merges/IMergingTransform.h @@ -0,0 +1,123 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// Base class for IMergingTransform. +/// It is needed to extract all non-template methods in single translation unit. +class IMergingTransformBase : public IProcessor +{ +public: + IMergingTransformBase( + size_t num_inputs, + const Block & input_header, + const Block & output_header, + bool have_all_inputs_); + + /// Methods to add additional input port. It is possible to do only before the first call of `prepare`. + void addInput(); + /// Need to be called after all inputs are added. (only if have_all_inputs was not specified). + void setHaveAllInputs(); + + Status prepare() override; + +protected: + virtual void onNewInput(); /// Is called when new input is added. Only if have_all_inputs = false. + virtual void onFinish() {} /// Is called when all data is processed. + + /// Processor state. + struct State + { + Chunk output_chunk; + Chunk input_chunk; + bool is_finished = false; + bool need_data = false; + size_t next_input_to_read = 0; + + Chunks init_chunks; + }; + + State state; + +private: + struct InputState + { + explicit InputState(InputPort & port_) : port(port_) {} + + InputPort & port; + bool is_initialized = false; + }; + + std::vector input_states; + std::atomic have_all_inputs; + bool is_initialized = false; + + IProcessor::Status prepareInitializeInputs(); +}; + +/// Implementation of MergingTransform using IMergingAlgorithm. +template +class IMergingTransform : public IMergingTransformBase +{ +public: + template + IMergingTransform( + size_t num_inputs, + const Block & input_header, + const Block & output_header, + bool have_all_inputs_, + Args && ... args) + : IMergingTransformBase(num_inputs, input_header, output_header, have_all_inputs_) + , algorithm(std::forward(args) ...) + { + } + + void work() override + { + if (!state.init_chunks.empty()) + algorithm.initialize(std::move(state.init_chunks)); + + if (state.input_chunk) + { + // std::cerr << "Consume chunk with " << state.input_chunk.getNumRows() + // << " for input " << state.next_input_to_read << std::endl; + algorithm.consume(std::move(state.input_chunk), state.next_input_to_read); + } + + IMergingAlgorithm::Status status = algorithm.merge(); + + if (status.chunk && status.chunk.hasRows()) + { + // std::cerr << "Got chunk with " << status.chunk.getNumRows() << " rows" << std::endl; + state.output_chunk = std::move(status.chunk); + } + + if (status.required_source >= 0) + { + // std::cerr << "Required data for input " << status.required_source << std::endl; + state.next_input_to_read = status.required_source; + state.need_data = true; + } + + if (status.is_finished) + { + // std::cerr << "Finished" << std::endl; + state.is_finished = true; + } + } + +protected: + Algorithm algorithm; + + /// Profile info. + Stopwatch total_stopwatch {CLOCK_MONOTONIC_COARSE}; + +private: + using IMergingTransformBase::state; +}; + +} diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp new file mode 100644 index 00000000000..c7597e4da62 --- /dev/null +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -0,0 +1,63 @@ +#include +#include +#include +#include + +#include + +namespace DB +{ + +MergingSortedTransform::MergingSortedTransform( + const Block & header, + size_t num_inputs, + SortDescription description_, + size_t max_block_size, + UInt64 limit_, + WriteBuffer * out_row_sources_buf_, + bool quiet_, + bool use_average_block_sizes, + bool have_all_inputs_) + : IMergingTransform( + num_inputs, header, header, have_all_inputs_, + header, + num_inputs, + std::move(description_), + max_block_size, + limit_, + out_row_sources_buf_, + use_average_block_sizes) + , quiet(quiet_) +{ +} + +void MergingSortedTransform::onNewInput() +{ + algorithm.addInput(); +} + +void MergingSortedTransform::onFinish() +{ + if (quiet) + return; + + const auto & merged_data = algorithm.getMergedData(); + + auto * log = &Logger::get("MergingSortedTransform"); + + double seconds = total_stopwatch.elapsedSeconds(); + + std::stringstream message; + message << std::fixed << std::setprecision(2) + << "Merge sorted " << merged_data.totalChunks() << " blocks, " << merged_data.totalMergedRows() << " rows" + << " in " << seconds << " sec."; + + if (seconds != 0) + message << ", " + << merged_data.totalMergedRows() / seconds << " rows/sec., " + << merged_data.totalAllocatedBytes() / 1000000.0 / seconds << " MB/sec."; + + LOG_DEBUG(log, message.str()); +} + +} diff --git a/src/Processors/Merges/MergingSortedTransform.h b/src/Processors/Merges/MergingSortedTransform.h new file mode 100644 index 00000000000..93bd36d8aec --- /dev/null +++ b/src/Processors/Merges/MergingSortedTransform.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/// Implementation of IMergingTransform via MergingSortedAlgorithm. +class MergingSortedTransform final : public IMergingTransform +{ +public: + MergingSortedTransform( + const Block & header, + size_t num_inputs, + SortDescription description, + size_t max_block_size, + UInt64 limit_ = 0, + WriteBuffer * out_row_sources_buf_ = nullptr, + bool quiet_ = false, + bool use_average_block_sizes = false, + bool have_all_inputs_ = true); + + String getName() const override { return "MergingSortedTransform"; } + +protected: + void onNewInput() override; + void onFinish() override; + +private: + bool quiet = false; +}; + +} diff --git a/src/Processors/Merges/ReplacingSortedTransform.h b/src/Processors/Merges/ReplacingSortedTransform.h new file mode 100644 index 00000000000..757e19e2cbe --- /dev/null +++ b/src/Processors/Merges/ReplacingSortedTransform.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/// Implementation of IMergingTransform via ReplacingSortedAlgorithm. +class ReplacingSortedTransform final : public IMergingTransform +{ +public: + ReplacingSortedTransform( + const Block & header, size_t num_inputs, + SortDescription description_, const String & version_column, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_ = nullptr, + bool use_average_block_sizes = false) + : IMergingTransform( + num_inputs, header, header, true, + header, + num_inputs, + std::move(description_), + version_column, + max_block_size, + out_row_sources_buf_, + use_average_block_sizes) + { + } + + String getName() const override { return "ReplacingSorted"; } +}; + +} diff --git a/src/Processors/Merges/SummingSortedTransform.h b/src/Processors/Merges/SummingSortedTransform.h new file mode 100644 index 00000000000..6fc22681132 --- /dev/null +++ b/src/Processors/Merges/SummingSortedTransform.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Implementation of IMergingTransform via SummingSortedAlgorithm. +class SummingSortedTransform final : public IMergingTransform +{ +public: + + SummingSortedTransform( + const Block & header, size_t num_inputs, + SortDescription description_, + /// List of columns to be summed. If empty, all numeric columns that are not in the description are taken. + const Names & column_names_to_sum, + size_t max_block_size) + : IMergingTransform( + num_inputs, header, header, true, + header, + num_inputs, + std::move(description_), + column_names_to_sum, + max_block_size) + { + } + + String getName() const override { return "SummingSortedTransform"; } +}; + +} diff --git a/src/Processors/Merges/VersionedCollapsingTransform.h b/src/Processors/Merges/VersionedCollapsingTransform.h new file mode 100644 index 00000000000..f593734c603 --- /dev/null +++ b/src/Processors/Merges/VersionedCollapsingTransform.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/// Implementation of IMergingTransform via VersionedCollapsingAlgorithm. +class VersionedCollapsingTransform final : public IMergingTransform +{ +public: + /// Don't need version column. It's in primary key. + VersionedCollapsingTransform( + const Block & header, size_t num_inputs, + SortDescription description_, const String & sign_column_, + size_t max_block_size, + WriteBuffer * out_row_sources_buf_ = nullptr, + bool use_average_block_sizes = false) + : IMergingTransform( + num_inputs, header, header, true, + header, + num_inputs, + std::move(description_), + sign_column_, + max_block_size, + out_row_sources_buf_, + use_average_block_sizes) + { + } + + String getName() const override { return "VersionedCollapsingTransform"; } +}; + +} diff --git a/src/Processors/NullSink.h b/src/Processors/NullSink.h index e4968daee29..5d304a0d68e 100644 --- a/src/Processors/NullSink.h +++ b/src/Processors/NullSink.h @@ -1,9 +1,11 @@ #pragma once #include +#include namespace DB { +/// Sink which closes input port and reads nothing. class NullSink : public IProcessor { public: @@ -19,4 +21,15 @@ public: InputPort & getPort() { return inputs.front(); } }; +/// Sink which reads everything and do nothing with it. +class EmptySink : public ISink +{ +public: + explicit EmptySink(Block header) : ISink(std::move(header)) {} + String getName() const override { return "EmptySink"; } + +protected: + void consume(Chunk) override {} +}; + } diff --git a/src/Processors/Pipe.cpp b/src/Processors/Pipe.cpp index 4461d714264..d9b21dbc854 100644 --- a/src/Processors/Pipe.cpp +++ b/src/Processors/Pipe.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -48,7 +49,7 @@ static void checkSource(const IProcessor & source) throw Exception("Source for pipe should have single output, but it doesn't have any", ErrorCodes::LOGICAL_ERROR); - if (source.getOutputs().size() > 2) + if (source.getOutputs().size() > 1) throw Exception("Source for pipe should have single or two outputs, but " + source.getName() + " has " + toString(source.getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR); } @@ -56,18 +57,22 @@ static void checkSource(const IProcessor & source) Pipe::Pipe(ProcessorPtr source) { - checkSource(*source); - output_port = &source->getOutputs().front(); + if (auto * source_from_input_stream = typeid_cast(source.get())) + { + totals = source_from_input_stream->getTotalsPort(); + extremes = source_from_input_stream->getExtremesPort(); + } + else if (source->getOutputs().size() != 1) + checkSource(*source); - if (source->getOutputs().size() > 1) - totals = &source->getOutputs().back(); + output_port = &source->getOutputs().front(); processors.emplace_back(std::move(source)); max_parallel_streams = 1; } -Pipe::Pipe(Processors processors_, OutputPort * output_port_, OutputPort * totals_) - : processors(std::move(processors_)), output_port(output_port_), totals(totals_) +Pipe::Pipe(Processors processors_, OutputPort * output_port_, OutputPort * totals_, OutputPort * extremes_) + : processors(std::move(processors_)), output_port(output_port_), totals(totals_), extremes(extremes_) { } diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index 60715d986af..42bbd4e06d0 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -47,8 +47,11 @@ public: void enableQuota(); + /// Totals and extremes port. void setTotalsPort(OutputPort * totals_) { totals = totals_; } + void setExtremesPort(OutputPort * extremes_) { extremes = extremes_; } OutputPort * getTotalsPort() const { return totals; } + OutputPort * getExtremesPort() const { return extremes; } size_t maxParallelStreams() const { return max_parallel_streams; } @@ -67,6 +70,7 @@ private: Processors processors; OutputPort * output_port = nullptr; OutputPort * totals = nullptr; + OutputPort * extremes = nullptr; /// It is the max number of processors which can be executed in parallel for each step. See QueryPipeline::Streams. size_t max_parallel_streams = 0; @@ -84,7 +88,7 @@ private: /// and therefore we can skip those checks. /// Note that Pipe represents a tree if it was created using public interface. But this constructor can't assert it. /// So, it's possible that TreeExecutorBlockInputStream could be unable to convert such Pipe to IBlockInputStream. - explicit Pipe(Processors processors_, OutputPort * output_port, OutputPort * totals); + explicit Pipe(Processors processors_, OutputPort * output_port, OutputPort * totals, OutputPort * extremes); friend class QueryPipeline; }; diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index ee6938a48a6..13787a3fd3b 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -60,6 +60,58 @@ void QueryPipeline::init(Pipe pipe) init(std::move(pipes)); } +static OutputPort * uniteExtremes(const std::vector & ports, const Block & header, Processors & processors) +{ + /// Here we calculate extremes for extremes in case we unite several pipelines. + /// Example: select number from numbers(2) union all select number from numbers(3) + + /// ->> Resize -> Extremes --(output port)----> Null + /// --(extremes port)--> ... + + auto resize = std::make_shared(header, ports.size(), 1); + auto extremes = std::make_shared(header); + auto sink = std::make_shared(header); + + auto * extremes_port = &extremes->getExtremesPort(); + + auto in = resize->getInputs().begin(); + for (const auto & port : ports) + connect(*port, *(in++)); + + connect(resize->getOutputs().front(), extremes->getInputPort()); + connect(extremes->getOutputPort(), sink->getPort()); + + processors.emplace_back(std::move(resize)); + processors.emplace_back(std::move(extremes)); + processors.emplace_back(std::move(sink)); + + return extremes_port; +} + +static OutputPort * uniteTotals(const std::vector & ports, const Block & header, Processors & processors) +{ + /// Calculate totals fro several streams. + /// Take totals from first sources which has any, skip others. + + /// ->> Concat -> Limit + + auto concat = std::make_shared(header, ports.size()); + auto limit = std::make_shared(header, 1, 0); + + auto * totals_port = &limit->getOutputPort(); + + auto in = concat->getInputs().begin(); + for (const auto & port : ports) + connect(*port, *(in++)); + + connect(concat->getOutputs().front(), limit->getInputPort()); + + processors.emplace_back(std::move(concat)); + processors.emplace_back(std::move(limit)); + + return totals_port; +} + void QueryPipeline::init(Pipes pipes) { if (initialized()) @@ -71,21 +123,22 @@ void QueryPipeline::init(Pipes pipes) /// Move locks from pipes to pipeline class. for (auto & pipe : pipes) { - for (auto & lock : pipe.getTableLocks()) + for (const auto & lock : pipe.getTableLocks()) table_locks.emplace_back(lock); - for (auto & context : pipe.getContexts()) + for (const auto & context : pipe.getContexts()) interpreter_context.emplace_back(context); - for (auto & storage : pipe.getStorageHolders()) + for (const auto & storage : pipe.getStorageHolders()) storage_holders.emplace_back(storage); } std::vector totals; + std::vector extremes; for (auto & pipe : pipes) { - auto & header = pipe.getHeader(); + const auto & header = pipe.getHeader(); if (current_header) assertBlocksHaveEqualStructure(current_header, header, "QueryPipeline"); @@ -98,6 +151,12 @@ void QueryPipeline::init(Pipes pipes) totals.emplace_back(totals_port); } + if (auto * port = pipe.getExtremesPort()) + { + assertBlocksHaveEqualStructure(current_header, port->getHeader(), "QueryPipeline"); + extremes.emplace_back(port); + } + streams.addStream(&pipe.getPort(), pipe.maxParallelStreams()); auto cur_processors = std::move(pipe).detachProcessors(); processors.insert(processors.end(), cur_processors.begin(), cur_processors.end()); @@ -108,15 +167,15 @@ void QueryPipeline::init(Pipes pipes) if (totals.size() == 1) totals_having_port = totals.back(); else - { - auto resize = std::make_shared(current_header, totals.size(), 1); - auto in = resize->getInputs().begin(); - for (auto & total : totals) - connect(*total, *(in++)); + totals_having_port = uniteTotals(totals, current_header, processors); + } - totals_having_port = &resize->getOutputs().front(); - processors.emplace_back(std::move(resize)); - } + if (!extremes.empty()) + { + if (extremes.size() == 1) + extremes_port = extremes.back(); + else + extremes_port = uniteExtremes(extremes, current_header, processors); } } @@ -342,43 +401,51 @@ void QueryPipeline::addTotals(ProcessorPtr source) assertBlocksHaveEqualStructure(current_header, source->getOutputs().front().getHeader(), "QueryPipeline"); totals_having_port = &source->getOutputs().front(); - processors.emplace_back(source); + processors.emplace_back(std::move(source)); } -void QueryPipeline::dropTotalsIfHas() +void QueryPipeline::dropTotalsAndExtremes() { - if (totals_having_port) + auto drop_port = [&](OutputPort *& port) { - auto null_sink = std::make_shared(totals_having_port->getHeader()); - connect(*totals_having_port, null_sink->getPort()); + auto null_sink = std::make_shared(port->getHeader()); + connect(*port, null_sink->getPort()); processors.emplace_back(std::move(null_sink)); - totals_having_port = nullptr; - } + port = nullptr; + }; + + if (totals_having_port) + drop_port(totals_having_port); + + if (extremes_port) + drop_port(extremes_port); } -void QueryPipeline::addExtremesTransform(ProcessorPtr transform) +void QueryPipeline::addExtremesTransform() { checkInitialized(); - if (!typeid_cast(transform.get())) - throw Exception("ExtremesTransform expected for QueryPipeline::addExtremesTransform.", - ErrorCodes::LOGICAL_ERROR); - if (extremes_port) throw Exception("Extremes transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); - if (getNumStreams() != 1) - throw Exception("Cant't add Extremes transform because pipeline is expected to have single stream, " - "but it has " + toString(getNumStreams()) + " streams.", ErrorCodes::LOGICAL_ERROR); + std::vector extremes; + extremes.reserve(streams.size()); - connect(*streams.front(), transform->getInputs().front()); + for (auto & stream : streams) + { + auto transform = std::make_shared(current_header); + connect(*stream, transform->getInputPort()); - auto & outputs = transform->getOutputs(); + stream = &transform->getOutputPort(); + extremes.push_back(&transform->getExtremesPort()); - streams.assign({ &outputs.front() }); - extremes_port = &outputs.back(); - current_header = outputs.front().getHeader(); - processors.emplace_back(std::move(transform)); + processors.emplace_back(std::move(transform)); + } + + if (extremes.size() == 1) + extremes_port = extremes.front(); + else + extremes_port = uniteExtremes(extremes, current_header, processors); } void QueryPipeline::addCreatingSetsTransform(ProcessorPtr transform) @@ -444,17 +511,24 @@ void QueryPipeline::setOutput(ProcessorPtr output) } void QueryPipeline::unitePipelines( - std::vector && pipelines, const Block & common_header, const Context & context) + std::vector && pipelines, const Block & common_header) { checkInitialized(); addSimpleTransform([&](const Block & header) { return std::make_shared( - header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + header, common_header, ConvertingTransform::MatchColumnsMode::Position); }); std::vector extremes; + std::vector totals; + + if (extremes_port) + extremes.push_back(extremes_port); + + if (totals_having_port) + totals.push_back(totals_having_port); for (auto & pipeline : pipelines) { @@ -463,13 +537,13 @@ void QueryPipeline::unitePipelines( pipeline.addSimpleTransform([&](const Block & header) { return std::make_shared( - header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + header, common_header, ConvertingTransform::MatchColumnsMode::Position); }); if (pipeline.extremes_port) { auto converting = std::make_shared( - pipeline.current_header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + pipeline.current_header, common_header, ConvertingTransform::MatchColumnsMode::Position); connect(*pipeline.extremes_port, converting->getInputPort()); extremes.push_back(&converting->getOutputPort()); @@ -479,17 +553,12 @@ void QueryPipeline::unitePipelines( /// Take totals only from first port. if (pipeline.totals_having_port) { - if (!totals_having_port) - { - auto converting = std::make_shared( - pipeline.current_header, common_header, ConvertingTransform::MatchColumnsMode::Position, context); + auto converting = std::make_shared( + pipeline.current_header, common_header, ConvertingTransform::MatchColumnsMode::Position); - connect(*pipeline.totals_having_port, converting->getInputPort()); - totals_having_port = &converting->getOutputPort(); - processors.push_back(std::move(converting)); - } - else - pipeline.dropTotalsIfHas(); + connect(*pipeline.totals_having_port, converting->getInputPort()); + totals.push_back(&converting->getOutputPort()); + processors.push_back(std::move(converting)); } processors.insert(processors.end(), pipeline.processors.begin(), pipeline.processors.end()); @@ -504,28 +573,18 @@ void QueryPipeline::unitePipelines( if (!extremes.empty()) { - size_t num_inputs = extremes.size() + (extremes_port ? 1u : 0u); - - if (num_inputs == 1) - extremes_port = extremes.front(); + if (extremes.size() == 1) + extremes_port = extremes.back(); else - { - /// Add extra processor for extremes. - auto resize = std::make_shared(current_header, num_inputs, 1); - auto input = resize->getInputs().begin(); + extremes_port = uniteExtremes(extremes, current_header, processors); + } - if (extremes_port) - connect(*extremes_port, *(input++)); - - for (auto & output : extremes) - connect(*output, *(input++)); - - auto transform = std::make_shared(current_header); - extremes_port = &transform->getOutputPort(); - - connect(resize->getOutputs().front(), transform->getInputPort()); - processors.emplace_back(std::move(transform)); - } + if (!totals.empty()) + { + if (totals.size() == 1) + totals_having_port = totals.back(); + else + totals_having_port = uniteTotals(totals, current_header, processors); } } @@ -577,7 +636,7 @@ void QueryPipeline::initRowsBeforeLimit() while (!queue.empty()) { - auto processor = queue.front().processor; + auto * processor = queue.front().processor; auto visited_limit = queue.front().visited_limit; queue.pop(); @@ -644,7 +703,7 @@ void QueryPipeline::initRowsBeforeLimit() Pipe QueryPipeline::getPipe() && { resize(1); - Pipe pipe(std::move(processors), streams.at(0), totals_having_port); + Pipe pipe(std::move(processors), streams.at(0), totals_having_port, extremes_port); pipe.max_parallel_streams = streams.maxParallelStreams(); for (auto & lock : table_locks) @@ -659,6 +718,9 @@ Pipe QueryPipeline::getPipe() && if (totals_having_port) pipe.setTotalsPort(totals_having_port); + if (extremes_port) + pipe.setExtremesPort(extremes_port); + return pipe; } diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 9ce12e75b91..45e38ffa715 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -99,7 +99,7 @@ public: void addSimpleTransform(const ProcessorGetterWithStreamKind & getter); void addPipe(Processors pipe); void addTotalsHavingTransform(ProcessorPtr transform); - void addExtremesTransform(ProcessorPtr transform); + void addExtremesTransform(); void addCreatingSetsTransform(ProcessorPtr transform); void setOutput(ProcessorPtr output); @@ -109,7 +109,8 @@ public: /// Add already calculated totals. void addTotals(ProcessorPtr source); - void dropTotalsIfHas(); + /// Forget about current totals and extremes. It is needed before aggregation, cause they will be calculated again. + void dropTotalsAndExtremes(); /// Will read from this stream after all data was read from other streams. void addDelayedStream(ProcessorPtr source); @@ -121,7 +122,7 @@ public: void enableQuotaForCurrentStreams(); - void unitePipelines(std::vector && pipelines, const Block & common_header, const Context & context); + void unitePipelines(std::vector && pipelines, const Block & common_header); PipelineExecutorPtr execute(); diff --git a/src/Processors/ResizeProcessor.cpp b/src/Processors/ResizeProcessor.cpp index fd3d6510aa8..3fcf4281105 100644 --- a/src/Processors/ResizeProcessor.cpp +++ b/src/Processors/ResizeProcessor.cpp @@ -173,7 +173,7 @@ IProcessor::Status ResizeProcessor::prepare(const PortNumbers & updated_inputs, output_ports.push_back({.port = &output, .status = OutputStatus::NotActive}); } - for (auto & output_number : updated_outputs) + for (const auto & output_number : updated_outputs) { auto & output = output_ports[output_number]; if (output.port->isFinished()) @@ -205,7 +205,7 @@ IProcessor::Status ResizeProcessor::prepare(const PortNumbers & updated_inputs, return Status::Finished; } - for (auto & input_number : updated_inputs) + for (const auto & input_number : updated_inputs) { auto & input = input_ports[input_number]; if (input.port->isFinished()) @@ -277,7 +277,7 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in output_ports.push_back({.port = &output, .status = OutputStatus::NotActive}); } - for (auto & output_number : updated_outputs) + for (const auto & output_number : updated_outputs) { auto & output = output_ports[output_number]; if (output.port->isFinished()) @@ -311,7 +311,7 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in std::queue inputs_with_data; - for (auto & input_number : updated_inputs) + for (const auto & input_number : updated_inputs) { auto & input = input_ports[input_number]; if (input.port->isFinished()) diff --git a/src/Processors/RowsBeforeLimitCounter.h b/src/Processors/RowsBeforeLimitCounter.h index abee5a09405..36ea4a557a8 100644 --- a/src/Processors/RowsBeforeLimitCounter.h +++ b/src/Processors/RowsBeforeLimitCounter.h @@ -17,7 +17,7 @@ public: uint64_t get() const { return rows_before_limit.load(std::memory_order_acquire); } - void setAppliedLimit() { has_applied_limit.store(true, std::memory_order::release); } + void setAppliedLimit() { has_applied_limit.store(true, std::memory_order_release); } bool hasAppliedLimit() const { return has_applied_limit.load(std::memory_order_acquire); } private: diff --git a/src/Processors/Sources/SourceFromInputStream.cpp b/src/Processors/Sources/SourceFromInputStream.cpp index 6f2a7eeb28a..7c88c6dfbeb 100644 --- a/src/Processors/Sources/SourceFromInputStream.cpp +++ b/src/Processors/Sources/SourceFromInputStream.cpp @@ -20,7 +20,7 @@ SourceFromInputStream::SourceFromInputStream(BlockInputStreamPtr stream_, bool f void SourceFromInputStream::init() { - auto & sample = getPort().getHeader(); + const auto & sample = getPort().getHeader(); for (auto & type : sample.getDataTypes()) if (typeid_cast(type.get())) has_aggregate_functions = true; @@ -28,11 +28,20 @@ void SourceFromInputStream::init() void SourceFromInputStream::addTotalsPort() { - if (has_totals_port) + if (totals_port) throw Exception("Totals port was already added for SourceFromInputStream.", ErrorCodes::LOGICAL_ERROR); outputs.emplace_back(outputs.front().getHeader(), this); - has_totals_port = true; + totals_port = &outputs.back(); +} + +void SourceFromInputStream::addExtremesPort() +{ + if (extremes_port) + throw Exception("Extremes port was already added for SourceFromInputStream.", ErrorCodes::LOGICAL_ERROR); + + outputs.emplace_back(outputs.front().getHeader(), this); + extremes_port = &outputs.back(); } IProcessor::Status SourceFromInputStream::prepare() @@ -47,23 +56,32 @@ IProcessor::Status SourceFromInputStream::prepare() if (!is_stream_finished && !isCancelled()) return Status::Ready; - if (has_totals_port) + if (totals_port && !totals_port->isFinished()) { - auto & totals_out = outputs.back(); - - if (totals_out.isFinished()) - return Status::Finished; - if (has_totals) { - if (!totals_out.canPush()) + if (!totals_port->canPush()) return Status::PortFull; - totals_out.push(std::move(totals)); + totals_port->push(std::move(totals)); has_totals = false; } - totals_out.finish(); + totals_port->finish(); + } + + if (extremes_port && !extremes_port->isFinished()) + { + if (has_extremes) + { + if (!extremes_port->canPush()) + return Status::PortFull; + + extremes_port->push(std::move(extremes)); + has_extremes = false; + } + + extremes_port->finish(); } } @@ -97,7 +115,7 @@ void SourceFromInputStream::work() if (rows_before_limit) { - auto & info = stream->getProfileInfo(); + const auto & info = stream->getProfileInfo(); if (info.hasAppliedLimit()) rows_before_limit->add(info.getRowsBeforeLimit()); } @@ -129,7 +147,7 @@ Chunk SourceFromInputStream::generate() { if (rows_before_limit) { - auto & info = stream->getProfileInfo(); + const auto & info = stream->getProfileInfo(); if (info.hasAppliedLimit()) rows_before_limit->add(info.getRowsBeforeLimit()); } @@ -138,17 +156,29 @@ Chunk SourceFromInputStream::generate() if (auto totals_block = stream->getTotals()) { - if (totals_block.rows() == 1) /// Sometimes we can get empty totals. Skip it. + if (totals_block.rows() > 0) /// Sometimes we can get empty totals. Skip it. { - totals.setColumns(totals_block.getColumns(), 1); + totals.setColumns(totals_block.getColumns(), totals_block.rows()); has_totals = true; } } + if (auto extremes_block = stream->getExtremes()) + { + if (extremes_block.rows() > 0) /// Sometimes we can get empty extremes. Skip it. + { + extremes.setColumns(extremes_block.getColumns(), extremes_block.rows()); + has_extremes = true; + } + } + is_stream_finished = true; return {}; } + if (isCancelled()) + return {}; + #ifndef NDEBUG assertBlocksHaveEqualStructure(getPort().getHeader(), block, "SourceFromInputStream"); #endif diff --git a/src/Processors/Sources/SourceFromInputStream.h b/src/Processors/Sources/SourceFromInputStream.h index 0fc92164059..88a045e65a2 100644 --- a/src/Processors/Sources/SourceFromInputStream.h +++ b/src/Processors/Sources/SourceFromInputStream.h @@ -23,6 +23,10 @@ public: BlockInputStreamPtr & getStream() { return stream; } void addTotalsPort(); + void addExtremesPort(); + + OutputPort * getTotalsPort() const { return totals_port; } + OutputPort * getExtremesPort() const { return extremes_port; } void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr counter) { rows_before_limit.swap(counter); } @@ -33,6 +37,13 @@ public: void setProgressCallback(const ProgressCallback & callback) final { stream->setProgressCallback(callback); } void addTotalRowsApprox(size_t value) final { stream->addTotalRowsApprox(value); } + /// Stop reading from stream if output port is finished. + void onUpdatePorts() override + { + if (getPort().isFinished()) + cancel(); + } + protected: void onCancel() override { stream->cancel(false); } @@ -44,9 +55,13 @@ private: RowsBeforeLimitCounterPtr rows_before_limit; Chunk totals; - bool has_totals_port = false; + OutputPort * totals_port = nullptr; bool has_totals = false; + Chunk extremes; + OutputPort * extremes_port = nullptr; + bool has_extremes = false; + bool is_generating_finished = false; bool is_stream_finished = false; bool is_stream_started = false; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 4a76e0f2273..6043c5689fa 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -39,11 +39,11 @@ namespace const AggregatedChunkInfo * getInfoFromChunk(const Chunk & chunk) { - auto & info = chunk.getChunkInfo(); + const auto & info = chunk.getChunkInfo(); if (!info) throw Exception("Chunk info was not set for chunk.", ErrorCodes::LOGICAL_ERROR); - auto * agg_info = typeid_cast(info.get()); + const auto * agg_info = typeid_cast(info.get()); if (!agg_info) throw Exception("Chunk should have AggregatedChunkInfo.", ErrorCodes::LOGICAL_ERROR); @@ -294,7 +294,6 @@ private: return Status::PortFull; } -private: AggregatingTransformParamsPtr params; ManyAggregatedDataVariantsPtr data; ConvertingAggregatedToChunksSource::SharedDataPtr shared_data; @@ -413,6 +412,9 @@ AggregatingTransform::~AggregatingTransform() = default; IProcessor::Status AggregatingTransform::prepare() { + /// There are one or two input ports. + /// The first one is used at aggregation step, the second one - while reading merged data from ConvertingAggregated + auto & output = outputs.front(); /// Last output is current. All other outputs should already be closed. auto & input = inputs.back(); @@ -432,7 +434,12 @@ IProcessor::Status AggregatingTransform::prepare() /// Finish data processing, prepare to generating. if (is_consume_finished && !is_generate_initialized) + { + /// Close input port in case max_rows_to_group_by was reached but not all data was read. + inputs.front().close(); + return Status::Ready; + } if (is_generate_initialized && !is_pipeline_created && !processors.empty()) return Status::ExpandPipeline; diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 469392e5840..9c1e9d4e2db 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -49,6 +49,21 @@ struct ManyAggregatedData using AggregatingTransformParamsPtr = std::shared_ptr; using ManyAggregatedDataPtr = std::shared_ptr; +/** Aggregates the stream of blocks using the specified key columns and aggregate functions. + * Columns with aggregate functions adds to the end of the block. + * If final = false, the aggregate functions are not finalized, that is, they are not replaced by their value, but contain an intermediate state of calculations. + * This is necessary so that aggregation can continue (for example, by combining streams of partially aggregated data). + * + * For every separate stream of data separate AggregatingTransform is created. + * Every AggregatingTransform reads data from the first port till is is not run out, or max_rows_to_group_by reached. + * When the last AggregatingTransform finish reading, the result of aggregation is needed to be merged together. + * This task is performed by ConvertingAggregatedToChunksTransform. + * Last AggregatingTransform expands pipeline and adds second input port, which reads from ConvertingAggregated. + * + * Aggregation data is passed by ManyAggregatedData structure, which is shared between all aggregating transforms. + * At aggregation step, every transform uses it's own AggregatedDataVariants structure. + * At merging step, all structures pass to ConvertingAggregatedToChunksTransform. + */ class AggregatingTransform : public IProcessor { public: diff --git a/src/Processors/Transforms/ConvertingTransform.cpp b/src/Processors/Transforms/ConvertingTransform.cpp index e801fe7cb26..c5bf5ad4d70 100644 --- a/src/Processors/Transforms/ConvertingTransform.cpp +++ b/src/Processors/Transforms/ConvertingTransform.cpp @@ -18,12 +18,11 @@ namespace ErrorCodes static ColumnPtr castColumnWithDiagnostic( const ColumnWithTypeAndName & src_elem, - const ColumnWithTypeAndName & res_elem, - const Context & context) + const ColumnWithTypeAndName & res_elem) { try { - return castColumn(src_elem, res_elem.type, context); + return castColumn(src_elem, res_elem.type); } catch (Exception & e) { @@ -36,14 +35,12 @@ static ColumnPtr castColumnWithDiagnostic( ConvertingTransform::ConvertingTransform( Block source_header_, Block result_header_, - MatchColumnsMode mode_, - const Context & context_) + MatchColumnsMode mode_) : ISimpleTransform(std::move(source_header_), std::move(result_header_), false) - , context(context_) , conversion(getOutputPort().getHeader().columns()) { - auto & source = getInputPort().getHeader(); - auto & result = getOutputPort().getHeader(); + const auto & source = getInputPort().getHeader(); + const auto & result = getOutputPort().getHeader(); size_t num_input_columns = source.columns(); size_t num_result_columns = result.columns(); @@ -74,9 +71,9 @@ ConvertingTransform::ConvertingTransform( /// Check constants. - if (auto * res_const = typeid_cast(res_elem.column.get())) + if (const auto * res_const = typeid_cast(res_elem.column.get())) { - if (auto * src_const = typeid_cast(src_elem.column.get())) + if (const auto * src_const = typeid_cast(src_elem.column.get())) { if (res_const->getField() != src_const->getField()) throw Exception("Cannot convert column " + backQuoteIfNeed(res_elem.name) + " because " @@ -91,14 +88,14 @@ ConvertingTransform::ConvertingTransform( /// Check conversion by dry run CAST function. - castColumnWithDiagnostic(src_elem, res_elem, context); + castColumnWithDiagnostic(src_elem, res_elem); } } void ConvertingTransform::transform(Chunk & chunk) { - auto & source = getInputPort().getHeader(); - auto & result = getOutputPort().getHeader(); + const auto & source = getInputPort().getHeader(); + const auto & result = getOutputPort().getHeader(); auto num_rows = chunk.getNumRows(); auto src_columns = chunk.detachColumns(); @@ -114,7 +111,7 @@ void ConvertingTransform::transform(Chunk & chunk) src_elem.column = src_columns[conversion[res_pos]]; auto res_elem = result.getByPosition(res_pos); - ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context); + ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem); if (!isColumnConst(*res_elem.column)) converted = converted->convertToFullColumnIfConst(); diff --git a/src/Processors/Transforms/ConvertingTransform.h b/src/Processors/Transforms/ConvertingTransform.h index b2412802ed6..45a6688c07a 100644 --- a/src/Processors/Transforms/ConvertingTransform.h +++ b/src/Processors/Transforms/ConvertingTransform.h @@ -31,8 +31,7 @@ public: ConvertingTransform( Block source_header_, Block result_header_, - MatchColumnsMode mode_, - const Context & context_); + MatchColumnsMode mode_); String getName() const override { return "Converting"; } @@ -40,8 +39,6 @@ protected: void transform(Chunk & chunk) override; private: - const Context & context; - /// How to construct result block. Position in source block, where to get each column. ColumnNumbers conversion; }; diff --git a/src/Processors/Transforms/DistinctTransform.cpp b/src/Processors/Transforms/DistinctTransform.cpp index f9383f1a5e5..fddfe663af5 100644 --- a/src/Processors/Transforms/DistinctTransform.cpp +++ b/src/Processors/Transforms/DistinctTransform.cpp @@ -25,7 +25,7 @@ DistinctTransform::DistinctTransform( auto pos = columns_.empty() ? i : header_.getPositionByName(columns_[i]); - auto & col = header_.getByPosition(pos).column; + const auto & col = header_.getByPosition(pos).column; if (!(col && isColumnConst(*col))) key_columns_pos.emplace_back(pos); diff --git a/src/Processors/Transforms/ExtremesTransform.cpp b/src/Processors/Transforms/ExtremesTransform.cpp index e7fce64219e..526bbc63374 100644 --- a/src/Processors/Transforms/ExtremesTransform.cpp +++ b/src/Processors/Transforms/ExtremesTransform.cpp @@ -61,7 +61,7 @@ void ExtremesTransform::transform(DB::Chunk & chunk) return; size_t num_columns = chunk.getNumColumns(); - auto & columns = chunk.getColumns(); + const auto & columns = chunk.getColumns(); if (extremes_columns.empty()) { diff --git a/src/Processors/Transforms/FilterTransform.cpp b/src/Processors/Transforms/FilterTransform.cpp index 27d77ddc3d6..9d4c97183b5 100644 --- a/src/Processors/Transforms/FilterTransform.cpp +++ b/src/Processors/Transforms/FilterTransform.cpp @@ -85,7 +85,7 @@ IProcessor::Status FilterTransform::prepare() } -void FilterTransform::removeFilterIfNeed(Chunk & chunk) +void FilterTransform::removeFilterIfNeed(Chunk & chunk) const { if (chunk && remove_filter_column) chunk.erase(filter_column_position); diff --git a/src/Processors/Transforms/FilterTransform.h b/src/Processors/Transforms/FilterTransform.h index 1652473aa3c..2eb321474de 100644 --- a/src/Processors/Transforms/FilterTransform.h +++ b/src/Processors/Transforms/FilterTransform.h @@ -38,7 +38,7 @@ private: bool are_prepared_sets_initialized = false; - void removeFilterIfNeed(Chunk & chunk); + void removeFilterIfNeed(Chunk & chunk) const; }; } diff --git a/src/Processors/Transforms/LimitByTransform.cpp b/src/Processors/Transforms/LimitByTransform.cpp index f9f7fbee3fe..8891ae11c03 100644 --- a/src/Processors/Transforms/LimitByTransform.cpp +++ b/src/Processors/Transforms/LimitByTransform.cpp @@ -15,7 +15,7 @@ LimitByTransform::LimitByTransform(const Block & header, size_t group_length_, s for (const auto & name : columns) { auto position = header.getPositionByName(name); - auto & column = header.getByPosition(position).column; + const auto & column = header.getByPosition(position).column; /// Ignore all constant columns. if (!(column && isColumnConst(*column))) diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 060d860b0b5..ee1836a4a14 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -191,6 +191,7 @@ void MergeSortingTransform::consume(Chunk chunk) { bool quiet = false; bool have_all_inputs = false; + bool use_average_block_sizes = false; external_merging_sorted = std::make_shared( header_without_constants, @@ -198,7 +199,9 @@ void MergeSortingTransform::consume(Chunk chunk) description, max_merged_block_size, limit, + nullptr, quiet, + use_average_block_sizes, have_all_inputs); processors.emplace_back(external_merging_sorted); diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index 12d289deaed..79dec00ba0d 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -248,11 +248,11 @@ IProcessor::Status GroupingAggregatedTransform::prepare() void GroupingAggregatedTransform::addChunk(Chunk chunk, size_t input) { - auto & info = chunk.getChunkInfo(); + const auto & info = chunk.getChunkInfo(); if (!info) throw Exception("Chunk info was not set for chunk in GroupingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - auto * agg_info = typeid_cast(info.get()); + const auto * agg_info = typeid_cast(info.get()); if (!agg_info) throw Exception("Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); @@ -275,7 +275,7 @@ void GroupingAggregatedTransform::work() { if (!single_level_chunks.empty()) { - auto & header = getInputs().front().getHeader(); /// Take header from input port. Output header is empty. + const auto & header = getInputs().front().getHeader(); /// Take header from input port. Output header is empty. auto block = header.cloneWithColumns(single_level_chunks.back().detachColumns()); single_level_chunks.pop_back(); auto blocks = params->aggregator.convertBlockToTwoLevel(block); @@ -302,8 +302,8 @@ MergingAggregatedBucketTransform::MergingAggregatedBucketTransform(AggregatingTr void MergingAggregatedBucketTransform::transform(Chunk & chunk) { - auto & info = chunk.getChunkInfo(); - auto * chunks_to_merge = typeid_cast(info.get()); + const auto & info = chunk.getChunkInfo(); + const auto * chunks_to_merge = typeid_cast(info.get()); if (!chunks_to_merge) throw Exception("MergingAggregatedSimpleTransform chunk must have ChunkInfo with type ChunksToMerge.", @@ -314,12 +314,12 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk) BlocksList blocks_list; for (auto & cur_chunk : *chunks_to_merge->chunks) { - auto & cur_info = cur_chunk.getChunkInfo(); + const auto & cur_info = cur_chunk.getChunkInfo(); if (!cur_info) throw Exception("Chunk info was not set for chunk in MergingAggregatedBucketTransform.", ErrorCodes::LOGICAL_ERROR); - auto * agg_info = typeid_cast(cur_info.get()); + const auto * agg_info = typeid_cast(cur_info.get()); if (!agg_info) throw Exception("Chunk should have AggregatedChunkInfo in MergingAggregatedBucketTransform.", ErrorCodes::LOGICAL_ERROR); @@ -374,11 +374,11 @@ bool SortingAggregatedTransform::tryPushChunk() void SortingAggregatedTransform::addChunk(Chunk chunk, size_t from_input) { - auto & info = chunk.getChunkInfo(); + const auto & info = chunk.getChunkInfo(); if (!info) throw Exception("Chunk info was not set for chunk in SortingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - auto * agg_info = typeid_cast(info.get()); + const auto * agg_info = typeid_cast(info.get()); if (!agg_info) throw Exception("Chunk should have AggregatedChunkInfo in SortingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 877b8b8811a..81341456faf 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -26,11 +26,11 @@ void MergingAggregatedTransform::consume(Chunk chunk) total_input_rows += chunk.getNumRows(); ++total_input_blocks; - auto & info = chunk.getChunkInfo(); + const auto & info = chunk.getChunkInfo(); if (!info) throw Exception("Chunk info was not set for chunk in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - auto * agg_info = typeid_cast(info.get()); + const auto * agg_info = typeid_cast(info.get()); if (!agg_info) throw Exception("Chunk should have AggregatedChunkInfo in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Processors/Transforms/MergingSortedTransform.cpp b/src/Processors/Transforms/MergingSortedTransform.cpp deleted file mode 100644 index b9e74277023..00000000000 --- a/src/Processors/Transforms/MergingSortedTransform.cpp +++ /dev/null @@ -1,331 +0,0 @@ -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -MergingSortedTransform::MergingSortedTransform( - const Block & header, - size_t num_inputs, - const SortDescription & description_, - size_t max_block_size_, - UInt64 limit_, - bool quiet_, - bool have_all_inputs_) - : IProcessor(InputPorts(num_inputs, header), {header}) - , description(description_), max_block_size(max_block_size_), limit(limit_), quiet(quiet_) - , have_all_inputs(have_all_inputs_) - , merged_data(header), source_chunks(num_inputs), cursors(num_inputs) -{ - auto & sample = outputs.front().getHeader(); - /// Replace column names in description to positions. - for (auto & column_description : description) - { - has_collation |= column_description.collator != nullptr; - if (!column_description.column_name.empty()) - { - column_description.column_number = sample.getPositionByName(column_description.column_name); - column_description.column_name.clear(); - } - } -} - -void MergingSortedTransform::addInput() -{ - if (have_all_inputs) - throw Exception("MergingSortedTransform already have all inputs.", ErrorCodes::LOGICAL_ERROR); - - inputs.emplace_back(outputs.front().getHeader(), this); - source_chunks.emplace_back(); - cursors.emplace_back(); -} - -void MergingSortedTransform::setHaveAllInputs() -{ - if (have_all_inputs) - throw Exception("MergingSortedTransform already have all inputs.", ErrorCodes::LOGICAL_ERROR); - - have_all_inputs = true; -} - -IProcessor::Status MergingSortedTransform::prepare() -{ - if (!have_all_inputs) - return Status::NeedData; - - auto & output = outputs.front(); - - /// Special case for no inputs. - if (inputs.empty()) - { - output.finish(); - return Status::Finished; - } - - /// Check can output. - - if (output.isFinished()) - { - for (auto & in : inputs) - in.close(); - - return Status::Finished; - } - - /// Do not disable inputs, so it will work in the same way as with AsynchronousBlockInputStream, like before. - bool is_port_full = !output.canPush(); - - /// Special case for single input. - if (inputs.size() == 1) - { - auto & input = inputs.front(); - if (input.isFinished()) - { - output.finish(); - return Status::Finished; - } - - input.setNeeded(); - - if (input.hasData()) - { - if (!is_port_full) - output.push(input.pull()); - - return Status::PortFull; - } - - return Status::NeedData; - } - - /// Push if has data. - if (merged_data.mergedRows() && !is_port_full) - output.push(merged_data.pull()); - - if (!is_initialized) - { - /// Check for inputs we need. - bool all_inputs_has_data = true; - auto it = inputs.begin(); - for (size_t i = 0; it != inputs.end(); ++i, ++it) - { - auto & input = *it; - if (input.isFinished()) - continue; - - if (!cursors[i].empty()) - { - // input.setNotNeeded(); - continue; - } - - input.setNeeded(); - - if (!input.hasData()) - { - all_inputs_has_data = false; - continue; - } - - auto chunk = input.pull(); - if (!chunk.hasRows()) - { - - if (!input.isFinished()) - all_inputs_has_data = false; - - continue; - } - - updateCursor(std::move(chunk), i); - } - - if (!all_inputs_has_data) - return Status::NeedData; - - if (has_collation) - queue_with_collation = SortingHeap(cursors); - else - queue_without_collation = SortingHeap(cursors); - - is_initialized = true; - return Status::Ready; - } - else - { - if (is_finished) - { - - if (is_port_full) - return Status::PortFull; - - for (auto & input : inputs) - input.close(); - - outputs.front().finish(); - - return Status::Finished; - } - - if (need_data) - { - auto & input = *std::next(inputs.begin(), next_input_to_read); - if (!input.isFinished()) - { - input.setNeeded(); - - if (!input.hasData()) - return Status::NeedData; - - auto chunk = input.pull(); - if (!chunk.hasRows() && !input.isFinished()) - return Status::NeedData; - - updateCursor(std::move(chunk), next_input_to_read); - - if (has_collation) - queue_with_collation.push(cursors[next_input_to_read]); - else - queue_without_collation.push(cursors[next_input_to_read]); - } - - need_data = false; - } - - if (is_port_full) - return Status::PortFull; - - return Status::Ready; - } -} - -void MergingSortedTransform::work() -{ - if (has_collation) - merge(queue_with_collation); - else - merge(queue_without_collation); -} - -template -void MergingSortedTransform::merge(TSortingHeap & queue) -{ - /// Returns MergeStatus which we should return if we are going to finish now. - auto can_read_another_row = [&, this]() - { - if (limit && merged_data.totalMergedRows() >= limit) - { - //std::cerr << "Limit reached\n"; - is_finished = true; - return false; - } - - return merged_data.mergedRows() < max_block_size; - }; - - /// Take rows in required order and put them into `merged_data`, while the rows are no more than `max_block_size` - while (queue.isValid()) - { - /// Shouldn't happen at first iteration, but check just in case. - if (!can_read_another_row()) - return; - - auto current = queue.current(); - - /** And what if the block is totally less or equal than the rest for the current cursor? - * Or is there only one data source left in the queue? Then you can take the entire block on current cursor. - */ - if (current.impl->isFirst() - && (queue.size() == 1 - || (queue.size() >= 2 && current.totallyLessOrEquals(queue.nextChild())))) - { - //std::cerr << "current block is totally less or equals\n"; - - /// If there are already data in the current block, we first return it. We'll get here again the next time we call the merge function. - if (merged_data.mergedRows() != 0) - { - //std::cerr << "merged rows is non-zero\n"; - return; - } - - /// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl) - size_t source_num = current.impl->order; - insertFromChunk(source_num); - queue.removeTop(); - return; - } - - //std::cerr << "total_merged_rows: " << total_merged_rows << ", merged_rows: " << merged_rows << "\n"; - //std::cerr << "Inserting row\n"; - merged_data.insertRow(current->all_columns, current->pos); - - if (out_row_sources_buf) - { - /// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl) - RowSourcePart row_source(current.impl->order); - out_row_sources_buf->write(row_source.data); - } - - if (!current->isLast()) - { -// std::cerr << "moving to next row\n"; - queue.next(); - } - else - { - /// We will get the next block from the corresponding source, if there is one. - queue.removeTop(); - -// std::cerr << "It was last row, fetching next block\n"; - need_data = true; - next_input_to_read = current.impl->order; - - if (limit && merged_data.totalMergedRows() >= limit) - is_finished = true; - - return; - } - } - is_finished = true; -} - -void MergingSortedTransform::insertFromChunk(size_t source_num) -{ - if (source_num >= cursors.size()) - throw Exception("Logical error in MergingSortedTrandform", ErrorCodes::LOGICAL_ERROR); - - //std::cerr << "copied columns\n"; - - auto num_rows = source_chunks[source_num].getNumRows(); - - UInt64 total_merged_rows_after_insertion = merged_data.mergedRows() + num_rows; - if (limit && total_merged_rows_after_insertion > limit) - { - num_rows = total_merged_rows_after_insertion - limit; - merged_data.insertFromChunk(std::move(source_chunks[source_num]), num_rows); - is_finished = true; - } - else - { - merged_data.insertFromChunk(std::move(source_chunks[source_num]), 0); - need_data = true; - next_input_to_read = source_num; - } - source_chunks[source_num] = Chunk(); - - if (out_row_sources_buf) - { - RowSourcePart row_source(source_num); - for (size_t i = 0; i < num_rows; ++i) - out_row_sources_buf->write(row_source.data); - } -} - - -} diff --git a/src/Processors/Transforms/MergingSortedTransform.h b/src/Processors/Transforms/MergingSortedTransform.h deleted file mode 100644 index 914ad543c75..00000000000 --- a/src/Processors/Transforms/MergingSortedTransform.h +++ /dev/null @@ -1,160 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -class MergingSortedTransform : public IProcessor -{ -public: - MergingSortedTransform( - const Block & header, - size_t num_inputs, - const SortDescription & description_, - size_t max_block_size, - UInt64 limit = 0, - bool quiet = false, - bool have_all_inputs = true); - - String getName() const override { return "MergingSortedTransform"; } - Status prepare() override; - void work() override; - - void addInput(); - void setHaveAllInputs(); - -protected: - - class MergedData - { - public: - explicit MergedData(const Block & header) - { - columns.reserve(header.columns()); - for (const auto & column : header) - columns.emplace_back(column.type->createColumn()); - } - - void insertRow(const ColumnRawPtrs & raw_columns, size_t row) - { - size_t num_columns = raw_columns.size(); - for (size_t i = 0; i < num_columns; ++i) - columns[i]->insertFrom(*raw_columns[i], row); - - ++total_merged_rows; - ++merged_rows; - } - - void insertFromChunk(Chunk && chunk, size_t limit_rows) - { - if (merged_rows) - throw Exception("Cannot insert to MergedData from Chunk because MergedData is not empty.", - ErrorCodes::LOGICAL_ERROR); - - auto num_rows = chunk.getNumRows(); - columns = chunk.mutateColumns(); - if (limit_rows && num_rows > limit_rows) - { - num_rows = limit_rows; - for (auto & column : columns) - column = (*column->cut(0, num_rows)).mutate(); - } - - total_merged_rows += num_rows; - merged_rows = num_rows; - } - - Chunk pull() - { - MutableColumns empty_columns; - empty_columns.reserve(columns.size()); - - for (const auto & column : columns) - empty_columns.emplace_back(column->cloneEmpty()); - - empty_columns.swap(columns); - Chunk chunk(std::move(empty_columns), merged_rows); - merged_rows = 0; - - return chunk; - } - - UInt64 totalMergedRows() const { return total_merged_rows; } - UInt64 mergedRows() const { return merged_rows; } - - private: - UInt64 total_merged_rows = 0; - UInt64 merged_rows = 0; - MutableColumns columns; - }; - - /// Settings - SortDescription description; - const size_t max_block_size; - UInt64 limit; - bool has_collation = false; - bool quiet = false; - - std::atomic have_all_inputs; - - MergedData merged_data; - - /// Used in Vertical merge algorithm to gather non-PK/non-index columns (on next step) - /// If it is not nullptr then it should be populated during execution - WriteBuffer * out_row_sources_buf = nullptr; - - /// Chunks currently being merged. - std::vector source_chunks; - - SortCursorImpls cursors; - - SortingHeap queue_without_collation; - SortingHeap queue_with_collation; - -private: - - /// Processor state. - bool is_initialized = false; - bool is_finished = false; - bool need_data = false; - size_t next_input_to_read = 0; - - template - void merge(TSortingHeap & queue); - - void insertFromChunk(size_t source_num); - - void updateCursor(Chunk chunk, size_t source_num) - { - auto num_rows = chunk.getNumRows(); - auto columns = chunk.detachColumns(); - for (auto & column : columns) - column = column->convertToFullColumnIfConst(); - - chunk.setColumns(std::move(columns), num_rows); - - auto & source_chunk = source_chunks[source_num]; - - if (source_chunk.empty()) - { - source_chunk = std::move(chunk); - cursors[source_num] = SortCursorImpl(source_chunk.getColumns(), description, source_num); - has_collation |= cursors[source_num].has_collation; - } - else - { - source_chunk = std::move(chunk); - cursors[source_num].reset(source_chunk.getColumns(), {}); - } - } -}; - -} diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index 6f4d36e125b..e56db6e3842 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -138,7 +138,7 @@ SortingTransform::SortingTransform( , max_merged_block_size(max_merged_block_size_) , limit(limit_) { - auto & sample = inputs.front().getHeader(); + const auto & sample = inputs.front().getHeader(); /// Replace column names to column position in sort_description. for (auto & column_description : description) @@ -368,7 +368,7 @@ void SortingTransform::enrichChunkWithConstants(Chunk & chunk) Columns column_with_constants; column_with_constants.reserve(num_result_columns); - auto & header = inputs.front().getHeader(); + const auto & header = inputs.front().getHeader(); size_t next_non_const_column = 0; for (size_t i = 0; i < num_result_columns; ++i) diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp index f9b6a56cdcb..6a21c5aabf0 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -21,7 +21,7 @@ void finalizeChunk(Chunk & chunk) auto columns = chunk.detachColumns(); for (auto & column : columns) - if (auto * agg_function = typeid_cast(column.get())) + if (const auto * agg_function = typeid_cast(column.get())) column = agg_function->convertToValues(); chunk.setColumns(std::move(columns), num_rows); @@ -122,11 +122,11 @@ void TotalsHavingTransform::transform(Chunk & chunk) /// Block with values not included in `max_rows_to_group_by`. We'll postpone it. if (overflow_row) { - auto & info = chunk.getChunkInfo(); + const auto & info = chunk.getChunkInfo(); if (!info) throw Exception("Chunk info was not set for chunk in TotalsHavingTransform.", ErrorCodes::LOGICAL_ERROR); - auto * agg_info = typeid_cast(info.get()); + const auto * agg_info = typeid_cast(info.get()); if (!agg_info) throw Exception("Chunk should have AggregatedChunkInfo in TotalsHavingTransform.", ErrorCodes::LOGICAL_ERROR); @@ -154,7 +154,7 @@ void TotalsHavingTransform::transform(Chunk & chunk) else { /// Compute the expression in HAVING. - auto & cur_header = final ? finalized_header : getInputPort().getHeader(); + const auto & cur_header = final ? finalized_header : getInputPort().getHeader(); auto finalized_block = cur_header.cloneWithColumns(finalized.detachColumns()); expression->execute(finalized_block); auto columns = finalized_block.getColumns(); diff --git a/src/Processors/tests/processors_test_merge.cpp b/src/Processors/tests/processors_test_merge.cpp index 41e4a591f41..11b0bfd1365 100644 --- a/src/Processors/tests/processors_test_merge.cpp +++ b/src/Processors/tests/processors_test_merge.cpp @@ -132,7 +132,7 @@ public: size_t ps = queue.top().second; queue.pop(); - auto & cur_col = chunks[ps].getColumns()[0]; + const auto & cur_col = chunks[ps].getColumns()[0]; col->insertFrom(*cur_col, positions[ps]); ++positions[ps]; diff --git a/src/Processors/tests/processors_test_merge_sorting_transform.cpp b/src/Processors/tests/processors_test_merge_sorting_transform.cpp index b9a0b0c4a5e..1a0c82f90d6 100644 --- a/src/Processors/tests/processors_test_merge_sorting_transform.cpp +++ b/src/Processors/tests/processors_test_merge_sorting_transform.cpp @@ -87,7 +87,7 @@ private: size_t rows = chunk.getNumRows(); UInt64 prev = current_number; - auto & col = chunk.getColumns().at(0); + const auto & col = chunk.getColumns().at(0); for (size_t row_num = 0; row_num < rows; ++row_num) { UInt64 val = col->getUInt(row_num); diff --git a/src/Processors/tests/processors_test_merging_sorted_transform.cpp b/src/Processors/tests/processors_test_merging_sorted_transform.cpp index 71d282585fd..477626d165d 100644 --- a/src/Processors/tests/processors_test_merging_sorted_transform.cpp +++ b/src/Processors/tests/processors_test_merging_sorted_transform.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Processors/ya.make b/src/Processors/ya.make new file mode 100644 index 00000000000..6469532a8e8 --- /dev/null +++ b/src/Processors/ya.make @@ -0,0 +1,133 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/msgpack + contrib/libs/protobuf_std +) + +SRCS( + Chunk.cpp + ConcatProcessor.cpp + DelayedPortsProcessor.cpp + Executors/ParallelPipelineExecutor.cpp + Executors/PipelineExecutor.cpp + Executors/SequentialPipelineExecutor.cpp + Executors/TreeExecutorBlockInputStream.cpp + ForkProcessor.cpp + Formats/IInputFormat.cpp + Formats/Impl/BinaryRowInputFormat.cpp + Formats/Impl/BinaryRowOutputFormat.cpp + Formats/Impl/ConstantExpressionTemplate.cpp + Formats/Impl/CSVRowInputFormat.cpp + Formats/Impl/CSVRowOutputFormat.cpp + Formats/Impl/JSONCompactEachRowRowInputFormat.cpp + Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp + Formats/Impl/JSONCompactRowOutputFormat.cpp + Formats/Impl/JSONEachRowRowInputFormat.cpp + Formats/Impl/JSONEachRowRowOutputFormat.cpp + Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp + Formats/Impl/JSONRowOutputFormat.cpp + Formats/Impl/MarkdownRowOutputFormat.cpp + Formats/Impl/MsgPackRowInputFormat.cpp + Formats/Impl/MsgPackRowOutputFormat.cpp + Formats/Impl/MySQLOutputFormat.cpp + Formats/Impl/NativeFormat.cpp + Formats/Impl/NullFormat.cpp + Formats/Impl/ODBCDriver2BlockOutputFormat.cpp + Formats/Impl/ODBCDriverBlockOutputFormat.cpp + Formats/Impl/PrettyBlockOutputFormat.cpp + Formats/Impl/PrettyCompactBlockOutputFormat.cpp + Formats/Impl/PrettySpaceBlockOutputFormat.cpp + Formats/Impl/ProtobufRowInputFormat.cpp + Formats/Impl/ProtobufRowOutputFormat.cpp + Formats/Impl/RegexpRowInputFormat.cpp + Formats/Impl/TabSeparatedRowInputFormat.cpp + Formats/Impl/TabSeparatedRowOutputFormat.cpp + Formats/Impl/TemplateBlockOutputFormat.cpp + Formats/Impl/TemplateRowInputFormat.cpp + Formats/Impl/TSKVRowInputFormat.cpp + Formats/Impl/TSKVRowOutputFormat.cpp + Formats/Impl/ValuesBlockInputFormat.cpp + Formats/Impl/ValuesRowOutputFormat.cpp + Formats/Impl/VerticalRowOutputFormat.cpp + Formats/Impl/XMLRowOutputFormat.cpp + Formats/IOutputFormat.cpp + Formats/IRowInputFormat.cpp + Formats/IRowOutputFormat.cpp + Formats/LazyOutputFormat.cpp + Formats/OutputStreamToOutputFormat.cpp + Formats/RowInputFormatWithDiagnosticInfo.cpp + IAccumulatingTransform.cpp + IInflatingTransform.cpp + IProcessor.cpp + ISimpleTransform.cpp + ISink.cpp + ISource.cpp + LimitTransform.cpp + Merges/AggregatingSortedTransform.h + Merges/Algorithms/AggregatingSortedAlgorithm.cpp + Merges/Algorithms/AggregatingSortedAlgorithm.h + Merges/Algorithms/CollapsingSortedAlgorithm.cpp + Merges/Algorithms/CollapsingSortedAlgorithm.h + Merges/Algorithms/FixedSizeDequeWithGaps.h + Merges/Algorithms/Graphite.h + Merges/Algorithms/GraphiteRollupSortedAlgorithm.cpp + Merges/Algorithms/GraphiteRollupSortedAlgorithm.h + Merges/Algorithms/IMergingAlgorithm.h + Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp + Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h + Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp + Merges/Algorithms/IMergingAlgorithmWithSharedChunks.h + Merges/Algorithms/MergedData.h + Merges/Algorithms/MergingSortedAlgorithm.cpp + Merges/Algorithms/MergingSortedAlgorithm.h + Merges/Algorithms/ReplacingSortedAlgorithm.cpp + Merges/Algorithms/ReplacingSortedAlgorithm.h + Merges/Algorithms/RowRef.h + Merges/Algorithms/SummingSortedAlgorithm.cpp + Merges/Algorithms/SummingSortedAlgorithm.h + Merges/Algorithms/VersionedCollapsingAlgorithm.cpp + Merges/Algorithms/VersionedCollapsingAlgorithm.h + Merges/CollapsingSortedTransform.h + Merges/GraphiteRollupSortedTransform.h + Merges/IMergingTransform.cpp + Merges/IMergingTransform.h + Merges/MergingSortedTransform.cpp + Merges/MergingSortedTransform.h + Merges/ReplacingSortedTransform.h + Merges/SummingSortedTransform.h + Merges/VersionedCollapsingTransform.h + Pipe.cpp + Port.cpp + QueryPipeline.cpp + ResizeProcessor.cpp + Sources/SinkToOutputStream.cpp + Sources/SourceFromInputStream.cpp + Sources/SourceWithProgress.cpp + Transforms/AddingMissedTransform.cpp + Transforms/AggregatingTransform.cpp + Transforms/ConvertingTransform.cpp + Transforms/CreatingSetsTransform.cpp + Transforms/CubeTransform.cpp + Transforms/DistinctTransform.cpp + Transforms/ExpressionTransform.cpp + Transforms/ExtremesTransform.cpp + Transforms/FillingTransform.cpp + Transforms/FilterTransform.cpp + Transforms/FinishSortingTransform.cpp + Transforms/InflatingExpressionTransform.cpp + Transforms/LimitByTransform.cpp + Transforms/LimitsCheckingTransform.cpp + Transforms/MaterializingTransform.cpp + Transforms/MergeSortingTransform.cpp + Transforms/MergingAggregatedMemoryEfficientTransform.cpp + Transforms/MergingAggregatedTransform.cpp + Transforms/PartialSortingTransform.cpp + Transforms/ReverseTransform.cpp + Transforms/RollupTransform.cpp + Transforms/SortingTransform.cpp + Transforms/TotalsHavingTransform.cpp +) + +END() diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 8eebaabf342..e2038c8c1db 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -455,7 +455,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata) const metadata.columns.rename(column_name, rename_to); RenameColumnData rename_data{column_name, rename_to}; RenameColumnVisitor rename_visitor(rename_data); - for (auto & column : metadata.columns) + for (const auto & column : metadata.columns) { metadata.columns.modify(column.name, [&](ColumnDescription & column_to_modify) { @@ -472,11 +472,21 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata) const throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); } -bool AlterCommand::isModifyingData() const +bool AlterCommand::isModifyingData(const StorageInMemoryMetadata & metadata) const { /// Possible change data representation on disk if (type == MODIFY_COLUMN) - return data_type != nullptr; + { + if (data_type == nullptr) + return false; + + /// It is allowed to ALTER data type to the same type as before. + for (const auto & column : metadata.columns.getAllPhysical()) + if (column.name == column_name) + return !column.type->equals(*data_type); + + return true; + } return type == ADD_COLUMN /// We need to change columns.txt in each part for MergeTree || type == DROP_COLUMN /// We need to change columns.txt in each part for MergeTree @@ -496,7 +506,7 @@ namespace /// The function works for Arrays and Nullables of the same structure. bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) { - if (from->getName() == to->getName()) + if (from->equals(*to)) return true; static const std::unordered_multimap ALLOWED_CONVERSIONS = @@ -723,7 +733,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con NameToNameMap renames_map; for (size_t i = 0; i < size(); ++i) { - auto & command = (*this)[i]; + const auto & command = (*this)[i]; const auto & column_name = command.column_name; if (command.type == AlterCommand::ADD_COLUMN) @@ -888,11 +898,11 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con validateColumnsDefaultsAndGetSampleBlock(default_expr_list, all_columns.getAll(), context); } -bool AlterCommands::isModifyingData() const +bool AlterCommands::isModifyingData(const StorageInMemoryMetadata & metadata) const { for (const auto & param : *this) { - if (param.isModifyingData()) + if (param.isModifyingData(metadata)) return true; } diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index be27ba6ac2b..4e40c1ee190 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -108,7 +108,7 @@ struct AlterCommand /// * column files (data and marks) /// * each part meta (columns.txt) /// in each part on disk (it's not lightweight alter). - bool isModifyingData() const; + bool isModifyingData(const StorageInMemoryMetadata & metadata) const; bool isRequireMutationStage(const StorageInMemoryMetadata & metadata) const; @@ -151,7 +151,7 @@ public: void apply(StorageInMemoryMetadata & metadata) const; /// At least one command modify data on disk. - bool isModifyingData() const; + bool isModifyingData(const StorageInMemoryMetadata & metadata) const; /// At least one command modify settings. bool isSettingsAlter() const; diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 76a55d059ee..74613fb3eeb 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -23,11 +23,13 @@ #include #include #include +#include #include #include #include #include + namespace DB { @@ -102,7 +104,7 @@ void ColumnDescription::readText(ReadBuffer & buf) ParserColumnDeclaration column_parser(/* require type */ true); String column_line; readEscapedStringUntilEOL(column_line, buf); - ASTPtr ast = parseQuery(column_parser, column_line, "column parser", 0); + ASTPtr ast = parseQuery(column_parser, column_line, "column parser", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (const auto * col_ast = ast->as()) { name = col_ast->name; @@ -452,7 +454,7 @@ Block validateColumnsDefaultsAndGetSampleBlock(ASTPtr default_expr_list, const N { auto syntax_analyzer_result = SyntaxAnalyzer(context).analyze(default_expr_list, all_columns); const auto actions = ExpressionAnalyzer(default_expr_list, syntax_analyzer_result, context).getActions(true); - for (auto & action : actions->getActions()) + for (const auto & action : actions->getActions()) if (action.type == ExpressionAction::Type::JOIN || action.type == ExpressionAction::Type::ARRAY_JOIN) throw Exception("Unsupported default value that requires ARRAY JOIN or JOIN action", ErrorCodes::THERE_IS_NO_DEFAULT_VALUE); diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index 72399a70128..4fbb041d124 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -61,6 +61,7 @@ public: /// TODO add ability to rename nested columns void rename(const String & column_from, const String & column_to); + /// NOTE Must correspond with Nested::flatten function. void flattenNested(); /// TODO: remove, insert already flattened Nested columns. bool operator==(const ColumnsDescription & other) const { return columns == other.columns; } diff --git a/src/Storages/ConstraintsDescription.cpp b/src/Storages/ConstraintsDescription.cpp index ad0cd76733a..8c38af0cd5e 100644 --- a/src/Storages/ConstraintsDescription.cpp +++ b/src/Storages/ConstraintsDescription.cpp @@ -5,6 +5,8 @@ #include #include +#include + namespace DB { @@ -28,7 +30,7 @@ ConstraintsDescription ConstraintsDescription::parse(const String & str) ConstraintsDescription res; ParserConstraintDeclarationList parser; - ASTPtr list = parseQuery(parser, str, 0); + ASTPtr list = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); for (const auto & constraint : list->children) res.constraints.push_back(std::dynamic_pointer_cast(constraint)); diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 01bf0798a63..e937d5e8a90 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -78,7 +77,7 @@ namespace StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( - StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_) + StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool_) /// It's important to initialize members before `thread` to avoid race. : storage(storage_) , pool(std::move(pool_)) @@ -92,7 +91,10 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( , max_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()} , log{&Logger::get(getLoggerName())} , monitor_blocker(monitor_blocker_) + , bg_pool(bg_pool_) { + task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); }); + task_handle->activateAndSchedule(); } @@ -100,12 +102,9 @@ StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor() { if (!quit) { - { - quit = true; - std::lock_guard lock{mutex}; - } + quit = true; cond.notify_one(); - thread.join(); + task_handle->deactivate(); } } @@ -122,12 +121,9 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() { if (!quit) { - { - quit = true; - std::lock_guard lock{mutex}; - } + quit = true; cond.notify_one(); - thread.join(); + task_handle->deactivate(); } Poco::File(path).remove(true); @@ -136,16 +132,11 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() void StorageDistributedDirectoryMonitor::run() { - setThreadName("DistrDirMonitor"); - std::unique_lock lock{mutex}; - const auto quit_requested = [this] { return quit.load(std::memory_order_relaxed); }; - - while (!quit_requested()) + while (!quit) { - auto do_sleep = true; - + bool do_sleep = true; if (!monitor_blocker.isCancelled()) { try @@ -167,15 +158,25 @@ void StorageDistributedDirectoryMonitor::run() LOG_DEBUG(log, "Skipping send data over distributed table."); } - if (do_sleep) - cond.wait_for(lock, sleep_time, quit_requested); - const auto now = std::chrono::system_clock::now(); if (now - last_decrease_time > decrease_error_count_period) { error_count /= 2; last_decrease_time = now; } + + if (do_sleep) + break; + } + + if (!quit) + { + /// If there is no error, then it will be scheduled by the DistributedBlockOutputStream, + /// so this is just in case, hence it is distributed_directory_monitor_max_sleep_time_ms + if (error_count) + task_handle->scheduleAfter(sleep_time.count()); + else + task_handle->scheduleAfter(max_sleep_time.count()); } } @@ -586,6 +587,13 @@ BlockInputStreamPtr StorageDistributedDirectoryMonitor::createStreamFromFile(con return std::make_shared(file_name); } +bool StorageDistributedDirectoryMonitor::scheduleAfter(size_t ms) +{ + if (quit) + return false; + return task_handle->scheduleAfter(ms); +} + void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map & files) { std::unordered_set file_indices_to_skip; @@ -714,8 +722,13 @@ std::string StorageDistributedDirectoryMonitor::getLoggerName() const void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_path) { std::lock_guard lock{mutex}; + + task_handle->deactivate(); + path = new_path; current_batch_file_path = path + "current_batch.txt"; + + task_handle->activateAndSchedule(); } } diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h index 475a3bc7bc6..61d51e5acfd 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DirectoryMonitor.h @@ -1,10 +1,9 @@ #pragma once #include -#include +#include #include -#include #include #include #include @@ -20,7 +19,7 @@ class StorageDistributedDirectoryMonitor { public: StorageDistributedDirectoryMonitor( - StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_); + StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool_); ~StorageDistributedDirectoryMonitor(); @@ -33,6 +32,9 @@ public: void shutdownAndDropAllData(); static BlockInputStreamPtr createStreamFromFile(const String & file_name); + + /// For scheduling via DistributedBlockOutputStream + bool scheduleAfter(size_t ms); private: void run(); bool processFiles(); @@ -67,7 +69,9 @@ private: std::condition_variable cond; Logger * log; ActionBlocker & monitor_blocker; - ThreadFromGlobalPool thread{&StorageDistributedDirectoryMonitor::run, this}; + + BackgroundSchedulePool & bg_pool; + BackgroundSchedulePoolTaskHolder task_handle; /// Read insert query and insert settings for backward compatible. static void readHeader(ReadBuffer & in, Settings & insert_settings, std::string & insert_query, ClientInfo & client_info, Logger * log); diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index af17a026927..e58876795eb 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -59,6 +61,26 @@ namespace ErrorCodes extern const int CANNOT_LINK; } +static void writeBlockConvert(const BlockOutputStreamPtr & out, const Block & block, const size_t repeats) +{ + if (!blocksHaveEqualStructure(out->getHeader(), block)) + { + ConvertingBlockInputStream convert( + std::make_shared(block), + out->getHeader(), + ConvertingBlockInputStream::MatchColumnsMode::Name); + auto adopted_block = convert.read(); + + for (size_t i = 0; i < repeats; ++i) + out->write(adopted_block); + } + else + { + for (size_t i = 0; i < repeats; ++i) + out->write(block); + } +} + DistributedBlockOutputStream::DistributedBlockOutputStream( const Context & context_, StorageDistributed & storage_, const ASTPtr & query_ast_, const ClusterPtr & cluster_, @@ -242,7 +264,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp for (size_t j = 0; j < current_block.columns(); ++j) { - auto & src_column = current_block.getByPosition(j).column; + const auto & src_column = current_block.getByPosition(j).column; auto & dst_column = job.current_shard_block.getByPosition(j).column; /// Zero permutation size has special meaning in IColumn::permute @@ -306,14 +328,12 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp InterpreterInsertQuery interp(query_ast, *job.local_context); auto block_io = interp.execute(); - assertBlocksHaveEqualStructure(block_io.out->getHeader(), shard_block, "flushing shard block for " + storage.getStorageID().getNameForLogs()); + job.stream = block_io.out; job.stream->writePrefix(); } - size_t num_repetitions = shard_info.getLocalNodeCount(); - for (size_t i = 0; i < num_repetitions; ++i) - job.stream->write(shard_block); + writeBlockConvert(job.stream, shard_block, shard_info.getLocalNodeCount()); } job.blocks_written += 1; @@ -547,13 +567,8 @@ void DistributedBlockOutputStream::writeToLocal(const Block & block, const size_ auto block_io = interp.execute(); - assertBlocksHaveEqualStructure(block_io.out->getHeader(), block, "flushing " + storage.getStorageID().getNameForLogs()); - block_io.out->writePrefix(); - - for (size_t i = 0; i < repeats; ++i) - block_io.out->write(block); - + writeBlockConvert(block_io.out, block, repeats); block_io.out->writeSuffix(); } @@ -574,8 +589,8 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: const std::string path(disk + data_path + dir_name + '/'); /// ensure shard subdirectory creation and notify storage - if (Poco::File(path).createDirectory()) - storage.requireDirectoryMonitor(disk, dir_name); + Poco::File(path).createDirectory(); + auto & directory_monitor = storage.requireDirectoryMonitor(disk, dir_name); const auto & file_name = toString(storage.file_names_increment.get()) + ".bin"; const auto & block_file_path = path + file_name; @@ -617,6 +632,9 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: stream.writePrefix(); stream.write(block); stream.writeSuffix(); + + auto sleep_ms = context.getSettingsRef().distributed_directory_monitor_sleep_time_ms; + directory_monitor.scheduleAfter(sleep_ms.totalMilliseconds()); } if (link(first_file_tmp_path.data(), block_file_path.data())) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 4d916ca1b46..91065fbf893 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include #include @@ -9,9 +12,6 @@ #include -#include -#include - namespace DB { @@ -28,6 +28,7 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int TABLE_IS_DROPPED; extern const int NOT_IMPLEMENTED; + extern const int DEADLOCK_AVOIDED; } IStorage::IStorage(StorageID storage_id_, ColumnsDescription virtuals_) : storage_id(std::move(storage_id_)), virtuals(std::move(virtuals_)) @@ -125,8 +126,13 @@ Block IStorage::getSampleBlockForColumns(const Names & column_names) const namespace { - using NamesAndTypesMap = ::google::dense_hash_map; - using UniqueStrings = ::google::dense_hash_set; +#if !defined(ARCADIA_BUILD) + using NamesAndTypesMap = google::dense_hash_map; + using UniqueStrings = google::dense_hash_set; +#else + using NamesAndTypesMap = google::sparsehash::dense_hash_map; + using UniqueStrings = google::sparsehash::dense_hash_set; +#endif String listOfColumns(const NamesAndTypesList & available_columns) { @@ -314,48 +320,64 @@ bool IStorage::isVirtualColumn(const String & column_name) const return getColumns().get(column_name).is_virtual; } -TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id) +RWLockImpl::LockHolder IStorage::tryLockTimed( + const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const +{ + auto lock_holder = rwlock->getLock(type, query_id, std::chrono::milliseconds(acquire_timeout.totalMilliseconds())); + if (!lock_holder) + { + const String type_str = type == RWLockImpl::Type::Read ? "READ" : "WRITE"; + throw Exception( + type_str + " locking attempt on \"" + getStorageID().getFullTableName() + + "\" has timed out! (" + toString(acquire_timeout.totalMilliseconds()) + "ms) " + "Possible deadlock avoided. Client should retry.", + ErrorCodes::DEADLOCK_AVOIDED); + } + return lock_holder; +} + +TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id, const SettingSeconds & acquire_timeout) { TableStructureReadLockHolder result; if (will_add_new_data) - result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Read, query_id); - result.structure_lock = structure_lock->getLock(RWLockImpl::Read, query_id); + result.new_data_structure_lock = tryLockTimed(new_data_structure_lock, RWLockImpl::Read, query_id, acquire_timeout); + result.structure_lock = tryLockTimed(structure_lock, RWLockImpl::Read, query_id, acquire_timeout); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); return result; } -TableStructureWriteLockHolder IStorage::lockAlterIntention(const String & query_id) +TableStructureWriteLockHolder IStorage::lockAlterIntention(const String & query_id, const SettingSeconds & acquire_timeout) { TableStructureWriteLockHolder result; - result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id); + result.alter_intention_lock = tryLockTimed(alter_intention_lock, RWLockImpl::Write, query_id, acquire_timeout); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); return result; } -void IStorage::lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id) +void IStorage::lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id, const SettingSeconds & acquire_timeout) { if (!lock_holder.alter_intention_lock) throw Exception("Alter intention lock for table " + getStorageID().getNameForLogs() + " was not taken. This is a bug.", ErrorCodes::LOGICAL_ERROR); if (!lock_holder.new_data_structure_lock) - lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); - lock_holder.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id); + lock_holder.new_data_structure_lock = tryLockTimed(new_data_structure_lock, RWLockImpl::Write, query_id, acquire_timeout); + lock_holder.structure_lock = tryLockTimed(structure_lock, RWLockImpl::Write, query_id, acquire_timeout); } -TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id) +TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id, const SettingSeconds & acquire_timeout) { TableStructureWriteLockHolder result; - result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id); + result.alter_intention_lock = tryLockTimed(alter_intention_lock, RWLockImpl::Write, query_id, acquire_timeout); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); - result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); - result.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id); + result.new_data_structure_lock = tryLockTimed(new_data_structure_lock, RWLockImpl::Write, query_id, acquire_timeout); + result.structure_lock = tryLockTimed(structure_lock, RWLockImpl::Write, query_id, acquire_timeout); return result; } @@ -370,7 +392,7 @@ void IStorage::alter( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 4cdfb3b29a3..5a7f71828ef 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -195,22 +195,26 @@ private: IndicesDescription indices; ConstraintsDescription constraints; +private: + RWLockImpl::LockHolder tryLockTimed( + const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const; + public: /// Acquire this lock if you need the table structure to remain constant during the execution of /// the query. If will_add_new_data is true, this means that the query will add new data to the table /// (INSERT or a parts merge). - TableStructureReadLockHolder lockStructureForShare(bool will_add_new_data, const String & query_id); + TableStructureReadLockHolder lockStructureForShare(bool will_add_new_data, const String & query_id, const SettingSeconds & acquire_timeout); /// Acquire this lock at the start of ALTER to lock out other ALTERs and make sure that only you /// can modify the table structure. It can later be upgraded to the exclusive lock. - TableStructureWriteLockHolder lockAlterIntention(const String & query_id); + TableStructureWriteLockHolder lockAlterIntention(const String & query_id, const SettingSeconds & acquire_timeout); /// Upgrade alter intention lock to the full exclusive structure lock. This is done by ALTER queries /// to ensure that no other query uses the table structure and it can be safely changed. - void lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id); + void lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id, const SettingSeconds & acquire_timeout); /// Acquire the full exclusive lock immediately. No other queries can run concurrently. - TableStructureWriteLockHolder lockExclusively(const String & query_id); + TableStructureWriteLockHolder lockExclusively(const String & query_id, const SettingSeconds & acquire_timeout); /** Returns stage to which query is going to be processed in read() function. * (Normally, the function only reads the columns from the list, but in other cases, diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index a5772a835bf..2363e7924ba 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -5,6 +5,9 @@ #include #include +#include + + namespace DB { @@ -42,7 +45,7 @@ IndicesDescription IndicesDescription::parse(const String & str) IndicesDescription res; ParserIndexDeclarationList parser; - ASTPtr list = parseQuery(parser, str, 0); + ASTPtr list = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); for (const auto & index : list->children) res.indices.push_back(std::dynamic_pointer_cast(index)); diff --git a/src/Storages/Kafka/KafkaBlockInputStream.cpp b/src/Storages/Kafka/KafkaBlockInputStream.cpp index 39f71912f8e..18f7e696419 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -190,7 +190,6 @@ Block KafkaBlockInputStream::readImpl() result_block.insert(column); return ConvertingBlockInputStream( - context, std::make_shared(result_block), getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Name) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index b65ff28e31d..62feea8fe34 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -335,7 +335,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() } // XXX: very fishy place with const casting. - auto new_position = reinterpret_cast(const_cast(current->get_payload().get_data())); + auto * new_position = reinterpret_cast(const_cast(current->get_payload().get_data())); BufferBase::set(new_position, current->get_payload().get_size(), 0); allowed = false; diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 049110a3294..569e5c24e1c 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -519,7 +519,7 @@ void StorageLiveView::drop(TableStructureWriteLockHolder &) void StorageLiveView::refresh(const Context & context) { - auto alter_lock = lockAlterIntention(context.getCurrentQueryId()); + auto alter_lock = lockAlterIntention(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); { std::lock_guard lock(mutex); if (getNewBlocks()) diff --git a/src/Storages/MergeTree/BackgroundProcessingPool.cpp b/src/Storages/MergeTree/BackgroundProcessingPool.cpp index d922d9bd302..58018324158 100644 --- a/src/Storages/MergeTree/BackgroundProcessingPool.cpp +++ b/src/Storages/MergeTree/BackgroundProcessingPool.cpp @@ -137,7 +137,7 @@ void BackgroundProcessingPool::threadFunction() } SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); - if (auto memory_tracker = CurrentThread::getMemoryTracker()) + if (auto * memory_tracker = CurrentThread::getMemoryTracker()) memory_tracker->setMetric(settings.memory_metric); pcg64 rng(randomSeed()); diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 6373c85a15d..70c06efff5c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -85,7 +85,8 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo try { - auto storage_lock = data.lockStructureForShare(false, RWLockImpl::NO_QUERY); + auto storage_lock = data.lockStructureForShare( + false, RWLockImpl::NO_QUERY, data.getSettings()->lock_acquire_timeout_for_background_operations); MergeTreeData::DataPartPtr part = findPart(part_name); @@ -254,23 +255,23 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( const ReservationPtr reservation, PooledReadWriteBufferFromHTTP & in) { - size_t files; readBinary(files, in); + auto disk = reservation->getDisk(); + static const String TMP_PREFIX = "tmp_fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; - String relative_part_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name; - String absolute_part_path = Poco::Path(data.getFullPathOnDisk(reservation->getDisk()) + relative_part_path + "/").absolute().toString(); - Poco::File part_file(absolute_part_path); + String part_relative_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name; + String part_download_path = data.getRelativeDataPath() + part_relative_path + "/"; - if (part_file.exists()) - throw Exception("Directory " + absolute_part_path + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + if (disk->exists(part_download_path)) + throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; - part_file.createDirectory(); + disk->createDirectories(part_download_path); MergeTreeData::DataPart::Checksums checksums; for (size_t i = 0; i < files; ++i) @@ -283,21 +284,21 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( /// File must be inside "absolute_part_path" directory. /// Otherwise malicious ClickHouse replica may force us to write to arbitrary path. - String absolute_file_path = Poco::Path(absolute_part_path + file_name).absolute().toString(); - if (!startsWith(absolute_file_path, absolute_part_path)) - throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + absolute_part_path + ")." + String absolute_file_path = Poco::Path(part_download_path + file_name).absolute().toString(); + if (!startsWith(absolute_file_path, Poco::Path(part_download_path).absolute().toString())) + throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")." " This may happen if we are trying to download part from malicious replica or logical error.", ErrorCodes::INSECURE_PATH); - WriteBufferFromFile file_out(absolute_file_path); - HashingWriteBuffer hashing_out(file_out); + auto file_out = disk->writeFile(part_download_path + file_name); + HashingWriteBuffer hashing_out(*file_out); copyData(in, hashing_out, file_size, blocker.getCounter()); if (blocker.isCancelled()) { /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, performing a poll with a not very large timeout. /// And now we check it only between read chunks (in the `copyData` function). - part_file.remove(true); + disk->removeRecursive(part_download_path); throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); } @@ -305,7 +306,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( readPODBinary(expected_hash, in); if (expected_hash != hashing_out.getHash()) - throw Exception("Checksum mismatch for file " + absolute_part_path + file_name + " transferred from " + replica_path, + throw Exception("Checksum mismatch for file " + fullPath(disk, part_download_path + file_name) + " transferred from " + replica_path, ErrorCodes::CHECKSUM_DOESNT_MATCH); if (file_name != "checksums.txt" && @@ -315,7 +316,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( assertEOF(in); - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, reservation->getDisk(), relative_part_path); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, reservation->getDisk(), part_relative_path); new_data_part->is_temp = true; new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 3e0caa67518..a5a7f1f4494 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -98,8 +98,8 @@ void IMergeTreeDataPart::MinMaxIndex::update(const Block & block, const Names & for (size_t i = 0; i < column_names.size(); ++i) { - Field min_value; - Field max_value; + FieldRef min_value; + FieldRef max_value; const ColumnWithTypeAndName & column = block.getByName(column_names[i]); column.column->getExtremes(min_value, max_value); @@ -731,35 +731,43 @@ void IMergeTreeDataPart::remove() const return; } - try + if (checksums.empty()) { - /// Remove each expected file in directory, then remove directory itself. - -#if !__clang__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-variable" -#endif - for (const auto & [file, _] : checksums.files) - disk->remove(to + "/" + file); -#if !__clang__ -# pragma GCC diagnostic pop -#endif - - for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->remove(to + "/" + file); - disk->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_PATH); - - disk->remove(to); - } - catch (...) - { - /// Recursive directory removal does many excessive "stat" syscalls under the hood. - - LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(disk, to) << " by removing files; fallback to recursive removal. Reason: " - << getCurrentExceptionMessage(false)); - + /// If the part is not completely written, we cannot use fast path by listing files. disk->removeRecursive(to + "/"); } + else + { + try + { + /// Remove each expected file in directory, then remove directory itself. + + #if !__clang__ + # pragma GCC diagnostic push + # pragma GCC diagnostic ignored "-Wunused-variable" + #endif + for (const auto & [file, _] : checksums.files) + disk->remove(to + "/" + file); + #if !__clang__ + # pragma GCC diagnostic pop + #endif + + for (const auto & file : {"checksums.txt", "columns.txt"}) + disk->remove(to + "/" + file); + disk->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_PATH); + + disk->remove(to); + } + catch (...) + { + /// Recursive directory removal does many excessive "stat" syscalls under the hood. + + LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(disk, to) << " by removing files; fallback to recursive removal. Reason: " + << getCurrentExceptionMessage(false)); + + disk->removeRecursive(to + "/"); + } + } } String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) const diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 8187799f4be..76e84cb620a 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -21,7 +21,7 @@ void IMergeTreeDataPartWriter::Stream::finalize() marks.next(); } -void IMergeTreeDataPartWriter::Stream::sync() +void IMergeTreeDataPartWriter::Stream::sync() const { plain_file->sync(); marks_file->sync(); diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 4eb842f9279..9679de8297f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -53,7 +53,7 @@ public: void finalize(); - void sync(); + void sync() const; void addToChecksums(IMergeTreeDataPart::Checksums & checksums); }; @@ -102,13 +102,12 @@ public: written_offset_columns = written_offset_columns_; } - using SkipIndices = std::vector; - const SkipIndices & getSkipIndices() { return skip_indices; } + const MergeTreeIndices & getSkipIndices() { return skip_indices; } void initSkipIndices(); void initPrimaryIndex(); - virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) = 0; + virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums) = 0; void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums); void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); @@ -126,7 +125,7 @@ protected: CompressionCodecPtr default_codec; - std::vector skip_indices; + MergeTreeIndices skip_indices; MergeTreeWriterSettings settings; diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 4978aeaaa58..8243983d837 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -32,6 +32,8 @@ IMergeTreeReader::IMergeTreeReader(const MergeTreeData::DataPartPtr & data_part_ , all_mark_ranges(all_mark_ranges_) , alter_conversions(storage.getAlterConversionsForPart(data_part)) { + for (const NameAndTypePair & column_from_part : data_part->getColumns()) + columns_from_part[column_from_part.name] = column_from_part.type; } IMergeTreeReader::~IMergeTreeReader() = default; @@ -183,6 +185,23 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns } } +NameAndTypePair IMergeTreeReader::getColumnFromPart(const NameAndTypePair & required_column) const +{ + auto it = columns_from_part.find(required_column.name); + if (it != columns_from_part.end()) + return {it->first, it->second}; + + if (alter_conversions.isColumnRenamed(required_column.name)) + { + String old_name = alter_conversions.getColumnOldName(required_column.name); + it = columns_from_part.find(old_name); + if (it != columns_from_part.end()) + return {it->first, it->second}; + } + + return required_column; +} + void IMergeTreeReader::performRequiredConversions(Columns & res_columns) { try @@ -209,10 +228,7 @@ void IMergeTreeReader::performRequiredConversions(Columns & res_columns) if (res_columns[pos] == nullptr) continue; - if (columns_from_part.count(name_and_type->name)) - copy_block.insert({res_columns[pos], columns_from_part[name_and_type->name], name_and_type->name}); - else - copy_block.insert({res_columns[pos], name_and_type->type, name_and_type->name}); + copy_block.insert({res_columns[pos], getColumnFromPart(*name_and_type).type, name_and_type->name}); } DB::performRequiredConversions(copy_block, columns, storage.global_context); diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index 622e11dae8b..02d8f67f9d0 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -4,7 +4,6 @@ #include #include - namespace DB { @@ -59,6 +58,9 @@ public: MergeTreeData::DataPartPtr data_part; protected: + /// Returns actual column type in part, which can differ from table metadata. + NameAndTypePair getColumnFromPart(const NameAndTypePair & required_column) const; + /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk @@ -67,8 +69,6 @@ protected: /// Columns that are read. NamesAndTypesList columns; - std::unordered_map columns_from_part; - UncompressedCache * uncompressed_cache; MarkCache * mark_cache; @@ -78,8 +78,13 @@ protected: MarkRanges all_mark_ranges; friend class MergeTreeRangeReader::DelayedStream; + +private: /// Alter conversions, which must be applied on fly if required MergeTreeData::AlterConversions alter_conversions; + + /// Actual data type of columns in part + std::unordered_map columns_from_part; }; } diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.h b/src/Storages/MergeTree/IMergedBlockOutputStream.h index ba04d7fa71b..7b808ef6784 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -25,7 +25,7 @@ public: protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; - IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); + IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns); /// Remove all columns marked expired in data_part. Also, clears checksums /// and columns array. Return set of removed files names. diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index e994d254958..24f460cab64 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -318,7 +318,7 @@ ASTPtr cloneASTWithInversionPushDown(const ASTPtr node, const bool need_inversio return result_node; } - const auto cloned_node = node->clone(); + auto cloned_node = node->clone(); if (func && inverse_relations.find(func->name) != inverse_relations.cend()) { @@ -338,44 +338,6 @@ inline bool Range::equals(const Field & lhs, const Field & rhs) { return applyVi inline bool Range::less(const Field & lhs, const Field & rhs) { return applyVisitor(FieldVisitorAccurateLess(), lhs, rhs); } -FieldWithInfinity::FieldWithInfinity(const Field & field_) - : field(field_), - type(Type::NORMAL) -{ -} - -FieldWithInfinity::FieldWithInfinity(Field && field_) - : field(std::move(field_)), - type(Type::NORMAL) -{ -} - -FieldWithInfinity::FieldWithInfinity(const Type type_) - : type(type_) -{ -} - -FieldWithInfinity FieldWithInfinity::getMinusInfinity() -{ - return FieldWithInfinity(Type::MINUS_INFINITY); -} - -FieldWithInfinity FieldWithInfinity::getPlusInfinity() -{ - return FieldWithInfinity(Type::PLUS_INFINITY); -} - -bool FieldWithInfinity::operator<(const FieldWithInfinity & other) const -{ - return type < other.type || (type == other.type && type == Type::NORMAL && field < other.field); -} - -bool FieldWithInfinity::operator==(const FieldWithInfinity & other) const -{ - return type == other.type && (type != Type::NORMAL || field == other.field); -} - - /** Calculate expressions, that depend only on constants. * For index to work when something like "WHERE Date = toDate(now())" is written. */ @@ -480,24 +442,41 @@ bool KeyCondition::getConstant(const ASTPtr & expr, Block & block_with_constants } -static void applyFunction( +static Field applyFunctionForField( const FunctionBasePtr & func, - const DataTypePtr & arg_type, const Field & arg_value, - DataTypePtr & res_type, Field & res_value) + const DataTypePtr & arg_type, + const Field & arg_value) { - res_type = func->getReturnType(); - Block block { { arg_type->createColumnConst(1, arg_value), arg_type, "x" }, - { nullptr, res_type, "y" } + { nullptr, func->getReturnType(), "y" } }; func->execute(block, {0}, 1, 1); - - block.safeGetByPosition(1).column->get(0, res_value); + return (*block.safeGetByPosition(1).column)[0]; } +static FieldRef applyFunction(FunctionBasePtr & func, const DataTypePtr & current_type, const FieldRef & field) +{ + /// Fallback for fields without block reference. + if (field.isExplicit()) + return applyFunctionForField(func, current_type, field); + + String result_name = "_" + func->getName() + "_" + toString(field.column_idx); + size_t result_idx; + const auto & block = field.block; + if (!block->has(result_name)) + { + result_idx = block->columns(); + field.block->insert({nullptr, func->getReturnType(), result_name}); + func->execute(*block, {field.column_idx}, result_idx, block->rows()); + } + else + result_idx = block->getPositionByName(result_name); + + return {field.block, field.row_idx, result_idx}; +} void KeyCondition::traverseAST(const ASTPtr & node, const Context & context, Block & block_with_constants) { @@ -569,12 +548,8 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( return false; // Apply the next transformation step - DataTypePtr new_type; - applyFunction(a.function_base, out_type, out_value, new_type, out_value); - if (!new_type) - return false; - - out_type.swap(new_type); + out_value = applyFunctionForField(a.function_base, out_type, out_value); + out_type = a.function_base->getReturnType(); expr_name = a.result_name; // Transformation results in a key expression, accept @@ -957,8 +932,8 @@ String KeyCondition::toString() const template static BoolMask forAnyHyperrectangle( size_t key_size, - const Field * key_left, - const Field * key_right, + const FieldRef * key_left, + const FieldRef * key_right, bool left_bounded, bool right_bounded, std::vector & hyperrectangle, @@ -1049,8 +1024,8 @@ static BoolMask forAnyHyperrectangle( BoolMask KeyCondition::checkInRange( size_t used_key_size, - const Field * left_key, - const Field * right_key, + const FieldRef * left_key, + const FieldRef * right_key, const DataTypes & data_types, bool right_bounded, BoolMask initial_mask) const @@ -1102,19 +1077,12 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( return {}; } - /// Apply the function. - DataTypePtr new_type; if (!key_range.left.isNull()) - applyFunction(func, current_type, key_range.left, new_type, key_range.left); + key_range.left = applyFunction(func, current_type, key_range.left); if (!key_range.right.isNull()) - applyFunction(func, current_type, key_range.right, new_type, key_range.right); + key_range.right = applyFunction(func, current_type, key_range.right); - if (!new_type) - { - return {}; - } - - current_type.swap(new_type); + current_type = func->getReturnType(); if (!monotonicity.is_positive) key_range.swapLeftAndRight(); @@ -1220,8 +1188,8 @@ BoolMask KeyCondition::checkInHyperrectangle( BoolMask KeyCondition::checkInRange( size_t used_key_size, - const Field * left_key, - const Field * right_key, + const FieldRef * left_key, + const FieldRef * right_key, const DataTypes & data_types, BoolMask initial_mask) const { @@ -1231,8 +1199,8 @@ BoolMask KeyCondition::checkInRange( bool KeyCondition::mayBeTrueInRange( size_t used_key_size, - const Field * left_key, - const Field * right_key, + const FieldRef * left_key, + const FieldRef * right_key, const DataTypes & data_types) const { return checkInRange(used_key_size, left_key, right_key, data_types, true, BoolMask::consider_only_can_be_true).can_be_true; @@ -1241,7 +1209,7 @@ bool KeyCondition::mayBeTrueInRange( BoolMask KeyCondition::checkAfter( size_t used_key_size, - const Field * left_key, + const FieldRef * left_key, const DataTypes & data_types, BoolMask initial_mask) const { @@ -1251,7 +1219,7 @@ BoolMask KeyCondition::checkAfter( bool KeyCondition::mayBeTrueAfter( size_t used_key_size, - const Field * left_key, + const FieldRef * left_key, const DataTypes & data_types) const { return checkInRange(used_key_size, left_key, nullptr, data_types, false, BoolMask::consider_only_can_be_true).can_be_true; @@ -1382,4 +1350,13 @@ size_t KeyCondition::getMaxKeyColumn() const return res; } +bool KeyCondition::hasMonotonicFunctionsChain() const +{ + for (const auto & element : rpn) + if (!element.monotonic_functions_chain.empty() + || (element.set_index && element.set_index->hasMonotonicFunctionsChain())) + return true; + return false; +} + } diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 8667e0aea27..3a3768f0e4c 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -15,10 +15,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_TYPE_OF_FIELD; -} class IFunction; using FunctionBasePtr = std::shared_ptr; @@ -26,6 +22,33 @@ using FunctionBasePtr = std::shared_ptr; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; +/** A field, that can be stored in two reperesenations: + * - A standalone field. + * - A field with reference to its position in a block. + * It's needed for execution of functions on ranges during + * index analysis. If function was executed once for field, + * its result would be cached for whole block for which field's reference points to. + */ +struct FieldRef : public Field +{ + FieldRef() = default; + + /// Create as explicit field without block. + template + FieldRef(T && value) : Field(std::forward(value)) {} + + /// Create as reference to field in block. + FieldRef(Block * block_, size_t row_idx_, size_t column_idx_) + : Field((*block_->getByPosition(column_idx_).column)[row_idx_]), + block(block_), row_idx(row_idx_), column_idx(column_idx_) {} + + bool isExplicit() const { return block == nullptr; } + + Block * block = nullptr; + size_t row_idx = 0; + size_t column_idx = 0; +}; + /** Range with open or closed ends; possibly unbounded. */ struct Range @@ -35,8 +58,8 @@ private: static bool less(const Field & lhs, const Field & rhs); public: - Field left; /// the left border, if any - Field right; /// the right border, if any + FieldRef left; /// the left border, if any + FieldRef right; /// the right border, if any bool left_bounded = false; /// bounded at the left bool right_bounded = false; /// bounded at the right bool left_included = false; /// includes the left border, if any @@ -46,11 +69,11 @@ public: Range() {} /// One point. - Range(const Field & point) + Range(const FieldRef & point) : left(point), right(point), left_bounded(true), right_bounded(true), left_included(true), right_included(true) {} /// A bounded two-sided range. - Range(const Field & left_, bool left_included_, const Field & right_, bool right_included_) + Range(const FieldRef & left_, bool left_included_, const FieldRef & right_, bool right_included_) : left(left_), right(right_), left_bounded(true), right_bounded(true), left_included(left_included_), right_included(right_included_) @@ -58,7 +81,7 @@ public: shrinkToIncludedIfPossible(); } - static Range createRightBounded(const Field & right_point, bool right_included) + static Range createRightBounded(const FieldRef & right_point, bool right_included) { Range r; r.right = right_point; @@ -68,7 +91,7 @@ public: return r; } - static Range createLeftBounded(const Field & left_point, bool left_included) + static Range createLeftBounded(const FieldRef & left_point, bool left_included) { Range r; r.left = left_point; @@ -84,7 +107,7 @@ public: */ void shrinkToIncludedIfPossible() { - if (left_bounded && !left_included) + if (left.isExplicit() && left_bounded && !left_included) { if (left.getType() == Field::Types::UInt64 && left.get() != std::numeric_limits::max()) { @@ -97,7 +120,7 @@ public: left_included = true; } } - if (right_bounded && !right_included) + if (right.isExplicit() && right_bounded && !right_included) { if (right.getType() == Field::Types::UInt64 && right.get() != std::numeric_limits::min()) { @@ -120,13 +143,13 @@ public: } /// x contained in the range - bool contains(const Field & x) const + bool contains(const FieldRef & x) const { return !leftThan(x) && !rightThan(x); } /// x is to the left - bool rightThan(const Field & x) const + bool rightThan(const FieldRef & x) const { return (left_bounded ? !(less(left, x) || (left_included && equals(x, left))) @@ -134,7 +157,7 @@ public: } /// x is to the right - bool leftThan(const Field & x) const + bool leftThan(const FieldRef & x) const { return (right_bounded ? !(less(x, right) || (right_included && equals(x, right))) @@ -195,42 +218,6 @@ public: String toString() const; }; - -/// Class that extends arbitrary objects with infinities, like +-inf for floats -class FieldWithInfinity -{ -public: - enum Type - { - MINUS_INFINITY = -1, - NORMAL = 0, - PLUS_INFINITY = 1 - }; - - explicit FieldWithInfinity(const Field & field_); - FieldWithInfinity(Field && field_); - - static FieldWithInfinity getMinusInfinity(); - static FieldWithInfinity getPlusInfinity(); - - bool operator<(const FieldWithInfinity & other) const; - bool operator==(const FieldWithInfinity & other) const; - - Field getFieldIfFinite() const - { - if (type != NORMAL) - throw Exception("Trying to get field of infinite type", ErrorCodes::BAD_TYPE_OF_FIELD); - return field; - } - -private: - Field field; - Type type; - - FieldWithInfinity(const Type type_); -}; - - /** Condition on the index. * * Consists of the conditions for the key belonging to all possible ranges or sets, @@ -261,8 +248,8 @@ public: /// one of the resulting mask components (see BoolMask::consider_only_can_be_XXX). BoolMask checkInRange( size_t used_key_size, - const Field * left_key, - const Field * right_key, + const FieldRef * left_key, + const FieldRef* right_key, const DataTypes & data_types, BoolMask initial_mask = BoolMask(false, false)) const; @@ -270,7 +257,7 @@ public: /// left_key must contain all the fields in the sort_descr in the appropriate order. BoolMask checkAfter( size_t used_key_size, - const Field * left_key, + const FieldRef * left_key, const DataTypes & data_types, BoolMask initial_mask = BoolMask(false, false)) const; @@ -278,15 +265,15 @@ public: /// This is more efficient than checkInRange(...).can_be_true. bool mayBeTrueInRange( size_t used_key_size, - const Field * left_key, - const Field * right_key, + const FieldRef * left_key, + const FieldRef * right_key, const DataTypes & data_types) const; /// Same as checkAfter, but calculate only may_be_true component of a result. /// This is more efficient than checkAfter(...).can_be_true. bool mayBeTrueAfter( size_t used_key_size, - const Field * left_key, + const FieldRef * left_key, const DataTypes & data_types) const; /// Checks that the index can not be used. @@ -295,6 +282,8 @@ public: /// Get the maximum number of the key element used in the condition. size_t getMaxKeyColumn() const; + bool hasMonotonicFunctionsChain() const; + /// Impose an additional condition: the value in the column `column` must be in the range `range`. /// Returns whether there is such a column in the key. bool addCondition(const String & column, const Range & range); @@ -374,8 +363,8 @@ public: private: BoolMask checkInRange( size_t used_key_size, - const Field * left_key, - const Field * right_key, + const FieldRef * left_key, + const FieldRef * right_key, const DataTypes & data_types, bool right_bounded, BoolMask initial_mask) const; diff --git a/src/Storages/MergeTree/LevelMergeSelector.cpp b/src/Storages/MergeTree/LevelMergeSelector.cpp index c6bec6540b4..5fedd42e72b 100644 --- a/src/Storages/MergeTree/LevelMergeSelector.cpp +++ b/src/Storages/MergeTree/LevelMergeSelector.cpp @@ -28,7 +28,7 @@ struct Estimator } } - LevelMergeSelector::PartsInPartition getBest() + LevelMergeSelector::PartsInPartition getBest() const { return LevelMergeSelector::PartsInPartition(best_begin, best_end); } diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 2917b4e795b..a2a3ca3a6cf 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -148,7 +148,7 @@ Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() if (read_result.num_rows == 0) read_result.columns.clear(); - auto & sample_block = task->range_reader.getSampleBlock(); + const auto & sample_block = task->range_reader.getSampleBlock(); if (read_result.num_rows != 0 && sample_block.columns() != read_result.columns.size()) throw Exception("Inconsistent number of columns got from MergeTreeRangeReader. " "Have " + toString(sample_block.columns()) + " in sample block " diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index aaacea6f1e7..9e0536159e5 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include @@ -94,15 +94,11 @@ namespace ErrorCodes extern const int CORRUPTED_DATA; extern const int BAD_TYPE_OF_FIELD; extern const int BAD_ARGUMENTS; - extern const int MEMORY_LIMIT_EXCEEDED; extern const int INVALID_PARTITION_VALUE; extern const int METADATA_MISMATCH; extern const int PART_IS_TEMPORARILY_LOCKED; extern const int TOO_MANY_PARTS; extern const int INCOMPATIBLE_COLUMNS; - extern const int CANNOT_ALLOCATE_MEMORY; - extern const int CANNOT_MUNMAP; - extern const int CANNOT_MREMAP; extern const int BAD_TTL_EXPRESSION; extern const int INCORRECT_FILE_NAME; extern const int BAD_DATA_PART_NAME; @@ -713,7 +709,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, } -void MergeTreeData::checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) +void MergeTreeData::checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const { const auto old_storage_policy = getStoragePolicy(); old_storage_policy->checkCompatibleWith(new_storage_policy); @@ -868,7 +864,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) for (const auto & disk_ptr : disks) defined_disk_names.insert(disk_ptr->getName()); - for (auto & [disk_name, disk] : global_context.getDiskSelector()->getDisksMap()) + for (const auto & [disk_name, disk] : global_context.getDiskSelector()->getDisksMap()) { if (defined_disk_names.count(disk_name) == 0 && disk->exists(relative_data_path)) { @@ -954,10 +950,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// Don't count the part as broken if there is not enough memory to load it. /// In fact, there can be many similar situations. /// But it is OK, because there is a safety guard against deleting too many parts. - if (e.code() == ErrorCodes::MEMORY_LIMIT_EXCEEDED - || e.code() == ErrorCodes::CANNOT_ALLOCATE_MEMORY - || e.code() == ErrorCodes::CANNOT_MUNMAP - || e.code() == ErrorCodes::CANNOT_MREMAP) + if (isNotEnoughMemoryErrorCode(e.code())) throw; broken = true; @@ -1209,7 +1202,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) void MergeTreeData::rollbackDeletingParts(const MergeTreeData::DataPartsVector & parts) { auto lock = lockParts(); - for (auto & part : parts) + for (const auto & part : parts) { /// We should modify it under data_parts_mutex part->assertState({DataPartState::Deleting}); @@ -1223,7 +1216,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa auto lock = lockParts(); /// TODO: use data_parts iterators instead of pointers - for (auto & part : parts) + for (const auto & part : parts) { auto it = data_parts_by_info.find(part->info); if (it == data_parts_by_info.end()) @@ -1250,7 +1243,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa part_log_elem.database_name = table_id.database_name; part_log_elem.table_name = table_id.table_name; - for (auto & part : parts) + for (const auto & part : parts) { part_log_elem.partition_id = part->info.partition_id; part_log_elem.part_name = part->name; @@ -1484,7 +1477,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S ErrorCodes::ILLEGAL_COLUMN); } } - else if (command.isModifyingData()) + else if (command.isModifyingData(getInMemoryMetadata())) { if (columns_alter_type_forbidden.count(command.column_name)) throw Exception("Trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN); @@ -1919,7 +1912,7 @@ void MergeTreeData::removePartsFromWorkingSet(const DataPartsVector & remove, bo { auto lock = (acquired_lock) ? DataPartsLock() : lockParts(); - for (auto & part : remove) + for (const auto & part : remove) { if (!data_parts_by_info.count(part->info)) throw Exception("Part " + part->getNameWithState() + " not found in data_parts", ErrorCodes::LOGICAL_ERROR); @@ -2144,7 +2137,7 @@ size_t MergeTreeData::getTotalActiveSizeInBytes() const { auto lock = lockParts(); - for (auto & part : getDataPartsStateRange(DataPartState::Committed)) + for (const auto & part : getDataPartsStateRange(DataPartState::Committed)) res += part->getBytesOnDisk(); } @@ -2158,7 +2151,7 @@ size_t MergeTreeData::getTotalActiveSizeInRows() const { auto lock = lockParts(); - for (auto & part : getDataPartsStateRange(DataPartState::Committed)) + for (const auto & part : getDataPartsStateRange(DataPartState::Committed)) res += part->rows_count; } @@ -2410,7 +2403,7 @@ static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) } } -MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path) +MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path) const { MutableDataPartPtr part = createPart(Poco::Path(relative_path).getFileName(), disk, relative_path); loadPartAndFixMetadataImpl(part); @@ -2810,7 +2803,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const ActiveDataPartSet active_parts(format_version); const auto disks = getStoragePolicy()->getDisks(); - for (auto & disk : disks) + for (const auto & disk : disks) { for (auto it = disk->iterateDirectory(relative_data_path + source_dir); it->isValid(); it->next()) { @@ -3062,7 +3055,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: if (!isEmpty()) { auto parts_lock = acquired_parts_lock ? MergeTreeData::DataPartsLock() : data.lockParts(); - auto owing_parts_lock = acquired_parts_lock ? acquired_parts_lock : &parts_lock; + auto * owing_parts_lock = acquired_parts_lock ? acquired_parts_lock : &parts_lock; auto current_time = time(nullptr); for (const DataPartPtr & part : precommitted_parts) @@ -3583,7 +3576,7 @@ ColumnDependencies MergeTreeData::getColumnDependencies(const NameSet & updated_ return res; } -bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, String * out_reason) +bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, String * out_reason) const { if (!canUseAdaptiveGranularity()) { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 125a90d26e0..288d44160c3 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -433,11 +433,6 @@ public: DataPartPtr getPartIfExists(const String & part_name, const DataPartStates & valid_states); DataPartPtr getPartIfExists(const MergeTreePartInfo & part_info, const DataPartStates & valid_states); - std::vector getSkipIndices() const - { - return std::vector(std::begin(skip_indices), std::end(skip_indices)); - } - /// Total size of active parts in bytes. size_t getTotalActiveSizeInBytes() const; @@ -565,7 +560,7 @@ public: bool hasAnyTTL() const override { return hasRowsTTL() || hasAnyMoveTTL() || hasAnyColumnTTL(); } /// Check that the part is not broken and calculate the checksums for it if they are not present. - MutableDataPartPtr loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path); + MutableDataPartPtr loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path) const; /** Create local backup (snapshot) for parts with specified prefix. * Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number, @@ -625,6 +620,8 @@ public: return storage_settings.get(); } + String getRelativeDataPath() const { return relative_data_path; } + /// Get table path on disk String getFullPathOnDisk(const DiskPtr & disk) const; @@ -881,7 +878,7 @@ protected: void setTTLExpressions(const ColumnsDescription & columns, const ASTPtr & new_ttl_table_ast, bool only_check = false); - void checkStoragePolicy(const StoragePolicyPtr & new_storage_policy); + void checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const; void setStoragePolicy(const String & new_storage_policy_name, bool only_check = false); @@ -961,7 +958,7 @@ private: /// Check selected parts for movements. Used by ALTER ... MOVE queries. CurrentlyMovingPartsTagger checkPartsForMove(const DataPartsVector & parts, SpacePtr space); - bool canUsePolymorphicParts(const MergeTreeSettings & settings, String * out_reason); + bool canUsePolymorphicParts(const MergeTreeSettings & settings, String * out_reason) const; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 2b732d879b0..2ab43f8f56c 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1,6 +1,6 @@ #include "MergeTreeDataMergerMutator.h" -#include +#include #include #include #include @@ -12,16 +12,19 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -221,16 +224,28 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( IMergeSelector::Partitions partitions; const String * prev_partition_id = nullptr; + /// Previous part only in boundaries of partition frame const MergeTreeData::DataPartPtr * prev_part = nullptr; bool has_part_with_expired_ttl = false; for (const MergeTreeData::DataPartPtr & part : data_parts) { + /// Check predicate only for first part in each partition. + if (!prev_part) + /* Parts can be merged with themselves for TTL needs for example. + * So we have to check if this part is currently being inserted with quorum and so on and so forth. + * Obviously we have to check it manually only for the first part + * of each partition because it will be automatically checked for a pair of parts. */ + if (!can_merge_callback(nullptr, part, nullptr)) + continue; + const String & partition_id = part->info.partition_id; if (!prev_partition_id || partition_id != *prev_partition_id || (prev_part && !can_merge_callback(*prev_part, part, nullptr))) { if (partitions.empty() || !partitions.back().empty()) partitions.emplace_back(); + /// New partition frame. prev_partition_id = &partition_id; + prev_part = nullptr; } IMergeSelector::Part part_info; @@ -663,7 +678,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /** Read from all parts, merge and write into a new one. * In passing, we calculate expression for sorting. */ - BlockInputStreams src_streams; + Pipes pipes; UInt64 watch_prev_elapsed = 0; /// We count total amount of bytes in parts @@ -690,18 +705,24 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor for (const auto & part : parts) { - auto input = std::make_unique( + auto input = std::make_unique( data, part, merging_column_names, read_with_direct_io, true); input->setProgressCallback( MergeProgressCallback(merge_entry, watch_prev_elapsed, horizontal_stage_progress)); - BlockInputStreamPtr stream = std::move(input); - if (data.hasPrimaryKey() || data.hasSkipIndices()) - stream = std::make_shared( - std::make_shared(stream, data.sorting_key_and_skip_indices_expr)); + Pipe pipe(std::move(input)); - src_streams.emplace_back(stream); + if (data.hasPrimaryKey() || data.hasSkipIndices()) + { + auto expr = std::make_shared(pipe.getHeader(), data.sorting_key_and_skip_indices_expr); + pipe.addSimpleTransform(std::move(expr)); + + auto materializing = std::make_shared(pipe.getHeader()); + pipe.addSimpleTransform(std::move(materializing)); + } + + pipes.emplace_back(std::move(pipe)); } Names sort_columns = data.sorting_key_columns; @@ -709,14 +730,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); - Block header = src_streams.at(0)->getHeader(); + Block header = pipes.at(0).getHeader(); for (size_t i = 0; i < sort_columns_size; ++i) sort_description.emplace_back(header.getPositionByName(sort_columns[i]), 1, 1); /// The order of the streams is important: when the key is matched, the elements go in the order of the source stream number. /// In the merged part, the lines with the same key must be in the ascending order of the identifier of original part, /// that is going in insertion order. - std::shared_ptr merged_stream; + ProcessorPtr merged_transform; /// If merge is vertical we cannot calculate it bool blocks_are_granules_size = (merge_alg == MergeAlgorithm::Vertical); @@ -725,45 +746,48 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor switch (data.merging_params.mode) { case MergeTreeData::MergingParams::Ordinary: - merged_stream = std::make_unique( - src_streams, sort_description, merge_block_size, 0, rows_sources_write_buf.get(), true, blocks_are_granules_size); + merged_transform = std::make_unique( + header, pipes.size(), sort_description, merge_block_size, 0, rows_sources_write_buf.get(), true, blocks_are_granules_size); break; case MergeTreeData::MergingParams::Collapsing: - merged_stream = std::make_unique( - src_streams, sort_description, data.merging_params.sign_column, + merged_transform = std::make_unique( + header, pipes.size(), sort_description, data.merging_params.sign_column, merge_block_size, rows_sources_write_buf.get(), blocks_are_granules_size); break; case MergeTreeData::MergingParams::Summing: - merged_stream = std::make_unique( - src_streams, sort_description, data.merging_params.columns_to_sum, merge_block_size); + merged_transform = std::make_unique( + header, pipes.size(), sort_description, data.merging_params.columns_to_sum, merge_block_size); break; case MergeTreeData::MergingParams::Aggregating: - merged_stream = std::make_unique( - src_streams, sort_description, merge_block_size); + merged_transform = std::make_unique( + header, pipes.size(), sort_description, merge_block_size); break; case MergeTreeData::MergingParams::Replacing: - merged_stream = std::make_unique( - src_streams, sort_description, data.merging_params.version_column, + merged_transform = std::make_unique( + header, pipes.size(), sort_description, data.merging_params.version_column, merge_block_size, rows_sources_write_buf.get(), blocks_are_granules_size); break; case MergeTreeData::MergingParams::Graphite: - merged_stream = std::make_unique( - src_streams, sort_description, merge_block_size, + merged_transform = std::make_unique( + header, pipes.size(), sort_description, merge_block_size, data.merging_params.graphite_params, time_of_merge); break; case MergeTreeData::MergingParams::VersionedCollapsing: - merged_stream = std::make_unique( - src_streams, sort_description, data.merging_params.sign_column, + merged_transform = std::make_unique( + header, pipes.size(), sort_description, data.merging_params.sign_column, merge_block_size, rows_sources_write_buf.get(), blocks_are_granules_size); break; } + Pipe merged_pipe(std::move(pipes), std::move(merged_transform)); + BlockInputStreamPtr merged_stream = std::make_shared(std::move(merged_pipe)); + if (deduplicate) merged_stream = std::make_shared(merged_stream, SizeLimits(), 0 /*limit_hint*/, Names()); @@ -773,6 +797,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergedBlockOutputStream to{ new_data_part, merging_columns, + data.skip_indices, compression_codec, merged_column_to_size, data_settings->min_merge_bytes_to_use_direct_io, @@ -860,13 +885,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeStageProgress column_progress(progress_before, column_sizes->columnWeight(column_name)); for (size_t part_num = 0; part_num < parts.size(); ++part_num) { - auto column_part_stream = std::make_shared( + auto column_part_source = std::make_shared( data, parts[part_num], column_names, read_with_direct_io, true); - column_part_stream->setProgressCallback( + column_part_source->setProgressCallback( MergeProgressCallback(merge_entry, watch_prev_elapsed, column_progress)); - column_part_streams[part_num] = std::move(column_part_stream); + column_part_streams[part_num] = std::make_shared( + Pipe(std::move(column_part_source))); } rows_sources_read_buf.seek(0, 0); @@ -875,9 +901,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergedColumnOnlyOutputStream column_to( new_data_part, column_gathered_stream.getHeader(), - false, compression_codec, - false, /// we don't need to recalc indices here /// because all of them were already recalculated and written /// as key part of vertical merge @@ -991,7 +1015,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor splitMutationCommands(source_part, commands_for_part, for_interpreter, for_file_renames); - UInt64 watch_prev_elapsed = 0; MergeStageProgress stage_progress(1.0); @@ -1043,8 +1066,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor /// All columns from part are changed and may be some more that were missing before in part if (isCompactPart(source_part) || source_part->getColumns().isSubsetOf(updated_header.getNamesAndTypesList())) { + auto part_indices = getIndicesForNewDataPart(data.skip_indices, for_file_renames); mutateAllPartColumns( new_data_part, + part_indices, in, time_of_mutation, compression_codec, @@ -1260,6 +1285,7 @@ void MergeTreeDataMergerMutator::splitMutationCommands( else if (is_compact_part && command.type == MutationCommand::Type::DROP_COLUMN) { removed_columns_from_compact_part.emplace(command.column_name); + for_file_renames.push_back(command); } else if (command.type == MutationCommand::Type::RENAME_COLUMN) { @@ -1439,6 +1465,22 @@ NamesAndTypesList MergeTreeDataMergerMutator::getColumnsForNewDataPart( return all_columns; } +MergeTreeIndices MergeTreeDataMergerMutator::getIndicesForNewDataPart( + const MergeTreeIndices & all_indices, + const MutationCommands & commands_for_removes) +{ + NameSet removed_indices; + for (const auto & command : commands_for_removes) + if (command.type == MutationCommand::DROP_INDEX) + removed_indices.insert(command.column_name); + + MergeTreeIndices new_indices; + for (const auto & index : all_indices) + if (!removed_indices.count(index->name)) + new_indices.push_back(index); + + return new_indices; +} std::set MergeTreeDataMergerMutator::getIndicesToRecalculate( BlockInputStreamPtr & input_stream, @@ -1503,6 +1545,7 @@ bool MergeTreeDataMergerMutator::shouldExecuteTTL(const Names & columns, const M void MergeTreeDataMergerMutator::mutateAllPartColumns( MergeTreeData::MutableDataPartPtr new_data_part, + const MergeTreeIndices & skip_indices, BlockInputStreamPtr mutating_stream, time_t time_of_mutation, const CompressionCodecPtr & compression_codec, @@ -1524,6 +1567,7 @@ void MergeTreeDataMergerMutator::mutateAllPartColumns( MergedBlockOutputStream out{ new_data_part, new_data_part->getColumns(), + skip_indices, compression_codec}; mutating_stream->readPrefix(); @@ -1560,7 +1604,6 @@ void MergeTreeDataMergerMutator::mutateSomePartColumns( if (mutating_stream == nullptr) throw Exception("Cannot mutate part columns with uninitialized mutations stream. It's a bug", ErrorCodes::LOGICAL_ERROR); - if (need_remove_expired_values) mutating_stream = std::make_shared(mutating_stream, data, new_data_part, time_of_mutation, true); @@ -1568,9 +1611,7 @@ void MergeTreeDataMergerMutator::mutateSomePartColumns( MergedColumnOnlyOutputStream out( new_data_part, mutation_header, - /* sync = */ false, compression_codec, - /* skip_offsets = */ false, std::vector(indices_to_recalc.begin(), indices_to_recalc.end()), nullptr, source_part->index_granularity, diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index b24b56a4780..f2f4ea95fd0 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -49,9 +49,8 @@ struct FutureMergedMutatedPart class MergeTreeDataMergerMutator { public: - using AllowedMergingPredicate = std::function; + using AllowedMergingPredicate = std::function; -public: MergeTreeDataMergerMutator(MergeTreeData & data_, size_t background_pool_size); /** Get maximum total size of parts to do merge, at current moment of time. @@ -160,6 +159,11 @@ private: NamesAndTypesList all_columns, const MutationCommands & commands_for_removes); + /// Get skip indcies, that should exists in the resulting data part. + static MergeTreeIndices getIndicesForNewDataPart( + const MergeTreeIndices & all_indices, + const MutationCommands & commands_for_removes); + bool shouldExecuteTTL(const Names & columns, const MutationCommands & commands) const; /// Return set of indices which should be recalculated during mutation also @@ -173,6 +177,7 @@ private: /// Override all columns of new part using mutating_stream void mutateAllPartColumns( MergeTreeData::MutableDataPartPtr new_data_part, + const MergeTreeIndices & skip_indices, BlockInputStreamPtr mutating_stream, time_t time_of_mutation, const CompressionCodecPtr & codec, diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 7bf52ee19d1..c75970f6cc1 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -324,7 +324,7 @@ void MinimalisticDataPartChecksums::serializeWithoutHeader(WriteBuffer & to) con writePODBinary(uncompressed_hash_of_compressed_files, to); } -String MinimalisticDataPartChecksums::getSerializedString() +String MinimalisticDataPartChecksums::getSerializedString() const { WriteBufferFromOwnString wb; serialize(wb); diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index a4eae3cc0e5..89e1ba1d78a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -126,7 +126,7 @@ struct MinimalisticDataPartChecksums void serialize(WriteBuffer & to) const; void serializeWithoutHeader(WriteBuffer & to) const; - String getSerializedString(); + String getSerializedString() const; static String getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic); void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) const; diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 12b2e106284..fa98a2c863f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -20,7 +20,6 @@ class MergeTreeDataPartCompact : public IMergeTreeDataPart public: static constexpr auto DATA_FILE_NAME = "data"; static constexpr auto DATA_FILE_EXTENSION = ".bin"; - static constexpr auto TEMP_FILE_SUFFIX = "_temp"; static constexpr auto DATA_FILE_NAME_WITH_EXTENSION = "data.bin"; MergeTreeDataPartCompact( diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index bd507866e9b..e33d4a97cac 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -22,8 +22,6 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( { using DataPart = MergeTreeDataPartCompact; String data_file_name = DataPart::DATA_FILE_NAME; - if (settings.is_writing_temp_files) - data_file_name += DataPart::TEMP_FILE_SUFFIX; stream = std::make_unique( data_file_name, @@ -145,7 +143,7 @@ void MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTy column.type->serializeBinaryBulkStateSuffix(serialize_settings, state); } -void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums) { if (columns_buffer.size() != 0) writeBlock(header.cloneWithColumns(columns_buffer.releaseColumns())); @@ -161,8 +159,6 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: } stream->finalize(); - if (sync) - stream->sync(); stream->addToChecksums(checksums); stream.reset(); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 598a4dd47fb..0aff55588aa 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -21,7 +21,7 @@ public: void write(const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) override; - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) override; + void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums) override; private: /// Write single granule of one column (rows between 2 marks) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 5c1c6e35a20..1e5640b4e23 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -39,9 +39,6 @@ void MergeTreeDataPartWriterWide::addStreams( { IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path) { - if (settings.skip_offsets && !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) - return; - String stream_name = IDataType::getFileNameForStream(name, substream_path); /// Shared offsets for Nested type. if (column_streams.count(stream_name)) @@ -69,8 +66,6 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( return [&, this] (const IDataType::SubstreamPath & substream_path) -> WriteBuffer * { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && settings.skip_offsets) - return nullptr; String stream_name = IDataType::getFileNameForStream(name, substream_path); @@ -135,8 +130,6 @@ void MergeTreeDataPartWriterWide::writeSingleMark( type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && settings.skip_offsets) - return; String stream_name = IDataType::getFileNameForStream(name, substream_path); @@ -177,8 +170,6 @@ size_t MergeTreeDataPartWriterWide::writeSingleGranule( type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && settings.skip_offsets) - return; String stream_name = IDataType::getFileNameForStream(name, substream_path); @@ -270,7 +261,7 @@ void MergeTreeDataPartWriterWide::writeColumn( next_index_offset = current_row - total_rows; } -void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums) { const auto & global_settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -300,8 +291,6 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch for (auto & stream : column_streams) { stream.second->finalize(); - if (sync) - stream.second->sync(); stream.second->addToChecksums(checksums); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 95e43cd31af..4e4f4806d53 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -24,7 +24,7 @@ public: void write(const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) override; - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) override; + void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums) override; IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 370286117ae..580c95b34dd 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -38,18 +38,9 @@ namespace std } #endif -#include -#include #include -#include #include -#include -#include -#include -#include #include -#include -#include #include #include #include @@ -58,7 +49,11 @@ namespace std #include #include #include -#include +#include +#include +#include +#include +#include #include #include #include @@ -1096,16 +1091,14 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( }; BlockInputStreamPtr merged; + ProcessorPtr merged_processor; switch (data.merging_params.mode) { case MergeTreeData::MergingParams::Ordinary: { - auto merged_processor = - std::make_shared(header, pipes.size(), sort_description, max_block_size); - Pipe pipe(std::move(pipes), std::move(merged_processor)); - pipes = Pipes(); - pipes.emplace_back(std::move(pipe)); - return pipes; + merged_processor = std::make_shared(header, pipes.size(), + sort_description, max_block_size); + break; } case MergeTreeData::MergingParams::Collapsing: @@ -1114,28 +1107,36 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( break; case MergeTreeData::MergingParams::Summing: - merged = std::make_shared(streams_to_merge(), + merged_processor = std::make_shared(header, pipes.size(), sort_description, data.merging_params.columns_to_sum, max_block_size); break; case MergeTreeData::MergingParams::Aggregating: - merged = std::make_shared(streams_to_merge(), sort_description, max_block_size); + merged_processor = std::make_shared(header, pipes.size(), + sort_description, max_block_size); break; case MergeTreeData::MergingParams::Replacing: /// TODO Make ReplacingFinalBlockInputStream - merged = std::make_shared(streams_to_merge(), + merged_processor = std::make_shared(header, pipes.size(), sort_description, data.merging_params.version_column, max_block_size); break; case MergeTreeData::MergingParams::VersionedCollapsing: /// TODO Make VersionedCollapsingFinalBlockInputStream - merged = std::make_shared( - streams_to_merge(), sort_description, data.merging_params.sign_column, max_block_size); + merged_processor = std::make_shared(header, pipes.size(), + sort_description, data.merging_params.sign_column, max_block_size); break; case MergeTreeData::MergingParams::Graphite: throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); } + if (merged_processor) + { + Pipe pipe(std::move(pipes), std::move(merged_processor)); + pipes = Pipes(); + pipes.emplace_back(std::move(pipe)); + } + if (merged) pipes.emplace_back(std::make_shared(merged)); @@ -1201,11 +1202,33 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( * If fits, split it into smaller ones and put them on the stack. If not, discard it. * If the segment is already of one mark length, add it to response and discard it. */ - std::vector ranges_stack{ {0, marks_count} }; + std::vector ranges_stack = { {0, marks_count} }; + + std::function create_field_ref; + /// If there are no monotonic functions, there is no need to save block reference. + /// Passing explicit field to FieldRef allows to optimize ranges and shows better performance. + if (key_condition.hasMonotonicFunctionsChain()) + { + auto index_block = std::make_shared(); + for (size_t i = 0; i < used_key_size; ++i) + index_block->insert({index[i], data.primary_key_data_types[i], data.primary_key_columns[i]}); + + create_field_ref = [index_block](size_t row, size_t column, FieldRef & field) + { + field = {index_block.get(), row, column}; + }; + } + else + { + create_field_ref = [&index](size_t row, size_t column, FieldRef & field) + { + index[column]->get(row, field); + }; + } /// NOTE Creating temporary Field objects to pass to KeyCondition. - Row index_left(used_key_size); - Row index_right(used_key_size); + std::vector index_left(used_key_size); + std::vector index_right(used_key_size); while (!ranges_stack.empty()) { @@ -1216,7 +1239,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( if (range.end == marks_count && !has_final_mark) { for (size_t i = 0; i < used_key_size; ++i) - index[i]->get(range.begin, index_left[i]); + create_field_ref(range.begin, i, index_left[i]); may_be_true = key_condition.mayBeTrueAfter( used_key_size, index_left.data(), data.primary_key_data_types); @@ -1228,8 +1251,8 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( for (size_t i = 0; i < used_key_size; ++i) { - index[i]->get(range.begin, index_left[i]); - index[i]->get(range.end, index_right[i]); + create_field_ref(range.begin, i, index_left[i]); + create_field_ref(range.end, i, index_right[i]); } may_be_true = key_condition.mayBeTrueInRange( @@ -1254,9 +1277,9 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( size_t end; for (end = range.end; end > range.begin + step; end -= step) - ranges_stack.push_back(MarkRange(end - step, end)); + ranges_stack.emplace_back(end - step, end); - ranges_stack.push_back(MarkRange(range.begin, end)); + ranges_stack.emplace_back(range.begin, end); } } } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index c560583259c..1814e40e10e 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -210,8 +211,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa const auto & date_lut = DateLUT::instance(); - DayNum min_month = date_lut.toFirstDayNumOfMonth(DayNum(min_date)); - DayNum max_month = date_lut.toFirstDayNumOfMonth(DayNum(max_date)); + auto min_month = date_lut.toNumYYYYMM(min_date); + auto max_month = date_lut.toNumYYYYMM(max_date); if (min_month != max_month) throw Exception("Logical error: part spans more than one month.", ErrorCodes::LOGICAL_ERROR); @@ -294,7 +295,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_codec = data.global_context.chooseCompressionCodec(0, 0); - MergedBlockOutputStream out(new_data_part, columns, compression_codec); + MergedBlockOutputStream out(new_data_part, columns, data.skip_indices, compression_codec); out.writePrefix(); out.writeWithPermutation(block, perm_ptr); diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 5d3b2945d47..f5c57659052 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -30,10 +30,7 @@ struct MergeTreeWriterSettings size_t aio_threshold; bool can_use_adaptive_granularity; bool blocks_are_granules_size; - /// true if we write temporary files during alter. - bool is_writing_temp_files = false; + size_t estimated_size = 0; - /// used when ALTERing columns if we know that array offsets are not altered. - bool skip_offsets = false; }; } diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 06b4d8b9a82..042050c15f1 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -21,8 +21,10 @@ namespace DB namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int ILLEGAL_COLUMN; extern const int INCORRECT_QUERY; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } MergeTreeIndexBloomFilter::MergeTreeIndexBloomFilter( @@ -73,7 +75,7 @@ static void assertIndexColumnsType(const Block & header) const DataTypes & columns_data_types = header.getDataTypes(); - for (auto & type : columns_data_types) + for (const auto & type : columns_data_types) { const IDataType * actual_type = BloomFilter::getPrimitiveType(type).get(); WhichDataType which(actual_type); @@ -100,8 +102,21 @@ std::unique_ptr bloomFilterIndexCreatorNew( assertIndexColumnsType(index_sample); double max_conflict_probability = 0.025; - if (node->type->arguments && !node->type->arguments->children.empty()) - max_conflict_probability = typeid_cast(*node->type->arguments->children[0]).value.get(); + const auto & arguments = node->type->arguments; + + if (arguments && arguments->children.size() > 1) + throw Exception("BloomFilter index cannot have more than one parameter.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (arguments && !arguments->children.empty()) + { + const auto & argument = arguments->children[0]->as(); + + if (!argument || (argument->value.safeGet() < 0 || argument->value.safeGet() > 1)) + throw Exception("The BloomFilter false positive must be a double number between 0 and 1.", ErrorCodes::BAD_ARGUMENTS); + + max_conflict_probability = argument->value.safeGet(); + } + const auto & bits_per_row_and_size_of_hash_functions = BloomFilterHash::calculationBestPractices(max_conflict_probability); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 88e3fc6662a..e823a190470 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -48,8 +48,8 @@ ColumnWithTypeAndName getPreparedSetInfo(const SetPtr & prepared_set) bool maybeTrueOnBloomFilter(const IColumn * hash_column, const BloomFilterPtr & bloom_filter, size_t hash_functions) { - const auto const_column = typeid_cast(hash_column); - const auto non_const_column = typeid_cast(hash_column); + const auto * const_column = typeid_cast(hash_column); + const auto * non_const_column = typeid_cast(hash_column); if (!const_column && !non_const_column) throw Exception("LOGICAL ERROR: hash column must be Const Column or UInt64 Column.", ErrorCodes::LOGICAL_ERROR); @@ -259,7 +259,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTIn( size_t row_size = column->size(); size_t position = header.getPositionByName(key_ast->getColumnName()); const DataTypePtr & index_type = header.getByPosition(position).type; - const auto & converted_column = castColumn(ColumnWithTypeAndName{column, type, ""}, index_type, context); + const auto & converted_column = castColumn(ColumnWithTypeAndName{column, type, ""}, index_type); out.predicate.emplace_back(std::make_pair(position, BloomFilterHash::hashWithColumn(index_type, converted_column, 0, row_size))); if (function_name == "in" || function_name == "globalIn") @@ -339,7 +339,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( if (which.isTuple() && function->name == "tuple") { const Tuple & tuple = get(value_field); - const auto value_tuple_data_type = typeid_cast(value_type.get()); + const auto * value_tuple_data_type = typeid_cast(value_type.get()); const ASTs & arguments = typeid_cast(*function->arguments).children; if (tuple.size() != arguments.size()) diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 4b3bd954496..6887b7c0d9d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -19,6 +19,15 @@ #include +#if defined(__SSE2__) +#include + +#if defined(__SSE4_2__) +#include +#endif + +#endif + namespace DB { @@ -199,8 +208,7 @@ bool MergeTreeConditionFullText::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx else if (element.function == RPNElement::FUNCTION_EQUALS || element.function == RPNElement::FUNCTION_NOT_EQUALS) { - rpn_stack.emplace_back( - granule->bloom_filters[element.key_column].contains(*element.bloom_filter), true); + rpn_stack.emplace_back(granule->bloom_filters[element.key_column].contains(*element.bloom_filter), true); if (element.function == RPNElement::FUNCTION_NOT_EQUALS) rpn_stack.back() = !rpn_stack.back(); @@ -606,8 +614,67 @@ bool SplitTokenExtractor::next(const char * data, size_t len, size_t * pos, size { *token_start = *pos; *token_len = 0; + while (*pos < len) { +#if defined(__SSE2__) + // NOTE: we assume that `data` string is padded from the right with 15 bytes. + const __m128i haystack = _mm_loadu_si128(reinterpret_cast(data + *pos)); + const size_t haystack_length = 16; + +#if defined(__SSE4_2__) + // With the help of https://www.strchr.com/strcmp_and_strlen_using_sse_4.2 + const auto alnum_chars_ranges = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, + '\xFF', '\x80', 'z', 'a', 'Z', 'A', '9', '0'); + // Every bit represents if `haystack` character is in the ranges (1) or not (0) + const int result_bitmask = _mm_cvtsi128_si32(_mm_cmpestrm(alnum_chars_ranges, 8, haystack, haystack_length, _SIDD_CMP_RANGES)); +#else + // NOTE: -1 and +1 required since SSE2 has no `>=` and `<=` instructions on packed 8-bit integers (epi8). + const auto number_begin = _mm_set1_epi8('0' - 1); + const auto number_end = _mm_set1_epi8('9' + 1); + const auto alpha_lower_begin = _mm_set1_epi8('a' - 1); + const auto alpha_lower_end = _mm_set1_epi8('z' + 1); + const auto alpha_upper_begin = _mm_set1_epi8('A' - 1); + const auto alpha_upper_end = _mm_set1_epi8('Z' + 1); + const auto zero = _mm_set1_epi8(0); + + // every bit represents if `haystack` character `c` statisfies condition: + // (c < 0) || (c > '0' - 1 && c < '9' + 1) || (c > 'a' - 1 && c < 'z' + 1) || (c > 'A' - 1 && c < 'Z' + 1) + // < 0 since _mm_cmplt_epi8 threats chars as SIGNED, and so all chars > 0x80 are negative. + const int result_bitmask = _mm_movemask_epi8(_mm_or_si128(_mm_or_si128(_mm_or_si128( + _mm_cmplt_epi8(haystack, zero), + _mm_and_si128(_mm_cmpgt_epi8(haystack, number_begin), _mm_cmplt_epi8(haystack, number_end))), + _mm_and_si128(_mm_cmpgt_epi8(haystack, alpha_lower_begin), _mm_cmplt_epi8(haystack, alpha_lower_end))), + _mm_and_si128(_mm_cmpgt_epi8(haystack, alpha_upper_begin), _mm_cmplt_epi8(haystack, alpha_upper_end)))); +#endif + if (result_bitmask == 0) + { + if (*token_len != 0) + // end of token started on previous haystack + return true; + + *pos += haystack_length; + continue; + } + + const auto token_start_pos_in_current_haystack = getTrailingZeroBitsUnsafe(result_bitmask); + if (*token_len == 0) + // new token + *token_start = *pos + token_start_pos_in_current_haystack; + else if (token_start_pos_in_current_haystack != 0) + // end of token starting in one of previous haystacks + return true; + + const auto token_bytes_in_current_haystack = getTrailingZeroBitsUnsafe(~(result_bitmask >> token_start_pos_in_current_haystack)); + *token_len += token_bytes_in_current_haystack; + + *pos += token_start_pos_in_current_haystack + token_bytes_in_current_haystack; + if (token_start_pos_in_current_haystack + token_bytes_in_current_haystack == haystack_length) + // check if there are leftovers in next `haystack` + continue; + + break; +#else if (isASCII(data[*pos]) && !isAlphaNumericASCII(data[*pos])) { /// Finish current token if any @@ -621,7 +688,16 @@ bool SplitTokenExtractor::next(const char * data, size_t len, size_t * pos, size ++*pos; ++*token_len; } +#endif } + +#if defined(__SSE2__) + // Could happen only if string is not padded with zeroes, and we accidentally hopped over end of data. + if (*token_start > len) + return false; + *token_len = std::min(len - *token_start, *token_len); +#endif + return *token_len > 0; } diff --git a/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp index 6e76333da9d..83f8884e03a 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp @@ -111,11 +111,11 @@ void MergeTreeIndexGranuleBloomFilter::serializeBinary(WriteBuffer & ostr) const ostr.write(reinterpret_cast(bloom_filter->getFilter().data()), bytes_size); } -void MergeTreeIndexGranuleBloomFilter::fillingBloomFilter(BloomFilterPtr & bf, const Block & granule_index_block, size_t index_hash_column) +void MergeTreeIndexGranuleBloomFilter::fillingBloomFilter(BloomFilterPtr & bf, const Block & granule_index_block, size_t index_hash_column) const { const auto & column = granule_index_block.getByPosition(index_hash_column); - if (const auto hash_column = typeid_cast(column.column.get())) + if (const auto * hash_column = typeid_cast(column.column.get())) { const auto & hash_column_vec = hash_column->getData(); diff --git a/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.h index 5524c904309..54e2c105db8 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.h @@ -27,7 +27,7 @@ private: size_t hash_functions; std::vector bloom_filters; - void fillingBloomFilter(BloomFilterPtr & bf, const Block & granule_index_block, size_t index_hash_column); + void fillingBloomFilter(BloomFilterPtr & bf, const Block & granule_index_block, size_t index_hash_column) const; }; diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp index 122f038fee6..220fc70c549 100644 --- a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp @@ -100,8 +100,8 @@ void MergeTreeIndexAggregatorMinMax::update(const Block & block, size_t * pos, s size_t rows_read = std::min(limit, block.rows() - *pos); - Field field_min; - Field field_max; + FieldRef field_min; + FieldRef field_max; for (size_t i = 0; i < index.columns.size(); ++i) { const auto & column = block.getByName(index.columns[i]).column; diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 8cc67adb097..25b204415ee 100644 --- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -269,11 +269,11 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx actions->execute(result); auto column = result.getByName(expression_ast->getColumnName()).column->convertToFullColumnIfLowCardinality(); - auto * col_uint8 = typeid_cast(column.get()); + const auto * col_uint8 = typeid_cast(column.get()); const NullMap * null_map = nullptr; - if (auto * col_nullable = checkAndGetColumn(*column)) + if (const auto * col_nullable = checkAndGetColumn(*column)) { col_uint8 = typeid_cast(&col_nullable->getNestedColumn()); null_map = &col_nullable->getNullMapData(); @@ -282,7 +282,7 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx if (!col_uint8) throw Exception("ColumnUInt8 expected as Set index condition result.", ErrorCodes::LOGICAL_ERROR); - auto & condition = col_uint8->getData(); + const auto & condition = col_uint8->getData(); for (size_t i = 0; i < column->size(); ++i) if ((!null_map || (*null_map)[i] == 0) && condition[i] & 1) diff --git a/src/Storages/MergeTree/MergeTreeIndices.h b/src/Storages/MergeTree/MergeTreeIndices.h index 007851f2912..d871a522e6c 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.h +++ b/src/Storages/MergeTree/MergeTreeIndices.h @@ -125,7 +125,7 @@ public: size_t granularity; }; -using MergeTreeIndices = std::vector; +using MergeTreeIndices = std::vector; class MergeTreeIndexFactory : private boost::noncopyable diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 0df125d69ff..a7107789bfd 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -96,7 +96,7 @@ void MergeTreeMarksLoader::loadMarks() auto key = mark_cache->hash(mrk_path); if (save_marks_in_cache) { - auto callback = std::bind(&MergeTreeMarksLoader::loadMarksImpl, this); + auto callback = [this]{ return loadMarksImpl(); }; marks = mark_cache->getOrSet(key, callback); } else diff --git a/src/Storages/MergeTree/MergeTreePartInfo.cpp b/src/Storages/MergeTree/MergeTreePartInfo.cpp index 43bd9538e3e..d30f6470bb1 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -120,8 +120,8 @@ void MergeTreePartInfo::parseMinMaxDatesFromPartName(const String & part_name, D min_date = date_lut.YYYYMMDDToDayNum(min_yyyymmdd); max_date = date_lut.YYYYMMDDToDayNum(max_yyyymmdd); - DayNum min_month = date_lut.toFirstDayNumOfMonth(min_date); - DayNum max_month = date_lut.toFirstDayNumOfMonth(max_date); + auto min_month = date_lut.toNumYYYYMM(min_date); + auto max_month = date_lut.toNumYYYYMM(max_date); if (min_month != max_month) throw Exception("Part name " + part_name + " contains different months", ErrorCodes::BAD_DATA_PART_NAME); diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index b74fb5d1f5d..f2b53fbb70a 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -370,7 +370,7 @@ size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & zero_tails.resize(0); zero_tails.reserve(rows_per_granule.size()); - auto filter_data = filter_vec.data(); + const auto * filter_data = filter_vec.data(); size_t total_zero_rows_in_tails = 0; @@ -390,8 +390,8 @@ size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & filter_vec, IColumn::Filter & new_filter_vec) { - auto filter_data = filter_vec.data(); - auto new_filter_data = new_filter_vec.data(); + const auto * filter_data = filter_vec.data(); + auto * new_filter_data = new_filter_vec.data(); for (auto i : ext::range(0, rows_per_granule.size())) { @@ -412,7 +412,7 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con while (end - begin >= 64) { end -= 64; - auto pos = end; + const auto * pos = end; UInt64 val = static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( _mm_loadu_si128(reinterpret_cast(pos)), @@ -498,7 +498,7 @@ MergeTreeRangeReader::MergeTreeRangeReader( if (prev_reader) sample_block = prev_reader->getSampleBlock(); - for (auto & name_and_type : merge_tree_reader->getColumns()) + for (const auto & name_and_type : merge_tree_reader->getColumns()) sample_block.insert({name_and_type.type->createColumn(), name_and_type.type, name_and_type.name}); if (prewhere) @@ -761,8 +761,8 @@ Columns MergeTreeRangeReader::continueReadingChain(ReadResult & result, size_t & columns.resize(merge_tree_reader->numColumnsInResult()); - auto & rows_per_granule = result.rowsPerGranule(); - auto & started_ranges = result.startedRanges(); + const auto & rows_per_granule = result.rowsPerGranule(); + const auto & started_ranges = result.startedRanges(); size_t next_range_to_start = 0; @@ -773,7 +773,7 @@ Columns MergeTreeRangeReader::continueReadingChain(ReadResult & result, size_t & && i == started_ranges[next_range_to_start].num_granules_read_before_start) { num_rows += stream.finalize(columns); - auto & range = started_ranges[next_range_to_start].range; + const auto & range = started_ranges[next_range_to_start].range; ++next_range_to_start; stream = Stream(range.begin, range.end, merge_tree_reader); } @@ -798,7 +798,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r if (!prewhere) return; - auto & header = merge_tree_reader->getColumns(); + const auto & header = merge_tree_reader->getColumns(); size_t num_columns = header.size(); if (result.columns.size() != num_columns) @@ -816,7 +816,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r if (prev_reader) { - for (auto & col : prev_reader->getSampleBlock()) + for (const auto & col : prev_reader->getSampleBlock()) { block.insert({result.columns[pos], col.type, col.name}); ++pos; @@ -866,7 +866,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r /// If there is a filter and without optimized if (result.getFilter() && last_reader_in_chain) { - auto result_filter = result.getFilter(); + const auto * result_filter = result.getFilter(); /// optimize is not called, need to check const 1 and const 0 size_t bytes_in_filter = result.countBytesInResultFilter(result_filter->getData()); if (bytes_in_filter == 0) @@ -879,7 +879,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r if (result.getFilter()) { /// filter might be shrinked while columns not - auto result_filter = result.getFilterOriginal(); + const auto * result_filter = result.getFilterOriginal(); filterColumns(result.columns, result_filter->getData()); result.need_filter = true; diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index e4f7275f4a5..a895149e12e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -78,15 +78,9 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( auto name_and_type = columns.begin(); for (size_t i = 0; i < columns_num; ++i, ++name_and_type) { - const auto & [name, type] = *name_and_type; + const auto & [name, type] = getColumnFromPart(*name_and_type); auto position = data_part->getColumnPosition(name); - if (!position && alter_conversions.isColumnRenamed(name)) - { - String old_name = alter_conversions.getColumnOldName(name); - position = data_part->getColumnPosition(old_name); - } - if (!position && typeid_cast(type.get())) { /// If array of Nested column is missing in part, @@ -118,7 +112,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, bool append = res_columns[i] != nullptr; if (!append) - res_columns[i] = column_it->type->createColumn(); + res_columns[i] = getColumnFromPart(*column_it).type->createColumn(); mutable_columns[i] = res_columns[i]->assumeMutable(); } @@ -132,15 +126,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, if (!res_columns[pos]) continue; - auto [name, type] = *name_and_type; - - if (alter_conversions.isColumnRenamed(name)) - { - String old_name = alter_conversions.getColumnOldName(name); - if (!data_part->getColumnPosition(name) && data_part->getColumnPosition(old_name)) - name = old_name; - } - + auto [name, type] = getColumnFromPart(*name_and_type); auto & column = mutable_columns[pos]; try diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index ad676b4db03..1a03acb5758 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -41,28 +41,10 @@ MergeTreeReaderWide::MergeTreeReaderWide( { try { - for (const NameAndTypePair & column_from_part : data_part->getColumns()) - columns_from_part[column_from_part.name] = column_from_part.type; - for (const NameAndTypePair & column : columns) { - if (columns_from_part.count(column.name)) - { - addStreams(column.name, *columns_from_part[column.name], profile_callback_, clock_type_); - } - else - { - if (alter_conversions.isColumnRenamed(column.name)) - { - String old_name = alter_conversions.getColumnOldName(column.name); - if (columns_from_part.count(old_name)) - addStreams(old_name, *columns_from_part[old_name], profile_callback_, clock_type_); - } - else - { - addStreams(column.name, *column.type, profile_callback_, clock_type_); - } - } + auto column_from_part = getColumnFromPart(column); + addStreams(column_from_part.name, *column_from_part.type, profile_callback_, clock_type_); } } catch (...) @@ -93,19 +75,7 @@ size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, si auto name_and_type = columns.begin(); for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) { - String name = name_and_type->name; - if (alter_conversions.isColumnRenamed(name)) - { - String original_name = alter_conversions.getColumnOldName(name); - if (!columns_from_part.count(name) && columns_from_part.count(original_name)) - name = original_name; - } - - DataTypePtr type; - if (columns_from_part.count(name)) - type = columns_from_part[name]; - else - type = name_and_type->type; + auto [name, type] = getColumnFromPart(*name_and_type); /// The column is already present in the block so we will append the values to the end. bool append = res_columns[pos] != nullptr; diff --git a/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp similarity index 64% rename from src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp rename to src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 9e0737810df..cfda7fdf562 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -1,4 +1,4 @@ -#include +#include #include namespace DB @@ -8,16 +8,17 @@ namespace ErrorCodes extern const int MEMORY_LIMIT_EXCEEDED; } -MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( +MergeTreeSequentialSource::MergeTreeSequentialSource( const MergeTreeData & storage_, - const MergeTreeData::DataPartPtr & data_part_, + MergeTreeData::DataPartPtr data_part_, Names columns_to_read_, bool read_with_direct_io_, bool take_column_types_from_storage, bool quiet) - : storage(storage_) - , data_part(data_part_) - , columns_to_read(columns_to_read_) + : SourceWithProgress(storage_.getSampleBlockForColumns(columns_to_read_)) + , storage(storage_) + , data_part(std::move(data_part_)) + , columns_to_read(std::move(columns_to_read_)) , read_with_direct_io(read_with_direct_io_) , mark_cache(storage.global_context.getMarkCache()) { @@ -27,14 +28,14 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( message << "Reading " << data_part->getMarksCount() << " marks from part " << data_part->name << ", total " << data_part->rows_count << " rows starting from the beginning of the part"; + if (columns_to_read.size() == 1) /// Print column name but don't pollute logs in case of many columns. + message << ", column " << columns_to_read.front(); LOG_TRACE(log, message.rdbuf()); } addTotalRowsApprox(data_part->rows_count); - header = storage.getSampleBlockForColumns(columns_to_read); - /// Add columns because we don't want to read empty blocks injectRequiredColumns(storage, data_part, columns_to_read); NamesAndTypesList columns_for_reader; @@ -62,49 +63,27 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( /* uncompressed_cache = */ nullptr, mark_cache.get(), reader_settings); } - -void MergeTreeSequentialBlockInputStream::fixHeader(Block & header_block) const -{ - /// Types may be different during ALTER (when this stream is used to perform an ALTER). - for (const auto & name_type : data_part->getColumns()) - { - if (header_block.has(name_type.name)) - { - auto & elem = header_block.getByName(name_type.name); - if (!elem.type->equals(*name_type.type)) - { - elem.type = name_type.type; - elem.column = elem.type->createColumn(); - } - } - } -} - -Block MergeTreeSequentialBlockInputStream::getHeader() const -{ - return header; -} - -Block MergeTreeSequentialBlockInputStream::readImpl() +Chunk MergeTreeSequentialSource::generate() try { - Block res; + const auto & header = getPort().getHeader(); + if (!isCancelled() && current_row < data_part->rows_count) { size_t rows_to_read = data_part->index_granularity.getMarkRows(current_mark); bool continue_reading = (current_mark != 0); - auto & sample = reader->getColumns(); + const auto & sample = reader->getColumns(); Columns columns(sample.size()); - size_t rows_readed = reader->readRows(current_mark, continue_reading, rows_to_read, columns); + size_t rows_read = reader->readRows(current_mark, continue_reading, rows_to_read, columns); - if (rows_readed) + if (rows_read) { - current_row += rows_readed; - current_mark += (rows_to_read == rows_readed); + current_row += rows_read; + current_mark += (rows_to_read == rows_read); bool should_evaluate_missing_defaults = false; - reader->fillMissingColumns(columns, should_evaluate_missing_defaults, rows_readed); + reader->fillMissingColumns(columns, should_evaluate_missing_defaults, rows_read); if (should_evaluate_missing_defaults) { @@ -113,20 +92,21 @@ try reader->performRequiredConversions(columns); - res = header.cloneEmpty(); - /// Reorder columns and fill result block. size_t num_columns = sample.size(); + Columns res_columns; + res_columns.reserve(num_columns); + auto it = sample.begin(); for (size_t i = 0; i < num_columns; ++i) { - if (res.has(it->name)) - res.getByName(it->name).column = std::move(columns[i]); + if (header.has(it->name)) + res_columns.emplace_back(std::move(columns[i])); ++it; } - res.checkNumberOfRows(); + return Chunk(std::move(res_columns), rows_read); } } else @@ -134,7 +114,7 @@ try finish(); } - return res; + return {}; } catch (...) { @@ -144,8 +124,7 @@ catch (...) throw; } - -void MergeTreeSequentialBlockInputStream::finish() +void MergeTreeSequentialSource::finish() { /** Close the files (before destroying the object). * When many sources are created, but simultaneously reading only a few of them, @@ -155,7 +134,6 @@ void MergeTreeSequentialBlockInputStream::finish() data_part.reset(); } - -MergeTreeSequentialBlockInputStream::~MergeTreeSequentialBlockInputStream() = default; +MergeTreeSequentialSource::~MergeTreeSequentialSource() = default; } diff --git a/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.h b/src/Storages/MergeTree/MergeTreeSequentialSource.h similarity index 73% rename from src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.h rename to src/Storages/MergeTree/MergeTreeSequentialSource.h index fd57a39fd7a..dac559913aa 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.h +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include @@ -9,40 +9,33 @@ namespace DB { /// Lightweight (in terms of logic) stream for reading single part from MergeTree -class MergeTreeSequentialBlockInputStream : public IBlockInputStream +class MergeTreeSequentialSource : public SourceWithProgress { public: - MergeTreeSequentialBlockInputStream( + MergeTreeSequentialSource( const MergeTreeData & storage_, - const MergeTreeData::DataPartPtr & data_part_, + MergeTreeData::DataPartPtr data_part_, Names columns_to_read_, bool read_with_direct_io_, bool take_column_types_from_storage, bool quiet = false ); - ~MergeTreeSequentialBlockInputStream() override; + ~MergeTreeSequentialSource() override; - String getName() const override { return "MergeTreeSequentialBlockInputStream"; } - - Block getHeader() const override; - - /// Closes readers and unlock part locks - void finish(); + String getName() const override { return "MergeTreeSequentialSource"; } size_t getCurrentMark() const { return current_mark; } size_t getCurrentRow() const { return current_row; } protected: - Block readImpl() override; + Chunk generate() override; private: const MergeTreeData & storage; - Block header; - /// Data part will not be removed if the pointer owns it MergeTreeData::DataPartPtr data_part; @@ -52,7 +45,7 @@ private: /// Should read using direct IO bool read_with_direct_io; - Logger * log = &Logger::get("MergeTreeSequentialBlockInputStream"); + Logger * log = &Logger::get("MergeTreeSequentialSource"); std::shared_ptr mark_cache; using MergeTreeReaderPtr = std::unique_ptr; @@ -65,8 +58,8 @@ private: size_t current_row = 0; private: - void fixHeader(Block & header_block) const; - + /// Closes readers and unlock part locks + void finish(); }; } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index bbd1fd6cbeb..dfe3643e60c 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -42,6 +42,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingUInt64, number_of_free_entries_in_pool_to_execute_mutation, 10, "When there is less than specified number of free entries in pool, do not execute part mutations. This is to leave free threads for regular merges and avoid \"Too many parts\"", 0) \ M(SettingSeconds, old_parts_lifetime, 8 * 60, "How many seconds to keep obsolete parts.", 0) \ M(SettingSeconds, temporary_directories_lifetime, 86400, "How many seconds to keep tmp_-directories.", 0) \ + M(SettingSeconds, lock_acquire_timeout_for_background_operations, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "For background operations like merges, mutations etc. How many seconds before failing to acquire table locks.", 0) \ \ /** Inserts settings. */ \ M(SettingUInt64, parts_to_delay_insert, 150, "If table contains at least that many active parts in single partition, artificially slow down insert into table.", 0) \ @@ -56,6 +57,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingUInt64, min_replicated_logs_to_keep, 100, "Keep about this number of last records in ZooKeeper log, even if they are obsolete. It doesn't affect work of tables: used only to diagnose ZooKeeper log before cleaning.", 0) \ M(SettingSeconds, prefer_fetch_merged_part_time_threshold, 3600, "If time passed after replication log entry creation exceeds this threshold and sum size of parts is greater than \"prefer_fetch_merged_part_size_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.", 0) \ M(SettingUInt64, prefer_fetch_merged_part_size_threshold, 10ULL * 1024 * 1024 * 1024, "If sum size of parts exceeds this threshold and time passed after replication log entry creation is greater than \"prefer_fetch_merged_part_time_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.", 0) \ + M(SettingBool, always_fetch_merged_part, 0, "If true, replica never merge parts and always download merged parts from other replicas.", 0) \ M(SettingUInt64, max_suspicious_broken_parts, 10, "Max broken parts, if more - deny automatic deletion.", 0) \ M(SettingUInt64, max_files_to_modify_in_alter_columns, 75, "Not apply ALTER if number of files for modification(deletion, addition) more than this.", 0) \ M(SettingUInt64, max_files_to_remove_in_alter_columns, 50, "Not apply ALTER, if number of files for deletion more than this.", 0) \ diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index fa29494d1c9..623da6318ff 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -82,8 +82,8 @@ static bool isConditionGood(const ASTPtr & condition) if (function->name != "equals") return false; - auto left_arg = function->arguments->children.front().get(); - auto right_arg = function->arguments->children.back().get(); + auto * left_arg = function->arguments->children.front().get(); + auto * right_arg = function->arguments->children.back().get(); /// try to ensure left_arg points to ASTIdentifier if (!left_arg->as() && right_arg->as()) @@ -202,10 +202,10 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, cond_it); total_size_of_moved_conditions += cond_it->columns_size; - /// Move all other conditions that depend on the same set of columns. + /// Move all other viable conditions that depend on the same set of columns. for (auto jt = where_conditions.begin(); jt != where_conditions.end();) { - if (jt->columns_size == cond_it->columns_size && jt->identifiers == cond_it->identifiers) + if (jt->viable && jt->columns_size == cond_it->columns_size && jt->identifiers == cond_it->identifiers) prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, jt++); else ++jt; diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 221170b7a32..2b482ac7c29 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -15,10 +15,11 @@ namespace ErrorCodes MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, + const MergeTreeIndices & skip_indices, CompressionCodecPtr default_codec, bool blocks_are_granules_size) : MergedBlockOutputStream( - data_part, columns_list_, default_codec, {}, + data_part, columns_list_, skip_indices, default_codec, {}, data_part->storage.global_context.getSettings().min_bytes_to_use_direct_io, blocks_are_granules_size) { @@ -27,6 +28,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, + const MergeTreeIndices & skip_indices, CompressionCodecPtr default_codec, const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size, size_t aio_threshold, @@ -49,7 +51,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( disk->createDirectories(part_path); - writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); + writer = data_part->getWriter(columns_list, skip_indices, default_codec, writer_settings); writer->initPrimaryIndex(); writer->initSkipIndices(); } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index ee453f41a31..5a92977640e 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -16,12 +16,14 @@ public: MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, + const MergeTreeIndices & skip_indices, CompressionCodecPtr default_codec, bool blocks_are_granules_size = false); MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, + const MergeTreeIndices & skip_indices, CompressionCodecPtr default_codec, const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size, size_t aio_threshold, diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 77bff8e4f02..892b4eccfbc 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -8,15 +8,14 @@ namespace ErrorCodes } MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( - const MergeTreeDataPartPtr & data_part, const Block & header_, bool sync_, - CompressionCodecPtr default_codec, bool skip_offsets_, + const MergeTreeDataPartPtr & data_part, + const Block & header_, + CompressionCodecPtr default_codec, const std::vector & indices_to_recalc, WrittenOffsetColumns * offset_columns_, const MergeTreeIndexGranularity & index_granularity, - const MergeTreeIndexGranularityInfo * index_granularity_info, - bool is_writing_temp_files) - : IMergedBlockOutputStream(data_part), - header(header_), sync(sync_) + const MergeTreeIndexGranularityInfo * index_granularity_info) + : IMergedBlockOutputStream(data_part), header(header_) { const auto & global_settings = data_part->storage.global_context.getSettings(); MergeTreeWriterSettings writer_settings( @@ -24,11 +23,13 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity(), global_settings.min_bytes_to_use_direct_io); - writer_settings.is_writing_temp_files = is_writing_temp_files; - writer_settings.skip_offsets = skip_offsets_; + writer = data_part->getWriter( + header.getNamesAndTypesList(), + indices_to_recalc, + default_codec, + std::move(writer_settings), + index_granularity); - writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, - default_codec,std::move(writer_settings), index_granularity); writer->setWrittenOffsetColumns(offset_columns_); writer->initSkipIndices(); } @@ -62,7 +63,7 @@ MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums(MergeTreeData::MutableD { /// Finish columns serialization. MergeTreeData::DataPart::Checksums checksums; - writer->finishDataSerialization(checksums, sync); + writer->finishDataSerialization(checksums); writer->finishSkipIndicesSerialization(checksums); auto columns = new_part->getColumns(); diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index c785bbaf6d0..2c5024bbcfe 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -11,17 +11,16 @@ class MergeTreeDataPartWriterWide; class MergedColumnOnlyOutputStream final : public IMergedBlockOutputStream { public: - /// skip_offsets: used when ALTERing columns if we know that array offsets are not altered. /// Pass empty 'already_written_offset_columns' first time then and pass the same object to subsequent instances of MergedColumnOnlyOutputStream /// if you want to serialize elements of Nested data structure in different instances of MergedColumnOnlyOutputStream. MergedColumnOnlyOutputStream( - const MergeTreeDataPartPtr & data_part, const Block & header_, bool sync_, - CompressionCodecPtr default_codec_, bool skip_offsets_, + const MergeTreeDataPartPtr & data_part, + const Block & header_, + CompressionCodecPtr default_codec_, const std::vector & indices_to_recalc_, WrittenOffsetColumns * offset_columns_ = nullptr, const MergeTreeIndexGranularity & index_granularity = {}, - const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr, - bool is_writing_temp_files = false); + const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr); Block getHeader() const override { return header; } void write(const Block & block) override; @@ -31,7 +30,6 @@ public: private: Block header; - bool sync; }; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index fda0a8eb5a8..72255081e6b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -147,11 +147,11 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) /// That is, do not insert the same data to the same partition twice. block_id = part->info.partition_id + "_" + toString(hash_value.words[0]) + "_" + toString(hash_value.words[1]); - LOG_DEBUG(log, "Wrote block with ID '" << block_id << "', " << block.rows() << " rows"); + LOG_DEBUG(log, "Wrote block with ID '" << block_id << "', " << current_block.block.rows() << " rows"); } else { - LOG_DEBUG(log, "Wrote block with " << block.rows() << " rows"); + LOG_DEBUG(log, "Wrote block with " << current_block.block.rows() << " rows"); } try diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 77a5bca7a92..b1164f6621c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -57,7 +57,8 @@ void ReplicatedMergeTreeCleanupThread::iterate() { /// TODO: Implement tryLockStructureForShare. - auto lock = storage.lockStructureForShare(false, ""); + auto lock = storage.lockStructureForShare( + false, RWLockImpl::NO_QUERY, storage.getSettings()->lock_acquire_timeout_for_background_operations); storage.clearOldTemporaryDirectories(); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 17b716d14c2..98b30498839 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -203,7 +203,9 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na else if (part->name == part_name) { auto zookeeper = storage.getZooKeeper(); - auto table_lock = storage.lockStructureForShare(false, RWLockImpl::NO_QUERY); + + auto table_lock = storage.lockStructureForShare( + false, RWLockImpl::NO_QUERY, storage.getSettings()->lock_acquire_timeout_for_background_operations); auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( part->getColumns(), part->checksums); @@ -246,9 +248,13 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na LOG_INFO(log, "Part " << part_name << " looks good."); } - catch (const Exception &) + catch (const Exception & e) { - /// TODO Better to check error code. + /// Don't count the part as broken if there is not enough memory to load it. + /// In fact, there can be many similar situations. + /// But it is OK, because there is a safety guard against deleting too many parts. + if (isNotEnoughMemoryErrorCode(e.code())) + throw; tryLogCurrentException(log, __PRETTY_FUNCTION__); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 796664d889c..a8133e3b06d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -16,6 +16,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int UNEXPECTED_NODE_IN_ZOOKEEPER; extern const int UNFINISHED; + extern const int ABORTED; } @@ -426,6 +427,8 @@ bool ReplicatedMergeTreeQueue::removeFromVirtualParts(const MergeTreePartInfo & void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) { std::lock_guard lock(pull_logs_to_queue_mutex); + if (pull_log_blocker.isCancelled()) + throw Exception("Log pulling is cancelled", ErrorCodes::ABORTED); String index_str = zookeeper->get(replica_path + "/log_pointer"); UInt64 index; @@ -878,7 +881,7 @@ size_t ReplicatedMergeTreeQueue::getConflictsCountForRange( { std::vector> conflicts; - for (auto & future_part_elem : future_parts) + for (const auto & future_part_elem : future_parts) { /// Do not check itself log entry if (future_part_elem.second->znode_name == entry.znode_name) @@ -1645,8 +1648,21 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( } bool ReplicatedMergeTreeMergePredicate::operator()( - const MergeTreeData::DataPartPtr & left, const MergeTreeData::DataPartPtr & right, - String * out_reason) const + const MergeTreeData::DataPartPtr & left, + const MergeTreeData::DataPartPtr & right, + String * out_reason) const +{ + if (left) + return canMergeTwoParts(left, right, out_reason); + else + return canMergeSinglePart(right, out_reason); +} + + +bool ReplicatedMergeTreeMergePredicate::canMergeTwoParts( + const MergeTreeData::DataPartPtr & left, + const MergeTreeData::DataPartPtr & right, + String * out_reason) const { /// A sketch of a proof of why this method actually works: /// @@ -1758,7 +1774,7 @@ bool ReplicatedMergeTreeMergePredicate::operator()( { if (out_reason) *out_reason = "There are " + toString(covered.size()) + " parts (from " + covered.front() - + " to " + covered.back() + ") that are still not present or beeing processed by " + + " to " + covered.back() + ") that are still not present or being processed by " + " other background process on this replica between " + left->name + " and " + right->name; return false; } @@ -1781,6 +1797,39 @@ bool ReplicatedMergeTreeMergePredicate::operator()( return true; } +bool ReplicatedMergeTreeMergePredicate::canMergeSinglePart( + const MergeTreeData::DataPartPtr & part, + String * out_reason) const +{ + if (part->name == inprogress_quorum_part) + { + if (out_reason) + *out_reason = "Quorum insert for part " + part->name + " is currently in progress"; + return false; + } + + if (prev_virtual_parts.getContainingPart(part->info).empty()) + { + if (out_reason) + *out_reason = "Entry for part " + part->name + " hasn't been read from the replication log yet"; + return false; + } + + std::lock_guard lock(queue.state_mutex); + + /// We look for containing parts in queue.virtual_parts (and not in prev_virtual_parts) because queue.virtual_parts is newer + /// and it is guaranteed that it will contain all merges assigned before this object is constructed. + String containing_part = queue.virtual_parts.getContainingPart(part->info); + if (containing_part != part->name) + { + if (out_reason) + *out_reason = "Part " + part->name + " has already been assigned a merge into " + containing_part; + return false; + } + + return true; +} + std::optional> ReplicatedMergeTreeMergePredicate::getDesiredMutationVersion(const MergeTreeData::DataPartPtr & part) const { @@ -1791,7 +1840,7 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir /// the part (checked by querying queue.virtual_parts), we can confidently assign a mutation to /// version X for this part. - /// We cannot mutate part if it's beeing inserted with quorum and it's not + /// We cannot mutate part if it's being inserted with quorum and it's not /// already reached. if (part->name == inprogress_quorum_part) return {}; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index fcb3dfb4b86..8e58c8b7af2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -356,6 +356,9 @@ public: /// A blocker that stops selects from the queue ActionBlocker actions_blocker; + /// A blocker that stops pulling entries from replication log to queue + ActionBlocker pull_log_blocker; + /// Adds a subscriber SubscriberHandler addSubscriber(SubscriberCallBack && callback); @@ -405,12 +408,22 @@ class ReplicatedMergeTreeMergePredicate public: ReplicatedMergeTreeMergePredicate(ReplicatedMergeTreeQueue & queue_, zkutil::ZooKeeperPtr & zookeeper); + /// Depending on the existence of left part checks a merge predicate for two parts or for single part. + bool operator()(const MergeTreeData::DataPartPtr & left, + const MergeTreeData::DataPartPtr & right, + String * out_reason = nullptr) const; + /// Can we assign a merge with these two parts? /// (assuming that no merge was assigned after the predicate was constructed) /// If we can't and out_reason is not nullptr, set it to the reason why we can't merge. - bool operator()( - const MergeTreeData::DataPartPtr & left, const MergeTreeData::DataPartPtr & right, - String * out_reason = nullptr) const; + bool canMergeTwoParts(const MergeTreeData::DataPartPtr & left, + const MergeTreeData::DataPartPtr & right, + String * out_reason = nullptr) const; + + /// Can we assign a merge this part and some other part? + /// For example a merge of a part and itself is needed for TTL. + /// This predicate is checked for the first part of each partitition. + bool canMergeSinglePart(const MergeTreeData::DataPartPtr & part, String * out_reason) const; /// Return nonempty optional of desired mutation version and alter version. /// If we have no alter (modify/drop) mutations in mutations queue, than we return biggest possible diff --git a/src/Storages/MergeTree/SimpleMergeSelector.cpp b/src/Storages/MergeTree/SimpleMergeSelector.cpp index a566e474a5f..ccb56188431 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.cpp +++ b/src/Storages/MergeTree/SimpleMergeSelector.cpp @@ -42,7 +42,7 @@ struct Estimator } } - SimpleMergeSelector::PartsInPartition getBest() + SimpleMergeSelector::PartsInPartition getBest() const { return SimpleMergeSelector::PartsInPartition(best_begin, best_end); } diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 52d9a2750c7..c12ed12c2e0 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -24,6 +24,22 @@ namespace ErrorCodes { extern const int CORRUPTED_DATA; extern const int UNKNOWN_PART_TYPE; + extern const int MEMORY_LIMIT_EXCEEDED; + extern const int CANNOT_ALLOCATE_MEMORY; + extern const int CANNOT_MUNMAP; + extern const int CANNOT_MREMAP; +} + + +bool isNotEnoughMemoryErrorCode(int code) +{ + /// Don't count the part as broken if there is not enough memory to load it. + /// In fact, there can be many similar situations. + /// But it is OK, because there is a safety guard against deleting too many parts. + return code == ErrorCodes::MEMORY_LIMIT_EXCEEDED + || code == ErrorCodes::CANNOT_ALLOCATE_MEMORY + || code == ErrorCodes::CANNOT_MUNMAP + || code == ErrorCodes::CANNOT_MREMAP; } diff --git a/src/Storages/MergeTree/checkDataPart.h b/src/Storages/MergeTree/checkDataPart.h index 9293c60ded4..9b905c5835e 100644 --- a/src/Storages/MergeTree/checkDataPart.h +++ b/src/Storages/MergeTree/checkDataPart.h @@ -19,4 +19,7 @@ IMergeTreeDataPart::Checksums checkDataPart( const MergeTreeDataPartType & part_type, bool require_checksums, std::function is_cancelled = []{ return false; }); + +bool isNotEnoughMemoryErrorCode(int code); + } diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index d48f8525162..f3569c344d9 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -137,7 +138,7 @@ void MutationCommands::readText(ReadBuffer & in) ParserAlterCommandList p_alter_commands; auto commands_ast = parseQuery( - p_alter_commands, commands_str.data(), commands_str.data() + commands_str.length(), "mutation commands list", 0); + p_alter_commands, commands_str.data(), commands_str.data() + commands_str.length(), "mutation commands list", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); for (ASTAlterCommand * command_ast : commands_ast->as().commands) { auto command = MutationCommand::parse(command_ast, true); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index ede162a180a..fab0c0ceb7b 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -76,6 +75,7 @@ StorageBuffer::StorageBuffer( , destination_id(destination_id_) , allow_materialized(allow_materialized_) , log(&Logger::get("StorageBuffer (" + table_id_.getFullTableName() + ")")) + , bg_pool(global_context.getBufferFlushSchedulePool()) { setColumns(columns_); setConstraints(constraints_); @@ -83,12 +83,7 @@ StorageBuffer::StorageBuffer( StorageBuffer::~StorageBuffer() { - // Should not happen if shutdown was called - if (flush_thread.joinable()) - { - shutdown_event.set(); - flush_thread.join(); - } + flush_handle->deactivate(); } @@ -168,7 +163,8 @@ Pipes StorageBuffer::read( if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); - auto destination_lock = destination->lockStructureForShare(false, context.getCurrentQueryId()); + auto destination_lock = destination->lockStructureForShare( + false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); const bool dst_has_same_structure = std::all_of(column_names.begin(), column_names.end(), [this, destination](const String& column_name) { @@ -224,7 +220,7 @@ Pipes StorageBuffer::read( pipe.getHeader(), header_after_adding_defaults, getColumns().getDefaults(), context)); pipe.addSimpleTransform(std::make_shared( - pipe.getHeader(), header, ConvertingTransform::MatchColumnsMode::Name, context)); + pipe.getHeader(), header, ConvertingTransform::MatchColumnsMode::Name)); } } } @@ -396,6 +392,9 @@ public: least_busy_lock = std::unique_lock(least_busy_buffer->mutex); } insertIntoBuffer(block, *least_busy_buffer); + least_busy_lock.unlock(); + + storage.reschedule(); } private: StorageBuffer & storage; @@ -457,16 +456,15 @@ void StorageBuffer::startup() << " Set appropriate system_profile to fix this."); } - flush_thread = ThreadFromGlobalPool(&StorageBuffer::flushThread, this); + + flush_handle = bg_pool.createTask(log->name() + "/Bg", [this]{ flushBack(); }); + flush_handle->activateAndSchedule(); } void StorageBuffer::shutdown() { - shutdown_event.set(); - - if (flush_thread.joinable()) - flush_thread.join(); + flush_handle->deactivate(); try { @@ -594,7 +592,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc ProfileEvents::increment(ProfileEvents::StorageBufferFlush); - LOG_TRACE(log, "Flushing buffer with " << rows << " rows, " << bytes << " bytes, age " << time_passed << " seconds."); + LOG_TRACE(log, "Flushing buffer with " << rows << " rows, " << bytes << " bytes, age " << time_passed << " seconds " << (check_thresholds ? "(bg)" : "(direct)") << "."); if (!destination_id) return; @@ -662,7 +660,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl << " have different type of column " << backQuoteIfNeed(column.name) << " (" << dst_col.type->getName() << " != " << column.type->getName() << "). Block of data is converted."); - column.column = castColumn(column, dst_col.type, global_context); + column.column = castColumn(column, dst_col.type); column.type = dst_col.type; } @@ -696,21 +694,42 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl } -void StorageBuffer::flushThread() +void StorageBuffer::flushBack() { - setThreadName("BufferFlush"); - - do + try { - try - { - flushAllBuffers(true); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } while (!shutdown_event.tryWait(1000)); + flushAllBuffers(true); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + + reschedule(); +} + +void StorageBuffer::reschedule() +{ + time_t min_first_write_time = std::numeric_limits::max(); + time_t rows = 0; + + for (auto & buffer : buffers) + { + std::lock_guard lock(buffer.mutex); + min_first_write_time = buffer.first_write_time; + rows += buffer.data.rows(); + } + + /// will be rescheduled via INSERT + if (!rows) + return; + + time_t current_time = time(nullptr); + time_t time_passed = current_time - min_first_write_time; + + size_t min = std::max(min_thresholds.time - time_passed, 1); + size_t max = std::max(max_thresholds.time - time_passed, 1); + flush_handle->scheduleAfter(std::min(min, max) * 1000); } void StorageBuffer::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) @@ -736,7 +755,7 @@ std::optional StorageBuffer::totalRows() const return underlying_rows; UInt64 rows = 0; - for (auto & buffer : buffers) + for (const auto & buffer : buffers) { std::lock_guard lock(buffer.mutex); rows += buffer.data.rows(); @@ -747,7 +766,7 @@ std::optional StorageBuffer::totalRows() const std::optional StorageBuffer::totalBytes() const { UInt64 bytes = 0; - for (auto & buffer : buffers) + for (const auto & buffer : buffers) { std::lock_guard lock(buffer.mutex); bytes += buffer.data.bytes(); @@ -757,7 +776,7 @@ std::optional StorageBuffer::totalBytes() const void StorageBuffer::alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); checkAlterIsPossible(params, context.getSettingsRef()); diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 93f95692b18..4c6c911e339 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -118,10 +118,6 @@ private: Poco::Logger * log; - Poco::Event shutdown_event; - /// Resets data by timeout. - ThreadFromGlobalPool flush_thread; - void flushAllBuffers(bool check_thresholds = true); /// Reset the buffer. If check_thresholds is set - resets only if thresholds are exceeded. void flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false); @@ -131,7 +127,11 @@ private: /// `table` argument is passed, as it is sometimes evaluated beforehand. It must match the `destination`. void writeBlockToDestination(const Block & block, StoragePtr table); - void flushThread(); + void flushBack(); + void reschedule(); + + BackgroundSchedulePool & bg_pool; + BackgroundSchedulePoolTaskHolder flush_handle; protected: /** num_shards - the level of internal parallelism (the number of independent buffers) diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 6f2d41d2b13..9b2c5784d85 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -1,48 +1,112 @@ -#include -#include -#include -#include -#include #include #include +#include +#include #include #include #include #include -#include -#include +#include #include #include +#include namespace DB { - namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int THERE_IS_NO_COLUMN; + extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE; +} + +namespace +{ + void checkNamesAndTypesCompatibleWithDictionary(const String & dictionary_name, const ColumnsDescription & columns, const DictionaryStructure & dictionary_structure) + { + auto dictionary_names_and_types = StorageDictionary::getNamesAndTypes(dictionary_structure); + std::set names_and_types_set(dictionary_names_and_types.begin(), dictionary_names_and_types.end()); + + for (const auto & column : columns.getOrdinary()) + { + if (names_and_types_set.find(column) == names_and_types_set.end()) + { + std::string message = "Not found column "; + message += column.name + " " + column.type->getName(); + message += " in dictionary " + backQuote(dictionary_name) + ". "; + message += "There are only columns "; + message += StorageDictionary::generateNamesAndTypesDescription(dictionary_names_and_types); + throw Exception(message, ErrorCodes::THERE_IS_NO_COLUMN); + } + } + } +} + + +NamesAndTypesList StorageDictionary::getNamesAndTypes(const DictionaryStructure & dictionary_structure) +{ + NamesAndTypesList dictionary_names_and_types; + + if (dictionary_structure.id) + dictionary_names_and_types.emplace_back(dictionary_structure.id->name, std::make_shared()); + + /// In old-style (XML) configuration we don't have this attributes in the + /// main attribute list, so we have to add them to columns list explicitly. + /// In the new configuration (DDL) we have them both in range_* nodes and + /// main attribute list, but for compatibility we add them before main + /// attributes list. + if (dictionary_structure.range_min) + dictionary_names_and_types.emplace_back(dictionary_structure.range_min->name, dictionary_structure.range_min->type); + + if (dictionary_structure.range_max) + dictionary_names_and_types.emplace_back(dictionary_structure.range_max->name, dictionary_structure.range_max->type); + + if (dictionary_structure.key) + { + for (const auto & attribute : *dictionary_structure.key) + dictionary_names_and_types.emplace_back(attribute.name, attribute.type); + } + + for (const auto & attribute : dictionary_structure.attributes) + { + /// Some attributes can be already added (range_min and range_max) + if (!dictionary_names_and_types.contains(attribute.name)) + dictionary_names_and_types.emplace_back(attribute.name, attribute.type); + } + + return dictionary_names_and_types; +} + + +String StorageDictionary::generateNamesAndTypesDescription(const NamesAndTypesList & list) +{ + std::stringstream ss; + bool first = true; + for (const auto & name_and_type : list) + { + if (!std::exchange(first, false)) + ss << ", "; + ss << name_and_type.name << ' ' << name_and_type.type->getName(); + } + return ss.str(); } StorageDictionary::StorageDictionary( const StorageID & table_id_, - const ColumnsDescription & columns_, - const Context & context, - bool attach, - const String & dictionary_name_) + const String & dictionary_name_, + const DictionaryStructure & dictionary_structure_) : IStorage(table_id_) , dictionary_name(dictionary_name_) - , logger(&Poco::Logger::get("StorageDictionary")) { - setColumns(columns_); + setColumns(ColumnsDescription{getNamesAndTypes(dictionary_structure_)}); +} - if (!attach) - { - const auto & dictionary = context.getExternalDictionariesLoader().getDictionary(dictionary_name); - const DictionaryStructure & dictionary_structure = dictionary->getStructure(); - checkNamesAndTypesCompatibleWithDictionary(dictionary_structure); - } + +void StorageDictionary::checkTableCanBeDropped() const +{ + throw Exception("Cannot detach dictionary " + backQuote(dictionary_name) + " as table, use DETACH DICTIONARY query.", ErrorCodes::CANNOT_DETACH_DICTIONARY_AS_TABLE); } Pipes StorageDictionary::read( @@ -62,44 +126,6 @@ Pipes StorageDictionary::read( return pipes; } -NamesAndTypesList StorageDictionary::getNamesAndTypes(const DictionaryStructure & dictionary_structure) -{ - NamesAndTypesList dictionary_names_and_types; - - if (dictionary_structure.id) - dictionary_names_and_types.emplace_back(dictionary_structure.id->name, std::make_shared()); - if (dictionary_structure.range_min) - dictionary_names_and_types.emplace_back(dictionary_structure.range_min->name, dictionary_structure.range_min->type); - if (dictionary_structure.range_max) - dictionary_names_and_types.emplace_back(dictionary_structure.range_max->name, dictionary_structure.range_max->type); - if (dictionary_structure.key) - for (const auto & attribute : *dictionary_structure.key) - dictionary_names_and_types.emplace_back(attribute.name, attribute.type); - - for (const auto & attribute : dictionary_structure.attributes) - dictionary_names_and_types.emplace_back(attribute.name, attribute.type); - - return dictionary_names_and_types; -} - -void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const -{ - auto dictionary_names_and_types = getNamesAndTypes(dictionary_structure); - std::set names_and_types_set(dictionary_names_and_types.begin(), dictionary_names_and_types.end()); - - for (const auto & column : getColumns().getOrdinary()) - { - if (names_and_types_set.find(column) == names_and_types_set.end()) - { - std::string message = "Not found column "; - message += column.name + " " + column.type->getName(); - message += " in dictionary " + dictionary_name + ". "; - message += "There are only columns "; - message += generateNamesAndTypesDescription(dictionary_names_and_types.begin(), dictionary_names_and_types.end()); - throw Exception(message, ErrorCodes::THERE_IS_NO_COLUMN); - } - } -} void registerStorageDictionary(StorageFactory & factory) { @@ -112,8 +138,11 @@ void registerStorageDictionary(StorageFactory & factory) args.engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args.engine_args[0], args.local_context); String dictionary_name = args.engine_args[0]->as().value.safeGet(); - return StorageDictionary::create( - args.table_id, args.columns, args.context, args.attach, dictionary_name); + const auto & dictionary = args.context.getExternalDictionariesLoader().getDictionary(dictionary_name); + const DictionaryStructure & dictionary_structure = dictionary->getStructure(); + checkNamesAndTypesCompatibleWithDictionary(dictionary_name, args.columns, dictionary_structure); + + return StorageDictionary::create(args.table_id, dictionary_name, dictionary_structure); }); } diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index fd6cb1902dc..7bb6fc22480 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -1,23 +1,12 @@ #pragma once #include -#include -#include #include -#include -#include -namespace Poco -{ -class Logger; -} - namespace DB { struct DictionaryStructure; -struct IDictionaryBase; -class ExternalDictionaries; class StorageDictionary final : public ext::shared_ptr_helper, public IStorage { @@ -25,6 +14,8 @@ class StorageDictionary final : public ext::shared_ptr_helper public: std::string getName() const override { return "Dictionary"; } + void checkTableCanBeDropped() const override; + Pipes read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -33,42 +24,18 @@ public: unsigned threads) override; static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); + static String generateNamesAndTypesDescription(const NamesAndTypesList & list); - template - static std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end) - { - std::string description; - { - WriteBufferFromString buffer(description); - bool first = true; - for (; begin != end; ++begin) - { - if (!first) - buffer << ", "; - first = false; - - buffer << begin->name << ' ' << begin->type->getName(); - } - } - - return description; - } + const String & dictionaryName() const { return dictionary_name; } private: - using Ptr = MultiVersion::Version; - String dictionary_name; - Poco::Logger * logger; - - void checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const; protected: StorageDictionary( const StorageID & table_id_, - const ColumnsDescription & columns_, - const Context & context, - bool attach, - const String & dictionary_name_); + const String & dictionary_name_, + const DictionaryStructure & dictionary_structure); }; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 2790d6e9bce..9c557ad5c8a 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -14,9 +14,12 @@ #include #include +#include + #include #include #include +#include #include #include @@ -73,6 +76,7 @@ namespace ErrorCodes extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int TOO_MANY_ROWS; extern const int UNABLE_TO_SKIP_UNUSED_SHARDS; + extern const int LOGICAL_ERROR; } namespace ActionLocks @@ -378,8 +382,77 @@ StoragePtr StorageDistributed::createWithOwnCluster( } -QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(const Context &context, QueryProcessingStage::Enum to_stage, const ASTPtr &query_ptr) const +bool StorageDistributed::canForceGroupByNoMerge(const Context &context, QueryProcessingStage::Enum to_stage, const ASTPtr & query_ptr) const { + const auto & settings = context.getSettingsRef(); + std::string reason; + + if (settings.distributed_group_by_no_merge) + return true; + /// Distributed-over-Distributed (see getQueryProcessingStageImpl()) + if (to_stage == QueryProcessingStage::WithMergeableState) + return false; + if (!settings.optimize_skip_unused_shards) + return false; + if (!has_sharding_key) + return false; + + const auto & select = query_ptr->as(); + + if (select.orderBy()) + return false; + + if (select.distinct) + { + for (auto & expr : select.select()->children) + { + const auto * id = expr->as(); + if (!id) + return false; + if (!sharding_key_expr->getSampleBlock().has(id->name)) + return false; + } + + reason = "DISTINCT " + backQuote(serializeAST(*select.select(), true)); + } + + // This can use distributed_group_by_no_merge but in this case limit stage + // should be done later (which is not the case right now). + if (select.limitBy() || select.limitLength()) + return false; + + const ASTPtr group_by = select.groupBy(); + if (!group_by) + { + if (!select.distinct) + return false; + } + else + { + // injective functions are optimized out in optimizeGroupBy() + // hence all we need to check is that column in GROUP BY matches sharding expression + auto & group_exprs = group_by->children; + if (group_exprs.empty()) + throw Exception("No ASTExpressionList in GROUP BY", ErrorCodes::LOGICAL_ERROR); + + const auto * id = group_exprs[0]->as(); + if (!id) + return false; + if (!sharding_key_expr->getSampleBlock().has(id->name)) + return false; + + reason = "GROUP BY " + backQuote(serializeAST(*group_by, true)); + } + + LOG_DEBUG(log, "Force distributed_group_by_no_merge for " << reason << " (injective)"); + return true; +} + +QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(const Context &context, QueryProcessingStage::Enum to_stage, const ASTPtr & query_ptr) const +{ + if (canForceGroupByNoMerge(context, to_stage, query_ptr)) + return QueryProcessingStage::Complete; + auto cluster = getOptimizedCluster(context, query_ptr); return getQueryProcessingStageImpl(context, to_stage, cluster); } @@ -463,7 +536,7 @@ void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, co void StorageDistributed::alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); checkAlterIsPossible(params, context.getSettingsRef()); @@ -577,15 +650,20 @@ void StorageDistributed::createDirectoryMonitors(const std::string & disk) } -void StorageDistributed::requireDirectoryMonitor(const std::string & disk, const std::string & name) +StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor(const std::string & disk, const std::string & name) { const std::string path(disk + relative_data_path + name); const std::string key(disk + name); std::lock_guard lock(cluster_nodes_mutex); auto & node_data = cluster_nodes_data[key]; - node_data.conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); - node_data.directory_monitor = std::make_unique(*this, path, node_data.conneciton_pool, monitors_blocker); + if (!node_data.directory_monitor) + { + node_data.conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); + node_data.directory_monitor = std::make_unique( + *this, path, node_data.conneciton_pool, monitors_blocker, global_context.getDistributedSchedulePool()); + } + return *node_data.directory_monitor; } size_t StorageDistributed::getShardCount() const @@ -648,12 +726,12 @@ ClusterPtr StorageDistributed::getOptimizedCluster(const Context & context, cons return cluster; } -void StorageDistributed::ClusterNodeData::flushAllData() +void StorageDistributed::ClusterNodeData::flushAllData() const { directory_monitor->flushAllData(); } -void StorageDistributed::ClusterNodeData::shutdownAndDropAllData() +void StorageDistributed::ClusterNodeData::shutdownAndDropAllData() const { directory_monitor->shutdownAndDropAllData(); } diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 81c6b54a63e..12cc94c7363 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -67,7 +67,9 @@ public: bool isRemote() const override { return true; } - QueryProcessingStage::Enum getQueryProcessingStage(const Context &, QueryProcessingStage::Enum /*to_stage*/, const ASTPtr &) const override; + /// Return true if distributed_group_by_no_merge may be applied. + bool canForceGroupByNoMerge(const Context &, QueryProcessingStage::Enum to_stage, const ASTPtr &) const; + QueryProcessingStage::Enum getQueryProcessingStage(const Context &, QueryProcessingStage::Enum to_stage, const ASTPtr &) const override; Pipes read( const Names & column_names, @@ -109,7 +111,7 @@ public: /// create directory monitors for each existing subdirectory void createDirectoryMonitors(const std::string & disk); /// ensure directory monitor thread and connectoin pool creation by disk and subdirectory name - void requireDirectoryMonitor(const std::string & disk, const std::string & name); + StorageDistributedDirectoryMonitor & requireDirectoryMonitor(const std::string & disk, const std::string & name); void flushClusterNodesAllData(); @@ -183,8 +185,8 @@ protected: std::unique_ptr directory_monitor; ConnectionPoolPtr conneciton_pool; - void flushAllData(); - void shutdownAndDropAllData(); + void flushAllData() const; + void shutdownAndDropAllData() const; }; std::unordered_map cluster_nodes_data; std::mutex cluster_nodes_mutex; diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index 1fd2d4ec2d8..70b84c076b7 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -57,7 +58,12 @@ void fillBufferWithRandomData(char * __restrict data, size_t size, pcg64 & rng) ColumnPtr fillColumnWithRandomData( - const DataTypePtr type, UInt64 limit, UInt64 max_array_length, UInt64 max_string_length, pcg64 & rng, const Context & context) + const DataTypePtr type, + UInt64 limit, + UInt64 max_array_length, + UInt64 max_string_length, + pcg64 & rng, + const Context & context) { TypeIndex idx = type->getTypeId(); @@ -205,7 +211,10 @@ ColumnPtr fillColumnWithRandomData( { auto column = ColumnUInt16::create(); column->getData().resize(limit); - fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(UInt16), rng); + + for (size_t i = 0; i < limit; ++i) + column->getData()[i] = rng() % (DATE_LUT_MAX_DAY_NUM + 1); /// Slow + return column; } case TypeIndex::UInt32: [[fallthrough]]; @@ -337,14 +346,24 @@ public: protected: Chunk generate() override { + /// To support Nested types, we will collect them to single Array of Tuple. + auto names_and_types = Nested::collect(block_header.getNamesAndTypesList()); + Columns columns; - columns.reserve(block_header.columns()); - DataTypes types = block_header.getDataTypes(); + columns.reserve(names_and_types.size()); - for (const auto & type : types) - columns.emplace_back(fillColumnWithRandomData(type, block_size, max_array_length, max_string_length, rng, context)); + Block compact_block; + for (const auto & elem : names_and_types) + { + compact_block.insert( + { + fillColumnWithRandomData(elem.type, block_size, max_array_length, max_string_length, rng, context), + elem.type, + elem.name + }); + } - return {std::move(columns), block_size}; + return {Nested::flatten(compact_block).getColumns(), block_size}; } private: diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index 8912680b1dd..9b17334e570 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -107,7 +107,7 @@ void registerStorageJoin(StorageFactory & factory) ASTs & engine_args = args.engine_args; - auto & settings = args.context.getSettingsRef(); + const auto & settings = args.context.getSettingsRef(); auto join_use_nulls = settings.join_use_nulls; auto max_rows_in_join = settings.max_rows_in_join; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index d974367bb93..a2f17d84e4e 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -185,7 +185,9 @@ Pipes StorageMaterializedView::read( const unsigned num_streams) { auto storage = getTargetTable(); - auto lock = storage->lockStructureForShare(false, context.getCurrentQueryId()); + auto lock = storage->lockStructureForShare( + false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); + if (query_info.order_by_optimizer) query_info.input_sorting_info = query_info.order_by_optimizer->getInputOrder(storage); @@ -200,7 +202,8 @@ Pipes StorageMaterializedView::read( BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const Context & context) { auto storage = getTargetTable(); - auto lock = storage->lockStructureForShare(true, context.getCurrentQueryId()); + auto lock = storage->lockStructureForShare( + true, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto stream = storage->write(query, context); stream->addTableLock(lock); return stream; @@ -258,7 +261,7 @@ void StorageMaterializedView::alter( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index b4cdabe4af2..20a0ee2dccb 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -31,6 +31,7 @@ public: bool supportsPrewhere() const override { return getTargetTable()->supportsPrewhere(); } bool supportsFinal() const override { return getTargetTable()->supportsFinal(); } bool supportsIndexForIn() const override { return getTargetTable()->supportsIndexForIn(); } + bool supportsParallelInsert() const override { return getTargetTable()->supportsParallelInsert(); } bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override { return getTargetTable()->mayBenefitFromIndexForIn(left_in_operand, query_context); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index f7af76e72b0..28ef2b2a7fc 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -140,7 +140,7 @@ std::optional StorageMemory::totalRows() const { UInt64 rows = 0; std::lock_guard lock(mutex); - for (auto & buffer : data) + for (const auto & buffer : data) rows += buffer.rows(); return rows; } @@ -149,7 +149,7 @@ std::optional StorageMemory::totalBytes() const { UInt64 bytes = 0; std::lock_guard lock(mutex); - for (auto & buffer : data) + for (const auto & buffer : data) bytes += buffer.bytes(); return bytes; } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 06c80a613c2..8dcc04cb680 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -97,7 +96,7 @@ StoragePtr StorageMerge::getFirstTable(F && predicate) const while (iterator->isValid()) { - auto & table = iterator->table(); + const auto & table = iterator->table(); if (table.get() != this && predicate(table)) return table; @@ -118,7 +117,8 @@ bool StorageMerge::isRemote() const bool StorageMerge::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const { /// It's beneficial if it is true for at least one table. - StorageListWithLocks selected_tables = getSelectedTables(query_context.getCurrentQueryId()); + StorageListWithLocks selected_tables = getSelectedTables( + query_context.getCurrentQueryId(), query_context.getSettingsRef()); size_t i = 0; for (const auto & table : selected_tables) @@ -146,7 +146,7 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage(const Context & while (iterator->isValid()) { - auto & table = iterator->table(); + const auto & table = iterator->table(); if (table.get() != this) { ++selected_table_size; @@ -195,7 +195,7 @@ Pipes StorageMerge::read( * This is necessary to correctly pass the recommended number of threads to each table. */ StorageListWithLocks selected_tables = getSelectedTables( - query_info.query, has_table_virtual_column, context.getCurrentQueryId()); + query_info.query, has_table_virtual_column, context.getCurrentQueryId(), context.getSettingsRef()); if (selected_tables.empty()) /// FIXME: do we support sampling in this case? @@ -232,7 +232,7 @@ Pipes StorageMerge::read( remaining_streams -= current_streams; current_streams = std::max(size_t(1), current_streams); - auto & storage = std::get<0>(table); + const auto & storage = std::get<0>(table); /// If sampling requested, then check that table supports it. if (query_info.query->as()->sampleSize() && !storage->supportsSampling()) @@ -258,7 +258,7 @@ Pipes StorageMerge::createSources(const SelectQueryInfo & query_info, const Quer const std::shared_ptr & modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams) { - auto & [storage, struct_lock, table_name] = storage_with_lock; + const auto & [storage, struct_lock, table_name] = storage_with_lock; SelectQueryInfo modified_query_info = query_info; modified_query_info.query = query_info.query->clone(); @@ -356,16 +356,17 @@ Pipes StorageMerge::createSources(const SelectQueryInfo & query_info, const Quer } -StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String & query_id) const +StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String & query_id, const Settings & settings) const { StorageListWithLocks selected_tables; auto iterator = getDatabaseIterator(); while (iterator->isValid()) { - auto & table = iterator->table(); + const auto & table = iterator->table(); if (table.get() != this) - selected_tables.emplace_back(table, table->lockStructureForShare(false, query_id), iterator->name()); + selected_tables.emplace_back( + table, table->lockStructureForShare(false, query_id, settings.lock_acquire_timeout), iterator->name()); iterator->next(); } @@ -374,7 +375,8 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String } -StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr & query, bool has_virtual_column, const String & query_id) const +StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables( + const ASTPtr & query, bool has_virtual_column, const String & query_id, const Settings & settings) const { StorageListWithLocks selected_tables; DatabaseTablesIteratorPtr iterator = getDatabaseIterator(); @@ -390,7 +392,8 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr if (storage.get() != this) { - selected_tables.emplace_back(storage, storage->lockStructureForShare(false, query_id), iterator->name()); + selected_tables.emplace_back( + storage, storage->lockStructureForShare(false, query_id, settings.lock_acquire_timeout), iterator->name()); virtual_column->insert(iterator->name()); } @@ -435,7 +438,7 @@ void StorageMerge::checkAlterIsPossible(const AlterCommands & commands, const Se void StorageMerge::alter( const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); StorageInMemoryMetadata storage_metadata = getInMemoryMetadata(); @@ -474,7 +477,7 @@ void StorageMerge::convertingSourceStream(const Block & header, const Context & Pipe & pipe, QueryProcessingStage::Enum processed_stage) { Block before_block_header = pipe.getHeader(); - pipe.addSimpleTransform(std::make_shared(before_block_header, header, ConvertingTransform::MatchColumnsMode::Name, context)); + pipe.addSimpleTransform(std::make_shared(before_block_header, header, ConvertingTransform::MatchColumnsMode::Name)); auto where_expression = query->as()->where(); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index b846663bc54..fbc1906b411 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -57,9 +57,10 @@ private: using StorageWithLockAndName = std::tuple; using StorageListWithLocks = std::list; - StorageListWithLocks getSelectedTables(const String & query_id) const; + StorageListWithLocks getSelectedTables(const String & query_id, const Settings & settings) const; - StorageMerge::StorageListWithLocks getSelectedTables(const ASTPtr & query, bool has_virtual_column, const String & query_id) const; + StorageMerge::StorageListWithLocks getSelectedTables( + const ASTPtr & query, bool has_virtual_column, const String & query_id, const Settings & settings) const; template StoragePtr getFirstTable(F && predicate) const; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index ce65f5748cc..5110258834e 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -223,7 +223,7 @@ void StorageMergeTree::alter( /// This alter can be performed at metadata level only if (commands.isSettingsAlter()) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); changeSettings(metadata.settings_ast, table_lock_holder); @@ -231,7 +231,7 @@ void StorageMergeTree::alter( } else { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); changeSettings(metadata.settings_ast, table_lock_holder); /// Reinitialize primary key because primary key column types might have changed. @@ -541,7 +541,8 @@ bool StorageMergeTree::merge( bool deduplicate, String * out_disable_reason) { - auto table_lock_holder = lockStructureForShare(true, RWLockImpl::NO_QUERY); + auto table_lock_holder = lockStructureForShare( + true, RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); FutureMergedMutatedPart future_part; @@ -551,8 +552,12 @@ bool StorageMergeTree::merge( { std::lock_guard lock(currently_processing_in_background_mutex); - auto can_merge = [this, &lock] (const DataPartPtr & left, const DataPartPtr & right, String *) + auto can_merge = [this, &lock] (const DataPartPtr & left, const DataPartPtr & right, String *) -> bool { + /// This predicate is checked for the first part of each partition. + /// (left = nullptr, right = "first part of partition") + if (!left) + return !currently_merging_mutating_parts.count(right); return !currently_merging_mutating_parts.count(left) && !currently_merging_mutating_parts.count(right) && getCurrentMutationVersion(left, lock) == getCurrentMutationVersion(right, lock); }; @@ -659,7 +664,8 @@ BackgroundProcessingPoolTaskResult StorageMergeTree::movePartsTask() bool StorageMergeTree::tryMutatePart() { - auto table_lock_holder = lockStructureForShare(true, RWLockImpl::NO_QUERY); + auto table_lock_holder = lockStructureForShare( + true, RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); size_t max_ast_elements = global_context.getSettingsRef().max_expanded_ast_elements; FutureMergedMutatedPart future_part; @@ -790,7 +796,8 @@ BackgroundProcessingPoolTaskResult StorageMergeTree::mergeMutateTask() { { /// TODO: Implement tryLockStructureForShare. - auto lock_structure = lockStructureForShare(false, ""); + auto lock_structure = lockStructureForShare( + false, RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); clearOldPartsFromFilesystem(); clearOldTemporaryDirectories(); } @@ -983,14 +990,16 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma case PartitionCommand::FREEZE_PARTITION: { - auto lock = lockStructureForShare(false, context.getCurrentQueryId()); + auto lock = lockStructureForShare( + false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); freezePartition(command.partition, command.with_name, context, lock); } break; case PartitionCommand::FREEZE_ALL_PARTITIONS: { - auto lock = lockStructureForShare(false, context.getCurrentQueryId()); + auto lock = lockStructureForShare( + false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); freezeAll(command.with_name, context, lock); } break; @@ -1008,7 +1017,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons /// This protects against "revival" of data for a removed partition after completion of merge. auto merge_blocker = merger_mutator.merges_blocker.cancel(); /// Waits for completion of merge and does not start new ones. - auto lock = lockExclusively(context.getCurrentQueryId()); + auto lock = lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); String partition_id = getPartitionIDFromQuery(partition, context); @@ -1055,8 +1064,8 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context) { - auto lock1 = lockStructureForShare(false, context.getCurrentQueryId()); - auto lock2 = source_table->lockStructureForShare(false, context.getCurrentQueryId()); + auto lock1 = lockStructureForShare(false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); + auto lock2 = source_table->lockStructureForShare(false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); Stopwatch watch; MergeTreeData & src_data = checkStructureAndGetMergeTreeData(source_table); @@ -1126,8 +1135,8 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & context) { - auto lock1 = lockStructureForShare(false, context.getCurrentQueryId()); - auto lock2 = dest_table->lockStructureForShare(false, context.getCurrentQueryId()); + auto lock1 = lockStructureForShare(false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); + auto lock2 = dest_table->lockStructureForShare(false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto dest_table_storage = std::dynamic_pointer_cast(dest_table); if (!dest_table_storage) diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index 3bde1cab4c7..8b98536e4d7 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -1,13 +1,16 @@ #pragma once -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_MYSQL -#include +# include -#include -#include -#include +# include +# include +# include namespace DB diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 878be5bbf2d..bafb3d9a9fb 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -48,7 +48,7 @@ void StorageNull::checkAlterIsPossible(const AlterCommands & commands, const Set void StorageNull::alter( const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 31456c8d1f1..930b20e3756 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1,4 +1,5 @@ -#include +#include + #include #include #include @@ -25,6 +26,8 @@ #include #include +#include + #include #include @@ -246,6 +249,11 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( createTableIfNotExists(); + /// We have to check granularity on other replicas. If it's fixed we + /// must create our new replica with fixed granularity and store this + /// information in /replica/metadata. + other_replicas_fixed_granularity = checkFixedGranualrityInZookeeper(); + checkTableStructure(zookeeper_path); Coordination::Stat metadata_stat; @@ -256,11 +264,14 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } else { + /// In old tables this node may missing or be empty String replica_metadata; bool replica_metadata_exists = current_zookeeper->tryGet(replica_path + "/metadata", replica_metadata); if (!replica_metadata_exists || replica_metadata.empty()) { + /// We have to check shared node granularity before we create ours. + other_replicas_fixed_granularity = checkFixedGranualrityInZookeeper(); ReplicatedMergeTreeTableMetadata current_metadata(*this); current_zookeeper->createOrUpdate(replica_path + "/metadata", current_metadata.toString(), zkutil::CreateMode::Persistent); } @@ -289,9 +300,6 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } createNewZooKeeperNodes(); - - - other_replicas_fixed_granularity = checkFixedGranualrityInZookeeper(); } @@ -472,7 +480,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column if (metadata_diff.sorting_key_changed) { ParserNotEmptyExpressionList parser(false); - auto new_sorting_key_expr_list = parseQuery(parser, metadata_diff.new_sorting_key, 0); + auto new_sorting_key_expr_list = parseQuery(parser, metadata_diff.new_sorting_key, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (new_sorting_key_expr_list->children.size() == 1) metadata.order_by_ast = new_sorting_key_expr_list->children[0]; @@ -500,7 +508,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column if (metadata_diff.ttl_table_changed) { ParserTTLExpressionList parser; - metadata.ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0); + metadata.ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); } } @@ -977,6 +985,14 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) LOG_TRACE(log, log_message.rdbuf()); } + const auto storage_settings_ptr = getSettings(); + + if (storage_settings_ptr->always_fetch_merged_part) + { + LOG_INFO(log, "Will fetch part " << entry.new_part_name << " because setting 'always_fetch_merged_part' is true"); + return false; + } + DataPartsVector parts; bool have_all_parts = true; for (const String & name : entry.source_parts) @@ -997,7 +1013,6 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) parts.push_back(part); } - const auto storage_settings_ptr = getSettings(); if (!have_all_parts) { /// If you do not have all the necessary parts, try to take some already merged part from someone. @@ -1038,7 +1053,8 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) ReservationPtr reserved_space = reserveSpacePreferringTTLRules(estimated_space_for_merge, ttl_infos, time(nullptr), max_volume_index); - auto table_lock = lockStructureForShare(false, RWLockImpl::NO_QUERY); + auto table_lock = lockStructureForShare( + false, RWLockImpl::NO_QUERY, storage_settings_ptr->lock_acquire_timeout_for_background_operations); FutureMergedMutatedPart future_merged_part(parts, entry.new_part_type); if (future_merged_part.name != entry.new_part_name) @@ -1173,7 +1189,8 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM /// Can throw an exception. ReservationPtr reserved_space = reserveSpace(estimated_space_for_result, source_part->disk); - auto table_lock = lockStructureForShare(false, RWLockImpl::NO_QUERY); + auto table_lock = lockStructureForShare( + false, RWLockImpl::NO_QUERY, storage_settings_ptr->lock_acquire_timeout_for_background_operations); MutableDataPartPtr new_part; Transaction transaction(*this); @@ -1264,7 +1281,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) ++total_fetches; SCOPE_EXIT({--total_fetches;}); - if (storage_settings_ptr->replicated_max_parallel_fetches_for_table && current_table_fetches >= storage_settings_ptr->replicated_max_parallel_fetches_for_table) + if (storage_settings_ptr->replicated_max_parallel_fetches_for_table + && current_table_fetches >= storage_settings_ptr->replicated_max_parallel_fetches_for_table) { throw Exception("Too many fetches from replicas for table, maximum: " + storage_settings_ptr->replicated_max_parallel_fetches_for_table.toString(), ErrorCodes::TOO_MANY_FETCHES); @@ -1406,7 +1424,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } catch (...) { - /** If you can not download the part you need for some merge, it's better not to try to get other parts for this merge, + /** If we can not download the part we need for some merge, it's better not to try to get other parts for this merge, * but try to get already merged part. To do this, move the action to get the remaining parts * for this merge at the end of the queue. */ @@ -1527,7 +1545,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) PartDescriptions parts_to_add; DataPartsVector parts_to_remove; - auto table_lock_holder_dst_table = lockStructureForShare(false, RWLockImpl::NO_QUERY); + auto table_lock_holder_dst_table = lockStructureForShare( + false, RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); for (size_t i = 0; i < entry_replace.new_part_names.size(); ++i) { @@ -1589,7 +1608,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) return 0; } - table_lock_holder_src_table = source_table->lockStructureForShare(false, RWLockImpl::NO_QUERY); + table_lock_holder_src_table = source_table->lockStructureForShare( + false, RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); DataPartStates valid_states{MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}; @@ -2203,7 +2223,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() FutureMergedMutatedPart future_merged_part; if (max_source_parts_size_for_merge > 0 && - merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, merge_pred)) + merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, merge_pred, nullptr)) { success = createLogEntryToMergeParts(zookeeper, future_merged_part.parts, future_merged_part.name, future_merged_part.type, deduplicate, force_ttl); @@ -2681,6 +2701,58 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) } +void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) +{ + auto zookeeper = getZooKeeper(); + + /// The name of the previous part for which the quorum was reached. + const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + + /// Delete information from "last_part" node. + + while (true) + { + Coordination::Stat added_parts_stat; + String old_added_parts = zookeeper->get(quorum_last_part_path, &added_parts_stat); + + ReplicatedMergeTreeQuorumAddedParts parts_with_quorum(format_version); + + if (!old_added_parts.empty()) + parts_with_quorum.fromString(old_added_parts); + + /// Delete information about particular partition. + if (!parts_with_quorum.added_parts.count(partition_id)) + { + /// There is no information about interested part. + break; + } + + parts_with_quorum.added_parts.erase(partition_id); + + String new_added_parts = parts_with_quorum.toString(); + + auto code = zookeeper->trySet(quorum_last_part_path, new_added_parts, added_parts_stat.version); + + if (code == Coordination::ZOK) + { + break; + } + else if (code == Coordination::ZNONODE) + { + /// Node is deleted. It is impossible, but it is Ok. + break; + } + else if (code == Coordination::ZBADVERSION) + { + /// Node was updated meanwhile. We must re-read it and repeat all the actions. + continue; + } + else + throw Coordination::Exception(code, quorum_last_part_path); + } +} + + bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const String & source_replica_path, bool to_detached, size_t quorum) { const auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); @@ -2712,7 +2784,8 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin TableStructureReadLockHolder table_lock_holder; if (!to_detached) - table_lock_holder = lockStructureForShare(true, RWLockImpl::NO_QUERY); + table_lock_holder = lockStructureForShare( + true, RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); /// Logging Stopwatch stopwatch; @@ -2890,6 +2963,7 @@ void StorageReplicatedMergeTree::shutdown() fetcher.blocker.cancelForever(); merger_mutator.merges_blocker.cancelForever(); parts_mover.moves_blocker.cancelForever(); + queue.pull_log_blocker.cancelForever(); restarting_thread.shutdown(); @@ -3013,7 +3087,7 @@ void StorageReplicatedMergeTree::foreachCommittedParts(const Func & func) const { auto max_added_blocks = getMaxAddedBlocks(); auto lock = lockParts(); - for (auto & part : getDataPartsStateRange(DataPartState::Committed)) + for (const auto & part : getDataPartsStateRange(DataPartState::Committed)) { if (part->isEmpty()) continue; @@ -3179,7 +3253,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer { /// TODO (relax this lock) - auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); + auto table_lock = lockExclusively(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); @@ -3206,7 +3280,8 @@ void StorageReplicatedMergeTree::alter( if (params.isSettingsAlter()) { - lockStructureExclusively(table_lock_holder, query_context.getCurrentQueryId()); + lockStructureExclusively( + table_lock_holder, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); /// We don't replicate storage_settings_ptr ALTER. It's local operation. /// Also we don't upgrade alter lock to table structure lock. StorageInMemoryMetadata metadata = getInMemoryMetadata(); @@ -3272,7 +3347,8 @@ void StorageReplicatedMergeTree::alter( if (ast_to_str(current_metadata.settings_ast) != ast_to_str(future_metadata.settings_ast)) { - lockStructureExclusively(table_lock_holder, query_context.getCurrentQueryId()); + lockStructureExclusively( + table_lock_holder, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); /// Just change settings current_metadata.settings_ast = future_metadata.settings_ast; changeSettings(current_metadata.settings_ast, table_lock_holder); @@ -3290,7 +3366,6 @@ void StorageReplicatedMergeTree::alter( alter_entry->alter_version = new_metadata_version; alter_entry->create_time = time(nullptr); - auto maybe_mutation_commands = params.getMutationCommands(current_metadata); alter_entry->have_mutation = !maybe_mutation_commands.empty(); @@ -3441,14 +3516,16 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part case PartitionCommand::FREEZE_PARTITION: { - auto lock = lockStructureForShare(false, query_context.getCurrentQueryId()); + auto lock = lockStructureForShare( + false, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); freezePartition(command.partition, command.with_name, query_context, lock); } break; case PartitionCommand::FREEZE_ALL_PARTITIONS: { - auto lock = lockStructureForShare(false, query_context.getCurrentQueryId()); + auto lock = lockStructureForShare( + false, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); freezeAll(command.with_name, query_context, lock); } break; @@ -3536,6 +3613,9 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPt waitForAllReplicasToProcessLogEntry(entry); } } + + /// Cleaning possibly stored information about parts from /quorum/last_part node in ZooKeeper. + cleanLastPartNode(partition_id); } @@ -3622,7 +3702,11 @@ void StorageReplicatedMergeTree::drop(TableStructureWriteLockHolder &) LOG_INFO(log, "Removing replica " << replica_path); replica_is_active_node = nullptr; + /// It may left some garbage if replica_path subtree are concurently modified zookeeper->tryRemoveRecursive(replica_path); + if (zookeeper->exists(replica_path)) + LOG_ERROR(log, "Replica was not completely removed from ZooKeeper, " + << replica_path << " still exists and may contain some garbage."); /// Check that `zookeeper_path` exists: it could have been deleted by another replica after execution of previous line. Strings replicas; @@ -3630,6 +3714,9 @@ void StorageReplicatedMergeTree::drop(TableStructureWriteLockHolder &) { LOG_INFO(log, "Removing table " << zookeeper_path << " (this might take several minutes)"); zookeeper->tryRemoveRecursive(zookeeper_path); + if (zookeeper->exists(zookeeper_path)) + LOG_ERROR(log, "Table was not completely removed from ZooKeeper, " + << zookeeper_path << " still exists and may contain some garbage."); } } @@ -4456,7 +4543,8 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() { /// Critical section is not required (since grabOldParts() returns unique part set on each call) - auto table_lock = lockStructureForShare(false, RWLockImpl::NO_QUERY); + auto table_lock = lockStructureForShare( + false, RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); auto zookeeper = getZooKeeper(); DataPartsVector parts = grabOldParts(); @@ -4479,7 +4567,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() auto remove_parts_from_filesystem = [log=log] (const DataPartsVector & parts_to_remove) { - for (auto & part : parts_to_remove) + for (const auto & part : parts_to_remove) { try { @@ -4751,8 +4839,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ const Context & context) { /// First argument is true, because we possibly will add new data to current table. - auto lock1 = lockStructureForShare(true, context.getCurrentQueryId()); - auto lock2 = source_table->lockStructureForShare(false, context.getCurrentQueryId()); + auto lock1 = lockStructureForShare(true, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); + auto lock2 = source_table->lockStructureForShare(false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); Stopwatch watch; MergeTreeData & src_data = checkStructureAndGetMergeTreeData(source_table); @@ -4928,10 +5016,10 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ } } -void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & context) +void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & query_context) { - auto lock1 = lockStructureForShare(false, context.getCurrentQueryId()); - auto lock2 = dest_table->lockStructureForShare(false, context.getCurrentQueryId()); + auto lock1 = lockStructureForShare(false, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); + auto lock2 = dest_table->lockStructureForShare(false, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); auto dest_table_storage = std::dynamic_pointer_cast(dest_table); if (!dest_table_storage) @@ -4946,7 +5034,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Stopwatch watch; MergeTreeData & src_data = dest_table_storage->checkStructureAndGetMergeTreeData(*this); auto src_data_id = src_data.getStorageID(); - String partition_id = getPartitionIDFromQuery(partition, context); + String partition_id = getPartitionIDFromQuery(partition, query_context); DataPartsVector src_all_parts = src_data.getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); DataPartsVector src_parts; @@ -5102,7 +5190,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta parts_to_remove.clear(); cleanup_thread.wakeup(); - if (context.getSettingsRef().replication_alter_partitions_sync > 1) + if (query_context.getSettingsRef().replication_alter_partitions_sync > 1) { lock2.release(); dest_table_storage->waitForAllReplicasToProcessLogEntry(entry); @@ -5117,11 +5205,14 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta log_znode_path = dynamic_cast(*op_results.back()).path_created; entry_delete.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); - if (context.getSettingsRef().replication_alter_partitions_sync > 1) + if (query_context.getSettingsRef().replication_alter_partitions_sync > 1) { lock1.release(); waitForAllReplicasToProcessLogEntry(entry_delete); } + + /// Cleaning possibly stored information about parts from /quorum/last_part node in ZooKeeper. + cleanLastPartNode(partition_id); } void StorageReplicatedMergeTree::getCommitPartOps( @@ -5212,6 +5303,9 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI /// Let's fetch new log entries firstly queue.pullLogsToQueue(getZooKeeper()); + /// This is significant, because the execution of this task could be delayed at BackgroundPool. + /// And we force it to be executed. + queue_task_handle->wake(); Poco::Event target_size_event; auto callback = [&target_size_event, queue_size] (size_t new_queue_size) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 01dd32614f9..17b454f2757 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -464,6 +464,9 @@ private: /// With the quorum being tracked, add a replica to the quorum for the part. void updateQuorum(const String & part_name); + /// Deletes info from quorum/last_part node for particular partition_id. + void cleanLastPartNode(const String & partition_id); + /// Creates new block number if block with such block_id does not exist std::optional allocateBlockNumber( const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 72ae46787c8..f7d68866715 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -112,7 +112,7 @@ StorageSet::StorageSet( const ConstraintsDescription & constraints_, const Context & context_) : StorageSetOrJoinBase{relative_path_, table_id_, columns_, constraints_, context_}, - set(std::make_shared(SizeLimits(), false)) + set(std::make_shared(SizeLimits(), false, true)) { Block header = getSampleBlock(); header = header.sortColumns(); @@ -137,7 +137,7 @@ void StorageSet::truncate(const ASTPtr &, const Context &, TableStructureWriteLo header = header.sortColumns(); increment = 0; - set = std::make_shared(SizeLimits(), false); + set = std::make_shared(SizeLimits(), false, true); set->setHeader(header); } @@ -151,7 +151,7 @@ void StorageSetOrJoinBase::restore() return; } - static const auto file_suffix = ".bin"; + static const char * file_suffix = ".bin"; static const auto file_suffix_size = strlen(".bin"); Poco::DirectoryIterator dir_end; diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 05feeb7d786..287e93fc99a 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -12,13 +12,12 @@ #include #include -#include - #include #include #include #include +#include namespace DB @@ -78,8 +77,15 @@ Pipes StorageView::read( /// It's expected that the columns read from storage are not constant. /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. for (auto & pipe : pipes) + { pipe.addSimpleTransform(std::make_shared(pipe.getHeader())); + /// And also convert to expected structure. + pipe.addSimpleTransform(std::make_shared( + pipe.getHeader(), getSampleBlockForColumns(column_names), + ConvertingTransform::MatchColumnsMode::Name)); + } + return pipes; } @@ -125,7 +131,7 @@ ASTPtr StorageView::getRuntimeViewQuery(ASTSelectQuery * outer_query, const Cont replaceTableNameWithSubquery(outer_query, runtime_view_query); if (context.getSettingsRef().joined_subquery_requires_alias && tables_with_columns.size() > 1) { - for (auto & pr : tables_with_columns) + for (const auto & pr : tables_with_columns) if (pr.table.table.empty() && pr.table.alias.empty()) throw Exception("Not unique subquery in FROM requires an alias (or joined_subquery_requires_alias=0 to disable restriction).", ErrorCodes::ALIAS_REQUIRED); diff --git a/src/Storages/System/StorageSystemBuildOptions.cpp b/src/Storages/System/StorageSystemBuildOptions.cpp index 73d1c91f422..a63afcf4ce5 100644 --- a/src/Storages/System/StorageSystemBuildOptions.cpp +++ b/src/Storages/System/StorageSystemBuildOptions.cpp @@ -18,11 +18,15 @@ NamesAndTypesList StorageSystemBuildOptions::getNamesAndTypes() void StorageSystemBuildOptions::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const { - for (auto it = auto_config_build; *it; it += 2) +#if !defined(ARCADIA_BUILD) + for (auto * it = auto_config_build; *it; it += 2) { res_columns[0]->insert(it[0]); res_columns[1]->insert(it[1]); } +#else + UNUSED(res_columns); +#endif } } diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index cbf6ada9ed3..43594f5355a 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -62,12 +62,12 @@ public: ColumnPtr databases_, ColumnPtr tables_, Storages storages_, - const std::shared_ptr & access_, - String query_id_) + const Context & context) : SourceWithProgress(header_) , columns_mask(std::move(columns_mask_)), max_block_size(max_block_size_) , databases(std::move(databases_)), tables(std::move(tables_)), storages(std::move(storages_)) - , query_id(std::move(query_id_)), total_tables(tables->size()), access(access_) + , total_tables(tables->size()), access(context.getAccess()) + , query_id(context.getCurrentQueryId()), lock_acquire_timeout(context.getSettingsRef().lock_acquire_timeout) { } @@ -103,7 +103,7 @@ protected: try { - table_lock = storage->lockStructureForShare(false, query_id); + table_lock = storage->lockStructureForShare(false, query_id, lock_acquire_timeout); } catch (const Exception & e) { @@ -227,10 +227,11 @@ private: ColumnPtr databases; ColumnPtr tables; Storages storages; - String query_id; size_t db_table_num = 0; size_t total_tables; std::shared_ptr access; + String query_id; + SettingSeconds lock_acquire_timeout; }; @@ -302,7 +303,7 @@ Pipes StorageSystemColumns::read( const DatabasePtr database = databases.at(database_name); offsets[i] = i ? offsets[i - 1] : 0; - for (auto iterator = database->getTablesWithDictionaryTablesIterator(context); iterator->isValid(); iterator->next()) + for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { const String & table_name = iterator->name(); storages.emplace(std::piecewise_construct, @@ -331,8 +332,8 @@ Pipes StorageSystemColumns::read( pipes.emplace_back(std::make_shared( std::move(columns_mask), std::move(header), max_block_size, - std::move(filtered_database_column), std::move(filtered_table_column), std::move(storages), - context.getAccess(), context.getCurrentQueryId())); + std::move(filtered_database_column), std::move(filtered_table_column), + std::move(storages), context)); return pipes; } diff --git a/src/Storages/System/StorageSystemContributors.cpp b/src/Storages/System/StorageSystemContributors.cpp index f1a3c68efc5..cd0f31975cc 100644 --- a/src/Storages/System/StorageSystemContributors.cpp +++ b/src/Storages/System/StorageSystemContributors.cpp @@ -19,7 +19,7 @@ NamesAndTypesList StorageSystemContributors::getNamesAndTypes() void StorageSystemContributors::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const { std::vector contributors; - for (auto it = auto_contributors; *it; ++it) + for (auto * it = auto_contributors; *it; ++it) contributors.emplace_back(*it); std::shuffle(contributors.begin(), contributors.end(), thread_local_rng); diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index c64fe7b52b6..f3fd51330d9 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -60,7 +60,7 @@ protected: while (StoragesInfo info = stream.next()) { const auto parts = info.data->getDetachedParts(); - for (auto & p : parts) + for (const auto & p : parts) { size_t i = 0; new_columns[i++]->insert(info.database); diff --git a/src/Storages/System/StorageSystemDictionaries.cpp b/src/Storages/System/StorageSystemDictionaries.cpp index a3b32324700..4c54353c44d 100644 --- a/src/Storages/System/StorageSystemDictionaries.cpp +++ b/src/Storages/System/StorageSystemDictionaries.cpp @@ -53,7 +53,7 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con const bool check_access_for_dictionaries = !access->isGranted(AccessType::SHOW_DICTIONARIES); const auto & external_dictionaries = context.getExternalDictionariesLoader(); - for (const auto & load_result : external_dictionaries.getCurrentLoadResults()) + for (const auto & load_result : external_dictionaries.getLoadResults()) { const auto dict_ptr = std::dynamic_pointer_cast(load_result.object); @@ -66,9 +66,10 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con else { short_name = load_result.name; - if (!load_result.repository_name.empty() && startsWith(short_name, load_result.repository_name + ".")) + String repository_name = load_result.config ? load_result.config->repository_name : ""; + if (!repository_name.empty() && startsWith(short_name, repository_name + ".")) { - database = load_result.repository_name; + database = repository_name; short_name = short_name.substr(database.length() + 1); } } @@ -81,7 +82,7 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con res_columns[i++]->insert(database); res_columns[i++]->insert(short_name); res_columns[i++]->insert(static_cast(load_result.status)); - res_columns[i++]->insert(load_result.origin); + res_columns[i++]->insert(load_result.config ? load_result.config->path : ""); std::exception_ptr last_exception = load_result.exception; diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index a6163f89182..bfa25a99838 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -38,7 +38,7 @@ static StorageSystemGraphite::Configs getConfigs(const Context & context) for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - auto & table = iterator->table(); + const auto & table = iterator->table(); const MergeTreeData * table_data = dynamic_cast(table.get()); if (!table_data) diff --git a/src/Storages/System/StorageSystemModels.cpp b/src/Storages/System/StorageSystemModels.cpp index 306829cf6de..9fae9803b96 100644 --- a/src/Storages/System/StorageSystemModels.cpp +++ b/src/Storages/System/StorageSystemModels.cpp @@ -28,13 +28,13 @@ NamesAndTypesList StorageSystemModels::getNamesAndTypes() void StorageSystemModels::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { const auto & external_models_loader = context.getExternalModelsLoader(); - auto load_results = external_models_loader.getCurrentLoadResults(); + auto load_results = external_models_loader.getLoadResults(); for (const auto & load_result : load_results) { res_columns[0]->insert(load_result.name); res_columns[1]->insert(static_cast(load_result.status)); - res_columns[2]->insert(load_result.origin); + res_columns[2]->insert(load_result.config ? load_result.config->path : ""); if (load_result.object) { diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index d8f564b0160..19c6f6b3d03 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -62,7 +62,7 @@ StoragesInfo::getParts(MergeTreeData::DataPartStateVector & state, bool has_stat } StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const Context & context) - : query_id(context.getCurrentQueryId()) + : query_id(context.getCurrentQueryId()), settings(context.getSettings()) { /// Will apply WHERE to subset of columns and then add more columns. /// This is kind of complicated, but we use WHERE to do less work. @@ -192,7 +192,7 @@ StoragesInfo StoragesInfoStream::next() try { /// For table not to be dropped and set of columns to remain constant. - info.table_lock = info.storage->lockStructureForShare(false, query_id); + info.table_lock = info.storage->lockStructureForShare(false, query_id, settings.lock_acquire_timeout); } catch (const Exception & e) { diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index b30f7c62914..be8e45146cb 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -36,6 +36,8 @@ public: private: String query_id; + Settings settings; + ColumnPtr database_column; ColumnPtr table_column; diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index a8d5fc2ec57..d50a5fe4185 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -226,7 +226,7 @@ protected: const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name); if (!tables_it || !tables_it->isValid()) - tables_it = database->getTablesWithDictionaryTablesIterator(context); + tables_it = database->getTablesIterator(context); const bool need_lock_structure = needLockStructure(database, getPort().getHeader()); @@ -239,19 +239,25 @@ protected: StoragePtr table = nullptr; TableStructureReadLockHolder lock; - try + if (need_lock_structure) { - if (need_lock_structure) + table = tables_it->table(); + if (table == nullptr) { - table = tables_it->table(); - lock = table->lockStructureForShare(false, context.getCurrentQueryId()); - } - } - catch (const Exception & e) - { - if (e.code() == ErrorCodes::TABLE_IS_DROPPED) + // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable()) continue; - throw; + } + try + { + lock = table->lockStructureForShare( + false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::TABLE_IS_DROPPED) + continue; + throw; + } } ++rows_count; diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index eb386c66d60..711612cf9e5 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -35,7 +35,7 @@ ColumnsDescription getStructureOfRemoteTable( std::string fail_messages; - for (auto & shard_info : shards_info) + for (const auto & shard_info : shards_info) { try { @@ -137,7 +137,7 @@ ColumnsDescription getStructureOfRemoteTableInShard( column.default_desc.kind = columnDefaultKindFromString(kind_name); String expr_str = (*default_expr)[i].get(); column.default_desc.expression = parseQuery( - expr_parser, expr_str.data(), expr_str.data() + expr_str.size(), "default expression", 0); + expr_parser, expr_str.data(), expr_str.data() + expr_str.size(), "default expression", 0, context.getSettingsRef().max_parser_depth); } res.add(column); diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index f5fab52285d..5ad26b70803 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -1,6 +1,3 @@ -#include -#include "config_core.h" - #include #include diff --git a/src/Storages/registerStorages.h b/src/Storages/registerStorages.h index 63a758f5b38..c9874551073 100644 --- a/src/Storages/registerStorages.h +++ b/src/Storages/registerStorages.h @@ -1,6 +1,9 @@ #pragma once -#include -#include "config_core.h" + +#if !defined(ARCADIA_BUILD) +# include +# include "config_core.h" +#endif namespace DB { diff --git a/src/Storages/tests/gtest_SplitTokenExtractor.cpp b/src/Storages/tests/gtest_SplitTokenExtractor.cpp new file mode 100644 index 00000000000..9255e5ca817 --- /dev/null +++ b/src/Storages/tests/gtest_SplitTokenExtractor.cpp @@ -0,0 +1,196 @@ +#include + +#include +#include + +#include + +#include +#include +#include +#include + +namespace +{ +using namespace DB; +using namespace std::literals::string_literals; +} + +struct SplitTokenExtractorTestCase +{ + const std::string_view description; + const std::string source; + const std::vector tokens; +}; + +std::ostream & operator<<(std::ostream & ostr, const SplitTokenExtractorTestCase & test_case) +{ + return ostr << test_case.description; +} + +class SplitTokenExtractorTest : public ::testing::TestWithParam +{ +public: + void SetUp() override + { + const auto & param = GetParam(); + const auto & source = param.source; + data = std::make_unique>(source.data(), source.data() + source.size()); + + // add predefined padding that forms tokens to ensure no reads past end of buffer. + const char extra_padding[] = "this is the end \xd1\x8d\xd1\x82\xd0\xbe\xd0\xba\xd0\xbe \xd0\xbd\xd0\xb5\xd1\x86"; + data->insert(data->end(), std::begin(extra_padding), std::end(extra_padding)); + + data->resize(data->size() - sizeof(extra_padding)); + } + + std::unique_ptr> data; +}; + +TEST_P(SplitTokenExtractorTest, next) +{ + const auto & param = GetParam(); + + SplitTokenExtractor token_extractor; + + size_t i = 0; + + size_t pos = 0; + size_t token_start = 0; + size_t token_len = 0; + + for (const auto & expected_token : param.tokens) + { + SCOPED_TRACE(++i); + ASSERT_TRUE(token_extractor.next(data->data(), data->size(), &pos, &token_start, &token_len)); + + EXPECT_EQ(expected_token, std::string_view(data->data() + token_start, token_len)) + << " token_start:" << token_start << " token_len: " << token_len; + } + ASSERT_FALSE(token_extractor.next(data->data(), data->size(), &pos, &token_start, &token_len)) + << "\n\t=> \"" << param.source.substr(token_start, token_len) << "\"" + << "\n\t" << token_start << ", " << token_len << ", " << pos << ", " << data->size(); +} + +INSTANTIATE_TEST_SUITE_P(NoTokens, + SplitTokenExtractorTest, + ::testing::ValuesIn(std::initializer_list{ + { + "Empty input sequence produces no tokens.", + "", + {} + }, + { + "Whitespace only", + " ", + {} + }, + { + "Whitespace only large string", + " \t\v\n\r \t\v\n\r \t\v\n\r \t\v\n\r \t\v\n\r \t\v\n\r \t\v\n\r \t\v\n\r \t\v\n\r \t\v\n\r", + {} + } + }) +); + +INSTANTIATE_TEST_SUITE_P(ShortSingleToken, + SplitTokenExtractorTest, + ::testing::ValuesIn(std::initializer_list{ + { + "Short single token", + "foo", + {"foo"} + }, + { + "Short single token surruonded by whitespace", + "\t\vfoo\n\r", + {"foo"} + } + }) +); + +INSTANTIATE_TEST_SUITE_P(UTF8, + SplitTokenExtractorTest, + ::testing::ValuesIn(std::initializer_list{ + { + "Single token with mixed ASCII and UTF-8 chars", + "abc\u0442" "123\u0447XYZ\u043A", + {"abc\u0442" "123\u0447XYZ\u043A"} + }, + { + "Multiple UTF-8 tokens", + "\u043F\u0440\u0438\u0432\u0435\u0442, \u043C\u0438\u0440!", + {"\u043F\u0440\u0438\u0432\u0435\u0442", "\u043C\u0438\u0440"} + }, + }) +); + +INSTANTIATE_TEST_SUITE_P(MultipleTokens, + SplitTokenExtractorTest, + ::testing::ValuesIn(std::initializer_list{ + { + "Multiple tokens separated by whitespace", + "\nabc 123\tXYZ\r", + { + "abc", "123", "XYZ" + } + }, + { + "Multiple tokens separated by non-printable chars", + "\0abc\1" "123\2XYZ\4"s, + { + "abc", "123", "XYZ" + } + }, + { + "ASCII table is split into numeric, upper case and lower case letters", + + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16" + "\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNO" + "PQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c" + "\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1" + "\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6" + "\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb" + "\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0" + "\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"s, + { + "0123456789", "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz", + "\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c" + "\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1" + "\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6" + "\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb" + "\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0" + "\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + } + } + }) +); + + +INSTANTIATE_TEST_SUITE_P(SIMD_Cases, + SplitTokenExtractorTest, + ::testing::ValuesIn(std::initializer_list{ + { + "First 16 bytes are empty, then a shor token", + " abcdef", + {"abcdef"} + }, + { + "Token crosses bounday of 16-byte chunk", + " abcdef", + {"abcdef"} + }, + { + "Token ends at the end of 16-byte chunk", + " abcdef", + {"abcdef"} + }, + { + "Token crosses bondaries of multiple 16-byte chunks", + "abcdefghijklmnopqrstuvwxyz", + {"abcdefghijklmnopqrstuvwxyz"} + }, + }) +); diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 3fd79079c8f..7f5340ab59a 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -68,7 +68,7 @@ using DiskImplementations = testing::Types; TYPED_TEST_SUITE(StorageLogTest, DiskImplementations); // Returns data written to table in Values format. -std::string writeData(int rows, DB::StoragePtr & table) +std::string writeData(int rows, DB::StoragePtr & table, DB::Context & context) { using namespace DB; @@ -96,23 +96,23 @@ std::string writeData(int rows, DB::StoragePtr & table) block.insert(column); } - BlockOutputStreamPtr out = table->write({}, getContext()); + BlockOutputStreamPtr out = table->write({}, context); out->write(block); return data; } // Returns all table data in Values format. -std::string readData(DB::StoragePtr & table) +std::string readData(DB::StoragePtr & table, DB::Context & context) { using namespace DB; Names column_names; column_names.push_back("a"); - QueryProcessingStage::Enum stage = table->getQueryProcessingStage(getContext()); + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); - BlockInputStreamPtr in = std::make_shared(std::move(table->read(column_names, {}, getContext(), stage, 8192, 1)[0])); + BlockInputStreamPtr in = std::make_shared(std::move(table->read(column_names, {}, context, stage, 8192, 1)[0])); Block sample; { @@ -123,7 +123,7 @@ std::string readData(DB::StoragePtr & table) std::ostringstream ss; WriteBufferFromOStream out_buf(ss); - BlockOutputStreamPtr output = FormatFactory::instance().getOutput("Values", out_buf, sample, getContext()); + BlockOutputStreamPtr output = FormatFactory::instance().getOutput("Values", out_buf, sample, context); copyData(*in, *output); @@ -135,15 +135,16 @@ std::string readData(DB::StoragePtr & table) TYPED_TEST(StorageLogTest, testReadWrite) { using namespace DB; + auto context_holder = getContext(); std::string data; // Write several chunks of data. - data += writeData(10, this->getTable()); + data += writeData(10, this->getTable(), context_holder.context); data += ","; - data += writeData(20, this->getTable()); + data += writeData(20, this->getTable(), context_holder.context); data += ","; - data += writeData(10, this->getTable()); + data += writeData(10, this->getTable(), context_holder.context); - ASSERT_EQ(data, readData(this->getTable())); + ASSERT_EQ(data, readData(this->getTable(), context_holder.context)); } diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 385e47201a5..df6ee2bd0a5 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -18,7 +18,7 @@ using namespace DB; /// NOTE How to do better? struct State { - Context context = getContext(); + Context & context; NamesAndTypesList columns{ {"column", std::make_shared()}, {"apply_id", std::make_shared()}, @@ -27,7 +27,7 @@ struct State {"create_time", std::make_shared()}, }; - State() + explicit State(Context & context_) : context(context_) { registerFunctions(); DatabasePtr database = std::make_shared("test"); @@ -38,17 +38,11 @@ struct State } }; -static State & state() -{ - static State res; - return res; -} - static void check(const std::string & query, const std::string & expected, const Context & context, const NamesAndTypesList & columns) { ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query, 1000); + ASTPtr ast = parseQuery(parser, query, 1000, 1000); SelectQueryInfo query_info; query_info.syntax_analyzer_result = SyntaxAnalyzer(context).analyzeSelect(ast, columns); query_info.query = ast; @@ -60,47 +54,62 @@ static void check(const std::string & query, const std::string & expected, const TEST(TransformQueryForExternalDatabase, InWithSingleElement) { + auto context_holder = getContext(); + State state(context_holder.context); + check("SELECT column FROM test.table WHERE 1 IN (1)", R"(SELECT "column" FROM "test"."table" WHERE 1)", - state().context, state().columns); + state.context, state.columns); check("SELECT column FROM test.table WHERE column IN (1, 2)", R"(SELECT "column" FROM "test"."table" WHERE "column" IN (1, 2))", - state().context, state().columns); + state.context, state.columns); check("SELECT column FROM test.table WHERE column NOT IN ('hello', 'world')", R"(SELECT "column" FROM "test"."table" WHERE "column" NOT IN ('hello', 'world'))", - state().context, state().columns); + state.context, state.columns); } TEST(TransformQueryForExternalDatabase, Like) { + auto context_holder = getContext(); + State state(context_holder.context); + check("SELECT column FROM test.table WHERE column LIKE '%hello%'", R"(SELECT "column" FROM "test"."table" WHERE "column" LIKE '%hello%')", - state().context, state().columns); + state.context, state.columns); check("SELECT column FROM test.table WHERE column NOT LIKE 'w%rld'", R"(SELECT "column" FROM "test"."table" WHERE "column" NOT LIKE 'w%rld')", - state().context, state().columns); + state.context, state.columns); } TEST(TransformQueryForExternalDatabase, Substring) { + auto context_holder = getContext(); + State state(context_holder.context); + check("SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", R"(SELECT "column" FROM "test"."table")", - state().context, state().columns); + state.context, state.columns); } TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) { + auto context_holder = getContext(); + State state(context_holder.context); + check("SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4", R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))", - state().context, state().columns); + state.context, state.columns); check("SELECT column FROM test.table WHERE toString(column) = '42' AND left(column, 10) = RIGHT(column, 10) AND column = 42", R"(SELECT "column" FROM "test"."table" WHERE ("column" = 42))", - state().context, state().columns); + state.context, state.columns); } TEST(TransformQueryForExternalDatabase, Issue7245) { + auto context_holder = getContext(); + State state(context_holder.context); + check("select apply_id from test.table where apply_type = 2 and create_time > addDays(toDateTime('2019-01-01 01:02:03'),-7) and apply_status in (3,4)", R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))", - state().context, state().columns); + state.context, state.columns); } diff --git a/src/Storages/tests/storage_log.cpp b/src/Storages/tests/storage_log.cpp index 5df1b5764f1..4a7e9b221a0 100644 --- a/src/Storages/tests/storage_log.cpp +++ b/src/Storages/tests/storage_log.cpp @@ -26,7 +26,8 @@ try names_and_types.emplace_back("a", std::make_shared()); names_and_types.emplace_back("b", std::make_shared()); - auto context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + auto context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); context.setPath("./"); diff --git a/src/Storages/tests/system_numbers.cpp b/src/Storages/tests/system_numbers.cpp index 9fc84d76f5c..6955c90b74e 100644 --- a/src/Storages/tests/system_numbers.cpp +++ b/src/Storages/tests/system_numbers.cpp @@ -27,7 +27,8 @@ try WriteBufferFromOStream out_buf(std::cout); - auto context = Context::createGlobal(); + SharedContextHolder shared_context = Context::createShared(); + auto context = Context::createGlobal(shared_context.get()); context.makeGlobalContext(); QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); diff --git a/src/Storages/ya.make b/src/Storages/ya.make new file mode 100644 index 00000000000..5ed6fcbf637 --- /dev/null +++ b/src/Storages/ya.make @@ -0,0 +1,167 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/sparsehash +) + +SRCS( + Distributed/DirectoryMonitor.cpp + Distributed/DistributedBlockOutputStream.cpp + LiveView/StorageLiveView.cpp + MergeTree/ActiveDataPartSet.cpp + MergeTree/AllMergeSelector.cpp + MergeTree/BackgroundProcessingPool.cpp + MergeTree/BoolMask.cpp + MergeTree/checkDataPart.cpp + MergeTree/DataPartsExchange.cpp + MergeTree/EphemeralLockInZooKeeper.cpp + MergeTree/IMergedBlockOutputStream.cpp + MergeTree/IMergeTreeDataPart.cpp + MergeTree/IMergeTreeDataPartWriter.cpp + MergeTree/IMergeTreeReader.cpp + MergeTree/KeyCondition.cpp + MergeTree/LevelMergeSelector.cpp + MergeTree/localBackup.cpp + MergeTree/MergedBlockOutputStream.cpp + MergeTree/MergedColumnOnlyOutputStream.cpp + MergeTree/MergeList.cpp + MergeTree/MergeTreeBaseSelectProcessor.cpp + MergeTree/MergeTreeBlockOutputStream.cpp + MergeTree/MergeTreeBlockReadUtils.cpp + MergeTree/MergeTreeData.cpp + MergeTree/MergeTreeDataMergerMutator.cpp + MergeTree/MergeTreeDataPartChecksum.cpp + MergeTree/MergeTreeDataPartCompact.cpp + MergeTree/MergeTreeDataPartTTLInfo.cpp + MergeTree/MergeTreeDataPartType.cpp + MergeTree/MergeTreeDataPartWide.cpp + MergeTree/MergeTreeDataPartWriterCompact.cpp + MergeTree/MergeTreeDataPartWriterWide.cpp + MergeTree/MergeTreeDataSelectExecutor.cpp + MergeTree/MergeTreeDataWriter.cpp + MergeTree/MergeTreeIndexAggregatorBloomFilter.cpp + MergeTree/MergeTreeIndexBloomFilter.cpp + MergeTree/MergeTreeIndexConditionBloomFilter.cpp + MergeTree/MergeTreeIndexFullText.cpp + MergeTree/MergeTreeIndexGranularity.cpp + MergeTree/MergeTreeIndexGranularityInfo.cpp + MergeTree/MergeTreeIndexGranuleBloomFilter.cpp + MergeTree/MergeTreeIndexMinMax.cpp + MergeTree/MergeTreeIndexReader.cpp + MergeTree/MergeTreeIndexSet.cpp + MergeTree/MergeTreeIndices.cpp + MergeTree/MergeTreeMarksLoader.cpp + MergeTree/MergeTreeMutationEntry.cpp + MergeTree/MergeTreePartInfo.cpp + MergeTree/MergeTreePartition.cpp + MergeTree/MergeTreePartsMover.cpp + MergeTree/MergeTreeRangeReader.cpp + MergeTree/MergeTreeReaderCompact.cpp + MergeTree/MergeTreeReaderStream.cpp + MergeTree/MergeTreeReaderWide.cpp + MergeTree/MergeTreeReadPool.cpp + MergeTree/MergeTreeReverseSelectProcessor.cpp + MergeTree/MergeTreeSelectProcessor.cpp + MergeTree/MergeTreeSequentialSource.cpp + MergeTree/MergeTreeSettings.cpp + MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp + MergeTree/MergeTreeWhereOptimizer.cpp + MergeTree/registerStorageMergeTree.cpp + MergeTree/ReplicatedMergeTreeAddress.cpp + MergeTree/ReplicatedMergeTreeAltersSequence.cpp + MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp + MergeTree/ReplicatedMergeTreeCleanupThread.cpp + MergeTree/ReplicatedMergeTreeLogEntry.cpp + MergeTree/ReplicatedMergeTreeMutationEntry.cpp + MergeTree/ReplicatedMergeTreePartCheckThread.cpp + MergeTree/ReplicatedMergeTreePartHeader.cpp + MergeTree/ReplicatedMergeTreeQueue.cpp + MergeTree/ReplicatedMergeTreeRestartingThread.cpp + MergeTree/ReplicatedMergeTreeTableMetadata.cpp + MergeTree/SimpleMergeSelector.cpp + MergeTree/TTLMergeSelector.cpp + System/attachSystemTables.cpp + System/StorageSystemAggregateFunctionCombinators.cpp + System/StorageSystemAsynchronousMetrics.cpp + System/StorageSystemBuildOptions.cpp + System/StorageSystemClusters.cpp + System/StorageSystemCollations.cpp + System/StorageSystemColumns.cpp + System/StorageSystemContributors.cpp + System/StorageSystemContributors.generated.cpp + System/StorageSystemDatabases.cpp + System/StorageSystemDataTypeFamilies.cpp + System/StorageSystemDetachedParts.cpp + System/StorageSystemDictionaries.cpp + System/StorageSystemDisks.cpp + System/StorageSystemEvents.cpp + System/StorageSystemFormats.cpp + System/StorageSystemFunctions.cpp + System/StorageSystemGraphite.cpp + System/StorageSystemMacros.cpp + System/StorageSystemMerges.cpp + System/StorageSystemMergeTreeSettings.cpp + System/StorageSystemMetrics.cpp + System/StorageSystemModels.cpp + System/StorageSystemMutations.cpp + System/StorageSystemNumbers.cpp + System/StorageSystemOne.cpp + System/StorageSystemParts.cpp + System/StorageSystemPartsBase.cpp + System/StorageSystemPartsColumns.cpp + System/StorageSystemProcesses.cpp + System/StorageSystemQuotas.cpp + System/StorageSystemQuotaUsage.cpp + System/StorageSystemReplicas.cpp + System/StorageSystemReplicationQueue.cpp + System/StorageSystemRowPolicies.cpp + System/StorageSystemSettings.cpp + System/StorageSystemStackTrace.cpp + System/StorageSystemStoragePolicies.cpp + System/StorageSystemTableEngines.cpp + System/StorageSystemTableFunctions.cpp + System/StorageSystemTables.cpp + System/StorageSystemZeros.cpp + System/StorageSystemZooKeeper.cpp + AlterCommands.cpp + ColumnDefault.cpp + ColumnsDescription.cpp + ConstraintsDescription.cpp + getStructureOfRemoteTable.cpp + IndicesDescription.cpp + IStorage.cpp + MutationCommands.cpp + PartitionCommands.cpp + ReadInOrderOptimizer.cpp + registerStorages.cpp + StorageBuffer.cpp + StorageDictionary.cpp + StorageDistributed.cpp + StorageFactory.cpp + StorageFile.cpp + StorageGenerateRandom.cpp + StorageInMemoryMetadata.cpp + StorageInput.cpp + StorageJoin.cpp + StorageLog.cpp + StorageLogSettings.cpp + StorageMaterializedView.cpp + StorageMemory.cpp + StorageMerge.cpp + StorageMergeTree.cpp + StorageMySQL.cpp + StorageNull.cpp + StorageReplicatedMergeTree.cpp + StorageSet.cpp + StorageStripeLog.cpp + StorageTinyLog.cpp + StorageURL.cpp + StorageValues.cpp + StorageView.cpp + StorageXDBC.cpp + transformQueryForExternalDatabase.cpp + VirtualColumnUtils.cpp +) + +END() diff --git a/src/TableFunctions/ITableFunctionXDBC.h b/src/TableFunctions/ITableFunctionXDBC.h index 262c237bac2..547fda3f734 100644 --- a/src/TableFunctions/ITableFunctionXDBC.h +++ b/src/TableFunctions/ITableFunctionXDBC.h @@ -4,7 +4,10 @@ #include #include #include -#include + +#if !defined(ARCADIA_BUILD) +# include +#endif namespace DB { diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index be707c3520d..0afc74f163e 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -1,25 +1,27 @@ -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #if USE_MYSQL +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include "registerTableFunctions.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "registerTableFunctions.h" - -#include +# include namespace DB diff --git a/src/TableFunctions/parseColumnsListForTableFunction.cpp b/src/TableFunctions/parseColumnsListForTableFunction.cpp index 9b775b70492..c419164ac38 100644 --- a/src/TableFunctions/parseColumnsListForTableFunction.cpp +++ b/src/TableFunctions/parseColumnsListForTableFunction.cpp @@ -19,9 +19,7 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con Expected expected; Tokens tokens(structure.c_str(), structure.c_str() + structure.size()); - IParser::Pos token_iterator(tokens); - const Settings & settings = context.getSettingsRef(); - token_iterator.max_depth = settings.max_parser_depth; + IParser::Pos token_iterator(tokens, context.getSettingsRef().max_parser_depth); ParserColumnDeclarationList parser; ASTPtr columns_list_raw; diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 3cde28685cc..8c54913676a 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -1,6 +1,9 @@ #pragma once -#include -#include "config_core.h" + +#if !defined(ARCADIA_BUILD) +# include +# include "config_core.h" +#endif namespace DB { diff --git a/src/TableFunctions/ya.make b/src/TableFunctions/ya.make new file mode 100644 index 00000000000..4d5d53ead88 --- /dev/null +++ b/src/TableFunctions/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Common +) + +SRCS( + ITableFunction.cpp + ITableFunctionFileLike.cpp + ITableFunctionXDBC.cpp + parseColumnsListForTableFunction.cpp + registerTableFunctions.cpp + TableFunctionFactory.cpp + TableFunctionFile.cpp + TableFunctionGenerateRandom.cpp + TableFunctionInput.cpp + TableFunctionMerge.cpp + TableFunctionMySQL.cpp + TableFunctionNumbers.cpp + TableFunctionRemote.cpp + TableFunctionURL.cpp + TableFunctionValues.cpp + TableFunctionZeros.cpp +) + +END() diff --git a/src/ya.make b/src/ya.make new file mode 100644 index 00000000000..95e5914bbe7 --- /dev/null +++ b/src/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +PEERDIR( + clickhouse/src/Access + clickhouse/src/AggregateFunctions + clickhouse/src/Client + clickhouse/src/Columns + clickhouse/src/Common + clickhouse/src/Compression + clickhouse/src/Core + clickhouse/src/Databases + clickhouse/src/DataStreams + clickhouse/src/DataTypes + clickhouse/src/Formats + clickhouse/src/Dictionaries + clickhouse/src/Disks + clickhouse/src/Functions + clickhouse/src/Interpreters + clickhouse/src/IO + clickhouse/src/Parsers + clickhouse/src/Processors + clickhouse/src/Storages + clickhouse/src/TableFunctions +) + +END() diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 27a71f9949c..26fef8cb476 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -110,7 +110,7 @@ def get_stacktraces_from_gdb(server_pid): # collect server stacktraces from system.stack_trace table def get_stacktraces_from_clickhouse(client): try: - return subprocess.check_call("{} --allow_introspection_functions=1 --query \"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace FROM system.stack_trace format Vertical\"".format(client), shell=True) + return subprocess.check_output("{} --allow_introspection_functions=1 --query \"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace FROM system.stack_trace format Vertical\"".format(client), shell=True) except Exception as ex: return "Error occured while receiving stack traces from client: {}".format(str(ex)) @@ -182,7 +182,8 @@ def run_tests_array(all_tests_with_params): if args.skip and any(s in name for s in args.skip): print(MSG_SKIPPED + " - skip") skipped_total += 1 - elif not args.zookeeper and 'zookeeper' in name: + elif not args.zookeeper and ('zookeeper' in name + or 'replica' in name): print(MSG_SKIPPED + " - no zookeeper") skipped_total += 1 elif not args.shard and ('shard' in name @@ -568,7 +569,7 @@ if __name__ == '__main__': if args.tmp is None: args.tmp = '/tmp/clickhouse-test' if args.queries is None: - print_err("Failed to detect path to the queries directory. Please specify it with '--queries' option.") + print("Failed to detect path to the queries directory. Please specify it with '--queries' option.", file=sys.stderr) exit(1) if args.tmp is None: args.tmp = args.queries @@ -578,7 +579,7 @@ if __name__ == '__main__': elif find_binary(args.binary): args.client = args.binary + ' client' else: - print("No 'clickhouse' binary found in PATH") + print("No 'clickhouse' binary found in PATH", file=sys.stderr) parser.print_help() exit(1) diff --git a/tests/config/graphite.xml b/tests/config/graphite.xml new file mode 100644 index 00000000000..9920c8e42cf --- /dev/null +++ b/tests/config/graphite.xml @@ -0,0 +1,28 @@ + + + Version + + sum + sum + + 0 + 600 + + + 172800 + 6000 + + + + max + + 0 + 600 + + + 172800 + 6000 + + + + diff --git a/tests/integration/README.md b/tests/integration/README.md index 6ac224c449f..c72c009a0d6 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -12,9 +12,9 @@ You must install latest Docker from https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#set-up-the-repository Don't use Docker from your system repository. -* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python-pip libpq-dev` +* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev` * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest` -* [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: `sudo -H pip install docker-compose docker dicttoxml kazoo PyMySQL psycopg2 pymongo tzlocal kafka-python protobuf pytest-timeout minio rpm-confluent-schemaregistry` +* [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: `sudo -H pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio rpm-confluent-schemaregistry` (highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python-pytest python-dicttoxml python-docker python-pymysql python-pymongo python-tzlocal python-kazoo python-psycopg2 python-kafka python-pytest-timeout python-minio` @@ -94,7 +94,7 @@ cd docker/test/integration docker build -t yandex/clickhouse-integration-test . ``` -The helper container used by the `runner` script is in `tests/integration/image/Dockerfile`. +The helper container used by the `runner` script is in `docker/test/integration/runner/Dockerfile`. ### Adding new tests diff --git a/tests/integration/test_access_control_on_cluster/configs/users.d/access_management.xml b/tests/integration/helpers/0_common_instance_users.xml similarity index 100% rename from tests/integration/test_access_control_on_cluster/configs/users.d/access_management.xml rename to tests/integration/helpers/0_common_instance_users.xml diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 5dc93cb338a..39c6322acb1 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -28,6 +28,8 @@ from .client import Client from .hdfs_api import HDFSApi HELPERS_DIR = p.dirname(__file__) +CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..") +DOCKER_COMPOSE_DIR = p.join(CLICKHOUSE_ROOT_DIR, "docker/test/integration/compose/") DEFAULT_ENV_NAME = 'env_file' SANITIZER_SIGN = "==================" @@ -139,12 +141,13 @@ class ClickHouseCluster: cmd += " client" return cmd - def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, + def add_instance(self, name, config_dir=None, main_configs=None, user_configs=None, macros=None, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, - hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", - stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=[]): + hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", + stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, + zookeeper_docker_compose_path=None): """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -161,97 +164,101 @@ class ClickHouseCluster: raise Exception("Can\'t add instance `%s': there is already an instance with the same name!" % name) instance = ClickHouseInstance( - self, self.base_dir, name, config_dir, main_configs, user_configs, macros, with_zookeeper, + self, self.base_dir, name, config_dir, main_configs or [], user_configs or [], macros or {}, + with_zookeeper, self.zookeeper_config_path, with_mysql, with_kafka, with_mongo, with_redis, with_minio, self.base_configs_dir, self.server_bin_path, self.odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=hostname, - env_variables=env_variables, image=image, stay_alive=stay_alive, ipv4_address=ipv4_address, + env_variables=env_variables or {}, image=image, stay_alive=stay_alive, ipv4_address=ipv4_address, ipv6_address=ipv6_address, - with_installed_binary=with_installed_binary, tmpfs=tmpfs) + with_installed_binary=with_installed_binary, tmpfs=tmpfs or []) self.instances[name] = instance if ipv4_address is not None or ipv6_address is not None: self.with_net_trics = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_net.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_net.yml')]) self.base_cmd.extend(['--file', instance.docker_compose_path]) cmds = [] if with_zookeeper and not self.with_zookeeper: + if not zookeeper_docker_compose_path: + zookeeper_docker_compose_path = p.join(DOCKER_COMPOSE_DIR, 'docker_compose_zookeeper.yml') + self.with_zookeeper = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_zookeeper.yml')]) + self.base_cmd.extend(['--file', zookeeper_docker_compose_path]) self.base_zookeeper_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_zookeeper.yml')] + self.project_name, '--file', zookeeper_docker_compose_path] cmds.append(self.base_zookeeper_cmd) if with_mysql and not self.with_mysql: self.with_mysql = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')]) self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')] cmds.append(self.base_mysql_cmd) if with_postgres and not self.with_postgres: self.with_postgres = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_postgres.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')]) self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_postgres.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')] cmds.append(self.base_postgres_cmd) if with_odbc_drivers and not self.with_odbc_drivers: self.with_odbc_drivers = True if not self.with_mysql: self.with_mysql = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')]) self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')] cmds.append(self.base_mysql_cmd) if not self.with_postgres: self.with_postgres = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_postgres.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')]) self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.project_name, '--file', - p.join(HELPERS_DIR, 'docker_compose_postgres.yml')] + p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')] cmds.append(self.base_postgres_cmd) if with_kafka and not self.with_kafka: self.with_kafka = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_kafka.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_kafka.yml')]) self.base_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_kafka.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_kafka.yml')] cmds.append(self.base_kafka_cmd) if with_hdfs and not self.with_hdfs: self.with_hdfs = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_hdfs.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_hdfs.yml')]) self.base_hdfs_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_hdfs.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_hdfs.yml')] cmds.append(self.base_hdfs_cmd) if with_mongo and not self.with_mongo: self.with_mongo = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_mongo.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mongo.yml')]) self.base_mongo_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_mongo.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mongo.yml')] cmds.append(self.base_mongo_cmd) if self.with_net_trics: for cmd in cmds: - cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_net.yml')]) + cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_net.yml')]) if with_redis and not self.with_redis: self.with_redis = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_redis.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_redis.yml')]) self.base_redis_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_redis.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_redis.yml')] if with_minio and not self.with_minio: self.with_minio = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_minio.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_minio.yml')]) self.base_minio_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_minio.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_minio.yml')] cmds.append(self.base_minio_cmd) return instance @@ -580,17 +587,17 @@ class ClickHouseInstance: self, cluster, base_path, name, custom_config_dir, custom_main_configs, custom_user_configs, macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_mongo, with_redis, with_minio, base_configs_dir, server_bin_path, odbc_bridge_bin_path, - clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables={}, + clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", - stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=[]): + stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None): self.name = name - self.base_cmd = cluster.base_cmd[:] + self.base_cmd = cluster.base_cmd self.docker_id = cluster.get_instance_docker_id(self.name) self.cluster = cluster self.hostname = hostname if hostname is not None else self.name - self.tmpfs = tmpfs[:] + self.tmpfs = tmpfs or [] self.custom_config_dir = p.abspath(p.join(base_path, custom_config_dir)) if custom_config_dir else None self.custom_main_config_paths = [p.abspath(p.join(base_path, c)) for c in custom_main_configs] self.custom_user_config_paths = [p.abspath(p.join(base_path, c)) for c in custom_user_configs] @@ -611,7 +618,7 @@ class ClickHouseInstance: self.path = p.join(self.cluster.instances_dir, name) self.docker_compose_path = p.join(self.path, 'docker_compose.yml') - self.env_variables = env_variables + self.env_variables = env_variables or {} if with_odbc_drivers: self.odbc_ini_path = os.path.dirname(self.docker_compose_path) + "/odbc.ini:/etc/odbc.ini" self.with_mysql = True @@ -923,6 +930,7 @@ class ClickHouseInstance: # The file is named with 0_ prefix to be processed before other configuration overloads. shutil.copy(p.join(HELPERS_DIR, '0_common_instance_config.xml'), self.config_d_dir) + shutil.copy(p.join(HELPERS_DIR, '0_common_instance_users.xml'), users_d_dir) # Generate and write macros file macros = self.macros.copy() @@ -1041,4 +1049,4 @@ class ClickHouseKiller(object): self.clickhouse_node.kill_clickhouse() def __exit__(self, exc_type, exc_val, exc_tb): - self.clickhouse_node.restore_clickhouse() \ No newline at end of file + self.clickhouse_node.restore_clickhouse() diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 82384c6e183..3ba8ae3f9fd 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -5,7 +5,7 @@ import os import docker -from .cluster import HELPERS_DIR +from .cluster import CLICKHOUSE_ROOT_DIR class PartitionManager: @@ -156,7 +156,7 @@ class _NetworkManager: def __init__( self, image_name='clickhouse_tests_helper', - image_path=p.join(HELPERS_DIR, 'helper_container'), + image_path=p.join(CLICKHOUSE_ROOT_DIR, 'docker', 'test', 'integration', 'helper_container'), container_expire_timeout=50, container_exit_timeout=60): self.container_expire_timeout = container_expire_timeout diff --git a/tests/integration/helpers/uclient.py b/tests/integration/helpers/uclient.py new file mode 100644 index 00000000000..6318802d81a --- /dev/null +++ b/tests/integration/helpers/uclient.py @@ -0,0 +1,36 @@ +import os +import sys +import time + +CURDIR = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(CURDIR)) + +import uexpect + +prompt = ':\) ' +end_of_block = r'.*\r\n.*\r\n' + +class client(object): + def __init__(self, command=None, name='', log=None): + self.client = uexpect.spawn(['/bin/bash','--noediting']) + if command is None: + command = '/usr/bin/clickhouse-client' + self.client.command = command + self.client.eol('\r') + self.client.logger(log, prefix=name) + self.client.timeout(20) + self.client.expect('[#\$] ', timeout=2) + self.client.send(command) + + def __enter__(self): + return self.client.__enter__() + + def __exit__(self, type, value, traceback): + self.client.reader['kill_event'].set() + # send Ctrl-C + self.client.send('\x03', eol='') + time.sleep(0.3) + self.client.send('quit', eol='\r') + self.client.send('\x03', eol='') + return self.client.__exit__(type, value, traceback) diff --git a/tests/integration/helpers/uexpect.py b/tests/integration/helpers/uexpect.py new file mode 100644 index 00000000000..f71b32a53e1 --- /dev/null +++ b/tests/integration/helpers/uexpect.py @@ -0,0 +1,206 @@ +# Copyright (c) 2019 Vitaliy Zakaznikov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import pty +import time +import sys +import re + +from threading import Thread, Event +from subprocess import Popen +from Queue import Queue, Empty + +class TimeoutError(Exception): + def __init__(self, timeout): + self.timeout = timeout + + def __str__(self): + return 'Timeout %.3fs' % float(self.timeout) + +class ExpectTimeoutError(Exception): + def __init__(self, pattern, timeout, buffer): + self.pattern = pattern + self.timeout = timeout + self.buffer = buffer + + def __str__(self): + s = 'Timeout %.3fs ' % float(self.timeout) + if self.pattern: + s += 'for %s ' % repr(self.pattern.pattern) + if self.buffer: + s += 'buffer %s ' % repr(self.buffer[:]) + s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) + return s + +class IO(object): + class EOF(object): + pass + + class Timeout(object): + pass + + EOF = EOF + TIMEOUT = Timeout + + class Logger(object): + def __init__(self, logger, prefix=''): + self._logger = logger + self._prefix = prefix + + def write(self, data): + self._logger.write(('\n' + data).replace('\n','\n' + self._prefix)) + + def flush(self): + self._logger.flush() + + def __init__(self, process, master, queue, reader): + self.process = process + self.master = master + self.queue = queue + self.buffer = None + self.before = None + self.after = None + self.match = None + self.pattern = None + self.reader = reader + self._timeout = None + self._logger = None + self._eol = '' + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def logger(self, logger=None, prefix=''): + if logger: + self._logger = self.Logger(logger, prefix=prefix) + return self._logger + + def timeout(self, timeout=None): + if timeout: + self._timeout = timeout + return self._timeout + + def eol(self, eol=None): + if eol: + self._eol = eol + return self._eol + + def close(self, force=True): + self.reader['kill_event'].set() + os.system('pkill -TERM -P %d' % self.process.pid) + if force: + self.process.kill() + else: + self.process.terminate() + os.close(self.master) + if self._logger: + self._logger.write('\n') + self._logger.flush() + + def send(self, data, eol=None): + if eol is None: + eol = self._eol + return self.write(data + eol) + + def write(self, data): + return os.write(self.master, data) + + def expect(self, pattern, timeout=None, escape=False): + self.match = None + self.before = None + self.after = None + if escape: + pattern = re.escape(pattern) + pattern = re.compile(pattern) + if timeout is None: + timeout = self._timeout + timeleft = timeout + while True: + start_time = time.time() + if self.buffer is not None: + self.match = pattern.search(self.buffer, 0) + if self.match is not None: + self.after = self.buffer[self.match.start():self.match.end()] + self.before = self.buffer[:self.match.start()] + self.buffer = self.buffer[self.match.end():] + break + if timeleft < 0: + break + try: + data = self.read(timeout=timeleft, raise_exception=True) + except TimeoutError: + if self._logger: + self._logger.write((self.buffer or '') + '\n') + self._logger.flush() + exception = ExpectTimeoutError(pattern, timeout, self.buffer) + self.buffer = None + raise exception + timeleft -= (time.time() - start_time) + if data: + self.buffer = (self.buffer + data) if self.buffer else data + if self._logger: + self._logger.write((self.before or '') + (self.after or '')) + self._logger.flush() + if self.match is None: + exception = ExpectTimeoutError(pattern, timeout, self.buffer) + self.buffer = None + raise exception + return self.match + + def read(self, timeout=0, raise_exception=False): + data = '' + timeleft = timeout + try: + while timeleft >= 0 : + start_time = time.time() + data += self.queue.get(timeout=timeleft) + if data: + break + timeleft -= (time.time() - start_time) + except Empty: + if data: + return data + if raise_exception: + raise TimeoutError(timeout) + pass + if not data and raise_exception: + raise TimeoutError(timeout) + + return data + +def spawn(command): + master, slave = pty.openpty() + process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1) + os.close(slave) + + queue = Queue() + reader_kill_event = Event() + thread = Thread(target=reader, args=(process, master, queue, reader_kill_event)) + thread.daemon = True + thread.start() + + return IO(process, master, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) + +def reader(process, out, queue, kill_event): + while True: + try: + data = os.read(out, 65536) + queue.put(data) + except: + if kill_event.is_set(): + break + raise diff --git a/tests/integration/helpers/zookeeper-ssl-entrypoint.sh b/tests/integration/helpers/zookeeper-ssl-entrypoint.sh new file mode 100755 index 00000000000..3ddb21881d6 --- /dev/null +++ b/tests/integration/helpers/zookeeper-ssl-entrypoint.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -e + +export ZOO_SERVER_CNXN_FACTORY=org.apache.zookeeper.server.NettyServerCnxnFactory +export ZOO_SSL_KEYSTORE_LOCATION=/conf/certs/zookeeper.p12 +export ZOO_SSL_KEYSTORE_PASSWORD=password +export ZOO_SSL_TRUSTSTORE_LOCATION=/conf/certs/truststore.p12 +export ZOO_SSL_TRUSTSTORE_PASSWORD=password + + +# Allow the container to be started with `--user` +if [[ "$1" = 'zkServer.sh' && "$(id -u)" = '0' ]]; then + chown -R zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_LOG_DIR" "$ZOO_CONF_DIR" + exec gosu zookeeper "$0" "$@" +fi + +# Generate the config only if it doesn't exist +if [[ ! -f "$ZOO_CONF_DIR/zoo.cfg" ]]; then + CONFIG="$ZOO_CONF_DIR/zoo.cfg" + { + echo "dataDir=$ZOO_DATA_DIR" + echo "dataLogDir=$ZOO_DATA_LOG_DIR" + + echo "tickTime=$ZOO_TICK_TIME" + echo "initLimit=$ZOO_INIT_LIMIT" + echo "syncLimit=$ZOO_SYNC_LIMIT" + + echo "autopurge.snapRetainCount=$ZOO_AUTOPURGE_SNAPRETAINCOUNT" + echo "autopurge.purgeInterval=$ZOO_AUTOPURGE_PURGEINTERVAL" + echo "maxClientCnxns=$ZOO_MAX_CLIENT_CNXNS" + echo "standaloneEnabled=$ZOO_STANDALONE_ENABLED" + echo "admin.enableServer=$ZOO_ADMINSERVER_ENABLED" + } >> "$CONFIG" + if [[ -z $ZOO_SERVERS ]]; then + ZOO_SERVERS="server.1=localhost:2888:3888;2181" + fi + + for server in $ZOO_SERVERS; do + echo "$server" >> "$CONFIG" + done + + if [[ -n $ZOO_4LW_COMMANDS_WHITELIST ]]; then + echo "4lw.commands.whitelist=$ZOO_4LW_COMMANDS_WHITELIST" >> "$CONFIG" + fi + + + if [[ -n $ZOO_SSL_QUORUM ]]; then + { + echo "sslQuorum=$ZOO_SSL_QUORUM" + echo "serverCnxnFactory=$ZOO_SERVER_CNXN_FACTORY" + echo "ssl.quorum.keyStore.location=$ZOO_SSL_QUORUM_KEYSTORE_LOCATION" + echo "ssl.quorum.keyStore.password=$ZOO_SSL_QUORUM_KEYSTORE_PASSWORD" + echo "ssl.quorum.trustStore.location=$ZOO_SSL_QUORUM_TRUSTSTORE_LOCATION" + echo "ssl.quorum.trustStore.password=$ZOO_SSL_QUORUM_TRUSTSTORE_PASSWORD" + } >> "$CONFIG" + fi + + if [[ -n $ZOO_PORT_UNIFICATION ]]; then + echo "portUnification=$ZOO_PORT_UNIFICATION" >> "$CONFIG" + fi + + if [[ -n $ZOO_SECURE_CLIENT_PORT ]]; then + { + echo "secureClientPort=$ZOO_SECURE_CLIENT_PORT" + echo "serverCnxnFactory=$ZOO_SERVER_CNXN_FACTORY" + echo "ssl.keyStore.location=$ZOO_SSL_KEYSTORE_LOCATION" + echo "ssl.keyStore.password=$ZOO_SSL_KEYSTORE_PASSWORD" + echo "ssl.trustStore.location=$ZOO_SSL_TRUSTSTORE_LOCATION" + echo "ssl.trustStore.password=$ZOO_SSL_TRUSTSTORE_PASSWORD" + } >> "$CONFIG" + fi + + if [[ -n $ZOO_CLIENT_PORT_UNIFICATION ]]; then + echo "client.portUnification=$ZOO_CLIENT_PORT_UNIFICATION" >> "$CONFIG" + fi +fi + +# Write myid only if it doesn't exist +if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then + echo "${ZOO_MY_ID:-1}" > "$ZOO_DATA_DIR/myid" +fi + +mkdir -p $(dirname $ZOO_SSL_KEYSTORE_LOCATION) +mkdir -p $(dirname $ZOO_SSL_TRUSTSTORE_LOCATION) + +if [[ ! -f "$ZOO_SSL_KEYSTORE_LOCATION" ]]; then + keytool -genkeypair -alias zookeeper -keyalg RSA -validity 365 -keysize 2048 -dname "cn=zookeeper" -keypass password -keystore $ZOO_SSL_KEYSTORE_LOCATION -storepass password -deststoretype pkcs12 +fi + +if [[ ! -f "$ZOO_SSL_TRUSTSTORE_LOCATION" ]]; then + keytool -importcert -alias zookeeper -file /clickhouse-config/client.crt -keystore $ZOO_SSL_TRUSTSTORE_LOCATION -storepass password -noprompt -deststoretype pkcs12 +fi + +exec "$@" diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index adb431deaa1..a7ca8c57da8 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -2,3 +2,5 @@ python_files = test*.py norecursedirs = _instances timeout = 300 +junit_duration_report = call +junit_suite_name = integration diff --git a/tests/integration/test_adaptive_granularity_different_settings/__init__.py b/tests/integration/test_adaptive_granularity_different_settings/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_adaptive_granularity_different_settings/test.py b/tests/integration/test_adaptive_granularity_different_settings/test.py new file mode 100644 index 00000000000..b066c437e06 --- /dev/null +++ b/tests/integration/test_adaptive_granularity_different_settings/test.py @@ -0,0 +1,49 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def test_attach_detach(start_cluster): + + node1.query(""" + CREATE TABLE test (key UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/test', '1') + ORDER BY tuple() + SETTINGS index_granularity_bytes = 0""") + + node1.query("INSERT INTO test VALUES (1), (2)") + + node2.query(""" + CREATE TABLE test (key UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/test', '2') + ORDER BY tuple()""") + + node2.query("INSERT INTO test VALUES (3), (4)") + + node1.query("SYSTEM SYNC REPLICA test") + node2.query("SYSTEM SYNC REPLICA test") + + assert node1.query("SELECT COUNT() FROM test") == "4\n" + assert node2.query("SELECT COUNT() FROM test") == "4\n" + + node1.query("DETACH TABLE test") + node2.query("DETACH TABLE test") + + node1.query("ATTACH TABLE test") + node2.query("ATTACH TABLE test") + + assert node1.query("SELECT COUNT() FROM test") == "4\n" + assert node2.query("SELECT COUNT() FROM test") == "4\n" diff --git a/tests/integration/test_allowed_client_hosts/configs/users.xml b/tests/integration/test_allowed_client_hosts/configs/users.xml deleted file mode 100644 index 3142ec5355a..00000000000 --- a/tests/integration/test_allowed_client_hosts/configs/users.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - default - - - - diff --git a/tests/integration/test_alter_codec/__init__.py b/tests/integration/test_alter_codec/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_alter_codec/configs/config.d/cluster.xml b/tests/integration/test_alter_codec/configs/config.d/cluster.xml new file mode 100644 index 00000000000..ec7c9b8e4f8 --- /dev/null +++ b/tests/integration/test_alter_codec/configs/config.d/cluster.xml @@ -0,0 +1,16 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + \ No newline at end of file diff --git a/tests/integration/test_alter_codec/configs/logs_config.xml b/tests/integration/test_alter_codec/configs/logs_config.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/integration/test_alter_codec/configs/logs_config.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/integration/test_alter_codec/test.py b/tests/integration/test_alter_codec/test.py new file mode 100644 index 00000000000..573fe7f0961 --- /dev/null +++ b/tests/integration/test_alter_codec/test.py @@ -0,0 +1,88 @@ +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + macros={"shard": 0, "replica": 1} ) + +node2 = cluster.add_instance('node2', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + macros={"shard": 0, "replica": 2} ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_alter_codec_pk(started_cluster): + try: + name = "test_alter_codec_pk" + node1.query(""" + CREATE TABLE {name} (id UInt64) Engine=MergeTree() ORDER BY id + """.format(name=name)) + + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(NONE)".format(name=name)) + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(Delta, LZ4)".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name)) + + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 ALIAS 3 CODEC(Delta, LZ4)".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 MATERIALIZED 3 CODEC(Delta, LZ4)".format(name=name)) + + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id Int64".format(name=name)) + + finally: + node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + + +def test_alter_codec_index(started_cluster): + try: + name = "test_alter_codec_index" + node1.query(""" + CREATE TABLE {name} (`id` UInt64, INDEX id_index id TYPE minmax GRANULARITY 1) Engine=MergeTree() ORDER BY tuple() + """.format(name=name)) + + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(NONE)".format(name=name)) + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(Delta, LZ4)".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name)) + + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 ALIAS 3 CODEC(Delta, LZ4)".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 MATERIALIZED 3 CODEC(Delta, LZ4)".format(name=name)) + + node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64".format(name=name)) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MODIFY COLUMN id Int64".format(name=name)) + + finally: + node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) diff --git a/tests/integration/test_always_fetch_merged/__init__.py b/tests/integration/test_always_fetch_merged/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_always_fetch_merged/test.py b/tests/integration/test_always_fetch_merged/test.py new file mode 100644 index 00000000000..63ab6f6b5ea --- /dev/null +++ b/tests/integration/test_always_fetch_merged/test.py @@ -0,0 +1,63 @@ +import pytest +import time +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def test_replica_always_download(started_cluster): + node1.query(""" + CREATE TABLE test_table( + key UInt64, + value String + ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table/replicated', '1') + ORDER BY tuple() + """) + node2.query(""" + CREATE TABLE test_table( + key UInt64, + value String + ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table/replicated', '2') + ORDER BY tuple() + SETTINGS always_fetch_merged_part=1 + """) + + # Stop merges on single node + node1.query("SYSTEM STOP MERGES") + + for i in range(0, 10): + node1.query("INSERT INTO test_table VALUES ({}, '{}')".format(i, i)) + + assert node1.query("SELECT COUNT() FROM test_table") == "10\n" + assert_eq_with_retry(node2, "SELECT COUNT() FROM test_table", "10\n") + + time.sleep(3) + + # Nothing is merged + assert node1.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1") == "10\n" + + assert node2.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1") == "10\n" + + node1.query("SYSTEM START MERGES") + + time.sleep(3) + + node1_parts = node1.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() + node2_parts = node2.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() + assert int(node1_parts) < 10 # something merged + assert int(node2_parts) < 10 diff --git a/tests/integration/test_authentication/test.py b/tests/integration/test_authentication/test.py index b7ffd1ed35b..483b59813e5 100644 --- a/tests/integration/test_authentication/test.py +++ b/tests/integration/test_authentication/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', config_dir="configs") +instance = cluster.add_instance('instance') @pytest.fixture(scope="module", autouse=True) diff --git a/tests/integration/test_cluster_copier/task0_description.xml b/tests/integration/test_cluster_copier/task0_description.xml index 6fa490cd02f..72eff8d464d 100644 --- a/tests/integration/test_cluster_copier/task0_description.xml +++ b/tests/integration/test_cluster_copier/task0_description.xml @@ -33,7 +33,7 @@ 3 4 5 6 1 2 0 - ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 3 ORDER BY d SETTINGS index_granularity = 16 + ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 d + 1 diff --git a/tests/integration/test_cluster_copier/task_no_arg.xml b/tests/integration/test_cluster_copier/task_no_arg.xml index d9d49011f3f..a6bd80c8480 100644 --- a/tests/integration/test_cluster_copier/task_no_arg.xml +++ b/tests/integration/test_cluster_copier/task_no_arg.xml @@ -32,7 +32,7 @@ default_cluster default copier_test1_1 - ENGINE = MergeTree PARTITION BY date ORDER BY date + ENGINE = MergeTree PARTITION BY date ORDER BY (date, sipHash64(date)) SAMPLE BY sipHash64(date) rand() diff --git a/tests/integration/test_cluster_copier/task_trivial.xml b/tests/integration/test_cluster_copier/task_trivial.xml index c23b9322470..27af6f64bf4 100644 --- a/tests/integration/test_cluster_copier/task_trivial.xml +++ b/tests/integration/test_cluster_copier/task_trivial.xml @@ -29,7 +29,7 @@ trivial - ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 5 ORDER BY d SETTINGS index_granularity = 16 + ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 d + 1 diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index d411eba3974..440f0fc016b 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -36,7 +36,7 @@ def ddl_check_query(instance, query, num_hosts=3): return contents -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") def started_cluster(): global cluster try: @@ -85,7 +85,9 @@ class Task1: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format(cluster_num)) - ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 3 ORDER BY d SETTINGS index_granularity = 16") + ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " + + "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)") instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1}) @@ -155,7 +157,7 @@ class Task_test_block_size: ddl_check_query(instance, """ CREATE TABLE test_block_size ON CLUSTER shard_0_0 (partition Date, d UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}') - ORDER BY d""", 2) + ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2) instance.query("INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format(self.rows)) @@ -260,14 +262,50 @@ def execute_task(task, cmd_options): # Tests -def test_copy_simple(started_cluster): - execute_task(Task1(started_cluster), []) +@pytest.mark.parametrize( + ('use_sample_offset'), + [ + False, + True + ] +) -def test_copy_with_recovering(started_cluster): - execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) +def test_copy_simple(started_cluster, use_sample_offset): + if use_sample_offset: + execute_task(Task1(started_cluster), ['--experimental-use-sample-offset', '1']) + else: + execute_task(Task1(started_cluster), []) -def test_copy_with_recovering_after_move_faults(started_cluster): - execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + +@pytest.mark.parametrize( + ('use_sample_offset'), + [ + False, + True + ] +) + +def test_copy_with_recovering(started_cluster, use_sample_offset): + if use_sample_offset: + execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), + '--experimental-use-sample-offset', '1']) + else: + execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + +@pytest.mark.parametrize( + ('use_sample_offset'), + [ + False, + True + ] +) + +def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offset): + if use_sample_offset: + execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), + '--experimental-use-sample-offset', '1']) + else: + execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) def test_copy_month_to_week_partition(started_cluster): execute_task(Task2(started_cluster), []) diff --git a/tests/integration/test_cluster_copier/trivial_test.py b/tests/integration/test_cluster_copier/trivial_test.py index 9b71b08674c..c0966f77deb 100644 --- a/tests/integration/test_cluster_copier/trivial_test.py +++ b/tests/integration/test_cluster_copier/trivial_test.py @@ -18,7 +18,7 @@ COPYING_FAIL_PROBABILITY = 0.33 MOVING_FAIL_PROBABILITY = 0.1 cluster = None -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") def started_cluster(): global cluster try: @@ -47,9 +47,12 @@ def started_cluster(): class TaskTrivial: - def __init__(self, cluster): + def __init__(self, cluster, use_sample_offset): self.cluster = cluster - self.zk_task_path="/clickhouse-copier/task_trivial" + if use_sample_offset: + self.zk_task_path="/clickhouse-copier/task_trivial_use_sample_offset" + else: + self.zk_task_path="/clickhouse-copier/task_trivial" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_trivial.xml'), 'r').read() @@ -63,7 +66,7 @@ class TaskTrivial: source.query("CREATE TABLE trivial (d UInt64, d1 UInt64 MATERIALIZED d+1) " "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') " - "PARTITION BY d % 5 ORDER BY d SETTINGS index_granularity = 16") + "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1}) @@ -127,14 +130,51 @@ def execute_task(task, cmd_options): # Tests -def test_trivial_copy(started_cluster): - execute_task(TaskTrivial(started_cluster), []) -def test_trivial_copy_with_copy_fault(started_cluster): - execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) +@pytest.mark.parametrize( + ('use_sample_offset'), + [ + False, + True + ] +) -def test_trivial_copy_with_move_fault(started_cluster): - execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) +def test_trivial_copy(started_cluster, use_sample_offset): + if use_sample_offset: + execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--experimental-use-sample-offset', '1']) + else: + print("AAAAA") + execute_task(TaskTrivial(started_cluster, use_sample_offset), []) + +@pytest.mark.parametrize( + ('use_sample_offset'), + [ + False, + True + ] +) + +def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset): + if use_sample_offset: + execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), + '--experimental-use-sample-offset', '1']) + else: + execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + +@pytest.mark.parametrize( + ('use_sample_offset'), + [ + False, + True + ] +) + +def test_trivial_copy_with_move_fault(started_cluster, use_sample_offset): + if use_sample_offset: + execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), + '--experimental-use-sample-offset', '1']) + else: + execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) if __name__ == '__main__': diff --git a/tests/integration/test_config_corresponding_root/configs/config.xml b/tests/integration/test_config_corresponding_root/configs/config.xml index 154ebf6c35e..4e130afa84d 100644 --- a/tests/integration/test_config_corresponding_root/configs/config.xml +++ b/tests/integration/test_config_corresponding_root/configs/config.xml @@ -37,7 +37,7 @@ true - + true true sslv2,sslv3 diff --git a/tests/integration/test_ddl_alter_query/__init__.py b/tests/integration/test_ddl_alter_query/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_ddl_alter_query/configs/remote_servers.xml b/tests/integration/test_ddl_alter_query/configs/remote_servers.xml new file mode 100644 index 00000000000..4c3de4b3905 --- /dev/null +++ b/tests/integration/test_ddl_alter_query/configs/remote_servers.xml @@ -0,0 +1,28 @@ + + + + + true + + node1 + 9000 + + + node2 + 9000 + + + + true + + node3 + 9000 + + + node4 + 9000 + + + + + diff --git a/tests/integration/test_ddl_alter_query/test.py b/tests/integration/test_ddl_alter_query/test.py new file mode 100644 index 00000000000..2a16c58c9e6 --- /dev/null +++ b/tests/integration/test_ddl_alter_query/test.py @@ -0,0 +1,47 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for i, node in enumerate([node1, node2]): + node.query("CREATE DATABASE testdb") + node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format(i)) + for i, node in enumerate([node3, node4]): + node.query("CREATE DATABASE testdb") + node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format(i)) + yield cluster + + finally: + cluster.shutdown() + + +def test_alter(started_cluster): + node1.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node3.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node2.query("SYSTEM SYNC REPLICA testdb.test_table") + node4.query("SYSTEM SYNC REPLICA testdb.test_table") + + node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", settings={"replication_alter_partitions_sync": "2"}) + + node1.query("SYSTEM SYNC REPLICA testdb.test_table") + node2.query("SYSTEM SYNC REPLICA testdb.test_table") + node3.query("SYSTEM SYNC REPLICA testdb.test_table") + node4.query("SYSTEM SYNC REPLICA testdb.test_table") + + assert node1.query("SELECT somecolumn FROM testdb.test_table LIMIT 1") == "0\n" + assert node2.query("SELECT somecolumn FROM testdb.test_table LIMIT 1") == "0\n" + assert node3.query("SELECT somecolumn FROM testdb.test_table LIMIT 1") == "0\n" + assert node4.query("SELECT somecolumn FROM testdb.test_table LIMIT 1") == "0\n" diff --git a/tests/integration/test_delayed_replica_failover/test.py b/tests/integration/test_delayed_replica_failover/test.py index 58734b2c39b..66a3a9e1a34 100644 --- a/tests/integration/test_delayed_replica_failover/test.py +++ b/tests/integration/test_delayed_replica_failover/test.py @@ -81,6 +81,14 @@ SELECT sum(x) FROM distributed SETTINGS max_replica_delay_for_distributed_queries=1 ''').strip() == '3' + # Regression for skip_unavailable_shards in conjunction with skip_unavailable_shards + assert instance_with_dist_table.query(''' +SELECT sum(x) FROM distributed SETTINGS + load_balancing='in_order', + skip_unavailable_shards=1, + max_replica_delay_for_distributed_queries=1 +''').strip() == '3' + # If we forbid stale replicas, the query must fail. with pytest.raises(Exception): print instance_with_dist_table.query(''' diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index 80424a3471a..647e36c71b3 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -23,7 +23,7 @@ create_table_mysql_template = """ """ create_clickhouse_dictionary_table_template = """ - CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` (`id` Int32, `value` String) ENGINE = Dictionary({}) + CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` (`id` UInt64, `value` String) ENGINE = Dictionary({}) """ @pytest.fixture(scope="module") @@ -31,18 +31,18 @@ def started_cluster(): try: #time.sleep(30) cluster.start() - + # Create a MySQL database mysql_connection = get_mysql_conn() create_mysql_db(mysql_connection, 'test') mysql_connection.close() - + # Create database in ClickHouse instance.query("CREATE DATABASE IF NOT EXISTS test") - + # Create database in ClickChouse using MySQL protocol (will be used for data insertion) instance.query("CREATE DATABASE clickhouse_mysql ENGINE = MySQL('mysql1:3306', 'test', 'root', 'clickhouse')") - + yield cluster finally: @@ -53,11 +53,11 @@ def test_load_mysql_dictionaries(started_cluster): # Load dictionaries query = instance.query query("SYSTEM RELOAD DICTIONARIES") - + for n in range(0, 5): # Create MySQL tables, fill them and create CH dict tables prepare_mysql_table('test', str(n)) - + # Check dictionaries are loaded and have correct number of elements for n in range(0, 100): # Force reload of dictionaries (each 10 iteration) @@ -73,7 +73,7 @@ def create_mysql_db(mysql_connection, name): def prepare_mysql_table(table_name, index): mysql_connection = get_mysql_conn() - + # Create table create_mysql_table(mysql_connection, table_name + str(index)) @@ -82,8 +82,8 @@ def prepare_mysql_table(table_name, index): query("INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(table_name + str(index), table_name + str(index))) assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name + str(index))).rstrip() == '10000' mysql_connection.close() - - #Create CH Dictionary tables based on MySQL tables + + #Create CH Dictionary tables based on MySQL tables query(create_clickhouse_dictionary_table_template.format(table_name + str(index), 'dict' + str(index))) def get_mysql_conn(): diff --git a/tests/integration/test_dictionary_custom_settings/__init__.py b/tests/integration/test_dictionary_custom_settings/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dictionary_custom_settings/configs/config.xml b/tests/integration/test_dictionary_custom_settings/configs/config.xml new file mode 100644 index 00000000000..1e4c14585a9 --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/configs/config.xml @@ -0,0 +1,30 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + /etc/clickhouse-server/config.d/*.xml + diff --git a/tests/integration/test_dictionary_custom_settings/configs/dictionaries/ClickHouseSourceConfig.xml b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/ClickHouseSourceConfig.xml new file mode 100644 index 00000000000..2191c8ded8a --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/ClickHouseSourceConfig.xml @@ -0,0 +1,48 @@ + + + test_clickhouse + + + + localhost + 9000 + default + + default + source
+ + + + 1 + + + + + 600 + + + + + + + + id + + + first + String + + + + second + String + + + + third + String + + + + + diff --git a/tests/integration/test_dictionary_custom_settings/configs/dictionaries/ExecutableSourceConfig.xml b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/ExecutableSourceConfig.xml new file mode 100644 index 00000000000..3191118c4e9 --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/ExecutableSourceConfig.xml @@ -0,0 +1,45 @@ + + + test_executable + + + + cat /etc/clickhouse-server/config.d/source.csv + CSVWithNames + + + + 0 + 0 + + + + + 600 + + + + + + + + id + + + first + String + + + + second + String + + + + third + String + + + + + diff --git a/tests/integration/test_dictionary_custom_settings/configs/dictionaries/FileSourceConfig.xml b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/FileSourceConfig.xml new file mode 100644 index 00000000000..ff7baf29be0 --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/FileSourceConfig.xml @@ -0,0 +1,45 @@ + + + test_file + + + + /etc/clickhouse-server/config.d/source.csv + CSVWithNames + + + + 0 + 0 + + + + + 600 + + + + + + + + id + + + first + String + + + + second + String + + + + third + String + + + + + diff --git a/tests/integration/test_dictionary_custom_settings/configs/dictionaries/HTTPSourceConfig.xml b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/HTTPSourceConfig.xml new file mode 100644 index 00000000000..dc03974c4b6 --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/HTTPSourceConfig.xml @@ -0,0 +1,54 @@ + + + test_http + + + http://localhost:5555/source.csv + CSVWithNames + + foo + bar + + +
+ api-key + secret +
+ + + + + 0 + 0 + + + + + 600 + + + + + + + + id + + + first + String + + + + second + String + + + + third + String + + + + + diff --git a/tests/integration/test_dictionary_custom_settings/configs/dictionaries/source.csv b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/source.csv new file mode 100644 index 00000000000..23d113e5225 --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/configs/dictionaries/source.csv @@ -0,0 +1,3 @@ +id,first,second,third +1,'a,"b,c +2,'d,"e,f diff --git a/tests/integration/test_quota/configs/users.xml b/tests/integration/test_dictionary_custom_settings/configs/users.xml similarity index 77% rename from tests/integration/test_quota/configs/users.xml rename to tests/integration/test_dictionary_custom_settings/configs/users.xml index 4412345a731..6061af8e33d 100644 --- a/tests/integration/test_quota/configs/users.xml +++ b/tests/integration/test_dictionary_custom_settings/configs/users.xml @@ -4,6 +4,7 @@ + @@ -11,7 +12,12 @@ ::/0 default - myQuota + default + + + + + diff --git a/tests/integration/test_dictionary_custom_settings/http_server.py b/tests/integration/test_dictionary_custom_settings/http_server.py new file mode 100644 index 00000000000..c7920a9024d --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/http_server.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +import argparse +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import socket +import ssl +import csv + + +# Decorator used to see if authentication works for external dictionary who use a HTTP source. +def check_auth(fn): + def wrapper(req): + auth_header = req.headers.get('authorization', None) + api_key = req.headers.get('api-key', None) + if not auth_header or auth_header != 'Basic Zm9vOmJhcg==' or not api_key or api_key != 'secret': + req.send_response(401) + else: + fn(req) + return wrapper + + +def start_server(server_address, data_path, schema, cert_path, address_family): + class TSVHTTPHandler(BaseHTTPRequestHandler): + @check_auth + def do_GET(self): + self.__send_headers() + self.__send_data() + + @check_auth + def do_POST(self): + ids = self.__read_and_decode_post_ids() + print "ids=", ids + self.__send_headers() + self.__send_data(ids) + + def __send_headers(self): + self.send_response(200) + self.send_header('Content-type', 'text/csv') + self.end_headers() + + def __send_data(self, only_ids = None): + with open(data_path, 'r') as fl: + reader = csv.reader(fl, delimiter='\t') + for row in reader: + if not only_ids or (row[0] in only_ids): + self.wfile.write('\t'.join(row) + '\n') + + def __read_and_decode_post_ids(self): + data = self.__read_and_decode_post_data() + return filter(None, data.split()) + + def __read_and_decode_post_data(self): + transfer_encoding = self.headers.get("Transfer-encoding") + decoded = "" + if transfer_encoding == "chunked": + while True: + s = self.rfile.readline() + chunk_length = int(s, 16) + if not chunk_length: + break + decoded += self.rfile.read(chunk_length) + self.rfile.readline() + else: + content_length = int(self.headers.get("Content-Length", 0)) + decoded = self.rfile.read(content_length) + return decoded + + if address_family == "ipv6": + HTTPServer.address_family = socket.AF_INET6 + httpd = HTTPServer(server_address, TSVHTTPHandler) + if schema == "https": + httpd.socket = ssl.wrap_socket(httpd.socket, certfile=cert_path, server_side=True) + httpd.serve_forever() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Simple HTTP server returns data from file") + parser.add_argument("--host", default="localhost") + parser.add_argument("--port", default=5555, type=int) + parser.add_argument("--data-path", required=True) + parser.add_argument("--schema", choices=("http", "https"), required=True) + parser.add_argument("--cert-path", default="./fake_cert.pem") + parser.add_argument('--address-family', choices=("ipv4", "ipv6"), default="ipv4") + + args = parser.parse_args() + + start_server((args.host, args.port), args.data_path, args.schema, args.cert_path, args.address_family) diff --git a/tests/integration/test_dictionary_custom_settings/test.py b/tests/integration/test_dictionary_custom_settings/test.py new file mode 100644 index 00000000000..97874879525 --- /dev/null +++ b/tests/integration/test_dictionary_custom_settings/test.py @@ -0,0 +1,62 @@ +import os +import pytest + +from helpers.cluster import ClickHouseCluster + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +config_dir = os.path.join(SCRIPT_DIR, './configs') +DICTIONARY_FILES = [ + 'configs/dictionaries/FileSourceConfig.xml', + 'configs/dictionaries/ExecutableSourceConfig.xml', + 'configs/dictionaries/source.csv', + 'configs/dictionaries/HTTPSourceConfig.xml', + 'configs/dictionaries/ClickHouseSourceConfig.xml' +] + +cluster = ClickHouseCluster(__file__, base_configs_dir=config_dir) +instance = cluster.add_instance('node', main_configs=DICTIONARY_FILES, config_dir=config_dir) + +def prepare(): + node = instance + path = "/source.csv" + + script_dir = os.path.dirname(os.path.realpath(__file__)) + node.copy_file_to_container(os.path.join(script_dir, './http_server.py'), '/http_server.py') + node.copy_file_to_container(os.path.join(script_dir, 'configs/dictionaries/source.csv'), './source.csv') + node.exec_in_container([ + "bash", + "-c", + "python2 /http_server.py --data-path={tbl} --schema=http --host=localhost --port=5555".format( + tbl=path) + ], detach=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + prepare() + yield cluster + finally: + cluster.shutdown() + +def test_work(start_cluster): + query = instance.query + + assert query("SELECT dictGetString('test_file', 'first', toUInt64(1))") == "\\\'a\n" + assert query("SELECT dictGetString('test_file', 'second', toUInt64(1))") == "\"b\n" + assert query("SELECT dictGetString('test_executable', 'first', toUInt64(1))") == "\\\'a\n" + assert query("SELECT dictGetString('test_executable', 'second', toUInt64(1))") == "\"b\n" + + caught_exception = '' + try: + instance.query("CREATE TABLE source (id UInt64, first String, second String, third String) ENGINE=TinyLog;") + instance.query("INSERT INTO default.source VALUES (1, 'aaa', 'bbb', 'cccc'), (2, 'ddd', 'eee', 'fff')") + instance.query("SELECT dictGetString('test_clickhouse', 'second', toUInt64(1))") + except Exception as e: + caught_exception = str(e) + + assert caught_exception.find("Limit for result exceeded") != -1 + + assert query("SELECT dictGetString('test_http', 'first', toUInt64(1))") == "\\\'a\n" + assert query("SELECT dictGetString('test_http', 'second', toUInt64(1))") == "\"b\n" \ No newline at end of file diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index 1f6577b9dd1..babceee7c76 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', config_dir='configs', stay_alive=True) +instance = cluster.add_instance('instance', stay_alive=True) @pytest.fixture(scope="module", autouse=True) @@ -22,7 +22,7 @@ def create_entities(): instance.query("CREATE USER u2 IDENTIFIED BY 'qwerty' HOST LOCAL DEFAULT ROLE rx") instance.query("CREATE SETTINGS PROFILE s2 SETTINGS PROFILE s1 TO u2") instance.query("CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a<1000 TO u1, u2") - instance.query("CREATE QUOTA q FOR INTERVAL 1 HOUR SET MAX QUERIES = 100 TO ALL EXCEPT rx") + instance.query("CREATE QUOTA q FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx") @pytest.fixture(autouse=True) @@ -41,13 +41,13 @@ def test_create(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE rx\n" assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" - assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES = 100 TO ALL EXCEPT rx\n" + assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx\n" assert instance.query("SHOW GRANTS FOR u1") == "" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx TO u2\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s1\n" assert instance.query("SHOW GRANTS FOR rx") == "" assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS PROFILE s1 TO u2\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" check() instance.restart_clickhouse() # Check persistency @@ -77,7 +77,7 @@ def test_alter(): assert instance.query("SHOW GRANTS FOR rx") == "GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION\n" assert instance.query("SHOW GRANTS FOR ry") == "GRANT rx TO ry WITH ADMIN OPTION\n" assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS PROFILE s1 TO u2\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" check() instance.restart_clickhouse() # Check persistency diff --git a/tests/integration/test_distributed_ddl/test_replicated_alter.py b/tests/integration/test_distributed_ddl/test_replicated_alter.py index 490587240eb..e66e731cbb1 100644 --- a/tests/integration/test_distributed_ddl/test_replicated_alter.py +++ b/tests/integration/test_distributed_ddl/test_replicated_alter.py @@ -68,7 +68,7 @@ ENGINE = Distributed(cluster, default, merge_for_alter, i) test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64") - test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN String s DEFAULT toString(i)") + test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)") assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) diff --git a/tests/integration/test_distributed_over_live_view/__init__.py b/tests/integration/test_distributed_over_live_view/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_distributed_over_live_view/configs/remote_servers.xml b/tests/integration/test_distributed_over_live_view/configs/remote_servers.xml new file mode 100644 index 00000000000..ebce4697529 --- /dev/null +++ b/tests/integration/test_distributed_over_live_view/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_distributed_over_live_view/configs/set_distributed_defaults.xml b/tests/integration/test_distributed_over_live_view/configs/set_distributed_defaults.xml new file mode 100644 index 00000000000..194eb1ebb87 --- /dev/null +++ b/tests/integration/test_distributed_over_live_view/configs/set_distributed_defaults.xml @@ -0,0 +1,35 @@ + + + + 3 + 1000 + 1 + + + 5 + 3000 + 1 + + + + + + + + ::/0 + + default + default + + + + + ::/0 + + delays + default + + + + + diff --git a/tests/integration/test_distributed_over_live_view/test.py b/tests/integration/test_distributed_over_live_view/test.py new file mode 100644 index 00000000000..f932379e5c6 --- /dev/null +++ b/tests/integration/test_distributed_over_live_view/test.py @@ -0,0 +1,230 @@ +from __future__ import print_function + +import sys +import time +import itertools +import timeit +import logging + +import pytest + +from helpers.uclient import client, prompt, end_of_block +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) + +NODES = {'node' + str(i): cluster.add_instance( + 'node' + str(i), + main_configs=['configs/remote_servers.xml'], + user_configs=['configs/set_distributed_defaults.xml'], +) for i in (1, 2)} + +CREATE_TABLES_SQL = ''' +DROP TABLE IF EXISTS lv_over_base_table; +DROP TABLE IF EXISTS distributed_table; +DROP TABLE IF EXISTS base_table; + +SET allow_experimental_live_view = 1; + +CREATE TABLE + base_table( + node String, + key Int32, + value Int32 + ) +ENGINE = Memory; + +CREATE LIVE VIEW lv_over_base_table AS SELECT * FROM base_table; + +CREATE TABLE + distributed_table +AS base_table +ENGINE = Distributed(test_cluster, default, base_table, rand()); +''' + +INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" + +@pytest.fixture(scope="function") +def started_cluster(): + try: + cluster.start() + for node_index, (node_name, node) in enumerate(NODES.items()): + node.query(CREATE_TABLES_SQL) + for i in range(0, 2): + sql = INSERT_SQL_TEMPLATE.format(node_id=node_name, key=i, value=i + (node_index * 10)) + node.query(sql) + yield cluster + + finally: + cluster.shutdown() + + +@pytest.mark.parametrize("node", NODES.values()[:1]) +@pytest.mark.parametrize("source", ["lv_over_distributed_table"]) +class TestLiveViewOverDistributedSuite: + def test_distributed_over_live_view_order_by_node(self, started_cluster, node, source): + log = sys.stdout + node0, node1 = NODES.values() + + select_query = "SELECT * FROM distributed_over_lv ORDER BY node, key FORMAT CSV" + + with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ + client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS distributed_over_lv") + client1.expect(prompt) + client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.expect(prompt) + + client1.send(select_query) + client1.expect('"node1",0,0\r\n.*"node1",1,1\r\n.*"node2",0,10\r\n.*"node2",1,11\r\n') + client1.expect(prompt) + + client1.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 2, 3)") + client1.expect(prompt) + client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") + client2.expect(prompt) + time.sleep(2) + client1.send(select_query) + client1.expect('"node1",0,0\r\n.*"node1",1,1\r\n.*"node1",1,3\r\n.*"node1",2,3\r\n.*"node1",3,3\r\n.*"node2",0,10\r\n.*"node2",1,11\r\n') + client1.expect(prompt) + + def test_distributed_over_live_view_order_by_key(self, started_cluster, node, source): + log = sys.stdout + node0, node1 = NODES.values() + + select_query = "SELECT * FROM distributed_over_lv ORDER BY key, node FORMAT CSV" + + with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ + client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS distributed_over_lv") + client1.expect(prompt) + client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.expect(prompt) + + client1.send(select_query) + client1.expect('"node1",0,0\r\n"node2",0,10\r\n"node1",1,1\r\n.*"node2",1,11\r\n') + client1.expect(prompt) + + client1.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 2, 3)") + client1.expect(prompt) + client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") + client2.expect(prompt) + time.sleep(2) + client1.send(select_query) + client1.expect('"node1",0,0\r\n.*"node2",0,10.*\r\n"node1",1,1\r\n.*"node1",1,3\r\n.*"node2",1,11\r\n.*"node1",2,3\r\n.*"node1",3,3\r\n') + client1.expect(prompt) + + def test_distributed_over_live_view_group_by_node(self, started_cluster, node, source): + log = sys.stdout + node0, node1 = NODES.values() + + select_query = "SELECT node, SUM(value) FROM distributed_over_lv GROUP BY node ORDER BY node FORMAT CSV" + + with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ + client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS distributed_over_lv") + client1.expect(prompt) + client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.expect(prompt) + + client1.send(select_query) + client1.expect('"node1",1\r\n"node2",21\r\n') + client1.expect(prompt) + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + time.sleep(2) + client1.send(select_query) + client1.expect('"node1",3\r\n.*"node2",21\r\n') + client1.expect(prompt) + + client1.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 3, 3)") + client1.expect(prompt) + client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") + client2.expect(prompt) + time.sleep(2) + client1.send(select_query) + client1.expect('"node1",12\r\n.*"node2",21\r\n') + client1.expect(prompt) + + def test_distributed_over_live_view_group_by_key(self, started_cluster, node, source): + log = sys.stdout + node0, node1 = NODES.values() + + select_query = "SELECT key, SUM(value) FROM distributed_over_lv GROUP BY key ORDER BY key FORMAT CSV" + + with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ + client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS distributed_over_lv") + client1.expect(prompt) + client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.expect(prompt) + + client1.send(select_query) + client1.expect("0,10\r\n1,12\r\n") + client1.expect(prompt) + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + time.sleep(2) + client1.send(select_query) + client1.expect("0,10\r\n1,12\r\n2,2\r\n") + client1.expect(prompt) + + client2.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 3, 3)") + client2.expect(prompt) + time.sleep(2) + client1.send(select_query) + client1.expect("0,10\r\n.*1,15\r\n.*2,2\r\n.*3,3\r\n") + client1.expect(prompt) + + def test_distributed_over_live_view_sum(self, started_cluster, node, source): + log = sys.stdout + node0, node1 = NODES.values() + + with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ + client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS distributed_over_lv") + client1.expect(prompt) + client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.expect(prompt) + + client1.send("SELECT sum(value) FROM distributed_over_lv") + client1.expect(r"22" + end_of_block) + client1.expect(prompt) + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + + time.sleep(2) + + client1.send("SELECT sum(value) FROM distributed_over_lv") + client1.expect(r"24" + end_of_block) + client1.expect(prompt) + + client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3), ('node1', 4, 4)") + client2.expect(prompt) + + time.sleep(2) + + client1.send("SELECT sum(value) FROM distributed_over_lv") + client1.expect(r"31" + end_of_block) + client1.expect(prompt) + diff --git a/tests/integration/test_enabling_access_management/__init__.py b/tests/integration/test_enabling_access_management/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_enabling_access_management/configs/users.d/extra_users.xml b/tests/integration/test_enabling_access_management/configs/users.d/extra_users.xml new file mode 100644 index 00000000000..7d87a29a915 --- /dev/null +++ b/tests/integration/test_enabling_access_management/configs/users.d/extra_users.xml @@ -0,0 +1,13 @@ + + + + + readonly + 1 + + + + default + + + diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py new file mode 100644 index 00000000000..abb8cd6c07a --- /dev/null +++ b/tests/integration/test_enabling_access_management/test.py @@ -0,0 +1,24 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance', config_dir="configs") + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_enabling_access_management(): + instance.query("CREATE USER Alex", user='default') + assert instance.query("SHOW CREATE USER Alex", user='default') == "CREATE USER Alex\n" + assert instance.query("SHOW CREATE USER Alex", user='readonly') == "CREATE USER Alex\n" + assert "Not enough privileges" in instance.query_and_get_error("SHOW CREATE USER Alex", user='xyz') + + assert "Cannot execute query in readonly mode" in instance.query_and_get_error("CREATE USER Robin", user='readonly') + assert "Not enough privileges" in instance.query_and_get_error("CREATE USER Robin", user='xyz') diff --git a/tests/integration/test_grant_and_revoke/configs/users.d/access_management.xml b/tests/integration/test_grant_and_revoke/configs/users.d/access_management.xml deleted file mode 100644 index 7e799cb7b10..00000000000 --- a/tests/integration/test_grant_and_revoke/configs/users.d/access_management.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - 1 - - - diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 25e0e9882de..6f4b0be5325 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -3,7 +3,7 @@ from helpers.cluster import ClickHouseCluster import re cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', config_dir="configs") +instance = cluster.add_instance('instance') @pytest.fixture(scope="module", autouse=True) diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index b9767e8b694..e48584bac84 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -132,6 +132,7 @@ def test_inserts_batching(started_cluster): for i in range(10, 13): instance.query("INSERT INTO distributed(d, x) VALUES ('2000-01-01', {})".format(i)) + instance.query("SYSTEM FLUSH DISTRIBUTED distributed") time.sleep(1.0) result = remote.query("SELECT _part, groupArray(x) FROM local2 GROUP BY _part ORDER BY _part") diff --git a/tests/integration/test_live_view_over_distributed/__init__.py b/tests/integration/test_live_view_over_distributed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_live_view_over_distributed/configs/remote_servers.xml b/tests/integration/test_live_view_over_distributed/configs/remote_servers.xml new file mode 100644 index 00000000000..ebce4697529 --- /dev/null +++ b/tests/integration/test_live_view_over_distributed/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_live_view_over_distributed/configs/set_distributed_defaults.xml b/tests/integration/test_live_view_over_distributed/configs/set_distributed_defaults.xml new file mode 100644 index 00000000000..194eb1ebb87 --- /dev/null +++ b/tests/integration/test_live_view_over_distributed/configs/set_distributed_defaults.xml @@ -0,0 +1,35 @@ + + + + 3 + 1000 + 1 + + + 5 + 3000 + 1 + + + + + + + + ::/0 + + default + default + + + + + ::/0 + + delays + default + + + + + diff --git a/tests/integration/test_live_view_over_distributed/test.py b/tests/integration/test_live_view_over_distributed/test.py new file mode 100644 index 00000000000..c7b9c452725 --- /dev/null +++ b/tests/integration/test_live_view_over_distributed/test.py @@ -0,0 +1,243 @@ +from __future__ import print_function + +import sys +import itertools +import timeit +import logging + +import pytest + +from helpers.uclient import client, prompt, end_of_block +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) + +NODES = {'node' + str(i): cluster.add_instance( + 'node' + str(i), + main_configs=['configs/remote_servers.xml'], + user_configs=['configs/set_distributed_defaults.xml'], +) for i in (1, 2)} + +CREATE_TABLES_SQL = ''' +DROP TABLE IF EXISTS lv_over_distributed_table; +DROP TABLE IF EXISTS distributed_table; +DROP TABLE IF EXISTS base_table; + +SET allow_experimental_live_view = 1; + +CREATE TABLE + base_table( + node String, + key Int32, + value Int32 + ) +ENGINE = Memory; + +CREATE TABLE + distributed_table +AS base_table +ENGINE = Distributed(test_cluster, default, base_table, rand()); + +CREATE LIVE VIEW lv_over_distributed_table AS SELECT * FROM distributed_table; +''' + +INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" + +@pytest.fixture(scope="function") +def started_cluster(): + try: + cluster.start() + for node_index, (node_name, node) in enumerate(NODES.items()): + node.query(CREATE_TABLES_SQL) + for i in range(0, 2): + sql = INSERT_SQL_TEMPLATE.format(node_id=node_name, key=i, value=i + (node_index * 10)) + node.query(sql) + yield cluster + + finally: + cluster.shutdown() + + +@pytest.mark.parametrize("node", NODES.values()[:1]) +@pytest.mark.parametrize("source", ["lv_over_distributed_table"]) +class TestLiveViewOverDistributedSuite: + def test_select_with_order_by_node(self, started_cluster, node, source): + r = node.query("SELECT * FROM {source} ORDER BY node, key".format(source=source)) + assert r == """node1\t0\t0 +node1\t1\t1 +node2\t0\t10 +node2\t1\t11 +""" + + def test_select_with_order_by_key(self, started_cluster, node, source): + assert node.query("SELECT * FROM {source} ORDER BY key, node".format(source=source)) \ + == """node1\t0\t0 +node2\t0\t10 +node1\t1\t1 +node2\t1\t11 +""" + + def test_select_with_group_by_node(self, started_cluster, node, source): + assert node.query("SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format(source=source)) \ + == "node1\t1\nnode2\t21\n" + + def test_select_with_group_by_key(self, started_cluster, node, source): + assert node.query("SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format(source=source)) \ + == "0\t10\n1\t12\n" + + def test_select_sum(self, started_cluster, node, source): + assert node.query("SELECT SUM(value) FROM {source}".format(source=source)) \ + == "22\n" + + def test_watch_live_view_order_by_node(self, started_cluster, node, source): + log = sys.stdout + command = " ".join(node.client.command) + args = dict(log=log, command=command) + + with client(name="client1> ", **args) as client1, client(name="client2> ", **args) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("SET allow_experimental_live_view = 1") + client1.expect(prompt) + client2.send("SET allow_experimental_live_view = 1") + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS lv") + client1.expect(prompt) + client1.send("CREATE LIVE VIEW lv AS SELECT * FROM distributed_table ORDER BY node, key") + client1.expect(prompt) + + client1.send("WATCH lv FORMAT CSV") + client1.expect('"node1",0,0,1\r\n.*"node1",1,1,1\r\n.*"node2",0,10,1\r\n.*"node2",1,11,1\r\n') + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + client1.expect('"node1",0,0,2\r\n.*"node1",1,1,2\r\n.*"node1",2,2,2\r\n.*"node2",0,10,2\r\n.*"node2",1,11,2\r\n') + + client2.send("INSERT INTO distributed_table VALUES ('node1', 0, 3), ('node3', 3, 3)") + client2.expect(prompt) + client1.expect('"node1",0,0,3\r\n.*"node1",0,3,3\r\n.*"node1",1,1,3\r\n.*"node1",2,2,3\r\n.*"node2",0,10,3\r\n.*"node2",1,11,3\r\n.*"node3",3,3,3\r\n') + + def test_watch_live_view_order_by_key(self, started_cluster, node, source): + log = sys.stdout + command = " ".join(node.client.command) + args = dict(log=log, command=command) + + with client(name="client1> ", **args) as client1, client(name="client2> ", **args) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("SET allow_experimental_live_view = 1") + client1.expect(prompt) + client2.send("SET allow_experimental_live_view = 1") + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS lv") + client1.expect(prompt) + client1.send("CREATE LIVE VIEW lv AS SELECT * FROM distributed_table ORDER BY key, node") + client1.expect(prompt) + + client1.send("WATCH lv FORMAT CSV") + client1.expect('"node1",0,0,1\r\n.*"node2",0,10,1\r\n.*"node1",1,1,1\r\n.*"node2",1,11,1\r\n') + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + client1.expect('"node1",0,0,2\r\n.*"node2",0,10,2\r\n.*"node1",1,1,2\r\n.*"node2",1,11,2\r\n.*"node1",2,2,2\r\n') + + client2.send("INSERT INTO distributed_table VALUES ('node1', 0, 3), ('node3', 3, 3)") + client2.expect(prompt) + client1.expect('"node1",0,0,3\r\n.*"node1",0,3,3\r\n.*"node2",0,10,3\r\n.*"node1",1,1,3\r\n.*"node2",1,11,3\r\n.*"node1",2,2,3\r\n.*"node3",3,3,3\r\n') + + def test_watch_live_view_group_by_node(self, started_cluster, node, source): + log = sys.stdout + command = " ".join(node.client.command) + args = dict(log=log, command=command) + + with client(name="client1> ", **args) as client1, client(name="client2> ", **args) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("SET allow_experimental_live_view = 1") + client1.expect(prompt) + client2.send("SET allow_experimental_live_view = 1") + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS lv") + client1.expect(prompt) + client1.send("CREATE LIVE VIEW lv AS SELECT node, SUM(value) FROM distributed_table GROUP BY node ORDER BY node") + client1.expect(prompt) + + client1.send("WATCH lv FORMAT CSV") + client1.expect('"node1",1,1\r\n.*"node2",21,1\r\n') + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + client1.expect('"node1",3,2\r\n.*"node2",21,2\r\n') + + client2.send("INSERT INTO distributed_table VALUES ('node1', 0, 3), ('node3', 3, 3)") + client2.expect(prompt) + client1.expect('"node1",6,3\r\n.*"node2",21,3\r\n.*"node3",3,3\r\n') + + def test_watch_live_view_group_by_key(self, started_cluster, node, source): + log = sys.stdout + command = " ".join(node.client.command) + args = dict(log=log, command=command) + sep = ' \xe2\x94\x82' + with client(name="client1> ", **args) as client1, client(name="client2> ", **args) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("SET allow_experimental_live_view = 1") + client1.expect(prompt) + client2.send("SET allow_experimental_live_view = 1") + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS lv") + client1.expect(prompt) + client1.send("CREATE LIVE VIEW lv AS SELECT key, SUM(value) FROM distributed_table GROUP BY key ORDER BY key") + client1.expect(prompt) + + client1.send("WATCH lv FORMAT CSV") + client1.expect("0,10,1\r\n.*1,12,1\r\n") + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + client1.expect("0,10,2\r\n.*1,12,2\r\n.*2,2,2\r\n") + + client2.send("INSERT INTO distributed_table VALUES ('node1', 0, 3), ('node1', 3, 3)") + client2.expect(prompt) + client1.expect("0,13,3\r\n.*1,12,3\r\n.*2,2,3\r\n.*3,3,3\r\n") + + + def test_watch_live_view_sum(self, started_cluster, node, source): + log = sys.stdout + command = " ".join(node.client.command) + args = dict(log=log, command=command) + + with client(name="client1> ", **args) as client1, client(name="client2> ", **args) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send("SET allow_experimental_live_view = 1") + client1.expect(prompt) + client2.send("SET allow_experimental_live_view = 1") + client2.expect(prompt) + + client1.send("DROP TABLE IF EXISTS lv") + client1.expect(prompt) + client1.send("CREATE LIVE VIEW lv AS SELECT sum(value) FROM distributed_table") + client1.expect(prompt) + + client1.send("WATCH lv") + client1.expect(r"22.*1" + end_of_block) + + client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") + client2.expect(prompt) + client1.expect(r"24.*2" + end_of_block) + + client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3), ('node1', 4, 4)") + client2.expect(prompt) + client1.expect(r"31.*3" + end_of_block) diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml index 5b292446c6b..d097675ca63 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml @@ -13,7 +13,7 @@ - +
s3 @@ -22,7 +22,7 @@ hdd - + diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index f69c09631e8..50cf532e9a4 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -67,7 +67,9 @@ def create_table(cluster, table_name, additional_settings=None): PARTITION BY dt ORDER BY (dt, id) SETTINGS - old_parts_lifetime=0, index_granularity=512 + storage_policy='s3', + old_parts_lifetime=0, + index_granularity=512 """.format(table_name) if additional_settings: @@ -84,7 +86,12 @@ def drop_table(cluster): minio = cluster.minio_client node.query("DROP TABLE IF EXISTS s3_test") - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 0 + try: + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 0 + finally: + # Remove extra objects to prevent tests cascade failing + for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + minio.remove_object(cluster.minio_bucket, obj.object_name) @pytest.mark.parametrize( @@ -210,7 +217,7 @@ def test_attach_detach_partition(cluster): assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-04'") - node.query("SET allow_drop_detached=1; ALTER TABLE s3_test DROP DETACHED PARTITION '2020-01-04'") + node.query("ALTER TABLE s3_test DROP DETACHED PARTITION '2020-01-04'", settings={"allow_drop_detached": 1}) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD @@ -245,8 +252,7 @@ def test_table_manipulations(cluster): assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 node.query("RENAME TABLE s3_renamed TO s3_test") - # TODO: Doesn't work with min_max index. - #assert node.query("SET check_query_single_value_result='false'; CHECK TABLE s3_test FORMAT Values") == "(1)" + assert node.query("CHECK TABLE s3_test FORMAT Values") == "(1)" node.query("DETACH TABLE s3_test") node.query("ATTACH TABLE s3_test") diff --git a/tests/integration/test_mysql_database_engine/test.py b/tests/integration/test_mysql_database_engine/test.py index 86e0b9df5fd..2791cc7b382 100644 --- a/tests/integration/test_mysql_database_engine/test.py +++ b/tests/integration/test_mysql_database_engine/test.py @@ -5,6 +5,7 @@ import pymysql.cursors import pytest from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) clickhouse_node = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql=True) @@ -92,7 +93,7 @@ def test_clickhouse_dml_for_mysql_database(started_cluster): with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `i``d` int(11) NOT NULL, PRIMARY KEY (`i``d`)) ENGINE=InnoDB;') - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") + clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', test_database, 'root', 'clickhouse')") assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '0' clickhouse_node.query("INSERT INTO `test_database`.`test_table`(`i\`d`) select number from numbers(10000)") @@ -116,7 +117,17 @@ def test_clickhouse_join_for_mysql_database(started_cluster): clickhouse_node.query("CREATE TABLE default.t1_remote_mysql AS mysql('mysql1:3306','test','t1_mysql_local','root','clickhouse')") clickhouse_node.query("CREATE TABLE default.t2_remote_mysql AS mysql('mysql1:3306','test','t2_mysql_local','root','clickhouse')") assert clickhouse_node.query("SELECT s.pays " - "FROM default.t1_remote_mysql AS s " - "LEFT JOIN default.t1_remote_mysql AS s_ref " - "ON (s_ref.opco = s.opco AND s_ref.service = s.service)") == '' + "FROM default.t1_remote_mysql AS s " + "LEFT JOIN default.t1_remote_mysql AS s_ref " + "ON (s_ref.opco = s.opco AND s_ref.service = s.service)") == '' mysql_node.query("DROP DATABASE test") + + +def test_bad_arguments_for_mysql_database_engine(started_cluster): + with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: + with pytest.raises(QueryRuntimeException) as exception: + mysql_node.query("CREATE DATABASE IF NOT EXISTS test_bad_arguments DEFAULT CHARACTER SET 'utf8'") + clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', test_bad_arguments, root, 'clickhouse')") + + assert 'Database engine MySQL requested literal argument.' in str(exception.value) + mysql_node.query("DROP DATABASE test_bad_arguments") diff --git a/tests/integration/test_mysql_protocol/clients/java/0.reference b/tests/integration/test_mysql_protocol/clients/java/0.reference new file mode 100644 index 00000000000..3e3e20d1ebb --- /dev/null +++ b/tests/integration/test_mysql_protocol/clients/java/0.reference @@ -0,0 +1,15 @@ +33jdbcnull +44cknull +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 diff --git a/tests/integration/test_mysql_protocol/clients/java/Dockerfile b/tests/integration/test_mysql_protocol/clients/java/Dockerfile new file mode 100644 index 00000000000..96713a68e66 --- /dev/null +++ b/tests/integration/test_mysql_protocol/clients/java/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install -y software-properties-common build-essential openjdk-8-jdk libmysql-java curl + +RUN rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/debconf \ + /tmp/* \ +RUN apt-get clean + +ARG ver=5.1.46 +RUN curl -L -o /mysql-connector-java-${ver}.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/${ver}/mysql-connector-java-${ver}.jar +ENV CLASSPATH=$CLASSPATH:/mysql-connector-java-${ver}.jar + +WORKDIR /jdbc +COPY Test.java Test.java +RUN javac Test.java diff --git a/tests/integration/test_mysql_protocol/clients/java/Test.java b/tests/integration/test_mysql_protocol/clients/java/Test.java new file mode 100644 index 00000000000..2e256d5dc44 --- /dev/null +++ b/tests/integration/test_mysql_protocol/clients/java/Test.java @@ -0,0 +1,77 @@ +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +class JavaConnectorTest { + private static final String CREATE_TABLE_SQL = "CREATE TABLE IF NOT EXISTS default.test1 (`age` Int32, `name` String, `int_nullable` Nullable(Int32)) Engine = Memory"; + private static final String INSERT_SQL = "INSERT INTO default.test1(`age`, `name`) VALUES(33, 'jdbc'),(44, 'ck')"; + private static final String SELECT_SQL = "SELECT * FROM default.test1"; + private static final String SELECT_NUMBER_SQL = "SELECT * FROM system.numbers LIMIT 13"; + private static final String DROP_TABLE_SQL = "DROP TABLE default.test1"; + + public static void main(String[] args) { + int i = 0; + String host = "127.0.0.1"; + String port = "9004"; + String user = "default"; + String password = ""; + String database = "default"; + while (i < args.length) { + switch (args[i]) { + case "--host": + host = args[++i]; + break; + case "--port": + port = args[++i]; + break; + case "--user": + user = args[++i]; + break; + case "--password": + password = args[++i]; + break; + case "--database": + database = args[++i]; + break; + default: + i++; + break; + } + } + + String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false", host, port, database); + + Connection conn = null; + Statement stmt = null; + try { + conn = DriverManager.getConnection(jdbcUrl, user, password); + stmt = conn.createStatement(); + stmt.executeUpdate(CREATE_TABLE_SQL); + stmt.executeUpdate(INSERT_SQL); + + ResultSet rs = stmt.executeQuery(SELECT_SQL); + while (rs.next()) { + System.out.print(rs.getString("age")); + System.out.print(rs.getString("name")); + System.out.print(rs.getString("int_nullable")); + System.out.println(); + } + + stmt.executeUpdate(DROP_TABLE_SQL); + + rs = stmt.executeQuery(SELECT_NUMBER_SQL); + while (rs.next()) { + System.out.print(rs.getString(1)); + System.out.println(); + } + + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } +} diff --git a/tests/integration/test_mysql_protocol/clients/java/docker_compose.yml b/tests/integration/test_mysql_protocol/clients/java/docker_compose.yml new file mode 100644 index 00000000000..dbe404232a0 --- /dev/null +++ b/tests/integration/test_mysql_protocol/clients/java/docker_compose.yml @@ -0,0 +1,8 @@ +version: '2.2' +services: + java1: + build: + context: ./ + network: host + # to keep container running + command: sleep infinity diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index 7987076c29a..f75a168d5db 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -79,6 +79,13 @@ def nodejs_container(): yield docker.from_env().containers.get(cluster.project_name + '_mysqljs1_1') +@pytest.fixture(scope='module') +def java_container(): + docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'java', 'docker_compose.yml') + subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) + yield docker.from_env().containers.get(cluster.project_name + '_java1_1') + + def test_mysql_client(mysql_client, server_address): # type: (Container, str) -> None code, (stdout, stderr) = mysql_client.exec_run(''' @@ -266,6 +273,35 @@ def test_mysqljs_client(server_address, nodejs_container): assert code == 1 +def test_java_client(server_address, java_container): + # type: (str, Container) -> None + with open(os.path.join(SCRIPT_DIR, 'clients', 'java', '0.reference')) as fp: + reference = fp.read() + + # database not exists exception. + code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' + 'abc'.format(host=server_address, port=server_port), demux=True) + assert code == 1 + + # empty password passed. + code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' + 'default'.format(host=server_address, port=server_port), demux=True) + assert code == 0 + assert stdout == reference + + # non-empty password passed. + code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database ' + 'default'.format(host=server_address, port=server_port), demux=True) + assert code == 0 + assert stdout == reference + + # double-sha1 password passed. + code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' + 'default'.format(host=server_address, port=server_port), demux=True) + assert code == 0 + assert stdout == reference + + def test_types(server_address): client = pymysql.connections.Connection(host=server_address, user='default', password='123', database='default', port=server_port) diff --git a/tests/integration/test_quorum_inserts/__init__.py b/tests/integration/test_quorum_inserts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_quorum_inserts/configs/users.d/settings.xml b/tests/integration/test_quorum_inserts/configs/users.d/settings.xml new file mode 100644 index 00000000000..29b762656aa --- /dev/null +++ b/tests/integration/test_quorum_inserts/configs/users.d/settings.xml @@ -0,0 +1,11 @@ + + + + + 10000000000 + 0 + 2 + 1 + + + diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py new file mode 100644 index 00000000000..27901842692 --- /dev/null +++ b/tests/integration/test_quorum_inserts/test.py @@ -0,0 +1,294 @@ +import time + +import pytest + +from helpers.test_tools import TSV +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +zero = cluster.add_instance("zero", + config_dir="configs", + macros={"cluster": "anime", "shard": "0", "replica": "zero"}, + with_zookeeper=True) + +first = cluster.add_instance("first", + config_dir="configs", + macros={"cluster": "anime", "shard": "0", "replica": "first"}, + with_zookeeper=True) + +second = cluster.add_instance("second", + config_dir="configs", + macros={"cluster": "anime", "shard": "0", "replica": "second"}, + with_zookeeper=True) + +def execute_on_all_cluster(query_): + for node in [zero, first, second]: + node.query(query_) + +@pytest.fixture(scope="module") +def started_cluster(): + global cluster + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_simple_add_replica(started_cluster): + execute_on_all_cluster("DROP TABLE IF EXISTS test_simple") + + create_query = "CREATE TABLE test_simple " \ + "(a Int8, d Date) " \ + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "PARTITION BY d ORDER BY a" + + zero.query(create_query) + first.query(create_query) + + first.query("SYSTEM STOP FETCHES test_simple") + + zero.query("INSERT INTO test_simple VALUES (1, '2011-01-01')", settings={'insert_quorum' : 1}) + + assert '1\t2011-01-01\n' == zero.query("SELECT * from test_simple") + assert '' == first.query("SELECT * from test_simple") + + first.query("SYSTEM START FETCHES test_simple") + + first.query("SYSTEM SYNC REPLICA test_simple", timeout=20) + + assert '1\t2011-01-01\n' == zero.query("SELECT * from test_simple") + assert '1\t2011-01-01\n' == first.query("SELECT * from test_simple") + + second.query(create_query) + second.query("SYSTEM SYNC REPLICA test_simple", timeout=20) + + assert '1\t2011-01-01\n' == zero.query("SELECT * from test_simple") + assert '1\t2011-01-01\n' == first.query("SELECT * from test_simple") + assert '1\t2011-01-01\n' == second.query("SELECT * from test_simple") + + execute_on_all_cluster("DROP TABLE IF EXISTS test_simple") + + + +def test_drop_replica_and_achieve_quorum(started_cluster): + execute_on_all_cluster("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum") + + create_query = "CREATE TABLE test_drop_replica_and_achieve_quorum " \ + "(a Int8, d Date) " \ + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "PARTITION BY d ORDER BY a" + + print("Create Replicated table with two replicas") + zero.query(create_query) + first.query(create_query) + + print("Stop fetches on one replica. Since that, it will be isolated.") + first.query("SYSTEM STOP FETCHES test_drop_replica_and_achieve_quorum") + + print("Insert to other replica. This query will fail.") + quorum_timeout = zero.query_and_get_error("INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (1, '2011-01-01')", + settings={'insert_quorum_timeout' : 5000}) + assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." + + assert TSV("1\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum", + settings={'select_sequential_consistency' : 0})) + + assert TSV("") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum", + settings={'select_sequential_consistency' : 1})) + + #TODO:(Mikhaylov) begin; maybe delete this lines. I want clickhouse to fetch parts and update quorum. + print("START FETCHES first replica") + first.query("SYSTEM START FETCHES test_drop_replica_and_achieve_quorum") + + print("SYNC first replica") + first.query("SYSTEM SYNC REPLICA test_drop_replica_and_achieve_quorum", timeout=20) + #TODO:(Mikhaylov) end + + print("Add second replica") + second.query(create_query) + + print("SYNC second replica") + second.query("SYSTEM SYNC REPLICA test_drop_replica_and_achieve_quorum", timeout=20) + + print("Quorum for previous insert achieved.") + assert TSV("1\t2011-01-01\n") == TSV(second.query("SELECT * FROM test_drop_replica_and_achieve_quorum", + settings={'select_sequential_consistency' : 1})) + + print("Now we can insert some other data.") + zero.query("INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (2, '2012-02-02')") + + assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV(first.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV(second.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + + execute_on_all_cluster("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum") + + +@pytest.mark.parametrize( + ('add_new_data'), + [ + False, + True + ] +) + +def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): + execute_on_all_cluster("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition") + + create_query = "CREATE TABLE test_quorum_insert_with_drop_partition " \ + "(a Int8, d Date) " \ + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "PARTITION BY d ORDER BY a " + + print("Create Replicated table with three replicas") + zero.query(create_query) + first.query(create_query) + second.query(create_query) + + print("Stop fetches for test_quorum_insert_with_drop_partition at first replica.") + first.query("SYSTEM STOP FETCHES test_quorum_insert_with_drop_partition") + + print("Insert with quorum. (zero and second)") + zero.query("INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(1, '2011-01-01')") + + print("Drop partition.") + zero.query("ALTER TABLE test_quorum_insert_with_drop_partition DROP PARTITION '2011-01-01'") + + if (add_new_data): + print("Insert to deleted partition") + zero.query("INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(2, '2011-01-01')") + + print("Resume fetches for test_quorum_insert_with_drop_partition at first replica.") + first.query("SYSTEM START FETCHES test_quorum_insert_with_drop_partition") + + print("Sync first replica with others.") + first.query("SYSTEM SYNC REPLICA test_quorum_insert_with_drop_partition") + + assert "20110101" not in first.query("SELECT * FROM system.zookeeper " \ + "where path='/clickhouse/tables/0/test_quorum_insert_with_drop_partition/quorum/last_part' " \ + "format Vertical") + + print("Select from updated partition.") + if (add_new_data): + assert TSV("2\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_quorum_insert_with_drop_partition")) + assert TSV("2\t2011-01-01\n") == TSV(second.query("SELECT * FROM test_quorum_insert_with_drop_partition")) + else: + assert TSV("") == TSV(zero.query("SELECT * FROM test_quorum_insert_with_drop_partition")) + assert TSV("") == TSV(second.query("SELECT * FROM test_quorum_insert_with_drop_partition")) + + execute_on_all_cluster("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition") + + +@pytest.mark.parametrize( + ('add_new_data'), + [ + False, + True + ] +) + +def test_insert_quorum_with_move_partition(started_cluster, add_new_data): + execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source") + execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination") + + create_source = "CREATE TABLE test_insert_quorum_with_move_partition_source " \ + "(a Int8, d Date) " \ + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "PARTITION BY d ORDER BY a " + + create_destination = "CREATE TABLE test_insert_quorum_with_move_partition_destination " \ + "(a Int8, d Date) " \ + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "PARTITION BY d ORDER BY a " + + print("Create source Replicated table with three replicas") + zero.query(create_source) + first.query(create_source) + second.query(create_source) + + print("Create destination Replicated table with three replicas") + zero.query(create_destination) + first.query(create_destination) + second.query(create_destination) + + print("Stop fetches for test_insert_quorum_with_move_partition_source at first replica.") + first.query("SYSTEM STOP FETCHES test_insert_quorum_with_move_partition_source") + + print("Insert with quorum. (zero and second)") + zero.query("INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(1, '2011-01-01')") + + print("Drop partition.") + zero.query("ALTER TABLE test_insert_quorum_with_move_partition_source MOVE PARTITION '2011-01-01' TO TABLE test_insert_quorum_with_move_partition_destination") + + if (add_new_data): + print("Insert to deleted partition") + zero.query("INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(2, '2011-01-01')") + + print("Resume fetches for test_insert_quorum_with_move_partition_source at first replica.") + first.query("SYSTEM START FETCHES test_insert_quorum_with_move_partition_source") + + print("Sync first replica with others.") + first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_move_partition_source") + + assert "20110101" not in first.query("SELECT * FROM system.zookeeper " \ + "where path='/clickhouse/tables/0/test_insert_quorum_with_move_partition_source/quorum/last_part' " \ + "format Vertical") + + print("Select from updated partition.") + if (add_new_data): + assert TSV("2\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + assert TSV("2\t2011-01-01\n") == TSV(second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + else: + assert TSV("") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + assert TSV("") == TSV(second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + + execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source") + execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination") + + +def test_insert_quorum_with_ttl(started_cluster): + execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_ttl") + + create_query = "CREATE TABLE test_insert_quorum_with_ttl " \ + "(a Int8, d Date) " \ + "Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') " \ + "PARTITION BY d ORDER BY a " \ + "TTL d + INTERVAL 5 second " \ + "SETTINGS merge_with_ttl_timeout=2 " + + print("Create Replicated table with two replicas") + zero.query(create_query) + first.query(create_query) + + print("Stop fetches for test_insert_quorum_with_ttl at first replica.") + first.query("SYSTEM STOP FETCHES test_insert_quorum_with_ttl") + + print("Insert should fail since it can not reach the quorum.") + quorum_timeout = zero.query_and_get_error("INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", + settings={'insert_quorum_timeout' : 5000}) + assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." + + print("Wait 10 seconds and TTL merge have to be executed. But it won't delete data.") + time.sleep(10) + assert TSV("1\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 0})) + + print("Resume fetches for test_insert_quorum_with_ttl at first replica.") + first.query("SYSTEM START FETCHES test_insert_quorum_with_ttl") + + print("Sync first replica.") + first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") + + zero.query("INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", + settings={'insert_quorum_timeout' : 5000}) + + + assert TSV("1\t2011-01-01\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 0})) + assert TSV("1\t2011-01-01\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 1})) + + print("Inserts should resume.") + zero.query("INSERT INTO test_insert_quorum_with_ttl(a, d) VALUES(2, '2012-02-02')") + + execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_ttl") diff --git a/tests/integration/test_quota/configs/users.d/access_management.xml b/tests/integration/test_quota/configs/users.d/access_management.xml deleted file mode 100644 index 7e799cb7b10..00000000000 --- a/tests/integration/test_quota/configs/users.d/access_management.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - 1 - - - diff --git a/tests/integration/test_disk_access_storage/configs/users.d/access_management.xml b/tests/integration/test_quota/configs/users.d/assign_myquota.xml similarity index 60% rename from tests/integration/test_disk_access_storage/configs/users.d/access_management.xml rename to tests/integration/test_quota/configs/users.d/assign_myquota.xml index 7e799cb7b10..8b98ade8aeb 100644 --- a/tests/integration/test_disk_access_storage/configs/users.d/access_management.xml +++ b/tests/integration/test_quota/configs/users.d/assign_myquota.xml @@ -1,7 +1,7 @@ - 1 + myQuota diff --git a/tests/integration/test_quota/configs/users.d/drop_default_quota.xml b/tests/integration/test_quota/configs/users.d/drop_default_quota.xml new file mode 100644 index 00000000000..5f53ecf5f49 --- /dev/null +++ b/tests/integration/test_quota/configs/users.d/drop_default_quota.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index 85d2ded16c1..ae68a34a03e 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -180,7 +180,7 @@ def test_reload_users_xml_by_timer(): def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000 TO default\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0/1000 errors=0 result_rows=0 result_bytes=0 read_rows=0/1000 read_bytes=0 execution_time=0" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE CURRENT")) @@ -193,7 +193,7 @@ def test_dcl_introspection(): # Add interval. copy_quota_xml('two_intervals.xml') assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES = 30000, MAX READ BYTES = 20000, MAX EXECUTION TIME = 120 TO default\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES 30000, READ BYTES 20000, EXECUTION TIME 120 TO default\n" expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*\n"\ "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0/30000 read_rows=0 read_bytes=0/20000 execution_time=0/120" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) @@ -201,8 +201,8 @@ def test_dcl_introspection(): # Drop interval, add quota. copy_quota_xml('two_quotas.xml') assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000 TO default\n" - assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY \\'client key or user name\\' FOR RANDOMIZED INTERVAL 1 HOUR MAX RESULT ROWS = 4000, MAX RESULT BYTES = 400000, MAX READ ROWS = 4000, MAX READ BYTES = 400000, MAX EXECUTION TIME = 60, FOR INTERVAL 1 MONTH MAX EXECUTION TIME = 1800\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" + assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY \\'client key or user name\\' FOR RANDOMIZED INTERVAL 1 HOUR MAX RESULT ROWS 4000, RESULT BYTES 400000, READ ROWS 4000, READ BYTES 400000, EXECUTION TIME 60, FOR INTERVAL 1 MONTH MAX EXECUTION TIME 1800\n" expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) @@ -212,9 +212,9 @@ def test_dcl_management(): assert instance.query("SHOW QUOTAS") == "" assert instance.query("SHOW QUOTA USAGE") == "" - instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH SET MAX QUERIES = 123 TO CURRENT_USER") + instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER") assert instance.query("SHOW QUOTAS") == "qA\n" - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 5 QUARTER MAX QUERIES = 123 TO default\n" + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 5 QUARTER MAX QUERIES 123 TO default\n" expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0/123 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) @@ -222,14 +222,14 @@ def test_dcl_management(): expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1/123 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES = 321, MAX ERRORS = 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME = 0.5") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 30 MINUTE MAX EXECUTION TIME = 0.5, FOR INTERVAL 5 QUARTER MAX QUERIES = 321, MAX ERRORS = 10 TO default\n" + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5") + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 30 MINUTE MAX EXECUTION TIME 0.5, FOR INTERVAL 5 QUARTER MAX QUERIES 321, ERRORS 10 TO default\n" expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*/0.5\n"\ "qA key=\\\\'\\\\' interval=\[.*\] queries=1/321 errors=0/10 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH UNSET TRACKING, FOR RANDOMIZED INTERVAL 16 MONTH SET TRACKING, FOR INTERVAL 1800 SECOND UNSET TRACKING") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING TO default\n" + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY TO default\n" expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) @@ -238,7 +238,7 @@ def test_dcl_management(): assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) instance.query("ALTER QUOTA qA RENAME TO qB") - assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING TO default\n" + assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY TO default\n" expected_usage = "qB key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) diff --git a/tests/integration/test_replicated_merge_tree_s3/__init__.py b/tests/integration/test_replicated_merge_tree_s3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml new file mode 100644 index 00000000000..b32770095fc --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml @@ -0,0 +1,21 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + +
+ s3 +
+ + + + + diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py new file mode 100644 index 00000000000..d6b6015a388 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -0,0 +1,106 @@ +import logging +import random +import string +import time + +import pytest +from helpers.cluster import ClickHouseCluster + +logging.getLogger().setLevel(logging.INFO) +logging.getLogger().addHandler(logging.StreamHandler()) + + +# Creates S3 bucket for tests and allows anonymous read-write access to it. +def prepare_s3_bucket(cluster): + minio_client = cluster.minio_client + + if minio_client.bucket_exists(cluster.minio_bucket): + minio_client.remove_bucket(cluster.minio_bucket) + + minio_client.make_bucket(cluster.minio_bucket) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + + cluster.add_instance("node1", config_dir="configs", macros={'cluster': 'test1'}, with_minio=True, with_zookeeper=True) + cluster.add_instance("node2", config_dir="configs", macros={'cluster': 'test1'}, with_zookeeper=True) + cluster.add_instance("node3", config_dir="configs", macros={'cluster': 'test1'}, with_zookeeper=True) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + prepare_s3_bucket(cluster) + logging.info("S3 bucket created") + + yield cluster + finally: + cluster.shutdown() + + +FILES_OVERHEAD = 1 +FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files +FILES_OVERHEAD_PER_PART = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + + +def random_string(length): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(length)) + + +def generate_values(date_str, count, sign=1): + data = [[date_str, sign*(i + 1), random_string(10)] for i in range(count)] + data.sort(key=lambda tup: tup[1]) + return ",".join(["('{}',{},'{}')".format(x, y, z) for x, y, z in data]) + + +def create_table(cluster): + create_table_statement = """ + CREATE TABLE s3_test ( + dt Date, + id Int64, + data String, + INDEX min_max (id) TYPE minmax GRANULARITY 3 + ) ENGINE=ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/s3', '{instance}') + PARTITION BY dt + ORDER BY (dt, id) + SETTINGS storage_policy='s3' + """ + + for node in cluster.instances.values(): + node.query(create_table_statement) + + +@pytest.fixture(autouse=True) +def drop_table(cluster): + yield + for node in cluster.instances.values(): + node.query("DROP TABLE IF EXISTS s3_test") + + minio = cluster.minio_client + # Remove extra objects to prevent tests cascade failing + for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + minio.remove_object(cluster.minio_bucket, obj.object_name) + + +def test_insert_select_replicated(cluster): + create_table(cluster) + + all_values = "" + for node_idx in range(1, 4): + node = cluster.instances["node" + str(node_idx)] + values = generate_values("2020-01-0" + str(node_idx), 4096) + node.query("INSERT INTO s3_test VALUES {}".format(values), settings={"insert_quorum": 3}) + if node_idx != 1: + all_values += "," + all_values += values + + for node_idx in range(1, 4): + node = cluster.instances["node" + str(node_idx)] + assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values", settings={"select_sequential_consistency": 1}) == all_values + + minio = cluster.minio_client + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 3 * (FILES_OVERHEAD + FILES_OVERHEAD_PER_PART * 3) diff --git a/tests/integration/test_row_policy/configs/users.d/access_management.xml b/tests/integration/test_row_policy/configs/users.d/access_management.xml deleted file mode 100644 index 7e799cb7b10..00000000000 --- a/tests/integration/test_row_policy/configs/users.d/access_management.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - 1 - - - diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index 7087e6aafae..75c75e3b20a 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -102,8 +102,8 @@ def test_join(): def test_cannot_trick_row_policy_with_keyword_with(): assert instance.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == "1\n1\n" + assert instance.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == "0\t0\n0\t1\n" + assert instance.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == "0\n0\n" assert instance.query("WITH 0 AS a SELECT b FROM mydb.filtered_table1") == "0\n1\n" @@ -113,6 +113,9 @@ def test_prewhere_not_supported(): assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table2 PREWHERE 1") assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table3 PREWHERE 1") + # However PREWHERE should still work for user without filtering. + assert instance.query("SELECT * FROM mydb.filtered_table1 PREWHERE 1", user="another") == "0\t0\n0\t1\n1\t0\n1\t1\n" + def test_single_table_name(): copy_policy_xml('tag_with_table_name.xml') diff --git a/tests/integration/test_settings_constraints_distributed/configs/config.d/remote_servers.xml b/tests/integration/test_settings_constraints_distributed/configs/remote_servers.xml similarity index 100% rename from tests/integration/test_settings_constraints_distributed/configs/config.d/remote_servers.xml rename to tests/integration/test_settings_constraints_distributed/configs/remote_servers.xml diff --git a/tests/integration/test_settings_constraints_distributed/configs/users.d/access_management.xml b/tests/integration/test_settings_constraints_distributed/configs/users.d/access_management.xml deleted file mode 100644 index 7e799cb7b10..00000000000 --- a/tests/integration/test_settings_constraints_distributed/configs/users.d/access_management.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - 1 - - - diff --git a/tests/integration/test_settings_constraints_distributed/test.py b/tests/integration/test_settings_constraints_distributed/test.py index a58c037a2fc..86456f8a099 100644 --- a/tests/integration/test_settings_constraints_distributed/test.py +++ b/tests/integration/test_settings_constraints_distributed/test.py @@ -8,12 +8,12 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', config_dir="configs") -node2 = cluster.add_instance('node2', config_dir="configs") -distributed = cluster.add_instance('distributed', config_dir="configs") +node1 = cluster.add_instance('node1') +node2 = cluster.add_instance('node2') +distributed = cluster.add_instance('distributed', main_configs=["configs/remote_servers.xml"], stay_alive=True) -@pytest.fixture(scope="module") +@pytest.fixture(scope="module", autouse=True) def started_cluster(): try: cluster.start() @@ -35,7 +35,15 @@ def started_cluster(): cluster.shutdown() -def test_select_clamps_settings(started_cluster): +@pytest.fixture(autouse=True) +def restart_distributed(): + # Magic: Distributed table tries to keep connections to shards open, and after changing shards' default settings + # we need to reset connections to force the shards to reset sessions and therefore to reset current settings + # to their new defaults. + distributed.restart_clickhouse() + + +def test_select_clamps_settings(): distributed.query("CREATE USER normal DEFAULT ROLE admin SETTINGS max_memory_usage = 80000000") distributed.query("CREATE USER wasteful DEFAULT ROLE admin SETTINGS max_memory_usage = 2000000000") distributed.query("CREATE USER readonly DEFAULT ROLE admin SETTINGS readonly = 1") @@ -89,10 +97,11 @@ def test_select_clamps_settings(started_cluster): 'node2\tmax_memory_usage\t10000000000\n'\ 'node2\treadonly\t1\n' -def test_insert_clamps_settings(started_cluster): +def test_insert_clamps_settings(): node1.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") node2.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") distributed.query("INSERT INTO proxy VALUES (toDate('2020-02-20'), 2, 2)") distributed.query("INSERT INTO proxy VALUES (toDate('2020-02-21'), 2, 2)", settings={"max_memory_usage": 5000000}) - assert distributed.query("SELECT COUNT() FROM proxy") == "4\n" + distributed.query("SYSTEM FLUSH DISTRIBUTED proxy") + assert_eq_with_retry(distributed, "SELECT COUNT() FROM proxy", "4") diff --git a/tests/integration/test_settings_profile/configs/users.d/access_management.xml b/tests/integration/test_settings_profile/configs/users.d/access_management.xml deleted file mode 100644 index 7e799cb7b10..00000000000 --- a/tests/integration/test_settings_profile/configs/users.d/access_management.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - 1 - - - diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 6866c6b3901..8b9d023d56f 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', config_dir="configs") +instance = cluster.add_instance('instance') @pytest.fixture(scope="module", autouse=True) @@ -31,22 +31,26 @@ def reset_after_test(): def test_settings_profile(): # Set settings and constraints via CREATE SETTINGS PROFILE ... TO user instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin") + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") instance.query("ALTER SETTINGS PROFILE xyz TO NONE") + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") # Set settings and constraints via CREATE USER ... SETTINGS PROFILE instance.query("ALTER USER robin SETTINGS PROFILE xyz") + assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin SETTINGS PROFILE xyz\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") instance.query("ALTER USER robin SETTINGS NONE") + assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") @@ -57,6 +61,8 @@ def test_settings_profile_from_granted_role(): instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000") instance.query("CREATE ROLE worker SETTINGS PROFILE xyz") instance.query("GRANT worker TO robin") + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" + assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker SETTINGS PROFILE xyz\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") @@ -68,17 +74,20 @@ def test_settings_profile_from_granted_role(): instance.query("ALTER ROLE worker SETTINGS NONE") instance.query("GRANT worker TO robin") + assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") # Set settings and constraints via CREATE SETTINGS PROFILE ... TO granted role instance.query("ALTER SETTINGS PROFILE xyz TO worker") + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO worker\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") instance.query("ALTER SETTINGS PROFILE xyz TO NONE") + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") @@ -87,6 +96,8 @@ def test_settings_profile_from_granted_role(): def test_inheritance_of_settings_profile(): instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY") instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE alpha") == "CREATE SETTINGS PROFILE alpha SETTINGS INHERIT xyz TO robin\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000002\n" assert "Setting max_memory_usage should not be changed" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") diff --git a/tests/integration/test_system_tables_non_lazy_load/configs/config_non_lazy.xml b/tests/integration/test_system_tables_non_lazy_load/configs/config_non_lazy.xml new file mode 100644 index 00000000000..2dfee707a7c --- /dev/null +++ b/tests/integration/test_system_tables_non_lazy_load/configs/config_non_lazy.xml @@ -0,0 +1,4 @@ + + + false + diff --git a/tests/integration/test_system_tables_non_lazy_load/test_system_tables_non_lazy_load.py b/tests/integration/test_system_tables_non_lazy_load/test_system_tables_non_lazy_load.py new file mode 100644 index 00000000000..18841b4c585 --- /dev/null +++ b/tests/integration/test_system_tables_non_lazy_load/test_system_tables_non_lazy_load.py @@ -0,0 +1,24 @@ +import time +import pytest +import os + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', config_dir="configs") + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def test_system_tables_non_lazy_load(start_cluster): + assert node1.query_and_get_error("SELECT * FROM system.part_log") == "" + assert node1.query_and_get_error("SELECT * FROM system.query_log") == "" + assert node1.query_and_get_error("SELECT * FROM system.query_thread_log") == "" + assert node1.query_and_get_error("SELECT * FROM system.text_log") == "" + assert node1.query_and_get_error("SELECT * FROM system.trace_log") == "" + assert node1.query_and_get_error("SELECT * FROM system.metric_log") == "" diff --git a/tests/integration/test_zookeeper_config/configs/zookeeper_config_with_ssl.xml b/tests/integration/test_zookeeper_config/configs/zookeeper_config_with_ssl.xml new file mode 100644 index 00000000000..fc03b609146 --- /dev/null +++ b/tests/integration/test_zookeeper_config/configs/zookeeper_config_with_ssl.xml @@ -0,0 +1,20 @@ + + + + zoo1 + 2281 + 1 + + + zoo2 + 2281 + 1 + + + zoo3 + 2281 + 1 + + 3000 + + diff --git a/tests/integration/test_zookeeper_config/configs_secure/client.crt b/tests/integration/test_zookeeper_config/configs_secure/client.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/integration/test_zookeeper_config/configs_secure/client.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/integration/test_zookeeper_config/configs_secure/client.key b/tests/integration/test_zookeeper_config/configs_secure/client.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/integration/test_zookeeper_config/configs_secure/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_zookeeper_config/configs_secure/conf.d/remote_servers.xml b/tests/integration/test_zookeeper_config/configs_secure/conf.d/remote_servers.xml new file mode 100644 index 00000000000..01865e33a85 --- /dev/null +++ b/tests/integration/test_zookeeper_config/configs_secure/conf.d/remote_servers.xml @@ -0,0 +1,17 @@ + + + + + + node1 + 9000 + + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_zookeeper_config/configs_secure/conf.d/ssl_conf.xml b/tests/integration/test_zookeeper_config/configs_secure/conf.d/ssl_conf.xml new file mode 100644 index 00000000000..5e6f5f37624 --- /dev/null +++ b/tests/integration/test_zookeeper_config/configs_secure/conf.d/ssl_conf.xml @@ -0,0 +1,16 @@ + + + + /etc/clickhouse-server/client.crt + /etc/clickhouse-server/client.key + true + true + sslv2,sslv3 + true + none + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index d9323ae16f3..4be99c8711d 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -1,7 +1,11 @@ from __future__ import print_function from helpers.cluster import ClickHouseCluster +import helpers import pytest import time +from tempfile import NamedTemporaryFile +from os import path as p, unlink + def test_chroot_with_same_root(): @@ -100,10 +104,58 @@ def test_identity(): with pytest.raises(Exception): cluster_2.start(destroy_dirs=False) node2.query(''' - CREATE TABLE simple (date Date, id UInt32) + CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '1', date, id, 8192); ''') finally: cluster_1.shutdown() cluster_2.shutdown() + + +def test_secure_connection(): + # We need absolute path in zookeeper volumes. Generate it dynamically. + TEMPLATE = ''' + zoo{zoo_id}: + image: zookeeper:3.5.6 + restart: always + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: {zoo_id} + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + ZOO_SECURE_CLIENT_PORT: 2281 + volumes: + - {helpers_dir}/zookeeper-ssl-entrypoint.sh:/zookeeper-ssl-entrypoint.sh + - {configs_dir}:/clickhouse-config + command: ["zkServer.sh", "start-foreground"] + entrypoint: /zookeeper-ssl-entrypoint.sh + ''' + configs_dir = p.abspath(p.join(p.dirname(__file__), 'configs_secure')) + helpers_dir = p.abspath(p.dirname(helpers.__file__)) + + cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_with_ssl.xml') + + docker_compose = NamedTemporaryFile(delete=False) + + docker_compose.write( + "version: '2.2'\nservices:\n" + + TEMPLATE.format(zoo_id=1, configs_dir=configs_dir, helpers_dir=helpers_dir) + + TEMPLATE.format(zoo_id=2, configs_dir=configs_dir, helpers_dir=helpers_dir) + + TEMPLATE.format(zoo_id=3, configs_dir=configs_dir, helpers_dir=helpers_dir) + ) + docker_compose.close() + + node1 = cluster.add_instance('node1', config_dir='configs_secure', with_zookeeper=True, + zookeeper_docker_compose_path=docker_compose.name) + node2 = cluster.add_instance('node2', config_dir='configs_secure', with_zookeeper=True, + zookeeper_docker_compose_path=docker_compose.name) + + try: + cluster.start() + + assert node1.query("SELECT count() FROM system.zookeeper WHERE path = '/'") == '2\n' + assert node2.query("SELECT count() FROM system.zookeeper WHERE path = '/'") == '2\n' + + finally: + cluster.shutdown() + unlink(docker_compose.name) diff --git a/tests/performance/IPv4.xml b/tests/performance/IPv4.xml index 8f5b61d70c9..b3f6cf52584 100644 --- a/tests/performance/IPv4.xml +++ b/tests/performance/IPv4.xml @@ -1,14 +1,5 @@ - - - 30000 - - - 5000 - 60000 - - CREATE TABLE IF NOT EXISTS ips_v4(ip String) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY tuple() - 10000 - - diff --git a/tests/performance/bit_operations_fixed_string.xml b/tests/performance/bit_operations_fixed_string.xml index 90df91f1025..c08761ba8fc 100644 --- a/tests/performance/bit_operations_fixed_string.xml +++ b/tests/performance/bit_operations_fixed_string.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - diff --git a/tests/performance/bit_operations_fixed_string_numbers.xml b/tests/performance/bit_operations_fixed_string_numbers.xml index 779aea19cdc..e10e665ac81 100644 --- a/tests/performance/bit_operations_fixed_string_numbers.xml +++ b/tests/performance/bit_operations_fixed_string_numbers.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - SELECT count() FROM numbers(10000000) WHERE NOT ignore(bitXor(reinterpretAsFixedString(number), reinterpretAsFixedString(number + 1))) SELECT count() FROM numbers(10000000) WHERE NOT ignore(bitXor(reinterpretAsFixedString(number), reinterpretAsFixedString(0xabcd0123cdef4567))) diff --git a/tests/performance/bloom_filter.xml b/tests/performance/bloom_filter.xml index 079b7a43da3..3d9096afb03 100644 --- a/tests/performance/bloom_filter.xml +++ b/tests/performance/bloom_filter.xml @@ -1,10 +1,5 @@ - - - 30000 - - DROP TABLE IF EXISTS test_bf CREATE TABLE test_bf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 8192) ENGINE = MergeTree() ORDER BY id diff --git a/tests/performance/bounding_ratio.xml b/tests/performance/bounding_ratio.xml index 0d0adfaea45..113c9c4dc14 100644 --- a/tests/performance/bounding_ratio.xml +++ b/tests/performance/bounding_ratio.xml @@ -1,11 +1,5 @@ - - - - 10000 - - SELECT boundingRatio(number, number) FROM numbers(1000000) SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM numbers(1000000) diff --git a/tests/performance/cidr.xml b/tests/performance/cidr.xml index 938734e3709..a83a7e19182 100644 --- a/tests/performance/cidr.xml +++ b/tests/performance/cidr.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - diff --git a/tests/performance/codecs_float_insert.xml b/tests/performance/codecs_float_insert.xml index 706a2f3c0a0..273b5c07b67 100644 --- a/tests/performance/codecs_float_insert.xml +++ b/tests/performance/codecs_float_insert.xml @@ -1,15 +1,5 @@ - - - 10 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/codecs_float_select.xml b/tests/performance/codecs_float_select.xml index 4c2f671a90e..8dc78e5f90f 100644 --- a/tests/performance/codecs_float_select.xml +++ b/tests/performance/codecs_float_select.xml @@ -1,15 +1,5 @@ - - - 10 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index 1226d9020a0..8efdc031b33 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -1,15 +1,5 @@ - - - 10 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/codecs_int_select.xml b/tests/performance/codecs_int_select.xml index 8054c2b2de4..a8ed2e90069 100644 --- a/tests/performance/codecs_int_select.xml +++ b/tests/performance/codecs_int_select.xml @@ -1,15 +1,5 @@ - - - 10 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/collations.xml b/tests/performance/collations.xml index 03d77fa5e27..17b2d36b7e3 100644 --- a/tests/performance/collations.xml +++ b/tests/performance/collations.xml @@ -1,15 +1,5 @@ - - - 5 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/column_column_comparison.xml b/tests/performance/column_column_comparison.xml index 7559e03e506..88ceda7bf83 100644 --- a/tests/performance/column_column_comparison.xml +++ b/tests/performance/column_column_comparison.xml @@ -8,16 +8,6 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/columns_hashing.xml b/tests/performance/columns_hashing.xml index ca330b0d435..414c85c3853 100644 --- a/tests/performance/columns_hashing.xml +++ b/tests/performance/columns_hashing.xml @@ -9,16 +9,6 @@ - - - 5 - 60000 - - - 10 - 150000 - - SELECT count() FROM numbers(1000000) WHERE NOT ignore(geohashEncode((number % 150)*1.1 - 75, (number * 3.14 % 300)*1.1 - 150)) diff --git a/tests/performance/general_purpose_hashes.xml b/tests/performance/general_purpose_hashes.xml index 458e646f3a7..d636de1ddaa 100644 --- a/tests/performance/general_purpose_hashes.xml +++ b/tests/performance/general_purpose_hashes.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - diff --git a/tests/performance/general_purpose_hashes_on_UUID.xml b/tests/performance/general_purpose_hashes_on_UUID.xml index 9b749ae79e0..e2e6e6bb12e 100644 --- a/tests/performance/general_purpose_hashes_on_UUID.xml +++ b/tests/performance/general_purpose_hashes_on_UUID.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - @@ -53,7 +43,7 @@ table_slow numbers(1000000) - numbers_mt(10000000) + numbers_mt(100000000) diff --git a/tests/performance/generate_table_function.xml b/tests/performance/generate_table_function.xml index c53ec285bf5..8de711304ae 100644 --- a/tests/performance/generate_table_function.xml +++ b/tests/performance/generate_table_function.xml @@ -1,10 +1,5 @@ - - - 10000 - - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 10000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 10000000); diff --git a/tests/performance/great_circle_dist.xml b/tests/performance/great_circle_dist.xml index 3b2aac65230..ff297cffc58 100644 --- a/tests/performance/great_circle_dist.xml +++ b/tests/performance/great_circle_dist.xml @@ -1,10 +1,5 @@ - - - 10000 - - SELECT count() FROM numbers(1000000) WHERE NOT ignore(greatCircleDistance((rand(1) % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand(2)) / 4294967296 - 180, (rand(3) % 180) * 1. - 90)) diff --git a/tests/performance/group_array_moving_sum.xml b/tests/performance/group_array_moving_sum.xml index 76b6b5bb980..465306d837d 100644 --- a/tests/performance/group_array_moving_sum.xml +++ b/tests/performance/group_array_moving_sum.xml @@ -1,12 +1,4 @@ - - - 30000 - - - 60000 - - 30000000000 diff --git a/tests/performance/h3.xml b/tests/performance/h3.xml index 104e777fcc5..ce00ebbc9ec 100644 --- a/tests/performance/h3.xml +++ b/tests/performance/h3.xml @@ -1,11 +1,5 @@ - - - - 10000 - - SELECT count() FROM zeros(100000) WHERE NOT ignore(geoToH3(37.62 + rand(1) / 0x100000000, 55.75 + rand(2) / 0x100000000, 15)) diff --git a/tests/performance/if_array_num.xml b/tests/performance/if_array_num.xml index 4ecd1e66daa..f3f418b809c 100644 --- a/tests/performance/if_array_num.xml +++ b/tests/performance/if_array_num.xml @@ -1,10 +1,5 @@ - - - 10000 - - SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [4, 5]) diff --git a/tests/performance/if_array_string.xml b/tests/performance/if_array_string.xml index 40302131665..6713822e5a4 100644 --- a/tests/performance/if_array_string.xml +++ b/tests/performance/if_array_string.xml @@ -1,10 +1,5 @@ - - - 10000 - - SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c']) diff --git a/tests/performance/if_string_const.xml b/tests/performance/if_string_const.xml index 7e273db36d8..69dd8f75463 100644 --- a/tests/performance/if_string_const.xml +++ b/tests/performance/if_string_const.xml @@ -1,10 +1,5 @@ - - - 10000 - - SELECT count() FROM zeros(1000000) WHERE NOT ignore(rand() % 2 ? 'hello' : 'world') SELECT count() FROM zeros(1000000) WHERE NOT ignore(rand() % 2 ? 'hello' : '') diff --git a/tests/performance/if_string_hits.xml b/tests/performance/if_string_hits.xml index ec9ea39f7cf..ca23d710185 100644 --- a/tests/performance/if_string_hits.xml +++ b/tests/performance/if_string_hits.xml @@ -1,15 +1,5 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/if_to_multiif.xml b/tests/performance/if_to_multiif.xml index 373318c316c..88fc38d48f0 100644 --- a/tests/performance/if_to_multiif.xml +++ b/tests/performance/if_to_multiif.xml @@ -1,10 +1,5 @@ - - - 10000 - - - - - 10 - 3000 - - - 100 - 60000 - - file('test_all_expr_matches.values', Values, 'd DateTime, i UInt32, s String, ni Nullable(UInt64), ns Nullable(String), ars Array(String)') diff --git a/tests/performance/inserts_arrays_lowcardinality.xml b/tests/performance/inserts_arrays_lowcardinality.xml index bca5c858576..40617fb9593 100644 --- a/tests/performance/inserts_arrays_lowcardinality.xml +++ b/tests/performance/inserts_arrays_lowcardinality.xml @@ -1,14 +1,4 @@ - - - 5 - 12000 - - - 50 - 60000 - - CREATE TABLE lot_of_string_arrays_src (`id` UInt64, `col00` Array(String), `col01` Array(String), `col02` Array(String), `col03` Array(String), `col04` Array(String), `col05` Array(String), `col06` Array(String), `col07` Array(String), `col08` Array(String), `col09` Array(String), `col10` Array(String), `col11` Array(String), `col12` Array(String), `col13` Array(String), `col14` Array(String), `col15` Array(String), `col16` Array(String), `col17` Array(String), `col18` Array(String), `col19` Array(String), `col20` Array(String), `col21` Array(String), `col22` Array(String), `col23` Array(String), `col24` Array(String), `col25` Array(String), `col26` Array(String), `col27` Array(String), `col28` Array(String), `col29` Array(String), `col30` Array(String), `col31` Array(String), `col32` Array(String), `col33` Array(String), `col34` Array(String), `col35` Array(String), `col36` Array(String), `col37` Array(String), `col38` Array(String), `col39` Array(String), `col40` Array(String), `col41` Array(String), `col42` Array(String), `col43` Array(String), `col44` Array(String), `col45` Array(String), `col46` Array(String), `col47` Array(String), `col48` Array(String), `col49` Array(String)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192; CREATE TABLE lot_of_string_arrays_dst_lowcardinality (`id` UInt64, `col00` Array(LowCardinality(String)), `col01` Array(LowCardinality(String)), `col02` Array(LowCardinality(String)), `col03` Array(LowCardinality(String)), `col04` Array(LowCardinality(String)), `col05` Array(LowCardinality(String)), `col06` Array(LowCardinality(String)), `col07` Array(LowCardinality(String)), `col08` Array(LowCardinality(String)), `col09` Array(LowCardinality(String)), `col10` Array(LowCardinality(String)), `col11` Array(LowCardinality(String)), `col12` Array(LowCardinality(String)), `col13` Array(LowCardinality(String)), `col14` Array(LowCardinality(String)), `col15` Array(LowCardinality(String)), `col16` Array(LowCardinality(String)), `col17` Array(LowCardinality(String)), `col18` Array(LowCardinality(String)), `col19` Array(LowCardinality(String)), `col20` Array(LowCardinality(String)), `col21` Array(LowCardinality(String)), `col22` Array(LowCardinality(String)), `col23` Array(LowCardinality(String)), `col24` Array(LowCardinality(String)), `col25` Array(LowCardinality(String)), `col26` Array(LowCardinality(String)), `col27` Array(LowCardinality(String)), `col28` Array(LowCardinality(String)), `col29` Array(LowCardinality(String)), `col30` Array(LowCardinality(String)), `col31` Array(LowCardinality(String)), `col32` Array(LowCardinality(String)), `col33` Array(LowCardinality(String)), `col34` Array(LowCardinality(String)), `col35` Array(LowCardinality(String)), `col36` Array(LowCardinality(String)), `col37` Array(LowCardinality(String)), `col38` Array(LowCardinality(String)), `col39` Array(LowCardinality(String)), `col40` Array(LowCardinality(String)), `col41` Array(LowCardinality(String)), `col42` Array(LowCardinality(String)), `col43` Array(LowCardinality(String)), `col44` Array(LowCardinality(String)), `col45` Array(LowCardinality(String)), `col46` Array(LowCardinality(String)), `col47` Array(LowCardinality(String)), `col48` Array(LowCardinality(String)), `col49` Array(LowCardinality(String))) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192; diff --git a/tests/performance/int_parsing.xml b/tests/performance/int_parsing.xml index 8a6475546bf..a9258875b5e 100644 --- a/tests/performance/int_parsing.xml +++ b/tests/performance/int_parsing.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - diff --git a/tests/performance/jit_large_requests.xml b/tests/performance/jit_large_requests.xml index 4b390c36e20..805b7f2edb1 100644 --- a/tests/performance/jit_large_requests.xml +++ b/tests/performance/jit_large_requests.xml @@ -1,10 +1,5 @@ - - - 100 - - CREATE TABLE jit_test ( diff --git a/tests/performance/jit_small_requests.xml b/tests/performance/jit_small_requests.xml index 7a4bedf6832..f90415371ce 100644 --- a/tests/performance/jit_small_requests.xml +++ b/tests/performance/jit_small_requests.xml @@ -1,15 +1,5 @@ - - - 5 - 10000 - - - 5000 - 60000 - - diff --git a/tests/performance/joins_in_memory.xml b/tests/performance/joins_in_memory.xml index 67c86f49ff2..f24e7425ecd 100644 --- a/tests/performance/joins_in_memory.xml +++ b/tests/performance/joins_in_memory.xml @@ -1,10 +1,5 @@ - - - 10 - - CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory diff --git a/tests/performance/joins_in_memory_pmj.xml b/tests/performance/joins_in_memory_pmj.xml index 19383467fa1..6742943151e 100644 --- a/tests/performance/joins_in_memory_pmj.xml +++ b/tests/performance/joins_in_memory_pmj.xml @@ -1,10 +1,5 @@ - - - 10 - - CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory diff --git a/tests/performance/json_extract_rapidjson.xml b/tests/performance/json_extract_rapidjson.xml index 9818abb8581..f8d40c1e58d 100644 --- a/tests/performance/json_extract_rapidjson.xml +++ b/tests/performance/json_extract_rapidjson.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - diff --git a/tests/performance/json_extract_simdjson.xml b/tests/performance/json_extract_simdjson.xml index fa18d43df3e..f9f6df5140e 100644 --- a/tests/performance/json_extract_simdjson.xml +++ b/tests/performance/json_extract_simdjson.xml @@ -1,15 +1,5 @@ - - - 3 - 10000 - - - 5 - 60000 - - diff --git a/tests/performance/least_greatest_hits.xml b/tests/performance/least_greatest_hits.xml new file mode 100644 index 00000000000..464656b0201 --- /dev/null +++ b/tests/performance/least_greatest_hits.xml @@ -0,0 +1,9 @@ + + + test.hits + + + SELECT count() FROM test.hits WHERE NOT ignore(least(URL, Referer)) + SELECT count() FROM test.hits WHERE NOT ignore(greatest(URL, Referer, Title)) + SELECT count() FROM test.hits WHERE NOT ignore(greatest(ClientIP, RemoteIP)) + diff --git a/tests/performance/leftpad.xml b/tests/performance/leftpad.xml index eb0b09c72ed..3f747054122 100644 --- a/tests/performance/leftpad.xml +++ b/tests/performance/leftpad.xml @@ -9,16 +9,6 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/linear_regression.xml b/tests/performance/linear_regression.xml index 87fa034d851..a23924939a2 100644 --- a/tests/performance/linear_regression.xml +++ b/tests/performance/linear_regression.xml @@ -1,10 +1,5 @@ - - - 10000 - - test.hits diff --git a/tests/performance/logical_functions_large.xml b/tests/performance/logical_functions_large.xml index e199094c43c..077c88fa32c 100644 --- a/tests/performance/logical_functions_large.xml +++ b/tests/performance/logical_functions_large.xml @@ -1,16 +1,6 @@ 1 - - - 60 - 30000 - - - 120 - 60000 - - 1 diff --git a/tests/performance/logical_functions_medium.xml b/tests/performance/logical_functions_medium.xml index 1c4fd2a24dc..990fe2b6040 100644 --- a/tests/performance/logical_functions_medium.xml +++ b/tests/performance/logical_functions_medium.xml @@ -1,16 +1,6 @@ 1 - - - 60 - 30000 - - - 120 - 60000 - - 1 diff --git a/tests/performance/logical_functions_small.xml b/tests/performance/logical_functions_small.xml index d3d7a2eecca..15c1e87a558 100644 --- a/tests/performance/logical_functions_small.xml +++ b/tests/performance/logical_functions_small.xml @@ -1,16 +1,6 @@ 1 - - - 15 - 20000 - - - 120 - 60000 - - 1 diff --git a/tests/performance/materialized_view_parallel_insert.xml b/tests/performance/materialized_view_parallel_insert.xml new file mode 100644 index 00000000000..4b71354dec3 --- /dev/null +++ b/tests/performance/materialized_view_parallel_insert.xml @@ -0,0 +1,32 @@ + + + hits_10m_single + + + + CREATE MATERIALIZED VIEW hits_mv ENGINE MergeTree + PARTITION BY toYYYYMM(EventDate) + ORDER BY (CounterID, EventDate, intHash32(UserID)) + SAMPLE BY intHash32(UserID) + SETTINGS + parts_to_delay_insert = 5000, + parts_to_throw_insert = 5000 + AS + -- don't select all columns to keep the run time down + SELECT CounterID, EventDate, UserID, Title + FROM hits_10m_single + -- do not select anything because we only need column types + LIMIT 0 + + SET max_insert_threads=8 + SYSTEM STOP MERGES + + + INSERT INTO hits_mv + SELECT CounterID, EventDate, UserID, Title + FROM hits_10m_single + + + SYSTEM START MERGES + DROP TABLE IF EXISTS hits_mv + diff --git a/tests/performance/math.xml b/tests/performance/math.xml index 6ab497749f1..78effabaf1e 100644 --- a/tests/performance/math.xml +++ b/tests/performance/math.xml @@ -1,16 +1,6 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/merge_table_streams.xml b/tests/performance/merge_table_streams.xml index 084fa2da575..01f0444e54c 100644 --- a/tests/performance/merge_table_streams.xml +++ b/tests/performance/merge_table_streams.xml @@ -4,15 +4,6 @@ hits_100m_single - - - 60000 - 3 - - - 30 - - diff --git a/tests/performance/merge_tree_huge_pk.xml b/tests/performance/merge_tree_huge_pk.xml index 78a6cf6308e..2332769b522 100644 --- a/tests/performance/merge_tree_huge_pk.xml +++ b/tests/performance/merge_tree_huge_pk.xml @@ -1,15 +1,5 @@ - - - 10 - 12000 - - - 50 - 60000 - - CREATE TABLE huge_pk ENGINE = MergeTree ORDER BY ( diff --git a/tests/performance/merge_tree_many_partitions.xml b/tests/performance/merge_tree_many_partitions.xml index 33bb12ed22b..d3a5d204d5a 100644 --- a/tests/performance/merge_tree_many_partitions.xml +++ b/tests/performance/merge_tree_many_partitions.xml @@ -3,16 +3,6 @@ CREATE TABLE bad_partitions (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions SELECT * FROM numbers(10000) - - - 5 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/merge_tree_many_partitions_2.xml b/tests/performance/merge_tree_many_partitions_2.xml index 42bb0ac29c9..6799153ed65 100644 --- a/tests/performance/merge_tree_many_partitions_2.xml +++ b/tests/performance/merge_tree_many_partitions_2.xml @@ -3,16 +3,6 @@ CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) - - - 5 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/merge_tree_simple_select.xml b/tests/performance/merge_tree_simple_select.xml index f38a5241cb5..4d449e878d3 100644 --- a/tests/performance/merge_tree_simple_select.xml +++ b/tests/performance/merge_tree_simple_select.xml @@ -1,11 +1,6 @@ - - - 10 - - CREATE TABLE simple_mergetree (EventDate Date, x UInt64) ENGINE = MergeTree ORDER BY x INSERT INTO simple_mergetree SELECT number, today() + intDiv(number, 10000000) FROM numbers_mt(100000000) diff --git a/tests/performance/mingroupby-orderbylimit1.xml b/tests/performance/mingroupby-orderbylimit1.xml index 34cd992558b..5b23432255d 100644 --- a/tests/performance/mingroupby-orderbylimit1.xml +++ b/tests/performance/mingroupby-orderbylimit1.xml @@ -1,10 +1,5 @@ - - - 30000 - - 1 @@ -12,53 +7,69 @@ - aggregationscale + aggregationscale_all + + 5555555 + 555555 + 111111 + 11111 + + + + aggregationscale_small 111111 11111 + + aggregationscale_big + + 5555555 + 555555 + + - create table mingroupby_orderbylimit1_{aggregationscale}_tuple + create table mingroupby_orderbylimit1_{aggregationscale_all}_tuple Engine=MergeTree order by tuple() AS - select toUInt64( number % {aggregationscale} ) key, toUInt64(1) value + select toUInt64( number % {aggregationscale_all} ) key, toUInt64(1) value from numbers(2000000); - create table mingroupby_orderbylimit1_{aggregationscale}_key_value + create table mingroupby_orderbylimit1_{aggregationscale_all}_key_value Engine=MergeTree order by (key,value) AS - select toUInt64( number % {aggregationscale} ) key, toUInt64(1) value + select toUInt64( number % {aggregationscale_all} ) key, toUInt64(1) value from numbers(2000000); SELECT key, min(value) - FROM mingroupby_orderbylimit1_{aggregationscale}_tuple + FROM mingroupby_orderbylimit1_{aggregationscale_big}_tuple group by key format Null; SELECT key, min(value) - FROM mingroupby_orderbylimit1_{aggregationscale}_key_value + FROM mingroupby_orderbylimit1_{aggregationscale_big}_key_value group by key format Null; SELECT key, value - FROM mingroupby_orderbylimit1_{aggregationscale}_tuple + FROM mingroupby_orderbylimit1_{aggregationscale_small}_tuple order by key, value limit 1 by key format Null; SELECT key, value - FROM mingroupby_orderbylimit1_{aggregationscale}_key_value + FROM mingroupby_orderbylimit1_{aggregationscale_small}_key_value order by key, value limit 1 by key format Null; - DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale}_tuple - DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale}_key_value + DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale_all}_tuple + DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale_all}_key_value diff --git a/tests/performance/modulo.xml b/tests/performance/modulo.xml index e31de5c1701..77b544ff389 100644 --- a/tests/performance/modulo.xml +++ b/tests/performance/modulo.xml @@ -1,10 +1,5 @@ - - - 10 - - SELECT number % 128 FROM numbers(300000000) FORMAT Null diff --git a/tests/performance/ngram_distance.xml b/tests/performance/ngram_distance.xml index e90f49155b1..f102b90466d 100644 --- a/tests/performance/ngram_distance.xml +++ b/tests/performance/ngram_distance.xml @@ -13,16 +13,6 @@ 20000000000 - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/number_formatting_formats.xml b/tests/performance/number_formatting_formats.xml index c2a9a9c081d..9c9eb601b5a 100644 --- a/tests/performance/number_formatting_formats.xml +++ b/tests/performance/number_formatting_formats.xml @@ -2,16 +2,6 @@ CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`) - - - 5 - 10000 - - - 100 - 60000 - - diff --git a/tests/performance/nyc_taxi.xml b/tests/performance/nyc_taxi.xml index 92a1dd59441..b7c1cf58146 100644 --- a/tests/performance/nyc_taxi.xml +++ b/tests/performance/nyc_taxi.xml @@ -1,15 +1,5 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/order_by_decimals.xml b/tests/performance/order_by_decimals.xml index 5479181fb08..4889137865d 100644 --- a/tests/performance/order_by_decimals.xml +++ b/tests/performance/order_by_decimals.xml @@ -5,16 +5,6 @@ - - - 5 - 10000 - - - 50 - 60000 - - SELECT toInt32(number) AS n FROM numbers(1000000) ORDER BY n DESC FORMAT Null SELECT toDecimal32(number, 0) AS n FROM numbers(1000000) ORDER BY n FORMAT Null diff --git a/tests/performance/order_by_read_in_order.xml b/tests/performance/order_by_read_in_order.xml index e37e4df4681..5749a49a3aa 100644 --- a/tests/performance/order_by_read_in_order.xml +++ b/tests/performance/order_by_read_in_order.xml @@ -1,15 +1,5 @@ - - - 5 - 10000 - - - 500 - 60000 - - diff --git a/tests/performance/order_by_single_column.xml b/tests/performance/order_by_single_column.xml index 148b14e8959..9b708ea393c 100644 --- a/tests/performance/order_by_single_column.xml +++ b/tests/performance/order_by_single_column.xml @@ -9,16 +9,6 @@ - - - 5 - 10000 - - - 50 - 60000 - - SELECT URL as col FROM hits_100m_single ORDER BY col LIMIT 1000,1 SELECT SearchPhrase as col FROM hits_100m_single ORDER BY col LIMIT 10000,1 diff --git a/tests/performance/parallel_insert.xml b/tests/performance/parallel_insert.xml index 7af2a1418d9..34c45e08bc0 100644 --- a/tests/performance/parallel_insert.xml +++ b/tests/performance/parallel_insert.xml @@ -1,9 +1,4 @@ - - - 2 - - default.hits_10m_single diff --git a/tests/performance/parse_engine_file.xml b/tests/performance/parse_engine_file.xml index fb10fa97915..f962cbf4369 100644 --- a/tests/performance/parse_engine_file.xml +++ b/tests/performance/parse_engine_file.xml @@ -4,16 +4,6 @@ INSERT INTO table_{format} SELECT * FROM test.hits LIMIT 100000 - - - 5 - 10000 - - - 100 - 60000 - - @@ -34,6 +24,7 @@ RowBinary Native Avro + MsgPack @@ -42,7 +33,7 @@ test.hits -SELECT * FROM table_{format} +SELECT * FROM table_{format} FORMAT Null DROP TABLE IF EXISTS table_{format} diff --git a/tests/performance/pre_limit_no_sorting.xml b/tests/performance/pre_limit_no_sorting.xml index e93aef049aa..a1e50f736b8 100644 --- a/tests/performance/pre_limit_no_sorting.xml +++ b/tests/performance/pre_limit_no_sorting.xml @@ -1,14 +1,4 @@ - - - 10 - 200 - - - 100 - 1000 - - SELECT sum(number) FROM (select number from system.numbers_mt limit 1000000000) diff --git a/tests/performance/prewhere.xml b/tests/performance/prewhere.xml index e3350d765ee..40a12a68bb9 100644 --- a/tests/performance/prewhere.xml +++ b/tests/performance/prewhere.xml @@ -1,15 +1,5 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/random_printable_ascii.xml b/tests/performance/random_printable_ascii.xml index 20868bb4c27..3eb1441a4cc 100644 --- a/tests/performance/random_printable_ascii.xml +++ b/tests/performance/random_printable_ascii.xml @@ -1,10 +1,5 @@ - - - 10000 - - SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomPrintableASCII(10)) diff --git a/tests/performance/range.xml b/tests/performance/range.xml index 95b8455057e..c676f9124ba 100644 --- a/tests/performance/range.xml +++ b/tests/performance/range.xml @@ -1,10 +1,5 @@ - - - 10000 - - SELECT range(number % 100) FROM numbers(10000000) FORMAT Null diff --git a/tests/performance/read_hits_with_aio.xml b/tests/performance/read_hits_with_aio.xml index 850fd0fbadc..1e9a81f7693 100644 --- a/tests/performance/read_hits_with_aio.xml +++ b/tests/performance/read_hits_with_aio.xml @@ -1,10 +1,5 @@ - - - 30000 - - hits_100m_single diff --git a/tests/performance/right.xml b/tests/performance/right.xml index 73030e52f21..ac889e21e73 100644 --- a/tests/performance/right.xml +++ b/tests/performance/right.xml @@ -4,14 +4,6 @@ hits_100m_single - - - 10000 - - - 20000 - - diff --git a/tests/performance/round_down.xml b/tests/performance/round_down.xml index f453467ab2d..c309a767843 100644 --- a/tests/performance/round_down.xml +++ b/tests/performance/round_down.xml @@ -1,13 +1,5 @@ - - - 10000 - - - 20000 - - SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundDuration(rand() % 65536)) diff --git a/tests/performance/round_methods.xml b/tests/performance/round_methods.xml index 0e560b2eae6..fac9c1908b0 100644 --- a/tests/performance/round_methods.xml +++ b/tests/performance/round_methods.xml @@ -1,13 +1,5 @@ - - - 10000 - - - 20000 - - SELECT count() FROM numbers(1000000) WHERE NOT ignore(round(toInt64(number), -2)) diff --git a/tests/performance/scalar.xml b/tests/performance/scalar.xml index e8e487a80da..b50aef8747c 100644 --- a/tests/performance/scalar.xml +++ b/tests/performance/scalar.xml @@ -1,14 +1,5 @@ - - - 30000 - - - 5000 - 60000 - - CREATE TABLE cdp_tags (tag_id String, mid_seqs AggregateFunction(groupBitmap, UInt32)) engine=MergeTree() ORDER BY (tag_id) SETTINGS index_granularity=1 diff --git a/tests/performance/select_format.xml b/tests/performance/select_format.xml index b8df874304f..82d5198a71a 100644 --- a/tests/performance/select_format.xml +++ b/tests/performance/select_format.xml @@ -2,16 +2,6 @@ CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}, '/dev/null') AS test.hits - - - 5 - 10000 - - - 100 - 60000 - - @@ -44,6 +34,7 @@ ODBCDriver2 MySQLWire Avro + MsgPack diff --git a/tests/performance/set.xml b/tests/performance/set.xml index 345d9c05573..576a26390d1 100644 --- a/tests/performance/set.xml +++ b/tests/performance/set.xml @@ -3,14 +3,6 @@ long - - - 10000 - - - 20000 - - diff --git a/tests/performance/set_hits.xml b/tests/performance/set_hits.xml index 09860aa1cd7..8b9ae1da83b 100644 --- a/tests/performance/set_hits.xml +++ b/tests/performance/set_hits.xml @@ -5,15 +5,6 @@ hits_100m_single - - - 8000 - - - 7000 - 20000 - - SELECT count() FROM hits_100m_single WHERE UserID IN (SELECT UserID FROM hits_100m_single WHERE AdvEngineID != 0) diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 090d8ac8c08..d9b263159b5 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -3,18 +3,16 @@ CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a INSERT INTO test_in SELECT number FROM numbers(500000000) - - - 8000 - - - 7000 - 20000 - - - SELECT count() FROM test_in WHERE a IN (SELECT rand(1) FROM zeros(100000)) SETTINGS max_rows_to_read = 1, read_overflow_mode = 'break' + SELECT count() FROM test_in WHERE a IN (SELECT rand(1) FROM numbers(100000)) SETTINGS max_rows_to_read = 1, read_overflow_mode = 'break' + + SELECT count() FROM test_in WHERE toInt64(a) IN (SELECT toInt64(rand(1)) FROM numbers(100000)) settings max_rows_to_read=1, read_overflow_mode='break' + + + SELECT count() FROM test_in WHERE -toInt64(a) IN (SELECT toInt64(rand(1)) FROM numbers(100000)) settings max_rows_to_read=1, read_overflow_mode='break' + + SELECT count() FROM test_in WHERE -toInt64(a) NOT IN (SELECT toInt64(rand(1)) FROM numbers(100000)) settings max_rows_to_read=1, read_overflow_mode='break' SELECT count() FROM numbers(1000) WHERE toString(number) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', @@ -27,5 +25,24 @@ '37674', '44977', '54370', '97381', '60218', '2423', '99591', '69913', '26507', '19708', '6279', '58955', '20126', '1495', '57894', '7638', '38700', '77148', '36844', '7539', '91452', '6914', '74349', '66850', '49104', '6516', '58535', '20851', '27859', '32881', '72919', '28203', '32882', '2419', '77583', '63822', '37703', '66793', '65784', '62281', '55867', '70703', '89344', '1498', '33770', '87176', '95636', '64891', '90736', '95521', '10989', '5237', '99010', '21106', '11422', '1831', '67239', '52557', '36468', '71713', '39637', '49574', '50455', '14953', '96900', '70852', '96982', '4341', '44585', '95651', '79669', '29652', '87294', '74692', '16221', '768', '35380', '21352', '50907', '27259', '11718', '5017', '55964', '94137', '52347', '10595', '12968', '85602', '97965', '18836', '90511', '70960', '97336', '44575', '23791', '42195', '64776', '29363', '42379', '1805', '28919', '6772', '78143', '54797', '27362', '56149', '59048', '38567', '6339', '27787', '42167', '45990', '95532', '54839', '26572', '38496', '89797', '6634', '16468', '24898', '66814', '98126', '31762', '36133', '64539', '43167', '87022', '61295', '30364', '89249', '25756', '63570', '91484', '10564', '79648', '5756', '41376', '61897', '40388', '88927', '62891', '79708', '25495', '22204', '33892', '36871', '19879', '58646', '57061', '73100', '75831', '20029', '67462', '54675', '7766', '2409', '24506', '7877', '11720', '86252', '9897', '8080', '70684', '74497', '2242', '24604', '31969', '83999', '56635', '5283', '64971', '79152', '27470', '89042', '22835', '21476', '50292', '56081', '96342', '32763', '84487', '64856', '79152', '64656', '72169', '69971', '93094', '52804', '80917', '53152', '56016', '28496', '79110', '17133', '12581', '91742', '78929', '2676', '46700', '59528', '93808', '4535', '54035', '40161', '62796', '3598', '97088', '13599', '36337', '73395', '17494', '86275', '62058', '61937', '87747', '94883', '90677', '88544', '72553', '50210', '75481', '64378', '74464', '21659', '30970', '71989', '84846', '72289', '88716', '39143', '8487', '4912', '91013', '18623', '19122', '36507', '76438', '7516', '67970', '72350', '69873', '33635', '55983', '69008', '49545', '3134', '60056', '52509', '63304', '15560', '23651', '81090', '7027', '8317', '33060', '37295', '51961', '53037', '97431', '40512', '23536', '25168', '78455', '85613', '12304', '40733', '99890', '51238', '55439', '96201', '73559', '92533', '90173', '16721', '6078', '29854', '38894', '31117', '63040', '86795', '81786', '21149', '38998', '61811', '48622', '73019', '59296', '13576', '92559', '36300', '77294', '26794', '50912', '98380', '13176', '57746', '75286', '15330', '40921', '7337', '4664', '20384', '4674', '44516', '27633', '31950', '88210', '54536', '9839', '80137', '77491', '18434', '45152', '96942', '41005', '76103', '34825', '86869', '14772', '13384', '21051', '37348', '34434', '97210', '54960', '26598', '60981', '41889', '6446', '64492', '95310', '86236', '81885', '35684', '16539', '98476', '32028', '96470', '6318', '99576', '93935', '48609', '86090', '2476', '65576', '80636', '44817', '99646', '98963', '20486', '26261', '27334', '72946', '82023', '33506', '80193', '13762', '98133', '21134', '33268', '63477', '74609', '30454', '51477', '93391', '96805', '68653', '2714', '63642', '51520', '22972', '13305', '96058', '42336', '74461', '31597', '12050', '81712', '37977', '25718', '4834', '56608', '75731', '406', '28585', '63924', '23702', '29849', '16941', '91921', '65842', '76525', '68534', '50902', '17609', '23852', '53703', '31286', '58526', '9633', '87596', '10654', '2085', '52766', '22135', '76524', '32295', '90072', '70078', '77786', '93741', '87320', '70309', '44024', '95286', '12361', '29682', '59766', '26685', '90686', '81691', '49704', '23431', '53955', '39023', '47261', '1530', '58265', '80065', '95620', '90621', '63760', '90676', '81653', '36397', '20252', '81754', '20256', '67098', '7838', '49408', '88400', '87941', '84533', '6570', '22567', '18850', '55472', '40129', '48425', '23497', '39308', '34698', '53092', '89480', '47785', '57282', '25508', '19006', '50604', '86917', '9436', '88921', '3168', '70537', '3185', '34988', '5462', '69482', '45768', '91955', '56898', '15307', '99731', '89292', '19356', '20646', '66712', '7281', '12856', '31174', '19577', '8726', '62971', '33008', '37118', '59055', '84101', '68445', '91957', '47526', '15627', '79914', '20013', '26147', '80821', '56372', '74205', '28531', '25352', '51775', '93948', '55212', '17863', '91521', '74911', '88160', '2360', '98260', '18294', '62402', '84268', '9580', '42668', '1467', '40059', '5221', '4216', '9917', '35420', '16496', '34369', '50253', '95234', '95114', '84193', '28322', '37031', '81284', '88628', '36782', '42572', '73347', '66188', '43342', '77285', '16513', '89064', '63066', '72645', '67075', '48208', '18181', '77898', '65795', '53707', '39856', '92883', '92567', '49733', '30236', '10273', '53029', '69773', '78379', '72108', '47696', '97557', '95184', '14688', '29853', '62694', '70431', '88435', '58799', '21883', '99866', '69178', '55870', '14414', '85274', '27321', '55555', '613', '15067', '88217', '73655', '99548', '13631', '78789', '36690', '7952', '60830', '77438', '40059', '95602', '43097', '3429', '93731', '90537', '2932', '35702', '16125', '6652', '39632', '39349', '9910', '38103', '78608', '73565', '48556', '28978', '7128', '82326', '53980', '28059', '28212', '87101', '77752', '99170', '56753', '30484', '71470', '32607', '24674', '32687', '25098', '94712', '64024', '48239', '90408', '17316', '99243', '3656', '67402', '48009', '98427', '52800', '56024', '4417', '89747', '93338', '18758', '56411', '44810', '82456', '30808', '75470', '67115', '66876', '53906', '78403', '56059', '34383', '60056', '89136', '7237', '11129', '21351', '78662', '43606', '37454', '45465', '9292', '38099', '81699', '50195', '49368', '47503', '44605', '6523', '81478', '37910', '397', '20256', '6835', '2787', '80383', '4241', '65986', '83870', '21205', '10879', '26593', '44357', '72604', '56131', '43423', '80206', '26240', '87198', '99445', '53504', '10632', '2465', '31793', '89575', '64184', '39988', '60049', '87100', '37151', '61585', '82180', '52065', '72519', '72935', '3201', '5862', '20560', '95339', '21661', '17533', '17182', '71189', '91564', '57999', '35490', '94773', '95056', '51583', '59394', '10727', '8655', '48123', '10701', '25314', '20100', '6533', '46435', '43188', '23001', '23018', '76637', '32018', '36603', '18701', '9550', '61550', '47541', '36500', '67507', '81574', '95490', '69169', '32584', '30045', '64699', '83539', '89396', '42517', '61979', '41528', '8271', '88377', '61423', '1158', '89724', '70789', '14886', '64823', '56675', '97747', '23990', '58495', '82064', '17062', '90258', '86854', '93304', '12925', '49975', '45074', '87155', '72223', '67344', '42733', '42516', '40110', '15444', '88285', '39371', '23198', '61544', '90205', '6192', '15718', '19803', '92712', '20081', '31397', '5555', '70463', '19521', '80401', '74097', '32060', '26495', '20507', '40473', '1449', '57215', '46142', '39303', '50359', '35898', '46908', '90752', '7823', '27416', '73770', '98790', '17907', '29999', '76417', '49926', '76752', '21608', '26524', '88209', '6000', '88897', '19541', '41451', '59538', '56560', '1456', '67828', '82407', '45722', '93344', '54279', '78594', '38354', '93807', '10929', '91560', '60681', '70615', '32527', '10108', '48303', '63134', '28500', '18257', '57081', '24801', '99077', '52197', '15390', '52300', '57116', '417', '7503', '20054', '75315', '81359', '69091', '18853', '2465', '25600', '13522', '74575', '12661', '83071', '15191', '27543', '21730', '60853', '18961', '14773', '89185', '33694', '51143', '1449', '68831', '78062', '65173', '32697', '41674', '9429', '22156', '96022', '46305', '97534', '5685', '48870', '89988', '20686', '66705', '6865', '94250', '16872', '13178', '7420', '73531', '92723', '60620', '48843', '74207', '60016', '50943', '62699', '63507', '76537', '87066', '76922', '24711', '34809', '5021', '31293', '53854', '77607', '52322', '10934', '50284', '87804', '36730', '86946', '80749', '43325', '97958', '7362', '39582', '10042', '42053', '66236', '69931', '23463', '87996', '33563', '4468', '32905', '50815', '79478', '28658', '46018', '23186', '26080', '13494', '6237', '42762', '86440', '77407', '10426', '62902', '73251', '36861', '92357', '98754', '1839', '46391', '11420', '27132', '93028', '39609', '42015', '68218', '54228', '5456', '38705', '64307', '49483', '878', '54360', '54480', '66684', '55089', '4537', '82073', '72602', '96238', '56708', '58625', '32991', '74205', '72868', '79086', '64250', '56376', '10621', '76607', '47706', '72760', '70303', '60715', '14644', '44186', '36264', '29489', '14184', '62699', '30567', '16700', '31222', '15650', '1500', '22950', '54628', '41004', '96094', '70028', '74178', '65328', '26605', '63076', '75271', '79285', '8151', '42101', '56362', '25961', '87864', '972', '29510', '2747', '8877', '9780', '61052', '84105', '15573', '27475', '44570', '25334', '18517', '44237', '84094', '67524', '76761', '65678', '79284', '2462', '42631', '22696', '19223', '29728', '67742', '11883', '59027', '12377', '80538', '2165', '17377', '15030', '49838', '23920', '26025', '68179', '75894', '43783', '97106', '75558', '35528', '52081', '16951', '68855', '402', '21459', '97550', '16948', '5369', '4641', '2663', '15233', '79974', '71093', '15234', '42690', '22322', '54282', '95845', '90010', '40530', '88298', '41885', '7079', '6098', '72786', '36603', '77378', '48393', '45723', '41996', '96025', '89297', '75586', '8422', '24360', '170', '46036', '46725', '67944', '74029', '73069', '45371', '99916', '71085', '42608', '89904', '6393', '51274', '42729', '58924', '82497', '64143', '88622', '18818', '89041', '56090', '21369', '78224', '90450', '45488', '58830', '4133', '98062', '81113', '11285', '51457', '3183', '38800', '65278', '42169', '28602', '52648', '44683', '75647', '11778', '32151', '33528', '23773', '68268', '23367', '70964', '23548', '35575', '67570', '77681', '74158', '25374', '62714', '43100', '4977', '51678', '83460', '29755', '15890', '64626', '54044', '14793', '64339', '94008', '97126', '49202', '33889', '12601', '12275', '56123', '94557', '68226', '67200', '9374', '70687', '29211', '8039', '14598', '74548', '37433', '98991', '29933', '37203', '23973', '96482', '64774', '58350', '61781', '31824', '57193', '26476', '21814', '32297', '32627', '44277', '33876', '55468', '81715', '82505', '61462', '20324', '84293', '40116', '51087', '43594', '6854', '59077', '39841', '26023', '22777', '66859', '82460', '89515', '41712', '33711', '71875', '10685', '12655', '50138', '31063', '37040', '95819', '38919', '27391', '29833', '34350', '65646', '7697', '2688', '41146', '13241', '50305', '86568', '24487', '78741', '96370', '21015', '31719', '39750', '25014', '72415', '8486', '90668', '51143', '49488', '21057', '92803', '53528', '39550', '76039', '44185', '32404', '30217', '19796', '38084', '49161', '80140', '20241', '39357', '68908', '93083', '77231', '6952', '36322', '50790', '623', '29730', '13616', '57546', '17434', '93811', '35148', '81419', '40250', '40329', '89126', '72402', '16053', '27107', '28919', '16829', '96582', '65057', '28416', '30801', '77742', '27420', '73118', '89352', '54706', '23035', '88413', '64608', '61930', '15037', '47327', '59596', '18700', '57576', '63628', '56823', '60091', '68209', '21001', '14962', '72257', '83802', '33721', '86343', '11133', '65737', '68477', '90725', '86869', '98403', '47393', '25356', '61372', '8873', '19888', '48836', '66005', '23531', '72520', '26461', '78508', '28213', '96394', '22983', '37856', '71814', '27425', '72753', '27511', '65471', '38592', '3683', '24652', '64505', '92543', '53201', '40639', '99542', '53425', '35321', '47669', '14134', '47727', '48202', '71931', '32119', '50086', '50266', '67159', '89317', '81905', '30315', '49154', '8690', '69365', '56881', '46473', '64100', '38365', '59377', '65630', '54871', '52745', '91536', '16106', '70066', '62063', '84530', '88103', '33599', '51063', '87299', '41880', '25335', '51252', '42788', '13568', '1721', '62424', '83308', '36787', '91536', '92555', '27600', '24030', '12267', '66336', '30242', '7183', '67624', '28471', '48593', '79766', '31178', '47818', '94522', '88855', '45262', '43670', '18065', '25062', '44558', '37189', '69225', '35216', '42683', '26289', '72816', '31947', '65871', '45715', '59452', '22014', '56669', '60331', '33450', '60601', '95047', '30789', '90107', '81565', '32266', '3252', '5446', '58756', '55370', '34034', '81071', '2560', '39054', '39564', '15010', '5389', '60002', '53320', '49545', '48444', '31415', '39278', '79879', '30148', '10186', '60358', '29011', '14419', '95159', '94815', '55251', '90910', '80582', '92304', '11697', '60061', '38577', '84439', '76196', '34542', '50963', '36294', '11123', '59763', '29873', '47383', '12979', '22119', '21723', '64725', '48377', '77132', '9817', '79920', '47653', '60069', '12924', '53808', '55962', '66969', '13757', '60615', '10994', '9138', '34119', '58436', '64407', '75170', '73524', '51864', '94183', '86847', '15585', '57616', '96267', '5340', '52929', '49096', '50291', '5559', '32382', '84077', '6598', '87921', '59719', '31726', '44772', '63373', '75420', '66829', '47275', '98264', '61387', '94945', '44540', '50098', '13078', '44729', '95332', '63555', '30782', '63203', '15071', '60996', '72812', '17418', '80215', '37610', '30670', '44674', '74822', '15471', '25236', '16266', '76213', '35820', '19567', '8715', '72003', '90606', '1434', '53545', '88170', '75014', '62287', '35436', '38669', '12927', '83877', '38622', '28313', '82884', '73969', '38671', '10450', '24158', '22941', '73162', '86548', '42482', '95315', '92016', '96156', '44012', '35962', '6366', '3881', '74300', '26248', '30182', '19164', '67105', '66771', '52587', '69894', '61820', '16551', '50743', '10096', '69030', '24451', '89165', '23929', '96291', '30685', '64413', '19913', '9049', '71383', '61684', '45384', '45927', '81840', '49521', '89594', '30055', '83430', '14930', '60316', '86585', '99375', '80170', '14207', '19584', '20067', '82874', '30159', '46647', '6942', '66777', '32638', '55662', '75470', '77622', '26893', '96149', '14373', '33252', '50574', '7945', '20696', '56662', '94348', '3384', '20956', '89668', '99052', '65131', '56847', '17589', '16419', '2670', '10705', '59587', '92902', '92424', '48570', '11034', '69149', '35733', '17315', '84966', '69353', '69590', '52834', '32561', '6049', '50156', '71676', '76423', '32361', '61509', '8845', '75709', '35956', '21912', '31188', '59083', '43459', '38614', '92206', '55645', '38737', '34193', '6451', '94163', '24326', '49976', '71600', '58024', '67160', '4365', '38270', '59558', '80834', '60739', '54318', '19738', '42196', '43191', '13463', '88914', '99239', '66869', '75691', '33085', '4323', '7170', '46184', '41423', '89835', '46877', '20349', '14365', '32727', '35322', '841', '23597', '43370', '57527', '73250', '32553', '71489', '44617', '98323', '37672', '59549', '96023', '63176', '13524', '15621', '30448', '28136', '45549', '3513', '64153', '19839', '24219', '41987', '51083', '90268', '52052', '31430', '4727', '99409', '43595', '82374', '61251', '51470', '66562', '98724', '23529', '53895', '67562', '87573', '89964', '30821', '15733', '33062', '86963', '33450', '75338', '32570', '14453', '38080', '36335', '84226', '52790', '42883', '61156', '42789', '57846', '60096', '29946', '80178', '15882', '1971', '60722', '62458', '8754', '59991', '89321', '584', '70565', '36458', '21226', '23561', '9837', '39364', '23065', '30675', '9306', '40085', '52082', '89976', '73283', '77851', '36174', '54470', '63250', '72111', '70853', '26723', '42590', '91230', '47512', '13983', '70898', '70927', '40721', '30642', '41628', '90010', '27306', '1933', '43304', '44499', '87890', '22201', '89249', '63935', '48438', '58588', '1061', '70061', '63075', '9676', '65820', '82156', '82668', '111', '54350', '10328', '23466', '98936', '18285', '53919', '32422', '84859', '58387', '24022', '32423', '6010', '56417', '49452', '69999', '14885', '47102', '59577', '24999', '75984', '96464', '59088', '85987', '71442', '88789', '4753', '8229', '76883', '15284', '90610', '40507', '78882', '55575', '25315', '7214', '70602', '4796', '35767', '54657', '42153', '16050', '93607', '99249', '77236', '59949', '52871', '47837', '33534', '30023', '89137', '99938', '35824', '50775', '30282', '82798', '53312', '65277', '68375', '91445', '58166', '43344', '6589', '82515', '34632', '78588', '152', '67554', '15877', '74334', '32783', '45147', '39483', '92067', '59029', '38298', '55229', '28268', '85140', '33451', '15424', '46695', '23201', '83329', '28372', '19518', '89198', '33305', '43892', '470', '37662', '9407', '14376', '80310', '21459', '72381', '80414', '88305', '69073', '63101', '91054', '47190', '48595', '24696', '41426', '35133', '94399', '21790', '55040', '73279', '20809', '67805', '94115', '58633', '78053', '89444', '4112', '8', '34517', '22106', '85934', '86814', '53333', '93437', '85062', '32791', '72744', '99843', '51161', '22730', '34908', '82918', '92566', '22467', '41226', '98518', '29235', '94042', '84371', '79100', '25214', '7764', '59427', '47891', '61092', '23775', '13641', '30837', '77377', '43032', '38441', '29462', '20300', '19070', '20982', '73987', '87836', '68062', '6419', '51563', '40084', '85694', '86677', '47142', '27222', '17844', '19158', '45120', '88524', '74724', '73229', '42470', '38751', '1132', '28603', '61188', '55021', '88825', '58005', '62411', '8843', '94852', '93664', '39253', '27473', '247', '43824', '1804', '8905', '11509', '95659', '7811', '80691', '15779', '49794', '8991', '76099', '29223', '36060', '85399', '41369', '22885', '38473', '22376', '50446', '89578', '25818', '61333', '78787', '47605', '83654', '99068', '52120', '48367', '86381', '19803', '72600', '31998', '37755', '88031', '83969', '42319', '27974', '35780', '93662', '46808', '60529', '15491', '10447', '48829', '33886', '68333', '44855', '86554', '64794', '66376', '58222', '14021', '52043', '56375', '1300', '38105', '89159', '97456', '26800', '93124', '3673', '32279', '30658', '84475', '3708', '93952', '39245', '91980', '55333', '79440', '64407', '46559', '60759', '10688', '49872', '45810', '87405', '66932', '56530', '57751', '9619', '27361', '6356', '65848', '7524', '20273', '22362', '20504', '28042', '39475', '51677', '85733', '32426', '54558', '17222', '56485', '34928', '90917', '70528', '51732', '61014', '98420', '67265', '41383', '3883', '47642', '53324', '93679', '93088', '57534', '44449', '46779', '81482', '54279', '80135', '11216', '92545', '18426', '96005', '57801', '21898', '5104', '83467', '72015', '43783', '89674', '57468', '96686', '95167', '38507', '95187', '64923', '71214', '42834', '93219', '47342', '24476', '84834', '29080', '86533', '30687', '68400', '26933', '37396', '65169', '89767', '20642', '53843', '85167', '77306', '46723', '68501', '4243', '35044', '15950', '40388', '53630', '76125', '10816', '83285', '4120', '11402', '91344', '95169') + + SELECT count() FROM numbers(10000) WHERE (number, number) IN ((17258, 93148), (4508, 52749), (68660, 70017), (77797, 23528), (1136, 37393), (53237, 15379), (68370, 73211), (15782, 54962), (59432, 45415), (68396, 920), (96154, 21016), (12700, 26887), (88016, 43191), (68153, 51575), (91315, 40005), (18070, 73178), (86, 631), (77717, 20324), (3227, 76188), (74960, 43147), (77538, 19628), (82292, 6525), (24293, 12566), (85244, 96287), (93982, 1329), (38064, 54723), (83999, 45810), (71921, 53673), (88638, 9669), (1959, 39535), (82235, 95796), (27907, 90975), (42383, 91015), (9948, 91514), (81712, 47309), (400, 25808), (31791, 46948), (39740, 36098), (25943, 84598), (99598, 52939), (77134, 15845), (40313, 72174), (85017, 94036), (36595, 14303), (83961, 68078), (55792, 72759), (73574, 43606), (9853, 63560), (28580, 56721), (74804, 41025), (32095, 55657), (52881, 63416), (91368, 90310), (23922, 38883), (30592, 10758), (66448, 61183), (31880, 96697), (11362, 20633), (75331, 2015), (71129, 8785), (1115, 70955), (7886, 83698), (18961, 84556), (16677, 43028), (37347, 70220), (31699, 71244), (10578, 96159), (67600, 39041), (78791, 86687), (21545, 54174), (68774, 37637), (46132, 81768), (98413, 20605), (2960, 23665), (31507, 35719), (96209, 18368), (60558, 38035), (21952, 3264), (11834, 86458), (21651, 17650), (86276, 36087), (18818, 24849), (61951, 3390), (59637, 62545), (30346, 72253), (36281, 2992), (78340, 49872), (94326, 93723), (3416, 94405), (12272, 8741), (22600, 22095), (57636, 37106), (38702, 14889), (70238, 11276), (17325, 60648), (16492, 41271), (52100, 1304), (93416, 7795), (57209, 71008), (48010, 36078), (20384, 74420), (77440, 34439), (69224, 45099), (30374, 33884), (49038, 90140), (1154, 84725), (64926, 86985), (91746, 73472), (59757, 75755), (45860, 71557), (45833, 36526), (74618, 73598), (91360, 65168), (58029, 30793), (56332, 14973), (99943, 96877), (97454, 6450), (64502, 77301), (73182, 31853), (76809, 83964), (82916, 86188), (78736, 65427), (36495, 7422), (76196, 2804), (96117, 61093), (9177, 26099), (52942, 63007), (48578, 47876), (50638, 89903), (7113, 97316), (35301, 12750), (47807, 7254), (38217, 55418), (56970, 41687), (20527, 62886), (358, 14021), (64018, 18582), (91740, 21683), (81967, 53589), (45437, 38450), (45476, 67752), (76851, 72072), (7304, 60091), (40097, 12897), (39906, 29247), (84262, 58734), (30857, 43791), (56087, 78929), (20498, 45954), (48726, 500), (62723, 43763), (28368, 30756), (74048, 52403), (15045, 95926), (75542, 55384), (52543, 22525), (56001, 6935), (11431, 46745), (77731, 7310), (36718, 59909), (32235, 91254), (92417, 25917), (21782, 79277), (46378, 87536), (35324, 26075), (6310, 76915), (1551, 69473), (50642, 68865), (55190, 72934), (49780, 21873), (99466, 29686), (90761, 13179), (72959, 57033), (20020, 90200), (46186, 79105), (73871, 52382), (59559, 38801), (59916, 16082), (33610, 94966), (46001, 45225), (86679, 26469), (77245, 91929), (32887, 36623), (11179, 46898), (87881, 68087), (45438, 47991), (24950, 94525), (91664, 51656), (43914, 47805), (15736, 96156), (56346, 20283), (85053, 48931), (17790, 26179), (96195, 55728), (43765, 54807), (44988, 89269), (55911, 99411), (52446, 47397), (28346, 65442), (96669, 68226), (66194, 26848), (37276, 55864), (14116, 41583), (18058, 16317), (93136, 85318), (35616, 86252), (29222, 29969), (33386, 85372), (71094, 44238), (27733, 31838), (64626, 16692), (52904, 97899), (97619, 12663), (50165, 4688), (67557, 44053), (69184, 66269), (73164, 89705), (39822, 15169), (65499, 72808), (30068, 63697), (30154, 64235), (97016, 58716), (94366, 36592), (1592, 16261), (87985, 52102), (12554, 23652), (15909, 25292), (2527, 91531), (92139, 36031), (28986, 30032), (3038, 56314), (32239, 26707), (15973, 34901), (70246, 39680), (82529, 38132), (45827, 74783), (53665, 64111), (55218, 84170), (20466, 16130), (55734, 71203), (31438, 96906), (66338, 85858), (35988, 68511), (78391, 15191), (80747, 59213), (5357, 11546), (16822, 16607), (36607, 41106), (74949, 30739), (45726, 64887), (1524, 54847), (37371, 89195), (28726, 27788), (22600, 44777), (53999, 63625), (84304, 98338), (49260, 76480), (74564, 53907), (89867, 97096), (60157, 61299), (17165, 10146), (56334, 36268), (62114, 49222), (22715, 23620), (42830, 11539), (41091, 69151), (75471, 68364), (18681, 43249), (42738, 63219), (35474, 98454), (76815, 46024), (66310, 36521), (86095, 77013), (63693, 77319), (80731, 63031), (95478, 92387), (23787, 63724), (46299, 68994), (4800, 2460), (9663, 80639), (77231, 85814), (81615, 11311), (35638, 27340), (13598, 14322), (30657, 17238), (90957, 96846), (69962, 52140), (41681, 65962), (96836, 58177), (36190, 11623), (4231, 40500), (43049, 41949), (71177, 98492), (30193, 39750), (19744, 33204), (63358, 30210), (45638, 58918), (43641, 38741), (35598, 40932), (33238, 36236), (50835, 20968), (25099, 34071), (84986, 88456), (35333, 1529), (79771, 23985), (647, 61658), (9424, 11743), (77766, 31528), (77811, 86973), (76403, 74377), (55568, 79251), (68858, 20762), (68520, 66773), (93598, 89823), (8080, 82539), (87760, 52247), (25191, 16905), (17837, 8339), (85177, 59050), (51680, 77374), (3287, 43018), (43479, 62141), (34909, 46322), (11869, 5885), (96193, 58417), (101, 47460), (34937, 88582), (83216, 88388), (28571, 15292), (66683, 62613), (34478, 8924), (2680, 89973), (62438, 44460), (11724, 4791), (5383, 72888), (88206, 67586), (8124, 21690), (28779, 75789), (66791, 4757), (6176, 47760), (6403, 78084), (78122, 35446), (99494, 73608), (39691, 89098), (59182, 19484), (25389, 98963), (96487, 3692), (76222, 67381), (21199, 50358), (95998, 58137), (28777, 43913), (14176, 60117), (52257, 81703), (14604, 13438), (71301, 14401), (19758, 66914), (15506, 29873), (87205, 29449), (93295, 15930), (63651, 11287), (19785, 15966), (30795, 75112), (69462, 37655), (18793, 85764), (36240, 31236), (98153, 73724), (72491, 4223), (66930, 35048), (25686, 13269), (13940, 13259), (69163, 11235), (1183, 86961), (54323, 67315), (85044, 60872), (48875, 3683), (43052, 92861), (87574, 32969), (92552, 80564), (94832, 47682), (72011, 80994), (60182, 917), (97788, 34169), (66432, 47940), (87468, 80954), (35385, 68758), (50555, 63710), (55311, 44337), (87065, 26514), (84581, 98736), (23212, 56499), (75120, 72447), (56087, 38285), (58171, 45629), (28401, 44319), (70432, 27883), (18891, 14646), (26206, 49924), (79957, 44914), (56064, 27529), (99090, 29197), (49435, 340), (53525, 65601), (76998, 88349), (50416, 70860), (42506, 75290), (34024, 13295), (86663, 46523), (88814, 231), (57809, 21), (84914, 84771), (43042, 66892), (17288, 33908), (4934, 63195), (50590, 1516), (97843, 80208), (20091, 86717), (71566, 15929), (19531, 23634), (41646, 45549), (89226, 82902), (96683, 63386), (31072, 53788), (51135, 41099), (78912, 65609), (36094, 23603), (88403, 51455), (73795, 47066), (26448, 82852), (22829, 2894), (30041, 92548), (27733, 20608), (70180, 19892), (51650, 63440), (76328, 13666), (40514, 6677), (2786, 51059), (40809, 16499), (10857, 82541), (78221, 61067), (17982, 51969), (85369, 66965), (47153, 47149), (43965, 75796), (82725, 60767), (42407, 97249), (51475, 81224), (60957, 89414), (33065, 21663), (36601, 5290), (95842, 67301), (64630, 60398), (55212, 35638), (41750, 44235), (75260, 82400), (91291, 25843), (6477, 8311), (14919, 52306), (66220, 33180), (45736, 2313), (37450, 64444), (98614, 61344), (75007, 50946), (56701, 28117), (66632, 5174), (92323, 76613), (6796, 73695), (33696, 76280), (86876, 5614), (50863, 67993), (36068, 17049), (91912, 34271), (70706, 1904), (97798, 41117), (68154, 72483), (83862, 25578), (61643, 17204), (69974, 64232), (77926, 19637), (64901, 88988), (71424, 91703), (91655, 17147), (46872, 56530), (44189, 98087), (95939, 54420), (72651, 68785), (67624, 84875), (92587, 87663), (65275, 81256), (53798, 2506), (14702, 3638), (71291, 50452), (14909, 13903), (66965, 26606), (14127, 60345), (35306, 1738), (77234, 10468), (53521, 41218), (80681, 82583), (44227, 26521), (32263, 21482), (82270, 56963), (50580, 80567), (11593, 22346), (20074, 26867), (73126, 28667), (62996, 24317), (20295, 57163), (1506, 57668), (69567, 45236), (43366, 26001), (88052, 40181), (1599, 89349), (36789, 1579), (39895, 46673), (30381, 3206), (31723, 5625), (19252, 31317), (16932, 77149), (48794, 34409), (55986, 30328), (47551, 75088), (57363, 78365), (95221, 63385), (26449, 5733), (96588, 53077), (52980, 41140), (8187, 85947), (36723, 26520), (23579, 38909), (33350, 19275), (63930, 19357), (43536, 59941), (31117, 77322), (44638, 94812), (44730, 99097), (95108, 48170), (57813, 49503), (79959, 89436), (86980, 62031), (8275, 44009), (36666, 94645), (22064, 38882), (40471, 16939), (31156, 11337), (13101, 96977), (17906, 26835), (89861, 51405), (73369, 67946), (99141, 58572), (27131, 98703), (15900, 43412), (51768, 93125), (78579, 46689), (23029, 13895), (60870, 55830), (22553, 8236), (76449, 96207), (83766, 51024), (27630, 50614), (53484, 90104), (77626, 21944), (46755, 41583), (53616, 34240), (94159, 44415), (13914, 90059), (44387, 89012), (27499, 64579), (83415, 30809), (77558, 82619), (88880, 9814), (8466, 4424), (43598, 91921), (24695, 3349), (46295, 65208), (51256, 82461), (49126, 93012), (16186, 96585), (43284, 22655), (93130, 90393), (77495, 34372), (85509, 65856), (86662, 61906), (50988, 44393), (29828, 17737), (91651, 35308), (29796, 49716), (14019, 87751), (29688, 71207), (82845, 19100), (11989, 50132), (21158, 99905), (54732, 42547), (32314, 12851), (46405, 43794), (87849, 45643), (53524, 21212), (61925, 75491), (12498, 21937), (30185, 69475), (48421, 52487), (15112, 90935), (33187, 17801), (61704, 25514), (17889, 23917), (18758, 57197), (7693, 47232), (47905, 24618), (11494, 78950), (95662, 54561), (8075, 33909), (90427, 46065), (73962, 19821), (50691, 79400), (58218, 4881), (94106, 2509), (60633, 55169), (49600, 83054), (23339, 13270), (70262, 58946), (48417, 97266), (27629, 46905), (74465, 75514), (41687, 2564), (12814, 19492), (78899, 30168), (17745, 35206), (37972, 35296), (22288, 80001), + (68026, 36558), (40187, 12234), (92380, 22866), (56488, 64402), (41404, 62562), (47802, 45287), (83302, 85215), (58999, 85776), (35158, 16804), (13416, 94146), (62953, 28243), (83290, 19103), (4564, 21789), (64468, 20927), (25582, 47206), (57810, 18693), (28938, 97986), (61704, 14838), (19214, 3232), (12911, 25438), (85802, 28837), (56506, 89458), (66392, 47773), (68190, 43841), (43044, 52214), (57886, 32830), (15943, 59771), (37081, 89294), (4032, 32960), (46931, 85790), (69656, 72737), (28217, 39872), (86170, 42776), (55116, 51495), (90485, 45274), (60773, 36788), (2193, 2636), (70222, 62086), (75720, 70712), (17549, 51460), (23609, 31515), (70254, 39825), (63762, 11061), (13107, 15394), (45916, 72130), (91558, 86662), (99524, 69106), (93073, 29881), (31724, 3007), (69051, 59452), (59701, 86760), (4967, 82028), (57404, 48226), (71829, 79910), (23714, 62439), (73881, 67618), (63269, 40085), (6164, 23415), (48156, 93907), (18627, 16570), (6676, 22991), (36916, 41488), (99079, 13264), (32533, 99243), (55505, 63339), (89564, 3290), (24886, 34916), (91310, 9343), (49779, 12740), (26320, 3406), (57661, 5702), (10765, 57881), (5518, 47638), (93148, 27438), (73451, 24477), (84075, 96822), (58883, 58883), (96812, 82388), (30659, 59654), (24498, 95808), (25591, 21834), (13090, 87704), (76495, 17249), (75975, 84318), (55459, 70426), (84256, 88604), (79438, 43104), (45331, 7495), (63619, 11123), (24772, 2601), (63343, 14138), (39957, 98339), (55595, 17823), (97676, 53933), (91867, 25023), (64677, 67859), (43737, 34315), (24800, 53968), (93157, 17507), (24264, 35273), (33889, 507), (10207, 40542), (40213, 57800), (38321, 74160), (42391, 7651), (80267, 94736), (52473, 79634), (17075, 2531), (8595, 75890), (31496, 50367), (16069, 79896), (70067, 200), (23420, 49517), (1628, 45646), (8916, 36794), (72294, 88976), (40603, 86008), (91871, 71098), (5447, 70998), (24152, 17561), (65046, 34951), (56950, 9292), (19244, 31385), (74693, 31813), (97343, 21572), (38834, 135), (79717, 62486), (38, 10308), (58035, 71344), (85802, 81079), (5943, 156), (38735, 38867), (3803, 99366), (15853, 19408), (62988, 62008), (8316, 44684), (17035, 71012), (48584, 2117), (75425, 37336), (2405, 50420), (43653, 28836), (12394, 69430), (54522, 4954), (33359, 148), (41018, 82851), (79995, 55417), (65008, 32342), (36547, 88185), (8131, 7054), (38980, 20146), (27976, 63039), (53119, 67009), (40043, 98393), (29333, 51980), (85818, 98405), (77956, 20099), (99747, 16916), (11597, 50181), (40961, 8262), (75103, 13912), (62339, 69155), (3869, 85481), (7053, 30956), (33563, 53272), (96178, 81751), (99365, 88728), (34447, 11164), (62856, 30939), (92486, 3357), (56605, 35330), (42180, 15137), (83946, 62984), (61869, 55711), (52880, 49871), (44588, 27387), (16332, 24496), (1781, 13508), (56674, 95773), (21328, 19628), (96455, 24155), (14302, 74435), (54053, 24590), (86642, 22177), (24089, 16186), (70281, 4601), (18552, 70708), (95442, 5895), (96714, 6293), (43803, 45857), (93257, 18497), (90032, 85086), (40566, 87233), (32674, 73822), (95599, 49334), (62745, 51898), (8245, 93882), (14093, 40977), (47215, 53001), (59737, 68452), (90937, 25354), (43805, 82571), (81953, 68572), (37298, 96262), (94899, 65066), (34772, 80762), (55469, 1186), (8734, 91665), (18622, 51150), (85200, 39575), (65381, 15979), (89734, 89656), (64712, 53691), (87187, 58256), (8476, 89694), (49935, 35239), (63730, 34982), (27687, 91571), (87543, 15350), (85208, 18781), (14783, 2574), (44699, 666), (56440, 87617), (32732, 49301), (76725, 3895), (10419, 90580), (34725, 69476), (14831, 81588), (93924, 38057), (38528, 99060), (57136, 44206), (74685, 99559), (43083, 87511), (43105, 35474), (35582, 17560), (5578, 98727), (78947, 53865), (32013, 95029), (61552, 42674), (52191, 49975), (71566, 16403), (78534, 16350), (18520, 80501), (29114, 46547), (11488, 5069), (89591, 82384), (13741, 42318), (74385, 58849), (49739, 63421), (83821, 6676), (51997, 93321), (36677, 81768), (37915, 73495), (47175, 6086), (39989, 83110), (6489, 48112), (88822, 20370), (12846, 13952), (28930, 20879), (25139, 84552), (76434, 2665), (55145, 31523), (21177, 18630), (81077, 96275), (61006, 30845), (77722, 62651), (61181, 72545), (93838, 84287), (59300, 19014), (75076, 97980), (76979, 1473), (48409, 13097), (51718, 5325), (36522, 72119), (60917, 18995), (61469, 42853), (34387, 37322), (38684, 28120), (64136, 8559), (15368, 99424), (97824, 7864), (33833, 72029), (7024, 9961), (49400, 66220), (63025, 97179), (6135, 98878), (19873, 8438), (3963, 35670), (65186, 89423), (26653, 65943), (83132, 67000), (82578, 35007), (42680, 60479), (71102, 98589), (74842, 94010), (22931, 33725), (46537, 42629), (75793, 48115), (21630, 92454), (97993, 81332), (25747, 31814), (91231, 65953), (91981, 12219), (64719, 16254), (60914, 8334), (15887, 96432), (42110, 28837), (7295, 83147), (50334, 7053), (3949, 33594), (1524, 98230), (17265, 98024), (75969, 36232), (89538, 5212), (13444, 55946), (69823, 81848), (32578, 74024), (52018, 98290), (59118, 40186), (61002, 16977), (69537, 44780), (92, 13937), (33715, 42663), (46347, 8312), (86196, 59301), (17128, 85014), (26429, 57682), (45888, 99588), (22750, 96110), (46809, 49251), (24521, 40071), (287, 22115), (11741, 36315), (22742, 17581), (35808, 3110), (98904, 30407), (4584, 13383), (28585, 69669), (94823, 29715), (9551, 36389), (77997, 45746), (49894, 55722), (23415, 69459), (58246, 85144), (74136, 18102), (97366, 85724), (34271, 51601), (47535, 70883), (59443, 90103), (45213, 45811), (62741, 86898), (17324, 50034), (62080, 25193), (89524, 4421), (13476, 51456), (69198, 56718), (58024, 22969), (65210, 67941), (32561, 44881), (62295, 67448), (66135, 95453), (9417, 20443), (82486, 23745), (19185, 99041), (40662, 91714), (3423, 58624), (4512, 74502), (67772, 98023), (69575, 75779), (69107, 62805), (517, 33801), (47406, 7581), (81108, 10546), (12976, 47001), (16742, 83811), (44593, 82124), (52731, 34642), (81725, 20555), (94126, 91919), (24800, 59302), (97253, 39249), (71692, 10769), (88721, 56321), (7019, 69771), (31464, 61774), (29597, 19263), (65557, 31875), (28653, 69636), (58074, 76848), (15906, 80620), (18259, 40193), (99991, 4769), (98935, 99269), (12123, 60124), (20787, 47346), (13526, 33592), (95370, 40350), (17479, 42884), (58368, 83218), (63290, 74406), (97030, 35102), (45298, 27660), (64593, 21262), (76268, 82641), (1107, 44044), (21427, 79959), (85180, 62412), (7359, 1318), (83618, 9762), (1425, 55804), (32874, 97943), (68191, 38742), (41715, 17902), (3771, 15032), (7848, 74950), (33881, 40904), (75295, 26151), (75775, 13760), (90262, 89822), (88169, 18679), (57506, 32356), (94983, 44281), (37385, 37432), (18248, 48162), (45573, 66278), (25277, 72788), (26977, 36778), (26254, 61758), (12860, 48026), (96819, 3339), (13134, 1173), (26822, 53374), (15989, 29698), (11258, 54515), (37866, 34928), (22996, 26577), (39952, 42732), (6754, 70595), (86245, 44669), (47044, 34170), (6789, 45220), (31706, 2090), (42582, 40023), (35147, 46591), (88210, 11307), (53644, 7680), (11280, 91075), (42961, 65122), (40066, 52185), (20050, 6154), (98440, 20393), (88992, 75432), (32386, 66731), (36952, 34149), (18453, 32715), (84413, 10378), (59440, 2374), (45354, 85009), (50382, 66510), (64428, 95401), (9336, 41760), (26317, 91416), (81941, 99504), (26600, 53522), (81069, 40236), (51126, 27911), (97144, 14243), (62738, 50287), (37372, 28962), (12053, 9090), (69492, 95524), (68141, 52931), (17276, 16487), (69227, 25949), (14143, 70193), (7077, 53032), (65463, 74082), (94997, 66496), (80443, 55832), (66796, 5970), (15852, 95662), (81559, 97272), (55851, 18977), (91142, 48976), (91143, 950), (79225, 31004), (61310, 20760), (74541, 90842), (80322, 11630), (84631, 544), (66785, 86591), (25650, 63252), (59635, 18586), (2964, 6741), (37091, 71148), (11984, 43077), (87505, 62049), (61925, 92290), (18808, 3937), (8300, 33268), (70850, 50661), (86024, 73730), (85161, 47116), (50193, 89155), (37773, 40845), (9251, 41688), (6940, 65399), (42479, 95630), (19401, 43102), (48069, 36040), (62760, 95013), (394, 2641), (32567, 29306), (13870, 58835), (98248, 47291), (49803, 4523), (40222, 12883), (53576, 73105), (88265, 23629), (67865, 67875), (33473, 27144), (80219, 53893), (74878, 47341), (78070, 84803), (30003, 5600), (41103, 6145), (83490, 81076), (55059, 66736), (45015, 10239), (79555, 85819), (81808, 34970), (19235, 85480), (91807, 52177), (40887, 87009), (5003, 2687), (64964, 88122), (765, 94893), (93573, 20504), (28854, 38438), (94244, 93475), (72996, 84801), (75427, 81692), (63161, 98637), (18814, 61343), (22863, 60110), (8949, 12694), (19675, 94313), (43857, 74073), (15737, 58218), (48895, 68474), (22220, 92926), (69055, 50282), (40532, 74934), (59062, 66405), (85784, 87704), (58494, 88222), (2260, 20401), (73112, 99666), (46739, 95433), (21179, 85119), (11545, 38801), (59993, 50866), (10086, 4709), (70560, 29611), (27095, 89017), (6896, 2279), (92506, 5013), (48600, 90491), (18782, 54638), (54337, 82734), (52054, 13481), (38297, 56559), (15998, 30591), (89789, 7522), (18149, 28725), (3532, 28625), (70934, 49617), (84599, 55664), (74229, 52269), (55431, 11893), (32807, 72543), (83882, 53025), (11490, 83442), (14844, 88612), (12526, 45953), (906, 2231), (68240, 95612), (18818, 31535), (57774, 91290), (67250, 67400), (77332, 23550), (42332, 57775), (28792, 11539), (19108, 34608), (12399, 38591), (7329, 10740), (84288, 50928), (29461, 17629), (63884, 88489), (47479, 61085), (75357, 57255), (60107, 94046), (32934, 66312), (28615, 42600), (55553, 85213), (57838, 91426), (9783, 11513), (73677, 28821), (75408, 75561), (22995, 59224), (74874, 54145), (18513, 75901), (46440, 69414), (36072, 22263), (60560, 73325), (69967, 93358), (75949, 98634), (3688, 57991), (43482, 94541), (40922, 31011), (57763, 74497), (93576, 96392), (83038, 80656), (47757, 87045), (14061, 53465), (65619, 33775), (11341, 6702), (6249, 87358), (15766, 85937), (13135, 93945), (24495, 95900), (80359, 1739), (15468, 73426), (49240, 44999), (82839, 90808), + (87438, 75613), (348, 73144), (99523, 85853), (21557, 70210), (64933, 1672), (38154, 17477), (97136, 67363), (96491, 8038), (97981, 3434), (54372, 27038), (88480, 86675), (21028, 21083), (43197, 4440), (31702, 78290), (66631, 24438), (11482, 17922), (90351, 39503), (46186, 32439), (73828, 6640), (56916, 26029), (62840, 1815), (20281, 28488), (18211, 30043), (65211, 93012), (43614, 58012), (90322, 77343), (64293, 94525), (59489, 39760), (93219, 78440), (74613, 9732), (38085, 19191), (58029, 48186), (88762, 1764), (28627, 21993), (49975, 41225), (70486, 43480), (82764, 96425), (27218, 78327), (17844, 73333), (70463, 37629), (10500, 33826), (97343, 66575), (82833, 51210), (77353, 45073), (27163, 39728), (78076, 46691), (80302, 39342), (77142, 1319), (87403, 80110), (53805, 27786), (50558, 74264), (83146, 31358), (11567, 4438), (30041, 54287), (91731, 18496), (57591, 93894), (72534, 59009), (98064, 59148), (69626, 66615), (20951, 43949), (61960, 68060), (48892, 67918), (61321, 56222), (75424, 77260), (4916, 81929), (68892, 81531), (28096, 28548), (62016, 107), (8593, 12030), (66743, 36772), (60174, 15106), (52844, 1923), (34768, 22065), (88988, 62910), (79214, 2998), (25675, 31376), (69959, 3614), (43885, 31708), (12206, 46548), (69924, 19343), (12984, 38980), (58250, 69438), (2580, 48684), (38112, 37124), (21842, 43150), (59384, 21921), (19908, 46678), (73396, 79529), (8274, 1557), (36975, 65519), (81069, 18712), (13692, 9148), (60617, 84762), (75749, 66154), (80375, 24553), (4257, 47056), (76880, 7687), (40714, 43448), (79112, 74791), (33119, 72730), (17670, 89183), (51614, 3921), (21247, 39857), (86756, 67673), (32792, 70035), (5917, 7197), (1762, 23130), (6455, 63664), (32806, 3729), (60469, 20511), (12522, 15149), (98106, 79338), (84754, 11162), (52058, 17973), (28789, 1521), (32766, 36325), (78914, 40453), (70297, 71854), (9313, 45190), (54559, 66227), (22342, 43860), (44152, 84294), (36913, 93173), (88523, 36338), (82234, 71140), (8328, 22947), (73250, 88125), (74356, 16820), (94472, 37349), (23126, 87806), (40315, 88729), (19935, 19145), (93312, 65719), (8477, 33108), (86660, 69525), (75557, 66964), (60437, 57494), (94419, 42524), (95372, 72274), (49866, 85685), (96808, 39404), (62961, 72507), (38634, 70815), (91379, 42430), (66359, 98699), (24382, 4186), (4003, 77760), (87840, 75265), (57641, 68871), (9773, 15942), (5664, 51289), (47923, 31308), (58632, 82468), (14097, 71829), (1838, 97710), (70433, 11364), (82363, 97879), (25257, 20615), (18249, 6758), (98581, 13639), (3290, 72449), (74546, 79380), (97254, 44448), (80316, 31760), (40516, 94809), (14444, 88981), (9693, 10259), (83795, 95485), (70201, 81014), (66644, 16761), (35529, 82718), (75774, 73476), (80139, 3957), (34803, 80689), (92085, 46499), (97871, 8004), (67369, 11354), (43578, 81596), (94695, 44963), (93741, 41629), (16005, 96652), (37918, 69012), (89832, 56041), (51798, 32386), (89749, 27647), (76279, 7990), (31746, 1346), (40841, 20480), (99942, 24473), (78495, 99194), (13588, 57088), (22183, 42297), (82707, 34435), (45026, 12747), (8000, 93211), (40453, 13025), (44100, 39880), (83900, 56474), (87691, 42802), (82000, 63867), (76627, 84731), (112, 92774), (34749, 97737), (59262, 57169), (95571, 44144), (74310, 68970), (63232, 92744), (53698, 21610), (39969, 75475), (39942, 28713), (81230, 50140), (97953, 96528), (86144, 48041), (96677, 49677), (22051, 48183), (33436, 54784), (5553, 11920), (67057, 17115), (57736, 72309), (8086, 85329), (72623, 94949), (13096, 48992), (63153, 56337), (30462, 1036), (75000, 24048), (62635, 50128), (91480, 83131), (25929, 79809), (96237, 76974), (59645, 20603), (31850, 29754), (91070, 36568), (30191, 33785), (86776, 67259), (49073, 39179), (16121, 73834), (84217, 52951), (95866, 47244), (63326, 73460), (134, 91953), (48189, 86069), (42971, 3700), (28643, 10479), (80151, 7446), (78798, 2655), (39135, 69364), (80244, 24904), (22818, 74964), (26753, 82419), (16858, 5212), (79891, 11215), (785, 46103), (12559, 24617), (73601, 71490), (70342, 7099), (73330, 6665), (11903, 28194), (16375, 37746), (86132, 51788), (90345, 68366), (5464, 78338), (23569, 83141), (17904, 94046), (35868, 60017), (22591, 93373), (70584, 72116), (49331, 34312), (16180, 91286), (58494, 65441), (9336, 52671), (32523, 26734), (40205, 83549), (28810, 96876), (44703, 38944), (46981, 37157), (8582, 7529), (59718, 71700), (62545, 73716), (6531, 23200), (30528, 59720), (57152, 84660), (16232, 67946), (60446, 45983), (68737, 54959), (57795, 73107), (26930, 35938), (9844, 44760), (3716, 79020), (99126, 8264), (66120, 16151), (50616, 25765), (93340, 95875), (34103, 88003), (14879, 99758), (49188, 6087), (89858, 42861), (36730, 72076), (25069, 26403), (98183, 48108), (3229, 5367), (59306, 80078), (61144, 58598), (72600, 98765), (57701, 23177), (10176, 11553), (82964, 13697), (7788, 28538), (39943, 97491), (56261, 17781), (2458, 1892), (6679, 45554), (42171, 66222), (24420, 44115), (35852, 41965), (50196, 49555), (34718, 60734), (6932, 61638), (69472, 56723), (489, 97620), (41335, 90578), (1333, 92787), (97883, 64754), (14208, 22097), (75776, 5938), (67446, 61518), (58743, 45162), (34749, 81243), (71451, 91991), (27804, 41836), (45274, 8039), (17593, 24498), (8801, 38559), (87460, 7109), (50075, 18284), (84043, 82146), (62932, 25018), (89647, 56768), (59920, 80801), (56357, 35142), (97376, 58181), (70715, 91103), (90829, 78985), (29776, 13275), (30546, 42320), (99266, 35340), (21234, 61062), (39239, 10745), (45990, 715), (47047, 6619), (4270, 94575), (90009, 72203), (25629, 2691), (67926, 89112), (46990, 61101), (22355, 69536), (1977, 56723), (54681, 34041), (83819, 7024), (81235, 7093), (16659, 87135), (49384, 32135), (42204, 17362), (90585, 70374), (51255, 1), (31600, 70085), (90189, 95778), (57349, 87789), (83384, 93771), (20718, 15529), (10644, 53591), (84103, 62101), (91340, 48382), (82854, 84420), (12561, 53517), (64835, 45362), (54154, 75841), (46498, 31175), (75035, 49552), (9306, 53403), (68851, 49139), (13463, 42107), (2322, 36695), (55953, 12098), (60656, 80482), (78987, 19632), (31228, 18523), (98972, 80489), (32367, 98405), (25139, 5164), (5692, 60610), (36535, 70097), (80542, 74320), (87984, 46750), (98201, 41341), (35217, 46813), (81795, 69057), (83927, 41032), (60149, 26087), (39954, 48361), (64485, 61448), (87185, 14580), (74559, 93251), (88544, 83366), (74015, 15864), (78623, 69719), (16941, 80710), (16315, 58313), (47277, 59107), (16067, 66290), (63906, 59891), (20754, 67817), (44428, 10652), (95960, 99045), (52163, 26221), (65566, 22057), (26836, 38898), (57107, 79274), (39020, 74857), (53540, 84159), (76646, 44324), (27967, 40171), (28710, 56332), (84036, 28711), (68742, 57241), (40535, 34737), (62681, 85386), (30472, 58405), (85086, 33013), (67059, 47481), (30441, 55098), (97892, 71991), (90296, 42905), (22441, 18863), (19606, 77242), (11206, 58380), (23901, 49962), (84094, 33761), (64400, 28093), (64228, 94543), (71874, 20871), (25385, 73117), (63398, 20999), (77547, 51893), (80783, 65858), (39807, 80754), (10336, 90318), (7826, 55346), (30206, 10711), (94411, 67364), (33509, 14329), (65350, 17006), (65999, 55699), (82753, 61081), (38851, 11896), (15155, 48635), (19985, 75204), (37144, 5344), (26173, 39587), (61111, 30966), (16180, 22987), (60707, 43599), (30136, 74118), (7880, 43857), (97445, 30233), (62700, 24828), (90914, 89452), (64131, 56925), (25259, 39132), (47104, 43950), (93891, 21913), (84573, 91029), (8604, 79858), (33141, 25534), (12468, 90413), (97063, 76359), (80826, 26061), (64013, 99099), (82158, 38882), (25799, 7564), (25477, 69847), (73374, 58520), (48230, 9453), (91424, 72273), (64893, 11750), (46753, 48434), (15974, 94633), (14872, 27027), (14527, 21313), (25660, 64644), (54196, 15138), (6313, 10911), (36168, 47170), (45346, 76), (10305, 60286), (65283, 39977), (21804, 37972), (65389, 86954), (90674, 64458), (15838, 22392), (43540, 42503), (49584, 67828), (56711, 87887), (40075, 73696), (23832, 91552), (39002, 65562), (20451, 64664), (70783, 92171), (29319, 57694), (56217, 44247), (52856, 57873), (80560, 90902), (31068, 11280), (46996, 34739), (57527, 4554), (8410, 25816), (12269, 38319), (88054, 49939), (337, 13231), (56432, 68236), (74841, 21476), (96006, 15712), (87145, 91660), (58090, 55111), (10310, 79789), (5734, 79710), (98992, 69026), (77033, 5734), (43338, 42635), (23898, 28669), (62708, 81652), (41279, 51722), (93444, 26355), (62046, 52199), (71492, 58736), (7379, 62581), (8592, 71885), (75026, 40387), (46696, 3939), (9787, 88907), (86356, 363), (97479, 20358), (77363, 65553), (44036, 22178), (98279, 64612), (3615, 411), (77003, 93018), (41605, 88489), (55992, 83614), (19493, 21633), (34639, 97064), (94602, 89289), (45853, 26299), (84170, 73386), (9221, 51439), (41513, 68166), (37170, 17690), (82511, 59246), (96674, 27574), (99301, 45675), (42716, 41520), (56623, 49130), (84100, 76804), (73855, 97007), (73303, 26912), (37151, 23837), (49190, 97104), (23487, 45628), (87763, 46550), (65111, 92605), (80481, 8151), (83949, 18930), (81749, 27244), (37449, 3023), (28303, 51545), (96441, 93242), (22082, 43254), (35135, 68407), (37712, 48709), (5111, 26774), (15532, 74246), (93605, 83583), (21491, 66472), (38922, 53076), (55455, 54432), (955, 44063), (311, 91630), (53554, 4522), (29927, 65668), (7525, 16035), (44093, 76745), (21481, 78198), (76875, 5306), (56126, 76437), (96534, 16880), (85600, 68336), (4479, 81002), (80414, 11593), (8186, 61147), (5624, 32879), (79312, 20995), (40407, 41512), (91261, 66022), (93228, 75364), (21136, 40111), (92148, 60681), (42549, 7944), (60157, 15040), (63562, 88365), (69056, 72713), (78263, 89223), (3776, 33039), (30042, 59984), (64567, 20977), (24720, 39157), (63582, 75653), (45363, 20249), (58093, 53833), (27918, 93306), (25791, 92686), (15904, 862), (72093, 19257), (64125, 88986), (41717, 27989), (43165, 98675), (76840, 48170), (64508, 3535), (91964, 33435), (96686, 88673), (66648, 64594), (17927, 30539), (73615, 22800), (18580, 48077), (59803, 48202), + (76805, 89886), (2744, 52965), (55596, 22519), (35358, 11629), (83029, 80047), (36120, 91930), (26066, 23035), (48857, 14268), (63020, 26197), (60623, 23252), (34911, 72754), (34808, 21593), (64067, 58963), (34509, 8739), (52686, 96405), (98282, 10463), (6495, 64680), (59016, 86968), (33928, 51222), (39609, 84992), (67603, 89875), (14723, 16144), (30751, 46856), (76874, 75024), (89584, 58806), (51278, 4113), (27187, 93483), (80039, 52159), (6132, 25127), (42358, 77498), (33838, 79064), (74147, 76851), (39752, 27366), (44888, 9809), (10887, 4135), (22303, 36417), (58690, 34613), (53998, 74014), (71567, 32438), (65110, 93406), (77365, 41299), (18044, 70636), (77346, 21236), (78408, 245), (57704, 34662), (75258, 64730), (96992, 15533), (56010, 60769), (69163, 4826), (88709, 20725), (33197, 69743), (97169, 83194), (75277, 53343), (14531, 64740), (19997, 4752), (74016, 55946), (55290, 63626), (32533, 32920), (32946, 74610), (12386, 33853), (34825, 35374), (28772, 32716), (17280, 42683), (54184, 34332), (29964, 16203), (65767, 61448), (29133, 35728), (6861, 14160), (65483, 40224), (78335, 76002), (3061, 40615), (11780, 87517), (46135, 73448), (10920, 72592), (15696, 28810), (44154, 64134), (59365, 27248), (76601, 39862), (68264, 30019), (48572, 54575), (59499, 85796), (35064, 23789), (57028, 83545), (33911, 8463), (21827, 67966), (15983, 69649), (13919, 20584), (82742, 67956), (75457, 45767), (55394, 62309), (6099, 67510), (58078, 9594), (24511, 83149), (24781, 79624), (39745, 777), (92023, 40085), (22889, 37179), (17919, 28607), (79865, 72682), (99829, 38190), (21273, 21278), (88299, 23433), (88887, 48163), (62993, 61567), (82107, 84224), (65049, 61245), (75113, 93564), (81562, 7874), (32314, 32313), (3979, 46996), (40558, 93278), (58758, 68163), (40502, 58941), (76961, 65762), (48032, 36117), (64712, 9137), (12092, 56665), (12315, 66581), (20954, 29083), (57317, 48290), (23534, 86828), (4869, 35950), (26993, 24840), (93007, 45049), (18009, 20350), (43053, 71248), (47320, 66119), (50898, 96627), (669, 40018), (89236, 44039), (47375, 63306), (61906, 6658), (2672, 84546), (59778, 72319), (14497, 71952), (42420, 87023), (96465, 46140), (32857, 22772), (4985, 35125), (61918, 28016), (90275, 24406), (49799, 10811), (74137, 63345), (26135, 86306), (92971, 65541), (40134, 95892), (38554, 46307), (48113, 16343), (63990, 66283), (17793, 49570), (21736, 79819), (13831, 27523), (8939, 93929), (96577, 4909), (38583, 32781), (13701, 24436), (43444, 56054), (17166, 32346), (57202, 26264), (82858, 75049), (46317, 95666), (54911, 68161), (3894, 38521), (26456, 30270), (65214, 35331), (41143, 13109), (85441, 48899), (93226, 25027), (77045, 81171), (30345, 79232), (71167, 40854), (58761, 56824), (89047, 85314), (31686, 81947), (74946, 60661), (49903, 13625), (76341, 69067), (46963, 88891), (97223, 5921), (52143, 9828), (17413, 42731), (30236, 93426), (14540, 17652), (52251, 97233), (41581, 30097), (28771, 46426), (36260, 45179), (4068, 16410), (3146, 95055), (5993, 88855), (46103, 30022), (26667, 18756), (54576, 13438), (12800, 11258), (80761, 44979), (59811, 76627), (77917, 87270), (46286, 28657), (30609, 86852), (15200, 28936), (86331, 34195), (98461, 55054), (91760, 62792), (91551, 70192), (96030, 78205), (8254, 27057), (600, 37830), (58635, 65506), (81661, 73708), (11225, 24255), (15830, 9029), (84384, 46190), (31344, 25765), (25670, 30716), (88507, 19484), (28207, 45941), (91874, 15786), (10094, 10934), (38013, 2179), (14558, 36415), (65079, 48850), (65486, 85046), (54958, 60275), (99800, 96623), (68895, 99829), (3708, 75830), (96368, 22631), (99411, 50094), (56888, 3883), (87288, 45604), (64512, 84543), (45565, 14170), (77114, 15132), (31800, 70333), (57775, 40548), (31788, 67511), (51929, 13684), (53736, 81543), (84251, 86303), (63823, 83258), (77539, 61381), (43570, 39418), (79859, 34773), (8595, 64524), (97242, 9283), (15530, 84591), (75535, 65546), (16516, 50162), (58815, 1815), (34897, 82920), (66215, 81262), (81487, 4902), (64039, 25703), (78006, 90468), (3081, 26910), (58159, 4777), (73715, 36375), (69189, 60971), (18169, 39587), (57960, 57668), (6582, 63707), (11155, 47930), (70829, 92266), (6294, 92305), (2188, 6419), (17141, 54972), (60240, 35276), (10788, 29414), (17464, 76377), (3994, 17227), (12039, 24992), (1340, 77467), (1212, 41758), (52186, 80763), (970, 78819), (92897, 68714), (6349, 77016), (22069, 77732), (78209, 72708), (71986, 56770), (8580, 87225), (97505, 63546), (67459, 39771), (50707, 57066), (68226, 54176), (65425, 27407), (57723, 19288), (56974, 90449), (55878, 1264), (46939, 79863), (34868, 4652), (39872, 78482), (92657, 20961), (99690, 28825), (33761, 52922), (73738, 64995), (92092, 3237), (2463, 45045), (43984, 69864), (60146, 5333), (58127, 79082), (84395, 73949), (50818, 68457), (48585, 47420), (60878, 67337), (16573, 30621), (46524, 14168), (87995, 44854), (73143, 77177), (33967, 37276), (95038, 17670), (69022, 16038), (58485, 90526), (1705, 1443), (97969, 40011), (14719, 42770), (8695, 27192), (47546, 51349), (75263, 24419), (25420, 66286), (39198, 41401), (77896, 85583), (28265, 76766), (88836, 48759), (47768, 39582), (65103, 3167), (92171, 85360), (1549, 79296), (71725, 16645), (87349, 29290), (66201, 61712), (43525, 70338), (99025, 63090), (3687, 79963), (63600, 92088), (2480, 1359), (31384, 63603), (29650, 24391), (8552, 82260), (16729, 29139), (26503, 4767), (88945, 19824), (66695, 95696), (84016, 35417), (71521, 22206), (88433, 55606), (66380, 81316), (30573, 36000), (85223, 20494), (99672, 82813), (65500, 78258), (55817, 98414), (43248, 53800), (62787, 21018), (48981, 36258), (41216, 98585), (18576, 18004), (27272, 72860), (76774, 87664), (26737, 11514), (24472, 42538), (5860, 81355), (29066, 10012), (75308, 28561), (23609, 10007), (10007, 19146), (15568, 1487), (80743, 85294), (11207, 90623), (61258, 63879), (34363, 59005), (74884, 2528), (26604, 52738), (33304, 1202), (20381, 18984), (81968, 92425), (4407, 84677), (2112, 79756), (46970, 4367), (36854, 23482), (88346, 75107), (10643, 31806), (21351, 5590), (69317, 53292), (76711, 10085), (70333, 90592), (88818, 822), (23927, 48141), (84710, 33870), (96932, 22686), (5783, 87468), (7785, 11585), (49497, 33764), (13506, 55969), (37840, 78455), (21532, 22292), (97306, 42065), (6579, 40749), (2593, 4995), (81985, 23611), (63888, 98317), (44975, 83777), (57688, 42688), (641, 45787), (7316, 1967), (43837, 18274), (89994, 32770), (4285, 50388), (84699, 41841), (19564, 20683), (76027, 62278), (26140, 11288), (39656, 79954), (16718, 17335), (11583, 21283), (55441, 32178), (6810, 87225), (27191, 54323), (53406, 31512), (48003, 80077), (78497, 29570), (39140, 66619), (12651, 44576), (1761, 88410), (47139, 20766), (39183, 69367), (80479, 23285), (1568, 78535), (18476, 35058), (93551, 81063), (12059, 60021), (23356, 26572), (79975, 35434), (82230, 67436), (20243, 92343), (47809, 10634), (69537, 60167), (3873, 77404), (1136, 27956), (17470, 24156), (35849, 19150), (74760, 37961), (36660, 44448), (36009, 96619), (87110, 84921), (16080, 60637), (36046, 17351), (96403, 99978), (11060, 68629), (36081, 23464), (4684, 11817), (50126, 82936), (55262, 54135), (53717, 66293), (58028, 28065), (92791, 99766), (46266, 77711), (61912, 65782), (38677, 41158), (4001, 46340), (70987, 12784), (14819, 42857), (78985, 99956), (79737, 42497), (55305, 7329), (64103, 24170), (49093, 22115), (2465, 97282), (29009, 15663), (80976, 86477), (16439, 56685), (53482, 15293), (5038, 5991), (67060, 84201), (54350, 38095), (67539, 68292), (26464, 64908), (92909, 12867), (83517, 26474), (76081, 85247), (23250, 66616), (20783, 34330), (43074, 10165), (93968, 70375), (83802, 70820), (19871, 63094), (35699, 36506), (23905, 2401), (27847, 31968), (76714, 44112), (62599, 32720), (10362, 81985), (35708, 2090), (13071, 39035), (71851, 59493), (62833, 48082), (77164, 22804), (6469, 43229), (92051, 3719), (51910, 77689), (91470, 63253), (57914, 57836), (98819, 97813), (35975, 488), (51431, 34061), (45414, 85971), (56563, 93517), (40789, 53103), (9242, 20814), (784, 22584), (8740, 56894), (28504, 378), (8287, 96930), (74232, 97496), (61565, 7904), (9779, 45122), (33767, 48471), (16766, 10722), (47764, 478), (14374, 30099), (89134, 19977), (60860, 93201), (71123, 29840), (57159, 34410), (82411, 99537), (11307, 3733), (70264, 43028), (30418, 19372), (46543, 31506), (33043, 98980), (21137, 63374), (85640, 36957), (6790, 60751), (78771, 43700), (33808, 38263), (27232, 35152), (39925, 5062), (3120, 65621), (39319, 6795), (77468, 94964), (10481, 43009), (24237, 2103), (16837, 55667), (43846, 2874), (78786, 66811), (92185, 62395), (26318, 87942), (6208, 80815), (66952, 71885), (51435, 25450), (21443, 69801), (92554, 81977), (58912, 82288), (59681, 46177), (60397, 65866), (72065, 13318), (2848, 73852), (7577, 83238), (209, 40659), (72103, 15266), (23365, 67286), (14600, 29269), (85541, 63289), (25427, 54812), (22967, 54965), (81525, 27126), (20473, 55455), (84067, 25794), (46798, 79332), (93825, 74677), (447, 5904), (65661, 92916), (54428, 76482), (1025, 34415), (63761, 30038), (93025, 15090), (98807, 93426), (57562, 59615), (84884, 30620), (75066, 71824), (51199, 37934), (95530, 15260), (513, 98278), (62995, 28267), (47535, 69776), (39266, 4696), (14742, 13225), (44268, 16548), (45976, 41680), (99638, 30285), (85609, 5578), (28156, 12884), (76188, 86573), (44639, 15480), (86789, 72636), (18702, 13337), (96638, 59398), (90988, 26909), (85069, 95193), (67262, 38337), (51694, 19659), (93341, 80988), (48733, 88460), (55630, 22866), (96203, 10316), (30644, 68318), (79292, 63136), (60185, 73681), (60474, 19078), (48721, 82811), (19713, 99527), (17537, 55235), (296, 8353), (58691, 72158), (66734, 92490), (87642, 7174), (78285, 35337), (19503, 14273), (10713, 64116), (85966, 98738), (56561, 99347), (14869, 89963), (95126, 30748), (89272, 79060), (69888, 7193), (32583, 74564), (95542, 52128), (78360, 42675), (86062, 68404), (38732, 21411), (92935, 45415), (99027, 83925), (73232, 37405), + (97079, 87722), (49453, 61633), (53860, 29672), (29513, 42837), (47802, 37592), (12469, 34249), (42792, 53643), (62613, 70068), (82526, 73215), (72285, 11113), (42500, 55478), (43717, 95240), (43797, 75361), (24335, 33138), (38245, 66964), (20717, 95342), (64691, 18064), (48154, 59381), (25699, 77506), (67122, 65358), (4706, 47639), (39616, 49499), (81811, 43780), (835, 85236), (55259, 36091), (66392, 64552), (18111, 14142), (3313, 24306), (16039, 50370), (13767, 95591), (3619, 17285), (91602, 86378), (65370, 31997), (73103, 79544), (56207, 1833), (83803, 12639), (99772, 62410), (48525, 27025), (4843, 17363), (90649, 69831), (38294, 83908), (62986, 50254), (80915, 86505), (77831, 38427), (65269, 10998), (18364, 35533), (90495, 89902), (18124, 73639), (37972, 64523), (59756, 55630), (96021, 46676), (62423, 13536), (57039, 79285), (22584, 69218), (82257, 47164), (85219, 84268), (99430, 87817), (54061, 57126), (95756, 20489), (27341, 67329), (26826, 81082), (86892, 10021), (76753, 35118), (45122, 27808), (16706, 37697), (54734, 85826), (61753, 99199), (54920, 42265), (79397, 32234), (95638, 71031), (66412, 11193), (1874, 46395), (43748, 82517), (99596, 63908), (83987, 8436), (52027, 64590), (2821, 71063), (38625, 47246), (64890, 18073), (78221, 45134), (51729, 8336), (7235, 32732), (69849, 4355), (66632, 29817), (87115, 20363), (95882, 30431), (56386, 94579), (90145, 75474), (41608, 31413), (9284, 22127), (60184, 97820), (70813, 48129), (84426, 2932), (74599, 76277), (19551, 2074), (83741, 31228), (24540, 69367), (66373, 48734), (1043, 52758), (55207, 67301), (68930, 95643), (60576, 84717), (87418, 61921), (40078, 11839), (26131, 84239), (57553, 30194), (85992, 83655), (9469, 26355), (7871, 58669), (66599, 94551), (73474, 91775), (49288, 37125), (64459, 41703), (20761, 11140), (18453, 75828), (7666, 3242), (3851, 64702), (99727, 5458), (66247, 1430), (53801, 94055), (18633, 22045), (5594, 41215), (23845, 66849), (51801, 46932), (63960, 28483), (41699, 56219), (40125, 68122), (1809, 49948), (39206, 71380), (29721, 38251), (10858, 91494), (33459, 29030), (230, 28395), (45666, 41675), (31763, 68954), (28498, 58275), (27329, 14755), (98543, 32649), (26836, 19108), (31349, 45705), (89891, 19691), (42716, 14912), (34712, 4213), (72574, 37957), (29648, 54691), (77851, 61441), (62360, 25693), (78942, 30840), (58808, 36761), (39400, 96765), (77844, 33261), (98738, 80688), (99234, 82018), (49364, 46291), (12030, 32499), (42946, 19435), (34021, 40823), (54070, 84294), (76238, 50858), (94576, 62781), (4823, 17885), (55861, 33029), (43468, 78885), (15775, 25990), (2481, 64154), (65398, 30074), (85540, 50290), (28926, 8236), (75476, 56523), (51363, 82037), (97952, 75624), (71526, 55313), (55143, 49146), (11616, 79576), (83981, 24799), (30197, 96196), (46658, 64430), (34062, 422), (91483, 37099), (88286, 27917), (92932, 21740), (66567, 3590), (35787, 37973), (22601, 93171), (91215, 51041), (41702, 3198), (78731, 89390), (46850, 34544), (49404, 10082), (6393, 62145), (39241, 12836), (10383, 56740), (50850, 84828), (39908, 95738), (78309, 9041), (7898, 34833), (80987, 44693), (66170, 90006), (57599, 82993), (33458, 68423), (67786, 72838), (20364, 89240), (66760, 10965), (77090, 39427), (8706, 77011), (76225, 66045), (24829, 16580), (29040, 10745), (25973, 68024), (42886, 90000), (5125, 90725), (71675, 42961), (88074, 15503), (7096, 77238), (12467, 7224), (94344, 88876), (91643, 6834), (99776, 88805), (88992, 21613), (27629, 72345), (99262, 53013), (84894, 7422), (86469, 44852), (44709, 82358), (90338, 63426), (70093, 65813), (18014, 91297), (38135, 18842), (83643, 25418), (61985, 23996), (63495, 91872), (65518, 85038), (19747, 18165), (60916, 55986), (73826, 68279), (26147, 73224), (79606, 35342), (84063, 47648), (41115, 19541), (3710, 46873), (79926, 96227), (77885, 38548), (31121, 40100), (17197, 42700), (89898, 65060), (70854, 69897), (47212, 78340), (8871, 20368), (81502, 76700), (6251, 92364), (20652, 81152), (60197, 15839), (57972, 74525), (51527, 19454), (43081, 37404), (11551, 38858), (73710, 36789), (99064, 62900), (16292, 61362), (5438, 13708), (65610, 23813), (14701, 66586), (96560, 25606), (6101, 50386), (76171, 13159), (453, 51097), (80871, 29995), (65414, 10781), (56201, 59852), (57872, 29305), (85387, 49642), (98332, 45308), (78934, 88320), (27288, 84343), (71539, 48945), (17348, 35121), (51107, 73807), (60739, 89966), (49803, 53931), (91339, 18943), (66923, 8421), (1314, 61554), (56175, 73062), (1215, 51606), (16335, 1289), (62888, 62858), (26620, 51572), (19757, 93782), (95016, 49464), (77323, 61370), (14358, 40637), (75344, 25972), (12606, 53465), (22258, 96976), (36276, 85574), (93544, 82426), (50554, 1078), (54132, 59093), (83143, 37847), (36778, 7827), (42525, 47562), (48399, 16894), (48812, 94330), (2197, 63615), (16681, 83461), (61452, 74775), (33179, 39047), (11357, 45663), (34429, 3118), (97453, 71262), (76788, 9452), (42050, 94414), (62412, 57976), (10366, 59637), (91804, 41509), (66410, 60144), (20517, 85756), (16402, 50019), (33062, 94128), (66717, 95079), (80801, 53200), (43638, 16443), (68690, 30310), (78330, 31709), (53288, 51831), (59612, 15340), (69197, 67661), (25729, 54544), (61292, 3077), (91198, 68925), (94243, 84911), (81175, 41496), (56087, 30799), (19192, 5965), (6401, 98593), (90599, 40564), (99636, 3224), (13595, 15079), (81758, 98510), (31894, 87256), (32717, 15856), (58045, 25722), (80820, 39359), (89127, 50007), (57129, 30173), (143, 82490), (67320, 89298), (78031, 70771), (56372, 13856), (53792, 93923), (77536, 97686), (34281, 88893), (7720, 38598), (42577, 15445), (97049, 25292), (5755, 75274), (68500, 88440), (65460, 91968), (82251, 62700), (58737, 28221), (33966, 31858), (89283, 6137), (95713, 57892), (64716, 41893), (23226, 72596), (77494, 1226), (57456, 97085), (56638, 73195), (86791, 95487), (38616, 3710), (27612, 88402), (86211, 61383), (74751, 67963), (35294, 45969), (20040, 65633), (46379, 55793), (5254, 33070), (96039, 46828), (53169, 86515), (61456, 32728), (19306, 28894), (266, 73943), (6298, 78481), (55429, 72954), (4013, 62916), (19413, 82227), (97767, 41262), (1754, 66034), (62168, 95138), (8502, 1232), (87736, 22730), (94519, 57526), (47840, 52733), (57430, 11063), (48203, 88666), (94187, 17306), (51254, 3414), (42847, 70850), (30517, 37298), (51002, 3187), (69065, 57819), (28158, 31948), (28641, 57066), (64401, 41541), (16769, 93104), (36026, 40379), (89972, 21007), (23129, 3532), (25076, 78327), (91716, 52771), (85414, 74704), (25078, 29180), (82397, 27229), (71809, 86957), (84698, 34879), (28573, 40238), (79803, 740), (9445, 37484), (30310, 44337), (48396, 61468), (55206, 32145), (85429, 8394), (57022, 16754), (20686, 31181), (49381, 90708), (86368, 80026), (12503, 98384), (17987, 17590), (87975, 47488), (1917, 40089), (28921, 51966), (75061, 47560), (47682, 85166), (44983, 43829), (65132, 39721), (63156, 48271), (43469, 44812), (92402, 52827), (93574, 93874), (49160, 14154), (17068, 32747), (24013, 48763), (44388, 91612), (50092, 46361), (12638, 67958), (22733, 48580), (89692, 17614), (76259, 80333), (60667, 81071), (11866, 60670), (44127, 52256), (91538, 16284), (3947, 5775), (24013, 87274), (61890, 35907), (40033, 65872), (82217, 28133), (92448, 66250), (69722, 79165), (45642, 94585), (19332, 52617), (68376, 74661), (67538, 36822), (44251, 51303), (21737, 91360), (89816, 80022), (62003, 95863), (91553, 89883), (81327, 60251), (79081, 77218), (83875, 50172), (64753, 45810), (31661, 24768), (37225, 43323), (7290, 90747), (83726, 40577), (67955, 25873), (64937, 91179), (43298, 1399), (63408, 89816), (7356, 46305), (45184, 57862), (53297, 20667), (31709, 43903), (62673, 94822), (27913, 75493), (24687, 80894), (98262, 61113), (53123, 55460), (35147, 99271), (27646, 34622), (73077, 30910), (20512, 11203), (33770, 99560), (34580, 35708), (67059, 99164), (72256, 23281), (92534, 87165), (52774, 85396), (56154, 92895), (24460, 62535), (56634, 4948), (26182, 65572), (99426, 1748), (31410, 43026), (24172, 44480), (9426, 61375), (21718, 32700), (91428, 80453), (27315, 90992), (2743, 10613), (31248, 75906), (99086, 51477), (44829, 14817), (89794, 9674), (88775, 1393), (94268, 66168), (76314, 51601), (30142, 76441), (39895, 74493), (67323, 13617), (54056, 77441), (64209, 30237), (81819, 2685), (74961, 36558), (54405, 11789), (47419, 18927), (95646, 97923), (6793, 98367), (84729, 55346), (26209, 51661), (88902, 47479), (29248, 6257), (90293, 56023), (13659, 85240), (33402, 22946), (98715, 46806), (59061, 14084), (63647, 85523), (16920, 66956), (33806, 17029), (5921, 55022), (56918, 8478), (26353, 8772), (73846, 62583), (66724, 13905), (81927, 13344), (8535, 73788), (42475, 864), (43233, 31055), (25826, 92107), (58400, 93558), (76480, 27917), (11783, 45463), (1994, 30049), (69277, 48157), (64234, 49337), (71741, 56314), (2005, 69772), (49003, 37695), (39968, 77681), (832, 10155), (57104, 19308), (96734, 11547), (30184, 42421), (48375, 44830), (93937, 72937), (90182, 73901), (15777, 13356), (23590, 41957), (28339, 72344), (57738, 11792), (46981, 22885), (89701, 63079), (18853, 8142), (71023, 86607), (43044, 84076), (15504, 63914), (10852, 24752), (84420, 8955), (52478, 86472), (31493, 34784), (63120, 80839), (99483, 7075), (63463, 9701), (72217, 81768), (92665, 41759), (61299, 30348), (75839, 10811), (40924, 73497), (16847, 11969), (62221, 40296), (86514, 6737), (71717, 21760), (72280, 22605), (84073, 26361), (35313, 38026), (88244, 41921), (2395, 44993), (43095, 4491), (65029, 92486), (77825, 37626), (86268, 93758), (12282, 96669), (70109, 3854), (88201, 48985), (67432, 22291), (71226, 78916), (93242, 69896), (45961, 28858), (37543, 83277), (33989, 19303), (75997, 92827), (81996, 50401), (47766, 89974), (6969, 58504), (57495, 67704), (42186, 15837), (16266, 31543), (53482, 57047), (6741, 53863), (69708, 62402), (47622, 71680), (56407, 59557), (35497, 43626), (94382, 72849), (85322, 82494), (48283, 84203), (60187, 43506), (30227, 72397), (8179, 46362), (91848, 6724), + (44252, 49652), (62329, 71957), (48136, 53803), (29408, 16811), (2807, 35831), (3982, 42400), (31819, 922), (60411, 29397), (63298, 32886), (56696, 94800), (40452, 78058), (72632, 46188), (6187, 49011), (41030, 50745), (37287, 2821), (85833, 58997), (69513, 2449), (67890, 79660), (10510, 65067), (81559, 31412), (55173, 70275), (30720, 65244), (39362, 55066), (82509, 54146), (37012, 50941), (7777, 73746), (55324, 68529), (64001, 14553), (75092, 7732), (30396, 68267), (17785, 63304), (60691, 75626), (73360, 24028), (49094, 44015), (38186, 58436), (85214, 87107), (28851, 30981), (97558, 16884), (50485, 61227), (66390, 14523), (91346, 12746), (27417, 80041), (70675, 62543), (30493, 34005), (93733, 65419), (84518, 73912), (95889, 56587), (25811, 87640), (67088, 17670), (61030, 10236), (63615, 23071), (95064, 2080), (67171, 84194), (29917, 74112), (69466, 26813), (65199, 34701), (73665, 70408), (79992, 52475), (85736, 3601), (37979, 55469), (19825, 77218), (31666, 7906), (74130, 1569), (51690, 48772), (17872, 80141), (26340, 29232), (5680, 4004), (72841, 60873), (65725, 86711), (39659, 76703), (72206, 4834), (25126, 62083), (85881, 81324), (36485, 89451), (66626, 2033), (23961, 13900), (68404, 32573), (6568, 75904), (5463, 40836), (82243, 72476), (41469, 85222), (53063, 80905), (89420, 89405), (64438, 54748), (84609, 19925), (58071, 76697), (92927, 37184), (86525, 43159), (90683, 47762), (4221, 94429), (68456, 34730), (5303, 27512), (3857, 87130), (25963, 49542), (68981, 6398), (16900, 38579), (44288, 3921), (41848, 6266), (14235, 18205), (93114, 74697), (31693, 12438), (93873, 3135), (39867, 42033), (39257, 83168), (91557, 2168), (25761, 99686), (33891, 81917), (69185, 25496), (32937, 32667), (53201, 77822), (41625, 73595), (75159, 2574), (41500, 24049), (18998, 17778), (55319, 14793), (98343, 55538), (50306, 92789), (8111, 36989), (12332, 90370), (50098, 64743), (518, 42569), (53218, 75332), (37930, 85988), (26622, 30733), (72656, 87213), (62011, 83799), (74325, 39325), (61424, 29292), (38885, 35172), (48893, 53493), (5365, 46863), (35192, 50130), (67377, 31556), (89291, 38089), (7639, 18196), (66102, 23782), (94128, 13012), (14739, 40637), (38124, 70538), (21143, 9114), (81628, 1281), (2385, 12814), (97713, 12931), (15392, 71469), (285, 2139), (75082, 86935), (53815, 6664), (21102, 26350), (7191, 63079), (77811, 54178), (94993, 43113), (23712, 9041), (5151, 47040), (48565, 21701), (68877, 29339), (41207, 91567), (52928, 39020), (90950, 13773), (9269, 45202), (76822, 23962), (66347, 97641), (39410, 6179), (544, 78664), (6168, 22720), (41653, 97913), (17421, 50726), (7841, 71433), (54395, 26607), (4269, 30336), (34319, 79247), (95612, 91933), (25657, 23904), (19990, 4678), (87858, 51586), (5060, 74367), (56282, 50346), (24967, 12978), (5413, 84898), (83630, 81450), (4445, 3246), (8686, 37833), (25753, 491), (64949, 79083), (91227, 99361), (15478, 9776), (19138, 1197), (90231, 21775), (1603, 3420), (80483, 91945), (26254, 47364), (39956, 39011), (21944, 68718), (17476, 26688), (35124, 95520), (80675, 86048), (37802, 24654), (35991, 30501), (58201, 32076), (73425, 48851), (32081, 62678), (43890, 72473), (55420, 55864), (92730, 83636), (98991, 57739), (16554, 8671), (97393, 74717), (52946, 40432), (54038, 57550), (28407, 59994), (51960, 69485), (54362, 59837), (56608, 44803), (87765, 65240), (83537, 20410), (12462, 63600), (29652, 60603), (33694, 86009), (6328, 2247), (1163, 66579), (14339, 11994), (84291, 50111), (68889, 88342), (10269, 72875), (66319, 32677), (13064, 29930), (32190, 83783), (45970, 36250), (76934, 56387), (86991, 52198), (595, 49931), (56487, 35023), (64301, 3898), (62751, 16083), (76962, 42724), (25144, 84082), (42577, 55365), (35491, 62043), (98690, 95354), (78142, 63556), (43994, 19756), (44371, 88026), (79528, 38378), (60538, 95096), (94043, 64914), (56328, 52212), (23125, 3718), (68265, 31800), (50257, 30242), (90023, 16264), (26536, 54846), (56408, 26069), (48835, 90685), (41440, 85486), (50439, 28977), (57270, 58062), (93385, 81991), (41747, 90412), (4397, 54135), (27833, 63238), (30047, 54689), (97424, 22639), (38666, 7212), (11002, 23871), (94214, 17282), (38570, 36544), (21281, 2550), (18772, 9450), (21573, 89501), (72980, 3658), (4270, 75217), (84311, 52335), (50238, 7027), (79880, 13083), (4453, 53612), (42094, 83535), (26820, 29389), (68926, 37356), (41068, 99341), (17804, 36252), (95293, 20765), (16191, 10046), (97430, 9242), (8161, 65278), (57448, 35098), (94391, 77130), (98916, 39103), (20350, 32970), (90646, 11890), (75362, 5602), (27639, 77663), (43360, 46034), (22402, 35513), (8137, 28845), (85004, 55522), (41422, 53954), (31450, 3352), (43577, 62316), (2055, 68345), (81241, 88166), (99688, 49801), (34150, 62685), (61452, 89940), (17271, 17048), (35360, 54074), (24514, 86758), (65291, 94634), (56371, 29354), (17017, 58323), (97782, 38903), (64924, 7863), (26686, 44764), (6946, 65671), (19164, 55697), (53736, 61363), (16698, 88605), (68917, 45223), (61620, 71030), (36833, 87), (25499, 92341), (64999, 60843), (10332, 23741), (44183, 74772), (5876, 83182), (2777, 13943), (30518, 90419), (95186, 2879), (67770, 28915), (63386, 30643), (32598, 27470), (22690, 14598), (33055, 49882), (66771, 87914), (1528, 77971), (35398, 19178), (66537, 4756), (45085, 80236), (67054, 78236), (98986, 54864), (22249, 20937), (63417, 4603), (77613, 1198), (6611, 68649), (73627, 56112), (73633, 97606), (99775, 39893), (56752, 97490), (32399, 69269), (62186, 76302), (94184, 83140), (75653, 83050), (61775, 15249), (47463, 90032), (21937, 11962), (62241, 55183), (98504, 4933), (66272, 28609), (67746, 30488), (54196, 35321), (26259, 63765), (11312, 41065), (15815, 63183), (18767, 28578), (88996, 89259), (5635, 82121), (82835, 33221), (23282, 57401), (32823, 83397), (9333, 22645), (93949, 88014), (53441, 1659), (7606, 35735), (52971, 83457), (77912, 97683), (33195, 462), (36642, 84826), (14627, 39789), (35158, 74576), (31178, 3298), (78999, 38000), (58392, 70574), (32797, 9393), (31745, 73068), (3463, 99157), (85198, 99633), (69136, 61700), (90269, 56346), (54888, 30161), (69578, 67718), (31362, 88940), (72151, 69802), (10518, 42128), (10389, 85928), (79024, 46427), (75298, 320), (61374, 92005), (52693, 92872), (90677, 97987), (4977, 7002), (34950, 40356), (1286, 84570), (83587, 44056), (60303, 67073), (90693, 54889), (3665, 65556), (36965, 91476), (36018, 1114), (64173, 73176), (74375, 37973), (40226, 4894), (15708, 44097), (58497, 62654), (88190, 8945), (89623, 42745), (34666, 46515), (69908, 60462), (85257, 89711), (95343, 25345), (9671, 68154), (43560, 12783), (56946, 86701), (97027, 16286), (73465, 82933), (60864, 98196), (65681, 78455), (95949, 98824), (60427, 79639), (62125, 96638), (44349, 55436), (23889, 98287), (7982, 57100), (89134, 62354), (90345, 13136), (87117, 55670), (87142, 56737), (12166, 62622), (25757, 83242), (22839, 70866), (55230, 87710), (52836, 58660), (52001, 11928), (32300, 43940), (82815, 15966), (25995, 35077), (85590, 71892), (65700, 97621), (93120, 45889), (39090, 18624), (40711, 54078), (21512, 71018), (84273, 83224), (94455, 11165), (44762, 81935), (32383, 12092), (57681, 45595), (34900, 10701), (75918, 15671), (80894, 47445), (84887, 65029), (20046, 53478), (35117, 350), (58605, 95757), (41129, 70721), (17956, 60019), (92256, 31486), (25628, 39061), (46442, 95181), (99796, 98961), (84447, 67126), (90905, 35020), (44920, 85041), (29355, 81130), (27574, 74365), (18795, 87608), (9134, 94033), (40788, 33406), (66374, 27589), (35828, 79467), (63337, 21310), (19850, 91859), (8457, 46910), (17247, 54352), (78157, 12984), (19014, 95404), (33696, 61297), (84179, 86141), (88420, 35793), (56841, 54309), (61086, 51639), (51422, 42981), (24495, 8959), (44560, 17680), (5010, 310), (7658, 47071), (99542, 84835), (32524, 20855), (74043, 44891), (86735, 20535), (73782, 59995), (93939, 67476), (3553, 2160), (28263, 96116), (43233, 84344), (82394, 41116), (94208, 54259), (19293, 78307), (97691, 89913), (69180, 92822), (54514, 17070), (41565, 19577), (56860, 57944), (79429, 90334), (14209, 63872), (8193, 47375), (84457, 5729), (35499, 29880), (71046, 96629), (78320, 16678), (52171, 9008), (10285, 77600), (89222, 56452), (74877, 45130), (45122, 7477), (27680, 36119), (53572, 67978), (84553, 96126), (79947, 45957), (62290, 6001), (24885, 66150), (74756, 8350), (19600, 26665), (26140, 12798), (346, 52401), (55107, 64736), (55227, 20192), (5250, 75970), (22597, 36117), (46057, 6297), (146, 34226), (62101, 45872), (50973, 81711), (28437, 90316), (87949, 33752), (97761, 97260), (33866, 10748), (43546, 29031), (94371, 94717), (23676, 36834), (78530, 40195), (9449, 90868), (4688, 18113), (6478, 91149), (5770, 53359), (67467, 45123), (99961, 85269), (16771, 9423), (86093, 12345), (57781, 91310), (64235, 75090), (44983, 14518), (97295, 86692), (9084, 63091), (95536, 58091), (19196, 84870), (30135, 5353), (8031, 34065), (30428, 11126), (64190, 14178), (31159, 97833), (52514, 36312), (39470, 95767), (56521, 58481), (67491, 58112), (93348, 30378), (68108, 71570), (1098, 967), (38546, 81282), (56825, 65979), (59069, 72390), (2824, 13703), (48333, 14500), (9893, 25478), (2911, 47477), (58177, 26157), (82958, 11140), (29280, 90771), (87961, 77195), (41212, 31814), (57596, 21467), (95701, 96920), (42264, 95301), (40893, 490), (85295, 66203), (82996, 55114), (7692, 97829), (29833, 90468), (11917, 11893), (29516, 81375), (89726, 44500), (8932, 49236), (46908, 11625), (37562, 86013), (77455, 10903), (71128, 96631), (14398, 25421), (22273, 24791), (53432, 89441), (65639, 13703), (87847, 91934), (79196, 83398), (93841, 8300), (98217, 31103), (74206, 10777), (87556, 26040), (3488, 46363), (5052, 37977), (13148, 84770), (57802, 44741), (52562, 89370), (45417, 28092), (10850, 53735), (55528, 13265), (49639, 60069), (24297, 65409), (17691, 74780), (14056, 91384), (26678, 79745), (20819, 65318), (5109, 36921), (32429, 44351), (49300, 87487), (91490, 17942), (91165, 36259), (19889, 92478), (32525, 24314), + (32207, 12383), (18946, 80847), (26629, 99576), (23663, 33753), (78661, 43289), (42971, 85652), (13148, 37633), (7460, 31621), (7106, 19344), (56623, 40597), (3931, 46383), (40928, 80864), (54411, 42893), (97994, 37208), (98704, 34056), (28221, 65071), (17035, 74621), (44980, 99037), (74685, 40941), (51689, 87959), (1603, 88894), (58816, 80911), (6600, 88186), (51598, 56572), (94570, 53971), (36829, 22411), (78272, 19485), (27782, 91187), (50738, 34908), (77246, 37199), (54696, 25448), (22518, 20075), (93517, 9215), (63893, 61738), (1803, 66542), (8740, 44384), (50214, 2536), (82978, 22366), (30379, 1906), (69739, 14688), (32367, 37176), (94639, 5870), (28089, 44553), (23425, 12581), (55612, 25746), (10872, 85235), (78490, 47300), (53808, 99715), (73003, 81707), (69544, 44704), (49921, 45937), (53055, 64486), (29788, 41235), (66977, 32399), (14272, 19156), (52628, 43512), (84512, 95549), (37407, 73000), (97454, 5645), (94560, 22177), (43121, 24751), (16392, 71632), (57061, 95049), (78509, 12622), (19152, 31950), (60772, 58980), (80874, 81688), (57277, 94240), (72106, 83377), (96685, 38772), (67254, 25148), (55927, 21049), (60288, 62456), (70931, 77751), (79197, 16179), (3655, 47431), (46830, 73545), (45050, 41379), (27458, 17406), (39099, 99237), (67490, 80382), (62723, 79115), (8353, 32032), (23488, 12738), (4013, 67892), (26464, 50462), (35943, 94375), (44186, 34987), (60927, 88755), (8465, 63630), (66062, 34368), (88036, 58046), (58249, 81975), (18237, 47105), (6733, 82935), (78749, 39033), (15131, 26574), (33524, 8929), (41839, 16312), (55673, 27094), (49538, 11713), (88371, 69776), (94252, 13464), (5832, 84375), (89357, 64815), (46801, 11748), (30666, 81254), (16789, 10599), (42935, 98646), (66143, 20772), (44556, 23422), (46018, 16312), (22259, 77171), (56983, 86832), (49921, 22125), (92296, 45486), (69764, 34023), (9637, 96817), (51206, 30615), (10348, 92983), (14472, 99323), (40262, 97192), (2191, 68285), (22924, 25679), (5067, 66212), (69487, 49838), (20052, 16120), (95935, 71293), (1776, 51796), (23090, 30667), (52957, 44078), (91342, 24546), (50425, 89005), (53426, 47583), (6584, 10139), (3118, 13516), (55369, 99320), (63349, 89873), (64571, 74984), (26517, 2840), (33325, 27702), (43660, 87008), (25645, 63104), (26144, 93808), (38030, 8593), (46594, 22518), (17472, 60358), (23471, 75904), (64653, 25665), (58930, 2800), (9714, 21659), (90877, 15172), (24169, 33165), (40301, 28982), (5835, 58106), (47425, 55922), (85182, 8833), (54791, 76061), (43334, 95233), (88895, 11611), (67974, 99700), (42359, 7619), (46091, 3188), (96126, 1150), (67300, 70263), (15504, 6792), (43488, 51128), (59882, 82061), (53674, 82213), (13942, 71867), (37229, 44172), (8234, 10116), (63955, 93663), (90556, 81422), (36156, 23916), (80004, 32883), (97483, 91439), (40380, 75895), (49030, 13388), (10112, 23706), (12693, 54634), (89114, 44239), (79449, 97096), (93945, 17715), (84610, 91409), (77676, 32755), (96528, 43811), (32771, 17368), (72726, 51584), (68373, 27912), (13120, 2918), (32440, 89244), (94188, 42530), (47136, 16569), (38629, 72378), (6704, 33393), (29123, 2559), (4782, 19801), (67017, 24504), (10924, 38797), (8704, 22146), (32103, 77336), (38053, 7912), (6562, 22610), (27942, 63645), (40426, 37003), (21523, 64033), (46299, 23083), (48752, 46938), (28350, 22135), (14308, 14980), (14573, 24398), (7228, 37742), (82025, 86225), (53716, 36196), (81985, 16387), (57584, 81974), (99131, 7574), (62847, 67483), (86114, 25787), (53211, 78386), (80778, 49571), (32747, 39211), (71223, 56413), (60888, 54163), (7192, 51136), (53334, 59228), (4421, 66662), (33413, 84705), (56666, 78956), (61801, 42678), (43053, 77774), (63820, 28596), (1077, 97015), (64021, 64781), (92706, 3502), (10677, 85713), (25226, 89936), (72837, 6353), (48761, 89256), (65947, 66655), (72098, 73523), (90795, 85045), (55030, 92320), (55194, 86101), (10745, 5734), (88677, 87133), (73661, 34366), (45949, 27357), (88662, 12417), (8489, 13536), (27549, 4543), (92939, 3478), (54350, 80071), (18846, 12230), (44026, 16272), (61412, 29030), (19174, 95986), (95690, 96752), (89204, 59178), (88740, 16634), (86216, 73138), (78274, 75895), (33616, 9671), (34033, 95200), (34068, 66630), (86998, 11248), (33926, 18414), (70477, 34726), (78159, 51516), (39498, 51001), (8129, 45722), (80034, 74923), (72313, 14607), (24587, 66307), (1705, 17417), (72039, 30982), (38633, 26195), (28199, 39357), (38581, 11807), (86457, 96618), (26962, 65955), (91876, 59903), (73754, 98220), (52133, 25146), (66110, 69603), (1793, 33205), (15034, 44701), (51124, 76916), (39534, 36662), (25979, 69668), (57384, 67311), (18545, 60762), (19046, 92399), (38249, 4653), (72527, 59270), (33966, 37148), (27590, 117), (70374, 4742), (16192, 75527), (80174, 75518), (48577, 98517), (94414, 14445), (67138, 13404), (92294, 29477), (20990, 61280), (6082, 29420), (43509, 34811), (58769, 96326), (47885, 53134), (85917, 57373), (64717, 50457), (91239, 92881), (76074, 79095), (59275, 17799), (75883, 62491), (39406, 78272), (70235, 10333), (42309, 18408), (53739, 27090), (98800, 48751), (99156, 65930), (6271, 73084), (83758, 55709), (76940, 71591), (37834, 27418), (46486, 84091), (82677, 5668), (55483, 14400), (64749, 41246), (56980, 81320), (46656, 22155), (85260, 3342), (56103, 57517), (15434, 77526), (90927, 99005), (30948, 35441), (75700, 35428), (46759, 50091), (58497, 88465), (4893, 48180), (69671, 63922), (53917, 42248), (49692, 75520), (70709, 69868), (54172, 66609), (61142, 39487), (70390, 96503), (70529, 96890), (50848, 54706), (62574, 78585), (40976, 40546), (61448, 25487), (58813, 79694), (7978, 46692), (84810, 47407), (93301, 86788), (63513, 40417), (50816, 14830), (40051, 22510), (58231, 10222), (22087, 58099), (9757, 76220), (75890, 86576), (78978, 41357), (93283, 24850), (97099, 31675), (73642, 95617), (12154, 57246), (67695, 45009), (24512, 19561), (32912, 57975), (67101, 86402), (97078, 61699), (35404, 84433), (18387, 5574), (1180, 56954), (31887, 11863), (35632, 74282), (97946, 71382), (6358, 42197), (3005, 63694), (33955, 21482), (81930, 4333), (91114, 65927), (47204, 48539), (79942, 34686), (77140, 46671), (24959, 43593), (26774, 12024), (17931, 38033), (18367, 18250), (40538, 12655), (10024, 2337), (13012, 93366), (33214, 99392), (48339, 83815), (2573, 98741), (82426, 99315), (29161, 8227), (75276, 43314), (11764, 85873), (67647, 12433), (31911, 90471), (9087, 22041), (10617, 3373), (82133, 44117), (31131, 3256), (99076, 13086), (96657, 56552), (23455, 55109), (5669, 90191), (63460, 25790), (85510, 22302), (12906, 98397), (18329, 44604), (15748, 14050), (21155, 62893), (37100, 83871), (25674, 92250), (53171, 44750), (46127, 54563), (44166, 66056), (64908, 74534), (78505, 79266), (93470, 55884), (11989, 62985), (37465, 50339), (22878, 79000), (95605, 98821), (99575, 49974), (41632, 45612), (12260, 36028), (50671, 19437), (91453, 49879), (12875, 83736), (64102, 71173), (48782, 46491), (83745, 47638), (88148, 76992), (48165, 43667), (92415, 18463), (21414, 88902), (78393, 87130), (98155, 18890), (58021, 92700), (44621, 68966), (7039, 68126), (56673, 25907), (33837, 57344), (87382, 7033), (67819, 29142), (15461, 61937), (5731, 90942), (95838, 74174), (82453, 30836), (77643, 4537), (70433, 44347), (31774, 69412), (72181, 53595), (92692, 68594), (18251, 47790), (58848, 6565), (1826, 80845), (44441, 73811), (16752, 29291), (96792, 64536), (18171, 37623), (55235, 61816), (20040, 83863), (32432, 85684), (4782, 77614), (63342, 59121), (38926, 85577), (84429, 6450), (95889, 92209), (79472, 39518), (30545, 68613), (16797, 18913), (57243, 45824), (8263, 41386), (44982, 25137), (51794, 44064), (40504, 59817), (11905, 14718), (68061, 89924), (43733, 93778), (16215, 98598), (84708, 29188), (63541, 86310), (44070, 14118), (289, 75590), (45870, 49980), (18753, 17336), (24732, 68239), (27871, 52742), (14696, 68693), (99672, 72079), (71245, 39241), (67389, 16601), (21752, 72806), (96913, 30350), (54185, 84992), (57931, 11103), (48704, 71007), (99423, 36848), (41315, 42065), (87326, 72929), (25384, 89587), (59014, 60053), (62267, 54542), (12610, 32715), (68975, 19985), (44486, 69639), (82302, 6338), (63215, 87639), (96904, 81511), (69636, 10922), (88663, 59485), (59135, 3754), (75988, 60625), (64120, 45694), (70324, 80636), (15785, 77984), (51352, 57485), (2805, 39888), (74622, 13408), (20818, 1699), (52553, 75450), (10931, 27164), (25541, 66990), (40600, 10237), (37780, 18746), (58451, 29777), (99485, 87814), (35490, 54578), (27880, 7151), (5621, 11448), (16276, 60088), (80061, 79868), (79495, 85729), (5786, 28078), (7505, 68947), (3754, 73982), (83095, 13882), (13625, 48305), (65566, 46143), (28436, 47606), (41318, 73446), (70295, 17014), (70052, 53076), (75069, 22060), (38749, 80432), (5609, 18240), (73251, 78035), (44085, 34842), (64413, 19036), (76628, 13364), (77620, 44200), (51408, 58569), (63273, 12356), (73860, 29016), (1540, 65413), (4154, 73503), (8522, 57385), (24021, 60714), (53183, 28103), (9298, 20623), (35572, 72750), (20062, 88412), (53090, 54056), (75500, 24693), (63463, 23751), (17602, 18766), (36512, 54793), (83806, 80218), (13965, 576), (18995, 7730), (28727, 8773), (90604, 21980), (89414, 40461), (61108, 29240), (30381, 96560), (32285, 26545), (38149, 85647), (51471, 24524), (42153, 32188), (12747, 84522), (96507, 99501), (39908, 54706), (37524, 46449), (98286, 72361), (32855, 96561), (67644, 50836), (73095, 33548), (3239, 57909), (59527, 51138), (61862, 15544), (84048, 4884), (17440, 53732), (3534, 42048), (66894, 37347), (34323, 11139), (64183, 38919), (87672, 76883), (99127, 44364), (41417, 86134), (51181, 7902), (55456, 98977), (52953, 51347), (54164, 45398), (36894, 60364), (16911, 44245), (92377, 11522), (22925, 43609), (52979, 70952), (98059, 42838), (18178, 76534), (8036, 92888), (88589, 19468), (35901, 19860), (65686, 87008), (32387, 43037), (87245, 20229), (316, 46084), (4111, 2785), (89736, 56537), (5724, 30008), (82292, 28294), (38892, 21915), (51202, 50642), (99865, 33343), + (95804, 70967), (12070, 43458), (6220, 56015), (74329, 56204), (21399, 75467), (88203, 43858), (38704, 47055), (19362, 60219), (14634, 72618), (41448, 6543), (87651, 18534), (34327, 75539), (3541, 25818), (20404, 80339), (44944, 55964), (45651, 49213), (65846, 85412), (51675, 22001), (74879, 97711), (32455, 26455), (5056, 69437), (46238, 41978), (11156, 81627), (40029, 70165), (87185, 7409), (53171, 90875), (38789, 40512), (61386, 8745), (81736, 4812), (71196, 92427), (13554, 32085), (41154, 59222), (58043, 66198), (21838, 40563), (33897, 65211), (28903, 30143), (21409, 47493), (26059, 95417), (73900, 48368), (18774, 30490), (73012, 44125), (58456, 61155), (60554, 28996), (8827, 36373), (30404, 97981), (30881, 46098), (13258, 80907), (72012, 13691), (23651, 30968), (2969, 50097), (33733, 79191), (10223, 90306), (98149, 11670), (45547, 25317), (77139, 63654), (55598, 73329), (22887, 43966), (73233, 39855), (71344, 61217), (4060, 79956), (73925, 30838), (7635, 14926), (53804, 37839), (69335, 59926), (34807, 32895), (78509, 73239), (36494, 91580), (66250, 40428), (92495, 92721), (13503, 8974), (1533, 7446), (84074, 88477), (17188, 48916), (88681, 88849), (96162, 65641), (71517, 35507), (53803, 30244), (54097, 55628), (58528, 34100), (15910, 46623), (89311, 69918), (34706, 33424), (52525, 7174), (36409, 22829), (16575, 59807), (26247, 36935), (58999, 98875), (73380, 90788), (11800, 64690), (71312, 64235), (35708, 76049), (65621, 59223), (94220, 15399), (32073, 46262), (38017, 57787), (85453, 57087), (57354, 18076), (15620, 9885), (54585, 11470), (86532, 11214), (90392, 36339), (41222, 63156), (41956, 46957), (11257, 59047), (7904, 91165), (33757, 89829), (36561, 76484), (24535, 70034), (58932, 52035), (44009, 86647), (21710, 66730), (19750, 19959), (12488, 23120), (3857, 92163), (1605, 57740), (9460, 45243), (63737, 91447), (97379, 73859), (24109, 75972), (88375, 99620), (11175, 97854), (78504, 60445), (63669, 89304), (37661, 27971), (56797, 37186), (49704, 37165), (58568, 72097), (66051, 28778), (37075, 75947), (38543, 51647), (75077, 39030), (77878, 37117), (75140, 60743), (18900, 6472), (48578, 88913), (26467, 92904), (55006, 10313), (20860, 923), (96408, 80209), (68515, 29809), (11686, 36655), (37673, 68001), (71190, 98343), (9866, 26660), (52912, 14692), (71503, 17364), (22754, 80041), (20097, 45934), (85767, 40813), (8326, 21813), (71122, 42467), (11149, 52952), (99239, 73844), (46925, 31154), (4219, 6051), (4349, 18054), (57355, 88606), (79627, 89547), (35391, 52278), (7921, 5095), (84435, 58925), (23209, 80993), (6392, 64395), (26778, 62624), (4484, 76470), (85595, 21682), (43801, 57907), (21360, 55981), (90765, 19734), (59965, 32734), (21698, 54400), (3202, 51301), (50398, 27389), (11195, 59086), (80810, 68107), (80246, 86109), (8264, 67640), (45129, 49779), (3741, 39591), (92596, 44524), (61323, 7060), (85650, 4974), (57933, 64074), (82675, 91822), (88157, 69528), (15295, 64351), (38370, 6787), (32886, 73349), (64461, 91272), (61885, 12074), (66868, 69079), (97050, 88063), (17423, 65097), (98155, 52111), (30406, 53315), (77111, 31673), (88833, 29556), (17834, 49818), (93996, 77258), (27859, 8227), (50422, 99996), (72852, 25549), (50960, 30223), (26847, 86597), (57065, 75358), (17500, 59618), (71963, 7160), (82261, 26132), (57310, 33759), (74850, 36130), (80679, 12358), (25146, 69695), (51169, 36809), (34345, 42666), (25529, 79964), (16957, 2056), (11384, 73397), (87339, 16821), (36838, 38944), (33609, 29887), (81930, 27309), (85427, 98289), (56553, 34578), (28022, 64719), (48137, 50016), (64006, 99858), (96683, 2581), (64215, 63294), (80480, 59268), (37552, 74897), (68924, 33400), (12320, 3485), (46342, 47198), (10486, 84214), (10637, 37979), (82102, 8674), (45272, 18937), (85813, 9828), (42952, 81515), (36109, 60929), (49908, 81308), (77480, 79720), (47309, 93694), (92133, 64187), (59797, 76215), (57715, 88948), (34251, 75387), (98032, 9443), (82908, 50870), (36667, 90760), (11497, 70446), (55263, 9996), (95530, 15624), (61374, 49255), (47466, 11077), (42810, 44265), (78332, 84057), (95033, 41926), (24384, 51662), (47128, 47040), (14895, 37303), (6995, 23571), (30773, 73534), (69169, 94373), (29022, 92890), (46660, 69109), (39814, 75285), (57314, 90534), (63237, 59127), (69470, 9152), (71565, 81652), (42371, 77809), (66206, 24490), (53101, 69390), (56648, 69898), (25813, 5835), (82233, 2630), (8386, 52693), (85310, 85937), (87403, 83920), (290, 80083), (9562, 46994), (15806, 73988), (54094, 2149), (6324, 79545), (97932, 65717), (33267, 33685), (49599, 23793), (97471, 77574), (36005, 7441), (80654, 94174), (63236, 7016), (93694, 87168), (14169, 63653), (79452, 21186), (6879, 93530), (44992, 4491), (55716, 78989), (81229, 83954), (63203, 84403), (50575, 77597), (97412, 87338), (26386, 54156), (35465, 54200), (42993, 1096), (19501, 87529), (23563, 89524), (85276, 24678), (28839, 73007), (54050, 80260), (41862, 45747), (45532, 4796), (59766, 78513), (42873, 27842), (14066, 14788), (42226, 19459), (66168, 97420), (73275, 78209), (755, 75149), (10424, 17589), (59752, 5362), (16612, 48631), (97558, 55845), (77602, 94389), (17033, 91672), (81010, 47646), (8792, 74197), (62409, 86387), (23940, 73202), (74326, 93461), (99700, 28417), (12373, 31471), (19780, 82462), (84100, 66414), (33739, 27879), (65869, 19590), (53768, 80619), (7632, 72890), (13016, 2777), (28709, 51472), (64803, 48135), (2016, 6998), (32601, 51082), (58866, 23304), (87096, 82571), (54567, 12251), (60574, 64548), (32946, 20929), (95087, 91320), (32702, 51680), (71934, 62082), (7598, 45620), (25713, 83796), (22509, 59083), (99368, 66914), (22928, 984), (74309, 35226), (38296, 93051), (28362, 36277), (81873, 9185), (79122, 50838), (31448, 35443), (25046, 97283), (51849, 76572), (83722, 31233), (75316, 70515), (67359, 69686), (80368, 87585), (51714, 92098), (5772, 72330), (96652, 97645), (10531, 86149), (4378, 65728), (80551, 93867), (46660, 8809), (83039, 80501), (38249, 12848), (48679, 54119), (77295, 29795), (65702, 69250), (87195, 3131), (26611, 29868), (88164, 4187), (42078, 63303), (46501, 95311), (7063, 38185), (94346, 35287), (75324, 16799), (49000, 44453), (309, 54608), (16024, 77175), (75300, 29332), (45928, 81737), (28049, 97890), (87565, 86417), (25253, 22620), (77837, 33503), (19857, 67728), (6316, 72927), (33001, 15626), (79133, 25621), (23468, 29945), (10889, 13574), (84771, 66067), (22974, 59756), (25169, 32080), (2763, 69489), (95845, 1960), (8989, 53327), (68781, 39147), (90201, 88456), (59526, 93326), (52344, 75391), (85184, 31369), (94710, 93662), (54889, 49149), (29952, 55792), (33265, 8947), (44096, 26042), (56712, 53763), (55427, 14296), (8731, 11564), (62987, 83), (53575, 58165), (63627, 82316), (50117, 30149), (20026, 57072), (83807, 29051), (77431, 60892), (89918, 41830), (80530, 98080), (98699, 33390), (86743, 62505), (22963, 23417), (76581, 89370), (3725, 51427), (26360, 93305), (65141, 27209), (7445, 52852), (56158, 4399), (87204, 7475), (82189, 11735), (62875, 99644), (94415, 59283), (79059, 42636), (27095, 24606), (22228, 41626), (15766, 33419), (36090, 64191), (33858, 28804), (40697, 70965), (1895, 48621), (55078, 8926), (84940, 99519), (50750, 83936), (63108, 59256), (88855, 52935), (62544, 38446), (76256, 21365), (86161, 70307), (9247, 80836), (91204, 88378), (88362, 91302), (69563, 58603), (76583, 5971), (59824, 38885), (40338, 61292), (99653, 64086), (52198, 77598), (19667, 6), (30378, 41690), (5130, 21465), (32633, 80628), (69869, 59610), (30453, 13017), (59836, 37885), (58131, 42797), (3428, 5984), (17299, 13377), (68491, 26265), (75600, 36301), (79469, 65256), (98438, 92022), (40135, 69628), (52373, 11613), (12460, 96481), (36740, 9873), (92746, 96113), (32532, 75810), (45871, 37825), (13323, 92457), (11853, 716), (59161, 91872), (33609, 76950), (50145, 9040), (82818, 64780), (92700, 31810), (5745, 30311), (19252, 34721), (91678, 50602), (48847, 98284), (55640, 96281), (60726, 42098), (4494, 48219), (61428, 76363), (91534, 87454), (73201, 9683), (72379, 68604), (30607, 71891), (71963, 41250), (17585, 79804), (64529, 84544), (44429, 46924), (30658, 76431), (18422, 23472), (88171, 11830), (66762, 7895), (74476, 53181), (15386, 51972), (55536, 25039), (68166, 15362), (24986, 96477), (88289, 76764), (48193, 92905), (95456, 25805), (78131, 37308), (32236, 81885), (3083, 33218), (29652, 41185), (42417, 13339), (71362, 33608), (25475, 24287), (20460, 82673), (94769, 76462), (12527, 82290), (21183, 45096), (20544, 30257), (69721, 12886), (4, 36149), (23696, 29088), (86144, 92852), (66152, 22322), (22307, 63335), (25128, 35257), (11046, 4279), (32665, 25699), (28784, 44818), (59793, 40459), (13846, 39333), (59816, 7467), (40106, 36228), (13565, 33667), (17482, 80138), (37698, 95869), (16491, 96398), (53057, 48448), (71592, 23507), (98947, 62014), (85328, 91232), (94951, 11033), (38509, 57252), (70111, 63297), (36914, 97310), (86662, 26580), (36754, 49831), (35316, 42861), (49326, 47462), (47854, 49167), (61206, 69078), (67416, 3064), (33631, 27193), (56786, 41757), (14000, 63627), (98455, 72204), (11796, 3153), (49366, 6099), (16114, 6729), (59942, 41512), (85308, 60723), (60186, 8930), (28450, 20028), (54132, 40793), (20190, 39153), (91793, 59468), (48855, 49249), (3252, 88308), (55024, 19544), (47999, 98668), (3625, 66538), (27774, 17048), (49182, 25298), (10689, 72768), (64034, 38635), (23327, 65875), (18376, 42416), (15686, 35974), (22283, 56738), (14419, 36171), (397, 11730), (50136, 7806), (37842, 47683), (606, 5298), (66652, 8471), (23268, 88344), (8181, 42348), (30015, 82155), (65670, 4409), (29161, 1269), (62267, 51181), (38076, 3852), (14048, 61904), (6132, 3284), (22486, 26772), (76678, 2511), (16212, 13228), (51402, 46487), (99360, 11669), (36934, 58173), (71427, 76232), (14509, 22573), (98403, 76905), (24839, 32100), (71097, 7056), (47286, 70888), (90192, 33568), (38798, 54148), (20350, 66499), (93010, 48622), (13510, 34716), (92779, 14508), (57010, 91250), (25212, 10989), (65459, 91132), + (66251, 59146), (28920, 29336), (86273, 50897), (81311, 53009), (22278, 85352), (54231, 26356), (94250, 51117), (21152, 38875), (39286, 22339), (10290, 37852), (43426, 20385), (13603, 2341), (2314, 12288), (48785, 80096), (16762, 92809), (1689, 16006), (46828, 44433), (58147, 68612), (31702, 34921), (23807, 72511), (63437, 35834), (40643, 62303), (78198, 57748), (11496, 43030), (91150, 85333), (91817, 95369), (1792, 59975), (14816, 41573), (9355, 32450), (77243, 5352), (17681, 11128), (69700, 89110), (37695, 98303), (23160, 29924), (83829, 29122), (13009, 81770), (64481, 60303), (6842, 78532), (16282, 32405), (14204, 67463), (41795, 83324), (98134, 38132), (93686, 89418), (83031, 18065), (96431, 28312), (15488, 90870), (90914, 35142), (96174, 98411), (93075, 61533), (47394, 30134), (46309, 98014), (8574, 38881), (42376, 78153), (45938, 95700), (14026, 14059), (94138, 70460), (62882, 3321), (52628, 25566), (42145, 97919), (57580, 8897), (1766, 74887), (68391, 51311), (16485, 36095), (55867, 988), (34600, 46771), (35786, 23210), (78683, 62604), (11002, 87427), (29747, 6771), (53803, 50526), (52691, 75371), (58028, 36198), (98041, 28071), (81822, 4121), (90995, 93843), (3709, 74313), (23318, 28167), (60162, 87145), (53243, 31382), (69762, 54059), (25151, 95865), (55550, 62947), (45463, 8036), (59949, 91675), (55331, 79612), (13806, 57517), (41328, 94361), (8388, 210), (43458, 1327), (16416, 50975), (64370, 86524), (97121, 65142), (82878, 64545), (58547, 33372), (67494, 4130), (59446, 79185), (83812, 58833), (32172, 93730), (10095, 30319), (94135, 92795), (64893, 12870), (14634, 70083), (23344, 42836), (19624, 29135), (28755, 57738), (58401, 49863), (76809, 91787), (51952, 33167), (31717, 78538), (64811, 55131), (61969, 12339), (4900, 48310), (67609, 55369), (37333, 37245), (91492, 19704), (77592, 62200), (1017, 41815), (41362, 13627), (46703, 92040), (4664, 46637), (25430, 87837), (10270, 32932), (46025, 34792), (80621, 38362), (79949, 31518), (77767, 59212), (11080, 69948), (17376, 92279), (73871, 86715), (83524, 21178), (94368, 61971), (11262, 30062), (12840, 52656), (15300, 64176), (70094, 42565), (84503, 66968), (65824, 63349), (74306, 82791), (85701, 1655), (25190, 70557), (82208, 58145), (97669, 92489), (62730, 52924), (46970, 15748), (71217, 57505), (57336, 87380), (13982, 71979), (55899, 1192), (6890, 69297), (39849, 72896), (60213, 22499), (53820, 79877), (61389, 24954), (1021, 81047), (95389, 90605), (8590, 26855), (92881, 46825), (59567, 91455), (87297, 1567), (86333, 82913), (16285, 32370), (38900, 77251), (96301, 92302), (2799, 91579), (59757, 56681), (49229, 38404), (16880, 51290), (38700, 80456), (51440, 32556), (4507, 83325), (29924, 40923), (9937, 58356), (70895, 52611), (80488, 49196), (93118, 6112), (44058, 20923), (70808, 72928), (78269, 76351), (81081, 63186), (12406, 50247), (91769, 37875), (36197, 11434), (72225, 93141), (59295, 54160), (39342, 10117), (12920, 23240), (91877, 61524), (84807, 5431), (63163, 52152), (90320, 97456), (14899, 12941), (90079, 98004), (46096, 68208), (12176, 72405), (28451, 61964), (89254, 1676), (76242, 5438), (29756, 92658), (14855, 41944), (1519, 65495), (74052, 65006), (1075, 31521), (84877, 98988), (91426, 69442), (39694, 43391), (68328, 3447), (50508, 61610), (30890, 18792), (60706, 87518), (6692, 83260), (92044, 79399), (86846, 53543), (28606, 61656), (35449, 52965), (43421, 13117), (67404, 55238), (80817, 79927), (8908, 46083), (22583, 19705), (13, 25509), (79771, 21210), (42719, 19320), (94091, 35389), (95995, 42652), (6417, 13464), (22290, 34089), (27119, 33408), (7, 68019), (55405, 50766), (6331, 50009), (33, 48235), (3162, 85313), (25897, 91775), (30813, 83408), (36062, 46449), (78820, 78894), (92054, 23008), (10637, 4237), (90702, 62678), (80427, 98641), (20834, 48068), (74443, 29196), (76088, 91823), (39151, 48574), (54050, 5374), (60643, 1979), (4535, 95086), (82101, 67586), (55746, 92488), (88192, 22615), (93282, 87547), (37804, 61180), (13445, 56654), (38913, 10130), (23472, 83441), (94125, 56393), (2967, 25078), (76652, 84599), (3586, 83503), (46272, 62119), (26002, 55790), (75085, 55875), (73786, 22608), (63101, 59758), (9688, 11910), (54041, 38376), (75595, 95722), (36047, 59518), (55041, 69102), (42836, 48225), (30046, 61157), (60608, 94951), (80257, 70300), (40282, 67847), (90735, 56299), (82814, 89675), (2998, 52357), (86333, 5088), (3414, 67855), (28134, 64041), (56871, 38384), (51335, 28574), (73451, 40395), (59696, 49729), (73379, 78187), (67561, 79682), (66114, 9153), (4613, 41137), (96832, 77637), (64216, 22050), (69871, 69883), (78030, 8142), (88000, 98796), (24965, 60211), (61727, 82326), (64623, 21387), (27307, 39907), (47314, 29509), (14676, 73902), (48864, 28948), (15974, 66359), (68154, 2013), (44016, 51609), (95421, 89481), (39985, 22127), (88096, 18483), (18539, 98797), (83613, 26308), (93997, 84284), (35958, 46289), (79263, 50800), (4873, 92142), (69509, 97567), (17733, 9050), (14632, 79901), (94211, 83353), (72112, 79110), (13974, 83858), (76735, 38529), (67084, 67183), (97827, 95135), (55771, 88062), (94630, 45565), (20922, 85088), (9180, 39703), (86146, 21086), (65955, 51015), (77660, 57572), (31577, 56429), (52636, 56363), (74365, 62124), (60971, 13103), (75234, 35529), (26035, 39091), (1145, 38641), (24088, 49898), (83610, 25448), (8868, 32413), (20877, 9306), (40824, 73012), (63105, 93113), (21170, 21234), (6564, 21210), (27020, 39734), (7431, 11643), (56609, 23596), (2023, 39429), (63574, 88918), (5188, 46658), (3290, 24577), (33271, 95049), (72950, 94811), (74876, 73210), (6294, 42155), (13767, 27366), (86746, 54073), (13001, 93376), (13517, 96879), (24380, 59684), (49105, 55403), (90533, 2745), (48651, 23491), (21409, 71113), (7142, 71273), (62952, 88544), (82865, 10938), (65089, 17672), (43173, 47506), (13708, 32610), (26547, 77317), (8676, 7117), (62512, 39512), (42233, 81433), (45416, 63643), (32392, 33632), (83887, 402), (53596, 54897), (59707, 85005), (96343, 20979), (30344, 65226), (34513, 24150), (38715, 11205), (20649, 7679), (27948, 80080), (77430, 49123), (42437, 4729), (90176, 34827), (11039, 59573), (47042, 38460), (48086, 34987), (65457, 5585), (45956, 6198), (22786, 24937), (54287, 77515), (12113, 84710), (29543, 36869), (8388, 60473), (2771, 53359), (35299, 64874), (34132, 86266), (37594, 64297), (33390, 65843), (53057, 46715), (41874, 83250), (26061, 30352), (93351, 16651), (25890, 16570), (67403, 19546), (8608, 51991), (90616, 10018), (46870, 54345), (81919, 3577), (99462, 3562), (83193, 66098), (92054, 70338), (11383, 90149), (42458, 2302), (77850, 47281), (62433, 4959), (52392, 33711), (2033, 4096), (91460, 6795), (27462, 76972), (73087, 40511), (42563, 5924), (21819, 53017), (31309, 62793), (19798, 60725), (90987, 77178), (9949, 43079), (13773, 40312), (97080, 61184), (59679, 78611), (22361, 83227), (70120, 95483), (15529, 5672), (25893, 99672), (61722, 69201), (55012, 37509), (77999, 85866), (77835, 83627), (36792, 80218), (54232, 58138), (74593, 485), (45698, 4153), (96513, 738), (33060, 53078), (4439, 25097), (62007, 64140), (86128, 99499), (66141, 4433), (41032, 54527), (59831, 61170), (1189, 68117), (63914, 26798), (72045, 2999), (6016, 42337), (40611, 39747), (88328, 44836), (88939, 25365), (40010, 43971), (21419, 31975), (27725, 21929), (49459, 74763), (32059, 96546), (70985, 98967), (33034, 36073), (19600, 80524), (59150, 56734), (30433, 38795), (55387, 47581), (45109, 73833), (59487, 5428), (23140, 57618), (36367, 31008), (45983, 85518), (28234, 83843), (73005, 5096), (65602, 83431), (97198, 10971), (4873, 51104), (49700, 89396), (18900, 20447), (76400, 68088), (28055, 59547), (94737, 96052), (57933, 96132), (26945, 6293), (18559, 1317), (66327, 78332), (47194, 67979), (78039, 58735), (26666, 96144), (91574, 52530), (3216, 43850), (48367, 78987), (78296, 89800), (25015, 1497), (70292, 37629), (39657, 59240), (22576, 93232), (70102, 98407), (35051, 5762), (13700, 82540), (36857, 7528), (84536, 86107), (1258, 90702), (9152, 1127), (3136, 57298), (73203, 93120), (61897, 32903), (20284, 49358), (49444, 65481), (91260, 84792), (5360, 37164), (79212, 26499), (59456, 37025), (67953, 53566), (50253, 71742), (68181, 90923), (88522, 71071), (89615, 46115), (31661, 17673), (37699, 90994), (41974, 9481), (51974, 46364), (47470, 75958), (38427, 78267), (90226, 16121), (33257, 3088), (73044, 79701), (14149, 36868), (6421, 63653), (35772, 56493), (46665, 83651), (82299, 56268), (77397, 71795), (15180, 87135), (73936, 38858), (27060, 14829), (85847, 76756), (63068, 61864), (92040, 43753), (53882, 60715), (12648, 18276), (17873, 94081), (55982, 24983), (52686, 32111), (22555, 13144), (37190, 15039), (47304, 86179), (56214, 83657), (53838, 78111), (80530, 1686), (2976, 26711), (55713, 3487), (21157, 5923), (74613, 76555), (61736, 7322), (99017, 13779), (64133, 61706), (8276, 97269), (99641, 87447), (26292, 62924), (96267, 37877), (98635, 88685), (65598, 68210), (34885, 1063), (42880, 15694), (4058, 72666), (1319, 91582), (41749, 822), (82490, 42783), (17489, 17617), (81109, 16778), (86262, 53756), (15547, 21144), (89799, 70465), (99358, 87334), (65501, 81932), (30351, 64230), (55974, 46985), (24343, 93628), (4624, 56038), (15183, 38765), (21562, 5240), (36215, 51533), (49649, 74392), (7607, 91204), (86233, 85341), (35494, 44178), (78808, 11896), (45446, 12897), (4810, 20096), (3643, 10550), (12022, 14391), (99014, 87891), (67214, 29437), (7015, 54810), (84991, 75251), (45526, 82687), (43608, 89083), (96191, 12911), (36902, 50415), (90548, 66839), (54007, 5480), (57191, 39709), (73364, 73556), (12671, 64072), (5047, 12887), (22326, 88360), (59259, 63167), (39237, 9311), (47043, 11512), (32399, 22634), (89485, 95279), (30489, 43327), (70966, 18184), (28514, 12686), (1717, 89938), (14871, 76911), (47862, 97093), (97572, 77612), (88600, 36501), (9240, 96264), (96629, 98292), (4838, 72694), (52835, 73039), (63836, 26228), (26035, 74526), (23617, 6104), (6243, 54603), (20038, 81195), (56789, 96284), + (97009, 18989), (36765, 94831), (42002, 40820), (31838, 16186), (91127, 70783), (1684, 38076), (40928, 41294), (97107, 76517), (1797, 24093), (99175, 29445), (5788, 64502), (66972, 23772), (97394, 95256), (26705, 92412), (10676, 6686), (28464, 74051), (17598, 51675), (76449, 50050), (98812, 59818), (92058, 6625), (49804, 99637), (28545, 32790), (72167, 93636), (29012, 95623), (58249, 54541), (98926, 71489), (75062, 9546), (47555, 5802), (61606, 30979), (10467, 46108), (30491, 52750), (52934, 38533), (54616, 37276), (55118, 2330), (60402, 14806), (25730, 88759), (5209, 30946), (92017, 41987), (59747, 62415), (6306, 53297), (35770, 31254), (13106, 99865), (63519, 41637), (7416, 68154), (70874, 95983), (20074, 2173), (33740, 86880), (45020, 65869), (7302, 53494), (98814, 2118), (88287, 80646), (18771, 30449), (44557, 37984), (45415, 36032), (36584, 58485), (57843, 52354), (85647, 12150), (33990, 89380), (98597, 41936), (19427, 53990), (1457, 69269), (68438, 14471), (67175, 16881), (82982, 78546), (47661, 36294), (63229, 48108), (34390, 76095), (32101, 57268), (25322, 3312), (17037, 2900), (14342, 70149), (10518, 1377), (13553, 89073), (97084, 80365), (48340, 87192), (84969, 21195), (1031, 72944), (78322, 68583), (67238, 48374), (77366, 55850), (4383, 46759), (51088, 76488), (92049, 55847), (32750, 16422), (51788, 53619), (56976, 88951), (73323, 78435), (3489, 15945), (84231, 22302), (25695, 881), (19391, 11660), (92578, 78349), (52677, 17339), (99132, 35975), (25089, 70476), (34144, 66544), (59589, 59839), (47333, 87623), (4200, 6542), (91781, 77813), (77419, 17342), (388, 90753), (43401, 7351), (81829, 74989), (22381, 95695), (90302, 61687), (13534, 54534), (43527, 47703), (47856, 1186), (10112, 16747), (9508, 26179), (96536, 57565), (91052, 48922), (38575, 45523), (11602, 85317), (22595, 21610), (97849, 15638), (95060, 20111), (96011, 27148), (37596, 10426), (68667, 49057), (68657, 90882), (52704, 33574), (54727, 59704), (15853, 48283), (47021, 60021), (96208, 11440), (6068, 49421), (50363, 17684), (53218, 19), (82568, 3816), (51, 7612), (7073, 59063), (92318, 38587), (72388, 64952), (33318, 5594), (72806, 18573), (1175, 88747), (67044, 86039), (17376, 41265), (26356, 89524), (55678, 15627), (61156, 15723), (43239, 59010), (29054, 15261), (57669, 21395), (68469, 14849), (92654, 69664), (23975, 20021), (73688, 12134), (58053, 23849), (56234, 18559), (25051, 68957), (15181, 91615), (87633, 31040), (17494, 78515), (71971, 60923), (44170, 21724), (7088, 72107), (12924, 47933), (98139, 6819), (57842, 2906), (90278, 3534), (61695, 82885), (58831, 15129), (32033, 34634), (63477, 31357), (29077, 28166), (86531, 89574), (46943, 79113), (15667, 63523), (65818, 93225), (83392, 50497), (7157, 88909), (95101, 17111), (8981, 35791), (28328, 82722), (70215, 66862), (23442, 39128), (21218, 27773), (76215, 39265), (56858, 62397), (3610, 8766), (7423, 48886), (12235, 88792), (44839, 55196), (4896, 10080), (87090, 73997), (56745, 71875), (58478, 57472), (79423, 8771), (76202, 95733), (81635, 48857), (82519, 5321), (48756, 37198), (20101, 61177), (79047, 81682), (23287, 88578), (50984, 80946), (855, 49074), (11452, 56655), (6289, 8916), (33570, 636), (28703, 80592), (11820, 92299), (75590, 80188), (91365, 2787), (22963, 43710), (3996, 48007), (59047, 96418), (96710, 52447), (65677, 99927), (60727, 15115), (8161, 65729), (75253, 45731), (88130, 79264), (52087, 32657), (89678, 22834), (49244, 36835), (74032, 5096), (94142, 39297), (51061, 21311), (59391, 43458), (90918, 95081), (6058, 37018), (29917, 54214), (65986, 5543), (99777, 69683), (9447, 11273), (54873, 47138), (1306, 68306), (15316, 55995), (69280, 73432), (63054, 28502), (67161, 11333), (40800, 26415), (72350, 50225), (7232, 67500), (60631, 40338), (96765, 51671), (29375, 11235), (52707, 93742), (1317, 55926), (51844, 29351), (35754, 16733), (23014, 56950), (33119, 47531), (27806, 39940), (20119, 73119), (18458, 45982), (16605, 28759), (43488, 86201), (43560, 82883), (85085, 1977), (7129, 64077), (36070, 80679), (78638, 97212), (71251, 5268), (33610, 63341), (16722, 15882), (53876, 11734), (35855, 14481), (55072, 7889), (37213, 73390), (28899, 55948), (51996, 77777), (26621, 51914), (44492, 36420), (36437, 24452), (158, 77935), (3730, 7692), (81372, 29050), (29491, 3912), (57098, 73393), (55630, 53165), (79003, 19421), (47306, 40128), (96332, 49995), (54107, 83940), (11203, 76678), (53854, 284), (96087, 22398), (25481, 69301), (2770, 77586), (53004, 44915), (56264, 41165), (12335, 22090), (47188, 40678), (41823, 71805), (81810, 47825), (98892, 50696), (74323, 93133), (82867, 40800), (28884, 29276), (55697, 20888), (96901, 37428), (95509, 78780), (6443, 65188), (17669, 43924), (29933, 97244), (1024, 71265), (47229, 99610), (72546, 40811), (43407, 41566), (44692, 53743), (76823, 95950), (2509, 57091), (85077, 96439), (87650, 87716), (3822, 84513), (90864, 75439), (22043, 21770), (98371, 75482), (63823, 48100), (29999, 40875), (97696, 33865), (50880, 16402), (21412, 70936), (7150, 85970), (73104, 57073), (40099, 89138), (6430, 36791), (47677, 23485), (21073, 86912), (65018, 6092), (89932, 58178), (1264, 51681), (42044, 40135), (21774, 12190), (13844, 87188), (25807, 89548), (74682, 31603), (18482, 41232), (906, 21841), (10897, 39238), (62134, 6083), (80871, 66850), (49659, 28587), (61260, 22769), (53795, 66637), (89211, 32453), (4271, 69472), (1082, 32636), (62012, 47870), (7376, 77166), (13150, 73275), (66410, 85959), (19018, 57311), (30561, 58838), (19981, 32087), (91580, 42990), (77139, 23774), (10055, 99890), (92160, 54537), (75088, 70397), (2361, 61066), (81368, 40411), (84946, 42564), (41591, 42929), (50399, 26116), (51504, 63772), (99828, 69531), (19977, 17806), (71259, 76567), (78606, 25105), (8859, 46469), (19542, 73988), (49804, 17349), (65360, 96632), (57740, 30812), (9258, 26185), (90573, 4815), (39397, 10229), (35300, 81687), (5681, 22828), (6274, 38340), (44281, 67912), (38385, 26507), (95324, 84765), (39288, 79779), (4513, 28158), (46060, 40293), (70348, 87460), (67754, 19569), (70481, 94407), (67636, 69316), (14167, 81744), (18928, 29706), (32034, 45677), (19801, 5902), (42401, 62712), (35220, 22823), (33690, 22046), (91281, 55953), (4197, 96279), (32901, 27074), (35861, 5478), (99166, 36251), (70898, 80994), (87491, 52808), (59681, 30266), (2944, 18646), (77874, 7536), (24534, 5416), (90962, 47145), (95367, 34461), (10075, 74120), (74276, 1837), (38399, 63918), (10526, 42662), (33001, 37773), (13642, 25741), (5373, 82728), (9830, 54625), (13, 18937), (90113, 52300), (20973, 89640), (44930, 1598), (8100, 47745), (80659, 78877), (74154, 33417), (18691, 37573), (12477, 76628), (73018, 89583), (59029, 24879), (85237, 31017), (27701, 11025), (24010, 76942), (15627, 25276), (89050, 16118), (19776, 19318), (24620, 1487), (50157, 95158), (53030, 72020), (85571, 10817), (29196, 92799), (63501, 22445), (26241, 54318), (33769, 80112), (25312, 50574), (70583, 56662), (74366, 82575), (48461, 80545), (18834, 39444), (36597, 42715), (86547, 38453), (41067, 60718), (33350, 25404), (5532, 41438), (45918, 84038), (39040, 63724), (37629, 66002), (58037, 36196), (21400, 17330), (46491, 22602), (88057, 89249), (34188, 35948), (8555, 42479), (32961, 76606), (22395, 93711), (8058, 19591), (72063, 64136), (83360, 12607), (20671, 78285), (51887, 65373), (71966, 39341), (55953, 99780), (56205, 98138), (45986, 44708), (62541, 28259), (49450, 12641), (5192, 36601), (55295, 44352), (83762, 4343), (96203, 8870), (21786, 73828), (88830, 43827), (96669, 84761), (53061, 50061), (52150, 47463), (32090, 17570), (71495, 45027), (30215, 23717), (37599, 19789), (85208, 48465), (83599, 78596), (82141, 37105), (74943, 37580), (10322, 11520), (550, 55062), (80506, 57713), (55489, 80453), (64174, 49274), (75521, 51005), (26033, 58801), (78661, 50218), (31043, 17090), (70612, 88736), (42991, 49396), (86331, 2300), (66156, 6331), (34534, 78649), (9828, 40937), (35852, 46533), (68435, 12070), (26443, 75772), (50802, 16400), (69413, 89875), (2372, 97252), (88334, 76754), (29504, 25950), (10792, 12935), (3655, 90667), (35951, 16244), (73312, 82435), (60914, 58748), (89407, 86955), (47210, 84195), (77084, 4213), (15419, 5719), (12483, 39092), (61890, 83360), (94785, 37525), (69219, 83790), (79946, 42672), (36898, 9693), (10103, 82234), (71679, 18297), (88171, 43635), (94828, 71283), (76485, 56217), (70048, 92642), (12567, 8082), (10011, 76160), (31060, 30787), (199, 65728), (371, 74295), (25505, 76411), (70522, 8776), (58390, 48835), (88649, 10695), (21754, 5143), (62512, 77235), (837, 57129), (86979, 33448), (21219, 78593), (10048, 22504), (82658, 80394), (79451, 81707), (5940, 72284), (1555, 70284), (65988, 40439), (48165, 92837), (33680, 39653), (96852, 38612), (72548, 82912), (78383, 68910), (69985, 12164), (24334, 14856), (27180, 67558), (62131, 29120), (28744, 40472), (79896, 25579), (7172, 42815), (81070, 85827), (76227, 51604), (70056, 61128), (58609, 12641), (30207, 33772), (7940, 82930), (45176, 28620), (43176, 9513), (55515, 5743), (92762, 34063), (26540, 95690), (69518, 13909), (24403, 65731), (41317, 12732), (78020, 50738), (64622, 2001), (43604, 83548), (46739, 79104), (57132, 2737), (16823, 37703), (59239, 68217), (49630, 75156), (19535, 63429), (19685, 63032), (93214, 97701), (21809, 44089), (42700, 64026), (37310, 12041), (84416, 39080), (70589, 59876), (14692, 82526), (36230, 38375), (7345, 48208), (65006, 85251), (31754, 26359), (65198, 7876), (38917, 16682), (39527, 24043), (28543, 48728), (38065, 98259), (57509, 71565), (7706, 85866), (13059, 22966), (91722, 83066), (45269, 68810), (82194, 85898), (78369, 11390), (79637, 70777), (87544, 74018), (73661, 4252), (98305, 21715), (19375, 35375), (28324, 48444), (97548, 89589), (73031, 60766), (34818, 1986), (6206, 26488), (67681, 6380), (98946, 84665), (98568, 37447), (48649, 89141), (72561, 2077), (47055, 63199), (3956, 93341), (69392, 51121), (71277, 40181), (8509, 18405), (77921, 54923), (92137, 60652), (30271, 32980), + (60001, 93265), (69240, 45514), (57985, 21681), (35096, 6101), (49506, 76284), (57253, 21325), (98280, 15826), (1605, 80735), (74930, 30151), (58035, 93534), (17384, 31491), (10971, 7060), (73339, 68186), (88781, 52103), (11265, 46720), (5115, 30293), (75078, 87304), (43439, 10072), (27578, 33361), (90981, 10158), (24465, 57967), (49938, 61510), (55637, 44188), (93963, 32210), (68414, 40035), (77261, 88709), (55900, 34360), (39675, 19672), (66218, 19215), (74660, 74886), (82862, 19255), (78221, 82950), (71132, 13770), (3866, 56030), (78929, 78110), (48142, 5965), (80128, 31846), (72670, 2531), (14391, 67079), (60542, 10720), (91632, 40402), (607, 14720), (49500, 13967), (70778, 83672), (46041, 37708), (83720, 574), (57212, 80758), (28854, 46858), (39588, 19052), (50803, 71521), (66830, 71799), (21002, 61070), (96634, 39051), (4459, 26782), (63650, 82860), (24099, 69429), (80492, 18147), (41087, 995), (41768, 79625), (48190, 75710), (43300, 78132), (53778, 79772), (30657, 89149), (75840, 5676), (75810, 52995), (36563, 23231), (25721, 55136), (15, 23168), (63731, 64535), (67136, 4648), (25464, 28946), (95215, 98695), (64093, 28899), (62809, 40760), (44188, 52915), (82358, 98065), (24244, 56401), (61636, 63848), (95428, 93970), (93539, 88332), (57401, 68773), (53502, 33067), (3147, 65487), (75811, 11287), (58826, 7349), (57494, 66861), (99762, 33031), (68299, 20951), (26524, 96915), (14698, 22392), (55683, 85344), (25084, 10755), (52343, 73138), (99839, 61148), (33383, 66190), (18810, 10835), (918, 55695), (2664, 46566), (93105, 21138), (27088, 16730), (70156, 4974), (10294, 62096), (30808, 18639), (25091, 43231), (17920, 90076), (40104, 54257), (98729, 93113), (92419, 16006), (14446, 8769), (24875, 34761), (99525, 34543), (67461, 47601), (91998, 5899), (19623, 58521), (97278, 63650), (8396, 20069), (29854, 22735), (33695, 25257), (70338, 3989), (14296, 96530), (13790, 1350), (33144, 55649), (29072, 26337), (68611, 7960), (81765, 15614), (16633, 80649), (89942, 31234), (50827, 73426), (2282, 12998), (8566, 65109), (65979, 23638), (24802, 56043), (15638, 54100), (51431, 80669), (30744, 9349), (80148, 87322), (49673, 22775), (2096, 72107), (29876, 11845), (99687, 4215), (68797, 41891), (11363, 79333), (22351, 68211), (41273, 34990), (11414, 58625), (83047, 28825), (44579, 31219), (8753, 35994), (8321, 20399), (97524, 48382), (81485, 69822), (38382, 42853), (97045, 81294), (95109, 9825), (24648, 92398), (23245, 28557), (9114, 30750), (66151, 5307), (28704, 42997), (76185, 18506), (39182, 28385), (12660, 487), (81939, 70723), (78752, 4369), (49729, 16078), (27760, 78394), (27935, 65985), (97170, 17974), (45484, 51849), (37792, 42775), (50788, 50243), (29730, 5771), (8337, 32970), (87282, 34536), (82654, 40374), (25936, 99591), (87105, 44686), (61326, 23201), (48961, 39310), (72404, 34074), (47084, 53440), (44566, 21551), (27348, 31756), (14445, 57694), (3090, 78935), (37165, 19519), (42821, 24648), (4987, 91817), (15902, 33578), (63028, 39291), (9567, 41818), (68851, 3311), (1641, 59714), (51932, 83190), (40656, 19271), (44882, 35988), (94241, 78634), (15970, 79214), (11262, 56304), (46638, 79764), (9206, 54131), (77132, 24211), (38417, 34891), (55196, 29011), (64734, 42845), (33857, 44248), (40460, 66582), (34542, 273), (84169, 65245), (83434, 25030), (10937, 1189), (35213, 89760), (48231, 5149), (49920, 97597), (56463, 13926), (35160, 4587), (91526, 85881), (38624, 54542), (17138, 93601), (16139, 72928), (93357, 75097), (84733, 99796), (39474, 98875), (97908, 58734), (97237, 91217), (77132, 32920), (85467, 32730), (83289, 18742), (2980, 4330), (44920, 11778), (30191, 82779), (34431, 45207), (75529, 10325), (4657, 36429), (97464, 18604), (29451, 14997), (40066, 22598), (49471, 66259), (65164, 98515), (64098, 47692), (87542, 84786), (41170, 49570), (11476, 14536), (82271, 79916), (45889, 42059), (84965, 90533), (88733, 20381), (86349, 66190), (83280, 32409), (38436, 35910), (94362, 2535), (13565, 12567), (72551, 58564), (886, 87999), (73958, 94309), (34030, 60715), (25432, 92697), (85922, 24244), (46779, 52990), (33807, 44621), (39513, 51405), (16668, 32353), (60629, 14813), (73095, 12227), (17801, 38366), (65927, 52833), (33630, 50134), (19975, 71026), (93145, 90036), (73555, 6383), (71789, 81791), (67429, 79036), (20944, 39911), (75838, 56480), (29835, 80194), (19570, 28901), (16023, 82165), (74259, 15283), (11749, 92274), (73537, 52362), (54537, 37595), (79004, 72436), (49229, 21498), (24364, 97373), (81149, 61392), (22421, 6442), (62138, 86081), (16098, 23700), (66604, 86467), (66505, 26263), (57215, 12659), (61267, 77421), (58024, 98298), (44273, 89699), (62339, 73107), (32540, 17768), (31574, 77296), (63336, 23439), (26723, 45141), (61380, 58474), (6188, 2084), (88225, 6631), (10970, 70428), (91223, 64312), (89643, 76635), (42796, 77154), (34827, 21131), (83014, 2253), (34547, 12384), (53922, 57591), (12066, 44400), (89686, 21284), (81875, 3306), (45972, 11207), (96807, 28447), (87662, 42334), (68578, 37404), (21195, 35583), (97730, 41852), (76425, 22959), (63724, 64143), (79182, 50739), (81472, 79682), (26980, 62332), (60691, 50596), (38526, 10488), (74569, 46431), (62677, 60189), (35133, 21147), (3271, 93368), (14651, 76080), (91349, 87952), (7873, 51144), (72431, 54715), (15600, 69868), (46998, 23805), (7519, 75777), (71758, 71673), (23282, 37297), (35646, 89327), (44890, 58293), (34223, 99155), (87015, 18477), (20511, 30210), (82120, 13850), (6382, 41946), (97741, 86276), (15521, 44501), (87240, 57472), (65334, 51695), (34599, 97721), (45366, 9007), (91971, 7384), (74698, 40735), (73706, 9025), (42569, 18928), (62946, 62362), (89350, 83255), (26044, 85080), (27725, 3814), (45787, 32352), (16115, 51701), (3339, 72851), (60161, 34406), (24699, 43473), (28681, 573), (95106, 16784), (74232, 5831), (55361, 1366), (7020, 37807), (52490, 8558), (91203, 64944), (90783, 64700), (76499, 12592), (97070, 9395), (46602, 12688), (79052, 31675), (29316, 93503), (14095, 3100), (15066, 86579), (24651, 10981), (90683, 70162), (60604, 89175), (55172, 34997), (61098, 17843), (2376, 21399), (38186, 60680), (58637, 11541), (11101, 69351), (70200, 67918), (49245, 85252), (4201, 24436), (56636, 87099), (21406, 33757), (29044, 43846), (93900, 68223), (34429, 92683), (76720, 79334), (4035, 30364), (20145, 44853), (46689, 21352), (49446, 77224), (71241, 85720), (27177, 88332), (60151, 49376), (67102, 74396), (242, 26346), (22004, 95064), (2709, 71452), (71673, 86838), (36918, 17777), (37414, 88149), (2356, 70290), (84416, 35509), (32285, 54585), (9937, 23465), (3385, 25949), (97379, 87409), (72714, 43570), (54831, 64883), (80885, 81082), (89673, 5765), (84585, 51627), (64497, 72931), (71948, 60922), (70011, 37446), (81022, 43696), (11413, 58709), (89710, 85350), (96004, 36589), (99270, 82834), (52367, 12117), (62108, 56693), (69090, 34612), (56109, 22008), (36036, 34493), (88561, 4651), (44403, 60712), (91912, 19116), (40545, 94625), (47973, 46963), (30704, 99285), (9596, 21976), (84723, 81323), (98123, 90434), (35842, 29225), (91271, 14003), (42320, 16108), (14171, 88861), (15139, 84030), (31008, 58743), (86149, 98215), (59768, 15217), (85628, 82450), (14256, 2254), (44961, 44326), (86313, 69953), (99151, 37777), (46152, 77035), (36868, 62695), (37233, 49305), (20706, 56446), (75773, 90240), (88130, 91556), (89753, 38172), (31191, 98583), (31412, 81334), (57662, 15890), (9195, 3322), (94618, 22144), (24709, 24446), (54382, 68062), (17085, 30960), (53538, 12827), (12512, 56172), (26611, 90242), (92812, 98619), (82930, 90399), (2299, 45362), (48505, 88997), (58709, 77683), (85901, 33672), (45070, 24491), (79005, 44787), (7154, 14290), (56684, 35166), (76459, 60609), (6006, 47402), (54755, 19015), (5139, 33177), (13725, 50051), (95929, 58683), (91114, 68401), (88846, 23283), (37153, 36426), (36564, 32035), (49852, 49501), (2052, 59285), (69847, 54966), (82614, 10395), (72068, 35582), (50427, 19892), (75612, 3545), (75480, 26985), (24227, 50112), (22718, 52206), (35801, 41679), (17753, 44416), (4688, 82170), (36935, 6606), (71549, 86720), (53592, 68769), (92671, 84617), (97726, 13655), (65006, 17474), (25287, 80826), (22202, 25703), (98743, 71876), (72914, 31392), (54184, 1294), (9353, 10518), (93822, 74538), (85863, 56357), (56088, 14448), (12427, 27727), (1846, 350), (96301, 21623), (55225, 23813), (76947, 60946), (65400, 17489), (80586, 79099), (56913, 46914), (49218, 53295), (14085, 52313), (71158, 14144), (3641, 69327), (43705, 93045), (67880, 95849), (88054, 66156), (17425, 69320), (16391, 271), (31768, 8817), (63156, 21471), (97831, 43504), (54300, 54402), (8740, 27894), (93805, 30183), (76233, 26322), (60307, 8348), (63019, 32334), (7086, 44838), (93127, 94482), (20146, 84889), (33674, 2513), (66860, 65753), (69174, 35906), (93800, 41975), (31206, 93599), (75620, 60674), (77082, 60773), (75564, 20627), (53060, 67879), (52568, 22722), (4102, 42037), (66549, 71191), (20817, 80946), (13469, 96144), (29852, 713), (72644, 98510), (12567, 54057), (28746, 72197), (80235, 51566), (11953, 73225), (42732, 91659), (19673, 29661), (94438, 41029), (21884, 91078), (29280, 68018), (973, 75825), (83476, 79334), (99077, 625), (63309, 91304), (11626, 33709), (33735, 6897), (12585, 17114), (45256, 57549), (65150, 2524), (10399, 34571), (32783, 6045), (11900, 12875), (4393, 3129), (61420, 63521), (11799, 9092), (45887, 36931), (84256, 71195), (51949, 32406), (34640, 11424), (84866, 46109), (23449, 51873), (22371, 86773), (32155, 25902), (75058, 66016), (21217, 19002), (44067, 59363), (55901, 87102), (71213, 82390), (88408, 22154), (18231, 86852), (96803, 72776), (58608, 53500), (40747, 8542), (60870, 41475), (39578, 66560), (49522, 25492), (3485, 13917), (60177, 43310), (70317, 69166), (71910, 92946), (14427, 320), (5224, 79108), (20986, 81899), (34637, 53572), (14919, 27426), (8052, 10082), (27936, 24128), (73732, 97758), (31869, 5719), (22135, 86985), (85977, 79381), (41587, 34012), (75912, 66428), (62117, 37249), (70197, 91261), + (99703, 96452), (33036, 30249), (33709, 99448), (30343, 2962), (54245, 80785), (13866, 99804), (49392, 62886), (8530, 38228), (86249, 60394), (96787, 88788), (42724, 88066), (97051, 85589), (41414, 88972), (4384, 2025), (46524, 87620), (5823, 36079), (78720, 13519), (59092, 88699), (24120, 6388), (51664, 71039), (28756, 43890), (13729, 27176), (86717, 82448), (65598, 34805), (74975, 52791), (65429, 97173), (92472, 53334), (18778, 40576), (72776, 48956), (61833, 85936), (81344, 48055), (89467, 91099), (82333, 15367), (6019, 98391), (52943, 15338), (29662, 21975), (71935, 47606), (54633, 72216), (51648, 52203), (84649, 31791), (26281, 82065), (83163, 43394), (79859, 55897), (84706, 38368), (97175, 88009), (92800, 82164), (79819, 13645), (42754, 6273), (97025, 79579), (35694, 44160), (96053, 24801), (60315, 14411), (25961, 82303), (1791, 53355), (15508, 43236), (35667, 84655), (73764, 82190), (14465, 72390), (92464, 30555), (14121, 31360), (27630, 78672), (35483, 47439), (79467, 73233), (33010, 49718), (43095, 2002), (3755, 4070), (25460, 21281), (52718, 27575), (97091, 78423), (76536, 86761), (10682, 27325), (55401, 99488), (34196, 88793), (88024, 37693), (4586, 62614), (74857, 37796), (19724, 59109), (17642, 34748), (87983, 11996), (6814, 68602), (63566, 16836), (21026, 42675), (3528, 81994), (58141, 81954), (79520, 99518), (52627, 62974), (42056, 1000), (85464, 63420), (30434, 86139), (59754, 88493), (41386, 76058), (27894, 78093), (63842, 72458), (6928, 37187), (34478, 5193), (57116, 24009), (24858, 84226), (462, 93589), (86510, 80240), (78044, 98234), (30812, 40060), (423, 46196), (47139, 89017), (26066, 21081), (45463, 37260), (34366, 23814), (45937, 79795), (89975, 20405), (21792, 43980), (13675, 11145), (98902, 47577), (20383, 49241), (97905, 16457), (85839, 23279), (33931, 13532), (91486, 54207), (46981, 41204), (49524, 10080), (21201, 3854), (70178, 71393), (26825, 35670), (91350, 92038), (82297, 81201), (47908, 26073), (6456, 40194), (45104, 62038), (75781, 55421), (49561, 35610), (47838, 93109), (25109, 10837), (16666, 99790), (54882, 29259), (28169, 76030), (99605, 78334), (19151, 86372), (48747, 64053), (78764, 38705), (43762, 3756), (22725, 14026), (85174, 6263), (76327, 1072), (92817, 63030), (59923, 28851), (39614, 77354), (54467, 80792), (37309, 8679), (42247, 28830), (60892, 36834), (54108, 62952), (33682, 52992), (87318, 63605), (3180, 91762), (24774, 12744), (4431, 74084), (11712, 7577), (31451, 18294), (68722, 16450), (52185, 92005), (97312, 33963), (43882, 84266), (39958, 52806), (4759, 32862), (33017, 81530), (88475, 9613), (72656, 43214), (27288, 91542), (67232, 47120), (12714, 59507), (17175, 58197), (1719, 97181), (2806, 5682), (22373, 80608), (72129, 33750), (97168, 75472), (16157, 5484), (93177, 59589), (7019, 19832), (63810, 80313), (23551, 45065), (59657, 37189), (12995, 84512), (44630, 12434), (28833, 8088), (56795, 10570), (22385, 4138), (39502, 41865), (67720, 31597), (54837, 8933), (19906, 12496), (69993, 30745), (94886, 98186), (6474, 37743), (62063, 5596), (52064, 33592), (57476, 84891), (88798, 43044), (55029, 97222), (73911, 11862), (1205, 76381), (13388, 4303), (73635, 55410), (97777, 32985), (94358, 49709), (62607, 76061), (62525, 8598), (10327, 11235), (18778, 991), (16602, 1008), (6311, 36329), (6464, 33473), (71418, 15828), (49777, 9630), (67644, 45616), (97118, 54851), (79498, 75515), (45056, 12838), (51110, 53572), (4941, 80166), (16052, 81909), (24336, 25459), (60437, 70317), (51544, 8762), (44978, 27145), (78113, 27193), (45148, 7278), (78858, 90800), (29864, 14191), (67835, 78395), (45893, 74512), (81523, 270), (51436, 75334), (39456, 65552), (89548, 42368), (8953, 17953), (47296, 52855), (36236, 70545), (99705, 30645), (58038, 43017), (95851, 60592), (50658, 98282), (48614, 74538), (29026, 6898), (57506, 97917), (33553, 11205), (92292, 53328), (37861, 10865), (70306, 43839), (52283, 94600), (68802, 95126), (92765, 91310), (2574, 45229), (12463, 45681), (73331, 83847), (22565, 66014), (52454, 22571), (17530, 55911), (7358, 36106), (54186, 27674), (55476, 166), (12926, 8902), (63782, 1616), (60085, 35021), (86605, 99026), (7432, 57161), (37268, 11832), (3860, 95662), (30339, 54518), (26163, 96043), (26092, 71977), (63899, 7792), (9804, 66302), (81779, 54515), (2861, 42418), (44736, 75647), (50912, 53848), (72120, 3651), (91526, 72999), (71642, 69795), (40664, 37695), (57357, 83958), (41666, 52277), (49209, 99223), (47274, 75571), (64326, 88981), (67848, 52903), (44013, 43796), (21678, 8103), (74827, 132), (88279, 49672), (10747, 85149), (74506, 99115), (89615, 95102), (14077, 79858), (55139, 17814), (5519, 48761), (62278, 39931), (58406, 38267), (58705, 86071), (63443, 11013), (53144, 22114), (78535, 70561), (85423, 36012), (3285, 72479), (73658, 19340), (18511, 10477), (60780, 75674), (88032, 88205), (35386, 23855), (49977, 32673), (93206, 71515), (3482, 73115), (48423, 52857), (11899, 30649), (59730, 15323), (89442, 87402), (439, 50184), (98106, 42819), (25941, 82842), (82980, 56368), (71420, 11196), (20860, 78799), (71131, 53182), (45795, 85122), (34635, 83218), (78160, 79377), (8534, 26595), (35248, 17819), (54060, 99480), (36087, 84130), (16001, 59701), (63490, 15318), (41259, 62966), (79583, 94979), (49892, 92957), (8542, 94075), (66339, 33975), (21084, 67889), (14718, 20930), (1760, 60204), (42318, 36873), (2306, 42786), (1667, 59179), (29425, 64430), (64436, 4268), (52666, 79958), (11248, 35590), (37352, 72786), (79273, 24409), (47965, 59976), (61729, 87562), (22673, 52091), (58668, 33485), (60556, 6194), (9974, 17532), (91645, 12490), (16690, 75552), (30983, 79572), (73896, 87577), (82325, 6966), (46769, 19266), (6691, 2816), (65074, 51276), (83174, 210), (38395, 19191), (83871, 939), (9543, 23700), (18316, 3882), (18604, 6191), (53431, 2917), (71291, 3352), (46424, 29885), (44395, 57382), (95028, 33049), (96372, 2057), (26911, 14241), (35535, 27007), (25527, 36480), (13612, 95407), (56176, 82284), (28597, 48562), (76438, 43785), (17631, 15131), (18728, 76676), (49606, 60663), (64878, 57243), (10925, 14798), (72592, 95876), (8144, 58293), (36923, 57214), (82893, 42678), (15533, 63570), (66619, 96607), (32491, 70158), (25175, 70456), (54365, 64639), (56529, 44275), (85598, 13051), (82052, 78051), (6802, 14950), (18964, 77404), (69285, 33899), (42496, 817), (80092, 73354), (44095, 57405), (6715, 98298), (82698, 65201), (25536, 48241), (60721, 38389), (72591, 3716), (76974, 58694), (39723, 66127), (21781, 27885), (57871, 26565), (57327, 1713), (43406, 68141), (65690, 21025), (77459, 59220), (78038, 40571), (44248, 86386), (17714, 38469), (14993, 99711), (18231, 62648), (8081, 2797), (73996, 60186), (98808, 9291), (93219, 32512), (45831, 71316), (9315, 81476), (31933, 97130), (20920, 2404), (31015, 11982), (1251, 81196), (35575, 4027), (81650, 61732), (10241, 79907), (53046, 95973), (68668, 48509), (656, 1992), (10061, 89424), (11047, 71227), (10089, 90496), (57613, 95766), (38775, 44707), (10140, 62292), (91276, 50321), (75890, 59520), (32977, 39392), (43150, 88441), (51314, 81453), (57440, 53885), (85712, 10329), (52723, 17377), (29648, 83895), (41142, 80706), (4167, 43740), (49249, 91247), (15721, 53638), (95764, 35654), (21318, 81252), (97676, 77111), (27613, 35504), (20973, 47975), (60292, 36376), (78092, 59608), (53682, 29368), (95342, 22174), (2289, 80842), (97276, 49917), (21421, 37624), (50678, 98609), (49240, 25174), (64603, 13614), (75402, 61541), (80112, 45397), (14289, 87479), (16888, 98577), (14999, 80972), (64689, 57967), (79391, 54976), (37446, 39412), (86682, 70785), (21709, 57340), (53757, 39880), (33201, 94136), (27854, 30229), (14865, 56637), (75573, 68915), (84594, 1520), (71261, 69867), (84203, 43708), (76407, 88476), (84395, 77050), (53934, 83635), (32870, 27302), (46646, 46399), (26417, 60940), (87513, 88720), (12542, 32018), (45969, 62284), (26733, 90539), (50556, 13388), (60882, 56222), (48253, 22792), (68209, 18501), (58073, 48960), (6360, 4482), (53997, 31868), (15727, 78331), (81299, 84008), (99495, 89688), (504, 3147), (93985, 77265), (78723, 60673), (30278, 63220), (33434, 26642), (71481, 73333), (49805, 59269), (90657, 93630), (28814, 83954), (91466, 63542), (43843, 69657), (11466, 24136), (63826, 6209), (29687, 95777), (65616, 74144), (62798, 883), (54174, 88934), (35129, 40939), (42224, 54784), (80876, 68479), (80812, 51383), (39443, 11689), (15438, 66488), (32856, 96950), (80525, 99128), (25968, 28334), (91925, 37414), (84823, 58164), (17795, 78409), (29912, 53931), (1369, 92329), (36176, 34133), (79057, 62016), (74457, 51743), (90604, 95280), (18107, 81811), (26994, 78521), (4762, 53724), (8229, 90056), (60962, 1202), (80485, 44934), (58917, 36716), (71770, 42524), (94551, 98259), (39500, 41373), (50410, 70615), (63782, 25832), (74940, 15450), (90724, 44531), (69088, 22535), (40581, 3354), (1756, 86005), (93128, 31200), (14139, 8370), (65646, 40141), (31537, 89271), (7635, 66721), (67463, 97745), (12391, 36807), (6311, 30232), (20818, 91220), (53449, 11632), (21802, 93687), (75581, 79739), (5514, 57540), (67063, 99383), (98230, 18983), (50929, 35978), (16069, 59906), (93594, 94696), (13344, 34575), (30331, 84457), (78047, 97558), (5507, 41722), (81016, 8762), (91323, 59598), (67184, 8555), (76659, 67276), (6511, 94844), (1063, 39110), (40795, 16854), (40694, 32666), (14734, 95257), (6818, 30227), (32211, 56294), (31129, 97716), (93866, 6117), (42144, 91974), (64560, 9708), (5803, 92811), (7240, 42166), (97514, 19116), (59904, 64965), (73031, 69137), (63313, 82886), (58529, 84361), (47365, 96118), (27999, 83969), (98211, 52135), (86460, 86408), (21315, 59646), (98469, 99928), (24453, 79050), (97595, 41780), (35575, 22842), (18219, 72673), (4001, 89495), (19563, 18097), (566, 5304), (37517, 63100), (11251, 58908), (2474, 55788), (83078, 65059), (68185, 94587), (33305, 36706), (81350, 72951), (37115, 41580), (6005, 44393), (4318, 67309), (21346, 75943), (57955, 33395), (75723, 89514), (2661, 42938), (89643, 48828), + (3409, 13157), (13667, 78006), (56518, 86796), (38971, 28234), (38060, 77284), (68626, 83168), (69324, 67649), (48994, 40388), (87024, 14379), (12475, 27192), (41271, 41939), (50304, 38146), (8682, 25307), (93898, 48129), (34600, 25005), (48022, 22298), (25683, 46046), (24839, 69086), (50505, 93439), (90497, 50297), (16054, 68182), (69768, 67239), (35909, 84118), (64986, 21267), (32550, 33801), (8884, 66830), (10690, 75679), (59327, 9389), (16594, 37706), (54110, 76448), (22052, 90416), (18104, 20971), (78917, 19589), (48799, 38897), (55663, 11112), (24255, 19761), (90224, 59023), (51893, 61626), (57840, 2070), (37436, 84968), (55242, 52606), (71391, 70783), (84796, 72467), (61673, 90112), (61880, 10015), (84560, 21382), (91734, 52105), (94237, 68942), (36055, 26633), (43694, 33404), (60266, 17069), (63045, 2539), (39314, 49430), (66845, 42602), (76834, 50742), (26684, 64231), (83102, 88337), (43541, 16088), (53516, 23759), (21092, 43649), (71242, 31515), (62253, 81538), (19670, 48947), (24097, 83600), (11636, 70960), (55314, 99976), (56296, 68456), (25537, 64586), (45649, 53747), (9712, 85370), (82073, 71622), (97381, 26322), (4927, 47703), (8999, 33464), (5875, 80932), (88591, 63062), (40824, 93164), (55753, 10733), (56196, 8875), (13427, 66714), (22780, 69557), (37212, 13763), (61668, 95614), (79050, 55391), (92892, 94037), (4942, 23252), (34025, 31194), (96074, 59364), (11520, 11652), (94439, 19618), (64758, 41196), (73397, 72389), (57282, 73894), (79700, 78450), (65052, 54392), (72453, 23422), (49093, 48337), (9863, 89449), (60397, 40836), (18147, 51195), (39787, 61479), (63062, 62242), (67547, 5426), (25316, 76298), (16168, 12116), (91884, 94406), (57385, 89464), (93062, 71189), (97612, 38802), (85128, 1852), (80643, 41827), (59109, 50692), (28077, 24715), (67583, 40643), (54073, 32408), (97704, 17820), (15884, 69143), (66420, 36961), (60442, 34685), (3109, 47213), (27157, 42282), (18909, 52184), (16638, 51215), (66044, 40079), (67856, 15021), (26942, 58162), (76866, 73547), (3999, 47414), (5114, 74975), (85820, 10632), (71972, 59810), (66488, 16840), (50779, 93807), (40068, 12260), (67715, 72553), (45275, 57705), (45019, 86665), (97033, 97611), (69136, 54593), (1879, 87594), (49292, 92238), (69064, 7240), (60753, 5294), (36350, 67918), (48440, 43264), (70554, 79751), (40310, 64039), (48171, 75441), (53305, 72396), (44027, 58150), (67485, 38325), (87333, 75101), (2509, 54526), (40794, 18097), (17674, 84451), (30175, 65221), (71744, 57300), (91603, 35695), (33369, 1387), (51808, 2008), (42659, 11574), (39492, 79947), (95089, 47048), (57022, 73399), (72707, 8845), (66843, 61434), (7174, 16342), (50709, 76713), (45560, 1378), (80907, 97774), (651, 93255), (42650, 82788), (2875, 24267), (73437, 67551), (6685, 81749), (43495, 12190), (63932, 69464), (86694, 91717), (91608, 59712), (50833, 91382), (18301, 39265), (48304, 34276), (24633, 56311), (10463, 40342), (17645, 64242), (52245, 96838), (6606, 87413), (75088, 39449), (6247, 97599), (57124, 39047), (49579, 80395), (52959, 14523), (73129, 3675), (78880, 16111), (90870, 27122), (11166, 67671), (48048, 41486), (65329, 98702), (4801, 92915), (88765, 51873), (59370, 1440), (41316, 88008), (92238, 79140), (18523, 32178), (36124, 49818), (84251, 37274), (92085, 96091), (65382, 7189), (30130, 32066), (86434, 46207), (5500, 11160), (88366, 36983), (48685, 47634), (93314, 57448), (95978, 97924), (20017, 85908), (82568, 31256), (44633, 78762), (41625, 11793), (77646, 2315), (67604, 29898), (1492, 55783), (33177, 5891), (93884, 41363), (90527, 43977), (36905, 59152), (37750, 19264), (58020, 1360), (22974, 3246), (41403, 80614), (4479, 37045), (27477, 47093), (79899, 65264), (12625, 46993), (78884, 10896), (49043, 36064), (22725, 18991), (4279, 11656), (58193, 27118), (48315, 26221), (86372, 4638), (86413, 43777), (74216, 60197), (89547, 68700), (618, 62861), (88483, 17054), (59378, 75565), (15001, 59042), (15858, 10735), (42583, 57159), (27836, 12597), (87465, 37449), (95717, 59515), (63369, 36932), (14736, 61367), (14779, 30481), (33108, 95749), (5304, 7878), (74344, 65457), (3519, 98197), (34669, 96365), (31376, 21260), (75715, 11580), (14317, 29528), (10151, 13185), (6975, 59407), (7910, 71250), (21761, 8936), (11683, 43452), (97556, 87506), (87371, 16179), (46806, 74511), (737, 82723), (1197, 77799), (9835, 85745), (66026, 16556), (24784, 20015), (53234, 58687), (4393, 27429), (11484, 48640), (24376, 998), (76115, 38), (72364, 3765), (97109, 9842), (36814, 37745), (58829, 16139), (4165, 93524), (51052, 5215), (69771, 11456), (13263, 75724), (60779, 11359), (9924, 2846), (70887, 4526), (66054, 93309), (15486, 40658), (18780, 1579), (89386, 24351), (33682, 62095), (661, 84125), (13480, 97300), (2408, 25599), (99168, 10687), (84860, 17842), (38292, 12704), (97468, 83719), (38360, 25940), (97314, 65483), (95688, 44394), (47420, 96156), (66394, 61773), (98637, 46400), (69962, 57775), (83857, 45874), (74811, 5741), (96157, 62317), (57902, 79905), (73760, 21345), (58109, 65145), (58678, 5406), (96538, 16329), (42244, 63758), (17613, 66642), (72398, 5307), (12872, 94227), (65046, 1439), (90217, 10756), (58185, 63169), (50247, 86728), (98822, 4390), (82566, 26006), (3574, 42077), (43701, 59837), (85579, 48641), (47792, 34429), (77410, 8515), (97749, 11160), (57462, 33927), (32836, 57768), (26898, 80133), (52210, 57444), (33639, 288), (30309, 29233), (14867, 24307), (67420, 29214), (29889, 86125), (68662, 93027), (30716, 64262), (9145, 66627), (83817, 78000), (49074, 52390), (52974, 72121), (84006, 2545), (27803, 68907), (79008, 5828), (74872, 89207), (26302, 23622), (26263, 25254), (65652, 38273), (11673, 21629), (70494, 86207), (59074, 85729), (2011, 38108), (70239, 9100), (37062, 67038), (52681, 44407), (79718, 59481), (39506, 32648), (76903, 76789), (45162, 63035), (13344, 3835), (96239, 59063), (24507, 82181), (77403, 89247), (73887, 41747), (5639, 27352), (46718, 96169), (94630, 83108), (37999, 68318), (97717, 53272), (80506, 42486), (7677, 75413), (45129, 2230), (11721, 21720), (88499, 48784), (23501, 58439), (70767, 72086), (78533, 90007), (72580, 80411), (36564, 98451), (89444, 69448), (48038, 26523), (5123, 78026), (29168, 41429), (36287, 72619), (75350, 27908), (7381, 96048), (30090, 51055), (73670, 11691), (39082, 72957), (84785, 52400), (80436, 94637), (17776, 61224), (36848, 71607), (90182, 50522), (78350, 71698), (91120, 27247), (39638, 38298), (22945, 23976), (52034, 72847), (97888, 33382), (20825, 5844), (303, 5273), (62672, 96465), (46951, 93410), (57258, 108), (11116, 34876), (40588, 20691), (61906, 68626), (48319, 90189), (20255, 4601), (80998, 30793), (28741, 2714), (42195, 25873), (17046, 46739), (15319, 54014), (11773, 11673), (73012, 41575), (21747, 59925), (16088, 24272), (38326, 8140), (68006, 23594), (78052, 77991), (45820, 79226), (11143, 88881), (59251, 76272), (18953, 10412), (67075, 57195), (1386, 54226), (42759, 59791), (5696, 15215), (79633, 55487), (3546, 11417), (97976, 8323), (55197, 53843), (12182, 16579), (55706, 87702), (88871, 45930), (98978, 50942), (55874, 87465), (53256, 28916), (90605, 43673), (23906, 75994), (12746, 66785), (91372, 53574), (72302, 69561), (38833, 59338), (98528, 22616), (10045, 25111), (34397, 69161), (86106, 23720), (82705, 67810), (48691, 14861), (41392, 64552), (83105, 6557), (62123, 39827), (94840, 54871), (35778, 26572), (16194, 14615), (27, 8380), (33704, 44750), (86884, 51173), (60330, 80082), (56180, 47098), (6249, 36349), (32268, 12733), (86808, 40162), (27643, 75454), (36783, 30874), (54827, 7607), (54697, 57582), (83316, 18631), (44081, 33952), (89780, 86498), (63456, 27382), (5654, 13927), (41301, 35640), (4502, 70840), (4238, 62879), (66516, 67091), (96156, 8355), (2902, 23789), (98837, 77623), (86898, 91764), (75147, 6127), (8274, 46182), (7422, 18222), (86509, 25754), (72400, 55881), (84459, 68079), (5654, 76270), (43017, 78770), (45609, 55504), (67256, 51832), (42204, 15653), (52363, 44095), (11760, 24927), (53286, 20903), (97482, 76094), (43809, 8144), (82974, 74859), (65921, 42364), (11938, 55421), (92195, 10809), (91371, 77402), (17644, 86490), (4424, 27504), (56380, 99969), (1731, 80243), (8879, 23354), (94574, 42743), (78875, 2965), (1029, 61788), (64768, 61914), (54149, 90155), (83048, 46305), (80640, 10062), (12372, 94961), (84549, 84061), (99122, 78998), (91039, 95054), (19628, 18651), (52348, 42569), (58086, 76882), (49881, 13151), (8064, 47655), (4457, 63438), (35500, 82994), (47962, 43653), (5050, 36247), (96631, 67316), (49101, 96157), (15785, 96404), (56283, 36312), (31695, 24307), (81897, 59720), (51016, 45776), (92118, 47354), (16024, 38406), (57737, 68887), (28408, 46608), (65055, 1161), (60430, 21682), (22659, 74001), (35202, 60211), (38308, 52926), (74264, 26330), (17038, 41144), (27775, 24875), (49666, 73774), (28516, 34297), (21873, 11504), (3676, 38168), (90003, 25107), (45103, 7031), (71883, 40867), (6274, 71251), (60307, 49056), (58402, 76742), (10010, 36003), (67059, 75300), (36996, 57460), (61413, 54691), (99608, 49572), (87512, 44123), (43942, 87151), (18543, 43029), (886, 83140), (75529, 69749), (54090, 8088), (43094, 35468), (76460, 45310), (64690, 46953), (29561, 93370), (97598, 21618), (28166, 8888), (12234, 771), (65670, 1561), (85467, 97276), (98868, 57050), (14619, 53684), (924, 82181), (65602, 36977), (7888, 8938), (32792, 1046), (18719, 99863), (34993, 11678), (28837, 64924), (83079, 48559), (5806, 92117), (79239, 8754), (89819, 16162), (99505, 94650), (8636, 27356), (6056, 5682), (1921, 15164), (45536, 10747), (56628, 62991), (51456, 54815), (13030, 47920), (27330, 48623), (21393, 96651), (51282, 5317), (78200, 42975), (18082, 38478), (32594, 4205), (49208, 22593), (80079, 11765), (65186, 60836), (11601, 9377), (95550, 52242), (86838, 96932), (17508, 91440), (73427, 92075), (70969, 37180), (7091, 48708), (35444, 42181), (91735, 29746), (11924, 89935), (34418, 61018), (68065, 96782), (30869, 53390), (99315, 78165), (97205, 90309), (88481, 24681), + (17847, 75359), (73636, 11167), (91431, 62190), (28478, 6770), (97305, 77873), (56987, 22599), (40879, 63647), (1974, 67839), (88651, 99581), (67087, 51740), (24861, 74794), (92730, 14161), (52639, 78795), (37163, 80079), (87445, 7053), (27234, 79447), (43352, 66903), (96267, 10657), (57181, 24266), (84236, 83872), (56551, 30387), (4972, 87269), (98922, 1624), (48398, 19886), (40570, 79648), (92239, 98232), (86789, 55696), (64626, 44726), (82391, 95837), (74414, 93972), (97542, 73957), (76038, 54044), (66087, 24170), (25731, 50138), (39800, 75428), (21730, 88783), (53518, 83466), (50784, 72242), (23969, 85392), (43772, 70437), (87869, 33052), (15531, 61368), (22519, 38591), (37709, 35030), (74030, 5134), (18667, 50524), (84691, 21182), (50932, 66070), (27898, 10408), (79554, 49192), (2517, 7746), (71312, 22108), (15511, 37874), (21623, 77591), (10809, 46240), (40913, 29935), (78969, 56158), (46274, 75491), (87163, 49581), (88417, 86223), (41666, 28061), (17503, 13021), (16476, 50547), (56184, 32943), (89243, 61225), (89844, 81092), (19846, 44485), (25999, 46684), (67283, 76235), (64393, 53397), (41565, 93212), (82296, 54617), (62466, 54078), (97942, 92201), (18205, 67342), (44364, 51466), (72744, 4796), (72526, 68979), (50015, 88033), (19680, 47415), (65486, 46344), (21531, 42236), (3452, 94720), (44902, 89762), (13814, 85842), (98337, 42500), (74870, 31262), (72964, 10884), (64408, 43927), (97024, 66176), (55999, 75231), (72794, 37516), (67787, 70137), (4960, 2377), (22048, 86320), (9964, 86377), (30766, 39958), (68684, 29197), (43864, 72887), (90838, 11639), (97183, 10594), (99649, 22933), (91770, 4554), (97882, 60479), (91041, 72702), (47685, 56715), (51900, 61103), (90821, 27638), (95748, 57574), (79106, 14366), (10560, 97465), (12009, 55707), (65879, 34837), (67934, 69351), (58075, 96424), (26898, 13830), (2074, 46887), (47317, 28361), (97566, 43461), (8150, 8518), (21546, 42240), (75546, 59160), (65267, 35547), (32762, 8539), (31559, 21127), (31913, 36056), (80988, 11524), (41145, 62648), (23225, 40076), (17544, 39206), (1553, 91048), (16940, 78156), (7990, 70275), (14929, 46391), (92515, 27289), (77071, 8542), (29061, 27539), (61002, 28411), (88830, 96749), (52211, 71717), (19450, 46195), (5895, 47099), (29098, 35266), (75499, 90090), (76936, 30217), (97220, 51472), (63272, 35659), (62885, 64591), (95983, 97721), (63118, 93206), (98618, 82875), (89990, 44842), (75664, 44348), (86664, 44960), (50835, 42026), (26637, 6312), (99637, 96642), (59176, 14821), (50542, 9553), (76633, 65029), (25409, 65198), (3352, 69819), (63711, 60406), (95437, 53364), (37840, 77585), (46865, 9097), (19420, 46524), (88308, 21695), (32788, 12773), (42860, 16774), (20890, 85130), (91438, 59944), (18399, 28949), (20946, 76691), (46617, 83686), (34107, 99019), (49597, 44158), (33564, 36586), (87590, 72014), (59107, 18546), (44421, 43424), (21333, 27720), (1135, 18434), (69321, 80520), (28921, 41288), (18914, 53626), (63676, 98378), (29612, 9956), (3908, 2160), (78213, 53772), (45709, 90664), (77042, 41054), (86883, 36641), (99497, 83975), (98822, 39177), (38175, 78142), (8236, 62618), (11422, 92281), (24292, 13446), (14489, 14652), (89100, 73028), (84314, 64770), (93912, 25955), (33612, 87481), (12263, 39940), (38184, 58202), (77635, 79922), (78339, 97371), (47955, 29169), (51554, 40923), (74047, 67244), (85365, 31911), (56786, 6734), (82342, 51354), (82818, 80508), (61409, 85614), (25675, 10634), (17091, 62250), (84446, 23721), (267, 14486), (71998, 69455), (46656, 67266), (32788, 84839), (42014, 98189), (49293, 8851), (66248, 62924), (19458, 74293), (90351, 62521), (83067, 20364), (47665, 30564), (4391, 85292), (96286, 26682), (38393, 19001), (54383, 21667), (57586, 88071), (61809, 69649), (52249, 43209), (43878, 30047), (85943, 32125), (14229, 12697), (92247, 72822), (97179, 86933), (63243, 84200), (42053, 43557), (95748, 9682), (43850, 72883), (3576, 48689), (1517, 48614), (52726, 32475), (93527, 61058), (96804, 21108), (78109, 63584), (23551, 15738), (38196, 68934), (48406, 42638), (52135, 73232), (64063, 91934), (16822, 37151), (14686, 92843), (11115, 26382), (98109, 99834), (86192, 45754), (43093, 97836), (34061, 5654), (71966, 21209), (34289, 45307), (24639, 91324), (58483, 71864), (88435, 91876), (31995, 36723), (60461, 90431), (81765, 37792), (44888, 29738), (76600, 12587), (37134, 48440), (69151, 38382), (51397, 43750), (4293, 98001), (27450, 28456), (99384, 24614), (981, 89456), (59721, 47304), (60616, 98154), (98744, 11293), (83417, 71831), (32729, 41250), (46234, 15350), (39317, 40016), (10667, 89674), (50749, 56788), (2889, 90006), (20397, 68884), (32976, 67095), (72045, 90810), (73914, 29464), (36641, 44630), (21878, 385), (95246, 23783), (57354, 29305), (26745, 84026), (36892, 58119), (1149, 95862), (86209, 56761), (84711, 48878), (25394, 56006), (31581, 30386), (34987, 34411), (41423, 44982), (74735, 61128), (58589, 76665), (42483, 94983), (54883, 69038), (4802, 55544), (47092, 31410), (6642, 91583), (93798, 25259), (58424, 38132), (43946, 97030), (93700, 68673), (99034, 34439), (66998, 75560), (64453, 50462), (12208, 87262), (8175, 69014), (91514, 85630), (23685, 748), (88106, 51369), (27510, 26419), (83983, 5604), (66703, 57061), (72865, 67896), (94029, 2147), (95321, 30188), (24240, 67512), (21532, 29481), (69257, 9492), (59958, 62068), (18491, 93721), (37525, 12039), (95626, 86236), (49136, 11328), (71565, 98832), (77767, 34118), (64925, 21897), (73541, 79422), (92001, 79259), (8309, 34891), (66449, 55242), (85889, 88359), (90559, 87521), (92585, 92004), (21799, 17117), (98134, 53406), (40978, 17010), (55063, 73235), (7601, 13970), (46646, 31351), (7085, 97956), (45998, 98665), (40992, 82168), (4552, 54096), (77766, 7561), (66864, 26750), (88596, 49245), (82930, 33251), (36666, 90013), (71198, 85282), (30600, 59309), (20617, 9735), (96180, 42996), (71846, 84832), (10602, 94421), (83237, 85278), (63713, 14908), (10950, 42931), (33645, 74769), (69501, 8603), (63596, 44221), (75895, 36447), (43990, 4721), (19047, 91357), (35225, 30088), (78898, 87190), (9961, 52820), (6085, 61213), (98275, 84058), (61081, 91315), (28381, 30910), (15828, 72673), (79008, 65337), (60317, 77055), (78534, 50282), (9752, 74831), (62049, 63249), (31455, 74843), (2530, 28581), (68894, 84015), (12791, 14820), (37148, 48411), (88220, 90241), (60582, 70112), (49890, 26900), (94049, 49226), (19582, 98549), (87964, 73614), (54195, 95007), (73593, 29007), (35362, 47461), (93549, 82201), (74305, 49248), (7030, 46073), (69488, 56250), (63045, 25229), (44038, 24680), (65848, 36124), (53985, 95308), (65103, 82209), (26563, 58468), (33153, 12174), (29634, 21978), (26461, 3090), (88683, 4514), (45782, 83561), (56318, 92969), (5587, 61631), (92117, 78865), (78775, 72654), (47395, 49073), (20541, 46928), (71357, 58929), (45184, 76268), (59359, 77092), (5298, 10020), (580, 98499), (9119, 16986), (45518, 86386), (63375, 76589), (31660, 86697), (35851, 65479), (69920, 42815), (30208, 56290), (26343, 37505), (56509, 63902), (85757, 64286), (78627, 79973), (53080, 73913), (49000, 2312), (47315, 57178), (13033, 26209), (53799, 58615), (54508, 22862), (33971, 13889), (22815, 37914), (37426, 99282), (54022, 81696), (17550, 28739), (99122, 99584), (97931, 6054), (46163, 94552), (58328, 70123), (79181, 99242), (64725, 8569), (69146, 39323), (63563, 27537), (96076, 21338), (22936, 20737), (69747, 58505), (52140, 91592), (17198, 19845), (11003, 48066), (1758, 76734), (39891, 17323), (25673, 35578), (83704, 61339), (71674, 51463), (20522, 73773), (37708, 70488), (55492, 76397), (36380, 58022), (31085, 23886), (84428, 56183), (61034, 65835), (78297, 58728), (992, 11080), (26764, 59393), (3987, 81148), (57894, 50133), (57393, 54819), (639, 83329), (37777, 47517), (15526, 79423), (88248, 9629), (84464, 59890), (56601, 14257), (67283, 14213), (16041, 41528), (43407, 69763), (51687, 8960), (12121, 10409), (60598, 29848), (7839, 25189), (40941, 3224), (96406, 58803), (61378, 82788), (34541, 19703), (98061, 97148), (742, 7655), (61975, 65313), (64854, 93019), (80244, 18168), (92943, 6105), (73969, 74411), (47856, 26644), (94126, 18220), (81074, 61925), (26953, 28483), (65894, 12345), (54749, 51964), (14147, 41570), (47147, 41809), (8340, 35829), (18897, 32342), (36808, 22226), (18644, 68853), (4559, 12594), (96970, 17395), (50909, 77942), (72027, 49411), (98425, 79478), (45899, 24474), (46824, 48975), (15479, 28870), (45397, 76361), (11420, 22096), (74134, 37467), (25579, 59803), (74687, 44244), (71744, 71562), (49666, 32239), (34931, 56027), (50850, 78241), (71040, 91240), (61550, 47475), (26200, 78373), (54646, 87616), (28950, 6489), (51887, 75308), (3912, 42532), (82742, 42016), (94497, 36681), (62494, 3933), (81131, 18766), (51641, 94651), (72285, 53143), (73943, 73716), (66264, 17309), (43156, 99528), (43697, 78216), (84733, 50821), (75899, 63920), (18455, 7660), (30431, 879), (92204, 44911), (30237, 57641), (33407, 98166), (47862, 73576), (68097, 8833), (2454, 49159), (85772, 67530), (19732, 4536), (93827, 39013), (47927, 1682), (69143, 51598), (87471, 99956), (66362, 70450), (34353, 94326), (15692, 81083), (36376, 91470), (71608, 32098), (38115, 56236), (38822, 64266), (87707, 52553), (97973, 23102), (49267, 68907), (22649, 95175), (33001, 75648), (39808, 55971), (5608, 41816), (40313, 20564), (72994, 53489), (16983, 19806), (46114, 57880), (78622, 59284), (78285, 63449), (84552, 19702), (34234, 63531), (50848, 34774), (87144, 18212), (68214, 1005), (60080, 42398), (60514, 65965), (75134, 65529), (74107, 21666), (59817, 68782), (58940, 16958), (19975, 92583), (38480, 83257), (44510, 44997), (59488, 37263), (71209, 47719), (52080, 94252), (87128, 14831), (74001, 37313), (47448, 76625), (20205, 93608), (30249, 75697), (82584, 44654), (54091, 4298), (47736, 84084), (34052, 55781), (76012, 73648), (63204, 52778), (33692, 54616), (91839, 55044), (71674, 28475), (8517, 3272), (35810, 89905), (35146, 62215), (77262, 18227), (34439, 87732), (82690, 83051), + (13687, 19633), (66238, 44809), (53103, 29620), (93365, 87319), (50385, 4755), (89627, 50902), (51790, 15255), (48989, 19631), (16759, 3431), (90814, 19064), (27831, 70254), (75037, 52472), (35010, 24454), (31155, 22730), (87603, 59121), (1323, 23506), (56458, 11866), (21201, 50026), (66671, 36029), (21700, 69519), (85239, 77201), (95141, 57606), (3242, 19997), (52852, 15715), (49676, 70126), (73970, 9214), (86637, 18585), (14571, 65017), (65446, 43663), (20121, 50571), (88634, 87895), (65246, 62716), (27591, 85239), (33623, 14256), (66412, 77902), (16008, 6217), (979, 1666), (22123, 54052), (21596, 7241), (42224, 60675), (23345, 4908), (33780, 16632), (45087, 11756), (10951, 69020), (46138, 47932), (69936, 20048), (5871, 96808), (45595, 67151), (11993, 21010), (20462, 16667), (41417, 88964), (97285, 44958), (20062, 41444), (50808, 18976), (43350, 88847), (34906, 58686), (18132, 2775), (45165, 45298), (75000, 4332), (62157, 9375), (88002, 89214), (84772, 59883), (78078, 35447), (37222, 79319), (43545, 20731), (30761, 13677), (27691, 25767), (97750, 154), (87313, 16187), (78338, 15942), (61168, 16886), (73848, 48436), (74553, 13738), (50645, 37279), (11352, 63404), (85650, 43058), (9553, 2374), (5443, 50670), (55599, 64207), (74136, 41008), (12098, 83456), (98240, 65916), (59743, 64811), (53244, 68834), (66230, 81024), (13734, 83786), (40505, 76939), (93029, 55536), (29943, 53386), (33838, 94514), (71750, 96644), (30017, 8255), (4931, 38730), (2317, 55953), (52389, 67128), (70794, 44324), (24102, 85008), (46435, 65609), (19704, 92026), (99929, 86339), (67622, 13853), (95573, 72963), (99732, 75024), (60437, 5554), (78047, 64444), (95426, 63150), (45798, 26879), (45561, 12951), (95873, 14825), (36695, 13697), (88796, 31855), (52103, 53754), (34496, 56371), (33575, 17436), (39528, 83385), (36153, 48372), (15, 80583), (39015, 36980), (44747, 11011), (7314, 54178), (37252, 52551), (92128, 52377), (66571, 16799), (35841, 92438), (85840, 1365), (39413, 15660), (14759, 58002), (40158, 36685), (68936, 15130), (69488, 30639), (40633, 23355), (93700, 49305), (23456, 96541), (24357, 99667), (25072, 2364), (34734, 17869), (72064, 40226), (22712, 41911), (77545, 19335), (33945, 13408), (63813, 44097), (21816, 77633), (93359, 9036), (47001, 85666), (21934, 56004), (19148, 75033), (88715, 15387), (79158, 53971), (59300, 19502), (66517, 80891), (13859, 63857), (81036, 38595), (8876, 37315), (26819, 54318), (32428, 13471), (2615, 24036), (56894, 82103), (59056, 50983), (15479, 66848), (75766, 27048), (52141, 81023), (94385, 34233), (93361, 79520), (30867, 80795), (46204, 9262), (7946, 4084), (78272, 87422), (33639, 20782), (31276, 45969), (77720, 1335), (16701, 70082), (64963, 19854), (7812, 8856), (16170, 81834), (22877, 7520), (7132, 80996), (1917, 27657), (24737, 91605), (8195, 31354), (22561, 70343), (67166, 74828), (39705, 79118), (43241, 67180), (63526, 22190), (67442, 18172), (38470, 51567), (48664, 43895), (90585, 51544), (11842, 9287), (4231, 61462), (13104, 65739), (52455, 27122), (32318, 92494), (77028, 57920), (52314, 80381), (96749, 22065), (50502, 39753), (99787, 70903), (97786, 28008), (30462, 94898), (59403, 38638), (29827, 16423), (23066, 87865), (61889, 7941), (23948, 78242), (32395, 9480), (36150, 31784), (83146, 10647), (68557, 26863), (93482, 94228), (28512, 89595), (89424, 17693), (68947, 47792), (77918, 25182), (31272, 59863), (10836, 23758), (76913, 75785), (97969, 35902), (55507, 58629), (918, 8356), (66592, 31055), (95309, 59336), (51917, 63380), (79369, 21373), (82766, 76689), (90893, 53940), (31697, 95784), (78683, 89271), (29009, 41573), (9294, 6063), (80573, 79144), (59240, 77056), (99089, 4209), (90406, 92148), (48879, 74391), (22792, 13661), (52282, 74677), (48330, 54418), (6927, 75154), (35007, 12002), (86885, 50845), (74200, 7643), (57915, 3525), (89763, 13455), (98115, 11696), (56291, 17002), (22153, 27947), (85961, 14263), (74214, 58127), (21084, 3965), (86912, 49029), (43018, 83709), (67265, 85823), (68600, 15226), (25932, 81914), (73795, 41791), (3158, 90348), (7664, 81621), (81479, 60118), (93519, 76982), (75440, 3594), (88965, 91886), (92353, 12231), (39378, 69012), (71183, 88536), (36041, 65794), (6724, 26825), (5123, 57858), (2529, 85540), (33448, 94400), (30462, 37765), (30025, 78210), (74349, 5545), (73389, 88842), (78939, 62188), (27948, 62059), (90829, 67200), (44234, 57891), (30700, 61417), (44025, 71190), (22931, 17654), (23476, 87369), (9619, 78967), (24405, 67927), (84754, 91358), (14649, 71404), (4410, 74364), (61417, 87647), (73297, 46647), (32082, 33309), (44590, 47039), (67518, 99908), (22881, 99699), (36657, 17859), (17556, 91128), (58478, 75834), (57048, 73419), (90902, 11840), (4126, 69894), (30361, 54290), (31297, 28595), (54659, 98844), (71265, 86805), (49708, 38180), (24851, 15906), (86094, 67606), (13603, 68975), (3601, 93501), (93480, 17205), (82020, 17859), (68339, 66849), (66423, 9217), (84188, 55471), (28167, 29187), (51166, 46084), (88799, 39664), (67897, 91643), (43015, 68398), (7181, 78174), (6810, 46804), (6034, 55550), (51391, 29469), (76157, 96247), (54973, 75826), (83858, 53576), (87315, 76427), (44051, 44348), (78313, 73128), (44289, 62279), (61897, 87062), (40767, 93761), (91660, 32072), (5223, 36184), (30709, 51066), (40581, 38123), (27886, 39104), (40119, 20754), (27423, 22588), (57235, 49005), (88733, 11711), (4282, 60158), (18632, 3837), (68953, 17510), (21256, 60637), (71727, 176), (25865, 77991), (71032, 40412), (91622, 20288), (49014, 99799), (13599, 5764), (30979, 2454), (40338, 41233), (57336, 50430), (72959, 57286), (85827, 44852), (71907, 33794), (95577, 89084), (72485, 24104), (20306, 76152), (26079, 29381), (46907, 70771), (50365, 90449), (47884, 95986), (88953, 37962), (17407, 91977), (91616, 97270), (85966, 44410), (90389, 12295), (80529, 64553), (21003, 37956), (32923, 30387), (78916, 68050), (15505, 9628), (59050, 38770), (30091, 35499), (30658, 70636), (3160, 24877), (8277, 49676), (74352, 93399), (54074, 53202), (1643, 13321), (71865, 70329), (15251, 5871), (63695, 36988), (96887, 2370), (43098, 14907), (51946, 670), (53294, 72621), (57964, 99634), (17761, 42977), (77571, 38986), (50417, 76385), (80499, 52080), (24301, 56623), (43462, 34196), (49381, 50953), (34053, 74119), (62887, 15142), (69260, 13561), (42848, 87275), (91274, 37268), (87880, 18740), (61314, 8320), (59707, 95633), (38875, 94226), (1415, 62574), (25521, 90678), (48366, 41803), (497, 78256), (26209, 34736), (32946, 44203), (85348, 59409), (25037, 86769), (12753, 78083), (7901, 34859), (70495, 40252), (93318, 95446), (64601, 60123), (18929, 93512), (90647, 85301), (24280, 1900), (82057, 44496), (72227, 33583), (7109, 24591), (37709, 22897), (1237, 16650), (3383, 97434), (72245, 64486), (10201, 46701), (70246, 27545), (98647, 31169), (21901, 75927), (50188, 21441), (89577, 37821), (98823, 69937), (19263, 97734), (92937, 44136), (626, 95680), (12956, 24497), (454, 41351), (40585, 99858), (26190, 82119), (83467, 58869), (39533, 27469), (84276, 26501), (29838, 24478), (78589, 73649), (85241, 37456), (37739, 27890), (30935, 18054), (44388, 47592), (46287, 11414), (41275, 7284), (61172, 59962), (69172, 14436), (11827, 43746), (70455, 93926), (35662, 19338), (24436, 53824), (28435, 33323), (54767, 30980), (91379, 46216), (2656, 98555), (17703, 75278), (52299, 81894), (88460, 76874), (29298, 37649), (87510, 3320), (55513, 94080), (77940, 95554), (15605, 18918), (49452, 85395), (85737, 18109), (73279, 56989), (17461, 32967), (19605, 43424), (93999, 7945), (29865, 94947), (39164, 19448), (88080, 7422), (436, 99800), (27482, 44994), (73495, 66844), (47160, 66210), (98801, 18553), (33222, 8808), (31037, 15092), (92621, 47184), (54888, 65801), (30873, 33071), (61182, 77132), (81992, 40879), (10268, 38847), (26012, 24149), (85131, 81923), (10950, 55371), (17788, 48443), (39646, 51743), (36019, 76895), (9763, 36927), (19582, 16771), (68211, 84210), (9412, 55904), (44002, 9077), (36583, 1638), (97906, 20583), (12863, 74173), (29698, 34767), (34469, 78263), (87561, 58797), (77844, 10556), (68239, 63155), (41560, 93635), (52548, 95710), (11161, 632), (91657, 97784), (93088, 89618), (42604, 94389), (4198, 51324), (96597, 97093), (77382, 89074), (41381, 89801), (25857, 67775), (41422, 59757), (39574, 93308), (85990, 91621), (49182, 92967), (84187, 73625), (92495, 93334), (12471, 33289), (49105, 76813), (26330, 94103), (70805, 33031), (49193, 18360), (18002, 29159), (70447, 50454), (44081, 3675), (96495, 16455), (98675, 5876), (2913, 30032), (91048, 91939), (50496, 3621), (7244, 20490), (89364, 30152), (43588, 25656), (42086, 70055), (90150, 95763), (31727, 49598), (92625, 29569), (54148, 81753), (77721, 57094), (67706, 83472), (26935, 9073), (29554, 1952), (33394, 13797), (77797, 84190), (65692, 40517), (86640, 91307), (56461, 17709), (30204, 2620), (72787, 87380), (51439, 56294), (25033, 30268), (27379, 59569), (11651, 62401), (69998, 96886), (17278, 56849), (71326, 41808), (50372, 4986), (1711, 73335), (71837, 63661), (78810, 60882), (62918, 6111), (76267, 27414), (12536, 747), (24221, 44906), (2136, 84763), (7736, 18765), (71822, 70209), (55828, 97876), (27029, 80557), (47068, 2260), (47441, 81681), (95204, 83508), (94992, 98690), (86479, 34930), (51759, 64962), (23363, 25795), (50675, 29266), (54059, 35176), (37985, 70844), (270, 33485), (88135, 28988), (97160, 54235), (71087, 50356), (42552, 11173), (58508, 4623), (15663, 61919), (7672, 63865), (14458, 76549), (94036, 65780), (76076, 98795), (65158, 94233), (38869, 33986), (25926, 48043), (8196, 61978), (93547, 83000), (13218, 70869), (40220, 153), (40683, 8335), (38208, 71925), (63199, 70045), (3199, 10537), (16665, 29551), (91333, 59035), (60598, 74055), (18820, 76617), (31822, 14637), (1508, 13279), (99959, 73482), (69854, 34920), (59650, 1074), (1538, 52026), (84941, 6002), (82892, 20921), (88023, 79990), (6945, 92700), (2389, 62469), (38515, 78260), (34721, 56417), (37208, 71058), (78134, 76540), (18831, 53863), + (81277, 26395), (57176, 69906), (55080, 20726), (65843, 9495), (27703, 24879), (60594, 53447), (72896, 95513), (87884, 50244), (25281, 16427), (14122, 6399), (65765, 45679), (38160, 38650), (76820, 91595), (74663, 37143), (35220, 16865), (54664, 88840), (7671, 19781), (7255, 87276), (92182, 47030), (98922, 49381), (21954, 20742), (54674, 19730), (74694, 81672), (31070, 97794), (22759, 3530), (6085, 72621), (17392, 26996), (82220, 41192), (48225, 41272), (42850, 54369), (10817, 25978), (18889, 81907), (33157, 51203), (64325, 5137), (54695, 50597), (32244, 99170), (64264, 23137), (22912, 47267), (70570, 51635), (96224, 20571), (91110, 65969), (2598, 66368), (80926, 52434), (1538, 77141), (10149, 21484), (4981, 58501), (5154, 55414), (66791, 35619), (9905, 90667), (83595, 75525), (31989, 37492), (41740, 48920), (33876, 10878), (48654, 50120), (85890, 46200), (67100, 30387), (76475, 38916), (1654, 1657), (93842, 71472), (14560, 97747), (24866, 74187), (11477, 2492), (94590, 95965), (29363, 58015), (58783, 5304), (27337, 87412), (17695, 41885), (91837, 75700), (271, 49400), (33438, 84607), (78520, 46982), (38817, 66995), (2031, 58147), (80354, 75561), (7752, 55141), (22715, 29219), (10069, 69521), (1711, 23325), (88218, 94290), (44895, 32844), (62152, 964), (32945, 57419), (19030, 51353), (53097, 36525), (76592, 2389), (53949, 27282), (10569, 90204), (94318, 48527), (13855, 27099), (41107, 11962), (39143, 81444), (66004, 38387), (55906, 61394), (34933, 33952), (53868, 82829), (86213, 80445), (40351, 56305), (97656, 90533), (48021, 7262), (49406, 38294), (57170, 7675), (7040, 8039), (65129, 76668), (7554, 74457), (65930, 2421), (12739, 37328), (17174, 68325), (63207, 21315), (70066, 44503), (19112, 8081), (80329, 65152), (76480, 39600), (75262, 45432), (24382, 67054), (75360, 90438), (21786, 19182), (2050, 82428), (34831, 92670), (69746, 44939), (6130, 58630), (62478, 14374), (35730, 71761), (20641, 66145), (86551, 64076), (37559, 79379), (8215, 82203), (33701, 9187), (22679, 38196), (17476, 25994), (4391, 62658), (87726, 32997), (22552, 65818), (18115, 42587), (64374, 97689), (64896, 49572), (98479, 75547), (881, 32287), (25305, 16000), (295, 67454), (67642, 87363), (50945, 31544), (95966, 9972), (44139, 28252), (83682, 88471), (34615, 665), (13190, 85527), (32195, 81130), (22310, 12209), (27910, 60272), (40133, 5905), (29352, 36686), (76443, 64096), (79522, 79616), (63465, 96631), (95604, 90216), (88548, 12411), (63732, 50843), (1478, 11113), (41472, 43329), (35207, 78855), (95093, 22581), (67812, 1348), (96472, 75947), (18984, 63538), (84782, 72178), (53524, 46634), (95525, 15476), (82732, 89272), (59603, 2032), (74431, 78243), (47712, 51835), (82671, 67076), (63302, 96469), (77655, 95524), (11099, 31832), (77967, 32724), (85070, 36190), (93260, 32825), (51348, 34162), (40912, 26841), (27227, 21822), (15979, 96933), (76990, 90541), (36121, 90437), (85571, 9605), (85987, 46881), (86738, 82375), (80188, 48238), (64608, 41916), (79866, 9442), (28100, 95607), (10136, 19628), (6078, 59019), (8971, 11986), (75703, 209), (81982, 98701), (30455, 66367), (92825, 84403), (73144, 4923), (62835, 75533), (90925, 56381), (83853, 86786), (66906, 28576), (69683, 74216), (2171, 22531), (6591, 52559), (59362, 88732), (49865, 38016), (82881, 9151), (71356, 27365), (91463, 45944), (71040, 12264), (62116, 27681), (84941, 82284), (57515, 60713), (45227, 74196), (77502, 17765), (54055, 29484), (33708, 46220), (37344, 75102), (80480, 37942), (95782, 36781), (14092, 82457), (25284, 82297), (85150, 50659), (90037, 99122), (49584, 48321), (8028, 65210), (8162, 32091), (1609, 22211), (38548, 1023), (35864, 43021), (54196, 30754), (88535, 90185), (5518, 97226), (79272, 57758), (61381, 86419), (38344, 86877), (72629, 24694), (12563, 11747), (81456, 44023), (55536, 65195), (67741, 12182), (64250, 29636), (45252, 9834), (16576, 40098), (65924, 11123), (14090, 83443), (95180, 92697), (92452, 7957), (21244, 29717), (50253, 62030), (21401, 70735), (86803, 89625), (17066, 52757), (74570, 89660), (11647, 74336), (22463, 68191), (41693, 67542), (12994, 21955), (3180, 27966), (37236, 51099), (68568, 62778), (2671, 68542), (42771, 4268), (7578, 12919), (33838, 61136), (87904, 48076), (3781, 57164), (83289, 14414), (29626, 91409), (92536, 13946), (88125, 34961), (71541, 41249), (21045, 85183), (21628, 62294), (17768, 22390), (5892, 10231), (85902, 84414), (44078, 87944), (64981, 31018), (85690, 11077), (4711, 9309), (39077, 26339), (85313, 99603), (20267, 31090), (92684, 11726), (17887, 17415), (23064, 83141), (40925, 87246), (17575, 71065), (29396, 86192), (10859, 2881), (50457, 42309), (86972, 41824), (80954, 76970), (11664, 94322), (4286, 93562), (9093, 32589), (81261, 96519), (67886, 3086), (74132, 43197), (94549, 26192), (55498, 75374), (93813, 32911), (6727, 51629), (93543, 55373), (48682, 83316), (32468, 77339), (60028, 4896), (34177, 67214), (35429, 16686), (72814, 39944), (82791, 97364), (21249, 36007), (85536, 84815), (32784, 34349), (2391, 45661), (62595, 30434), (56513, 65051), (22536, 99421), (80603, 62636), (23609, 19082), (33262, 63428), (45307, 96049), (75079, 62835), (52361, 19400), (43845, 89033), (54257, 33298), (31133, 35836), (56725, 10158), (73650, 10456), (77548, 28249), (57998, 79546), (97895, 93196), (472, 18004), (67178, 2684), (34612, 36352), (84970, 98816), (32474, 5895), (60388, 75957), (81643, 26457), (30070, 55744), (14725, 58396), (66451, 69932), (59264, 59569), (98045, 63704), (11925, 87468), (24701, 6280), (47797, 84240), (998, 13575), (37865, 72036), (71339, 10339), (28248, 7481), (33945, 97829), (85194, 8904), (39669, 73192), (24326, 65934), (14438, 72729), (76676, 35176), (97604, 95931), (36032, 61056), (65788, 20291), (73067, 60718), (75742, 27362), (45393, 74792), (49586, 13048), (57961, 68534), (78171, 47271), (76064, 41030), (88738, 10155), (87757, 63134), (95783, 76438), (77935, 85529), (79345, 69555), (94483, 58668), (9432, 40485), (43585, 376), (61737, 97784), (1593, 37078), (11129, 23793), (94341, 64063), (41896, 70446), (77473, 33532), (66748, 65844), (17802, 27920), (22891, 12901), (40435, 43348), (27956, 16989), (29550, 63725), (49453, 61216), (25570, 97985), (39599, 83449), (96373, 12817), (48357, 40473), (99949, 33182), (85434, 45421), (28033, 46182), (80932, 26691), (99522, 29180), (70868, 31903), (63217, 51504), (70267, 29484), (36096, 28371), (45710, 2823), (85697, 3912), (25410, 70358), (73558, 82834), (28617, 80585), (81874, 57512), (76797, 45045), (22091, 84124), (68372, 42189), (66320, 83215), (23013, 22530), (33735, 97800), (54318, 99061), (18025, 39513), (69059, 39780), (58425, 39790), (10222, 96791), (17061, 3138), (92416, 5773), (20023, 31736), (47846, 74511), (52598, 26398), (3738, 44490), (53346, 79223), (31991, 57557), (34592, 98693), (86926, 20881), (14108, 87654), (4889, 34982), (77283, 73201), (13668, 22633), (50763, 95152), (97292, 54613), (22782, 30081), (21536, 46551), (45965, 4172), (75537, 79254), (41267, 86819), (57723, 75200), (60404, 27252), (80305, 57892), (50103, 24015), (69409, 13132), (30717, 56384), (54568, 79877), (8584, 8359), (41416, 64392), (32898, 92944), (5262, 35907), (88652, 77951), (16619, 2797), (42494, 65581), (21009, 18555), (29084, 28068), (29464, 35568), (7288, 52066), (32378, 48725), (95300, 62168), (52295, 97110), (36862, 87786), (12224, 86684), (85883, 34167), (45369, 76651), (95460, 42855), (34354, 87238), (7214, 96375), (58720, 87947), (55009, 10506), (98506, 6081), (46068, 81113), (99065, 58657), (81820, 28081), (90611, 57235), (7661, 72835), (93195, 90418), (34452, 20146), (39857, 78712), (47185, 52074), (9879, 51807), (92457, 92377), (36082, 17723), (21905, 43398), (35902, 54380), (21791, 6804), (50774, 35200), (11540, 83934), (54574, 57602), (99878, 35568), (28171, 68210), (58132, 17222), (91755, 80585), (18944, 65785), (78535, 70725), (42599, 65291), (62274, 97160), (76487, 23798), (34741, 36988), (7286, 61119), (36824, 29665), (82478, 60722), (90995, 28025), (71438, 40230), (83152, 45104), (1812, 91619)) + + DROP TABLE IF EXISTS test_in diff --git a/tests/performance/simple_join_query.xml b/tests/performance/simple_join_query.xml index 8f62ffdfd00..98f2b1eaebf 100644 --- a/tests/performance/simple_join_query.xml +++ b/tests/performance/simple_join_query.xml @@ -1,18 +1,10 @@ - - - 30000 - - - 60000 - - CREATE TABLE join_table(A Int64, S0 String, S1 String, S2 String, S3 String) ENGINE = MergeTree ORDER BY A - INSERT INTO join_table SELECT number AS A, toString(arrayMap(x->x, range(100))) S0, S0 AS S1, S0 AS S2, S0 AS S3 from numbers(500000) + INSERT INTO join_table SELECT number AS A, toString(arrayMap(x->x, range(100))) S0, S0 AS S1, S0 AS S2, S0 AS S3 from numbers_mt(5000000) SELECT COUNT() FROM join_table LEFT JOIN join_table USING A SELECT COUNT() FROM join_table LEFT JOIN (SELECT A FROM join_table) AS right USING A diff --git a/tests/performance/slices_hits.xml b/tests/performance/slices_hits.xml index 1745df3328c..4a6813579bf 100644 --- a/tests/performance/slices_hits.xml +++ b/tests/performance/slices_hits.xml @@ -1,15 +1,5 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/sort.xml b/tests/performance/sort.xml index 652dd7f4670..e5781548a37 100644 --- a/tests/performance/sort.xml +++ b/tests/performance/sort.xml @@ -1,10 +1,5 @@ - - - 10 - - CREATE TABLE rand_unlimited_10m_8 (key UInt8) Engine = Memory diff --git a/tests/performance/string_join.xml b/tests/performance/string_join.xml index 6aa2c576b4e..477f62c1327 100644 --- a/tests/performance/string_join.xml +++ b/tests/performance/string_join.xml @@ -1,10 +1,5 @@ - - - 10 - - diff --git a/tests/performance/string_set.xml b/tests/performance/string_set.xml index 7890ab11a4a..bbbfe2d3c2b 100644 --- a/tests/performance/string_set.xml +++ b/tests/performance/string_set.xml @@ -1,10 +1,5 @@ - - - 10 - - diff --git a/tests/performance/string_sort.xml b/tests/performance/string_sort.xml index 6a4e68270f9..71b56bdb9d6 100644 --- a/tests/performance/string_sort.xml +++ b/tests/performance/string_sort.xml @@ -5,16 +5,6 @@ - - - 5 - 10000 - - - 50 - 60000 - - diff --git a/tests/performance/sum_map.xml b/tests/performance/sum_map.xml index a88983fdbea..4f9ce56488c 100644 --- a/tests/performance/sum_map.xml +++ b/tests/performance/sum_map.xml @@ -1,10 +1,5 @@ - - - 30000 - - 1 diff --git a/tests/performance/synthetic_hardware_benchmark.xml b/tests/performance/synthetic_hardware_benchmark.xml index fc910077c9f..256fd623b3c 100644 --- a/tests/performance/synthetic_hardware_benchmark.xml +++ b/tests/performance/synthetic_hardware_benchmark.xml @@ -1,10 +1,5 @@ - - - 12000 - - 30000000000 diff --git a/tests/performance/trim_numbers.xml b/tests/performance/trim_numbers.xml index 62e26f8245a..35cd479d48c 100644 --- a/tests/performance/trim_numbers.xml +++ b/tests/performance/trim_numbers.xml @@ -1,13 +1,5 @@ - - - 10000 - - - 20000 - - diff --git a/tests/performance/trim_urls.xml b/tests/performance/trim_urls.xml index f29d878682f..276a12bc570 100644 --- a/tests/performance/trim_urls.xml +++ b/tests/performance/trim_urls.xml @@ -4,14 +4,6 @@ hits_100m_single - - - 10000 - - - 20000 - - diff --git a/tests/performance/trim_whitespace.xml b/tests/performance/trim_whitespace.xml index 8ec4aeaa54e..049387bbc0c 100644 --- a/tests/performance/trim_whitespace.xml +++ b/tests/performance/trim_whitespace.xml @@ -10,11 +10,6 @@ from numbers_mt(100000000); - - - 30000 - - diff --git a/tests/performance/uniq.xml b/tests/performance/uniq.xml index 0b7c8e58c86..2766c95e6a7 100644 --- a/tests/performance/uniq.xml +++ b/tests/performance/uniq.xml @@ -5,12 +5,6 @@ 30000000000 - - - 5000 - 20000 - - diff --git a/tests/performance/url_hits.xml b/tests/performance/url_hits.xml index f9383eb3910..c8cf119a7d7 100644 --- a/tests/performance/url_hits.xml +++ b/tests/performance/url_hits.xml @@ -4,14 +4,6 @@ hits_100m_single - - - 10000 - - - 20000 - - diff --git a/tests/performance/vectorize_aggregation_combinators.xml b/tests/performance/vectorize_aggregation_combinators.xml index 88870b56d1f..47ac0719bb5 100644 --- a/tests/performance/vectorize_aggregation_combinators.xml +++ b/tests/performance/vectorize_aggregation_combinators.xml @@ -1,14 +1,6 @@ - - - 30000 - - - 60000 - - diff --git a/tests/performance/visit_param_extract_raw.xml b/tests/performance/visit_param_extract_raw.xml index ca46c79c9b5..7be780d5d42 100644 --- a/tests/performance/visit_param_extract_raw.xml +++ b/tests/performance/visit_param_extract_raw.xml @@ -1,9 +1,4 @@ - - - 10000 - - diff --git a/tests/performance/website.xml b/tests/performance/website.xml index c21f09c57d8..6ed60c0860a 100644 --- a/tests/performance/website.xml +++ b/tests/performance/website.xml @@ -5,15 +5,6 @@ hits_100m_single - - - 60000 - 3 - - - 30 - - diff --git a/tests/queries/0_stateless/00061_merge_tree_alter.reference b/tests/queries/0_stateless/00061_merge_tree_alter.reference index 571affd7231..b609bc257f1 100644 --- a/tests/queries/0_stateless/00061_merge_tree_alter.reference +++ b/tests/queries/0_stateless/00061_merge_tree_alter.reference @@ -1,14 +1,14 @@ d Date k UInt64 i32 Int32 -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32) ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 10 42 d Date k UInt64 i32 Int32 n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `n.ui8` Array(UInt8), `n.s` Array(String)) ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 8 40 [1,2,3] ['12','13','14'] 2015-01-01 10 42 [] [] d Date @@ -17,7 +17,7 @@ i32 Int32 n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date)) ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 7 39 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 10 42 [] [] [] @@ -28,7 +28,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), `s` String DEFAULT \'0\') ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date), \n `s` String DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 2015-01-01 7 39 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 0 2015-01-01 8 40 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 0 @@ -39,7 +39,7 @@ i32 Int32 n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `n.ui8` Array(UInt8), `n.s` Array(String), `s` Int64 DEFAULT \'0\') ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` Int64 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 [10,20,30] ['asd','qwe','qwe'] 100500 2015-01-01 7 39 [10,20,30] ['120','130','140'] 0 2015-01-01 8 40 [1,2,3] ['12','13','14'] 0 @@ -51,7 +51,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `n.ui8` Array(UInt8), `n.s` Array(String), `s` UInt32 DEFAULT \'0\', `n.d` Array(Date)) ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\', \n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 [10,20,30] ['asd','qwe','qwe'] 100500 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 7 39 [10,20,30] ['120','130','140'] 0 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 8 40 [1,2,3] ['12','13','14'] 0 ['0000-00-00','0000-00-00','0000-00-00'] @@ -65,7 +65,7 @@ k UInt64 i32 Int32 n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `n.s` Array(String), `s` UInt32 DEFAULT \'0\') ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 ['asd','qwe','qwe'] 100500 2015-01-01 7 39 ['120','130','140'] 0 2015-01-01 8 40 ['12','13','14'] 0 @@ -74,7 +74,7 @@ d Date k UInt64 i32 Int32 s UInt32 DEFAULT \'0\' -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `s` UInt32 DEFAULT \'0\') ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 100500 2015-01-01 7 39 0 2015-01-01 8 40 0 @@ -85,7 +85,7 @@ i32 Int32 s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `s` UInt32 DEFAULT \'0\', `n.s` Array(String), `n.d` Array(Date)) ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `s` UInt32 DEFAULT \'0\', \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 100500 [] [] 2015-01-01 7 39 0 [] [] 2015-01-01 8 40 0 [] [] @@ -94,7 +94,7 @@ d Date k UInt64 i32 Int32 s UInt32 DEFAULT \'0\' -CREATE TABLE default.alter_00061 (`d` Date, `k` UInt64, `i32` Int32, `s` UInt32 DEFAULT \'0\') ENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 100500 2015-01-01 7 39 0 2015-01-01 8 40 0 diff --git a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference index 6f2eb080286..fa5e65d2d60 100644 --- a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference +++ b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference @@ -1,22 +1,22 @@ d Date k UInt64 i32 Int32 -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 10 42 d Date k UInt64 i32 Int32 dt DateTime -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 9 41 1992-01-01 08:00:00 2015-01-01 10 42 0000-00-00 00:00:00 d Date @@ -25,14 +25,14 @@ i32 Int32 dt DateTime n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] @@ -43,7 +43,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -51,7 +51,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] [] @@ -64,7 +64,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), `s` String DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date), \n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -73,7 +73,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), `s` String DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date), \n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 0 @@ -86,7 +86,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `s` Int64 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -94,7 +94,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `s` Int64 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 @@ -108,7 +108,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `s` UInt32 DEFAULT \'0\', `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\', \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -117,7 +117,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `s` UInt32 DEFAULT \'0\', `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\', \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 ['0000-00-00','0000-00-00','0000-00-00'] @@ -129,14 +129,14 @@ i32 Int32 dt DateTime n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.s` Array(String), `s` UInt32 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.s` Array(String), `s` UInt32 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 ['12','13','14'] 0 @@ -147,13 +147,13 @@ k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `s` UInt32 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `s` UInt32 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 @@ -166,7 +166,7 @@ dt DateTime s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `s` UInt32 DEFAULT \'0\', `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\', \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -174,7 +174,7 @@ dt DateTime s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `s` UInt32 DEFAULT \'0\', `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\', \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 [] [] 2015-01-01 7 39 2014-07-14 13:26:50 0 [] [] 2015-01-01 8 40 2012-12-12 12:12:12 0 [] [] @@ -185,13 +185,13 @@ k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `s` UInt32 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `s` UInt32 DEFAULT \'0\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 @@ -202,13 +202,13 @@ k UInt64 i32 Int32 dt Date s DateTime DEFAULT \'0000-00-00 00:00:00\' -CREATE TABLE test.replicated_alter1 (`d` Date, `k` UInt64, `i32` Int32, `dt` Date, `s` DateTime DEFAULT \'0000-00-00 00:00:00\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` Date, \n `s` DateTime DEFAULT \'0000-00-00 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt Date s DateTime DEFAULT \'0000-00-00 00:00:00\' -CREATE TABLE test.replicated_alter2 (`d` Date, `k` UInt64, `i32` Int32, `dt` Date, `s` DateTime DEFAULT \'0000-00-00 00:00:00\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` Date, \n `s` DateTime DEFAULT \'0000-00-00 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 1970-01-02 06:55:00 2015-01-01 7 39 2014-07-14 0000-00-00 00:00:00 2015-01-01 8 40 2012-12-12 0000-00-00 00:00:00 diff --git a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference index 1957f3a9604..e3dbeb81c7c 100644 --- a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference +++ b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference @@ -1,2 +1,6 @@ 1 1 1 1 +1 1 +1 1 +1 1 +1 1 diff --git a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql index 7b7a0914c0c..78f1fb68385 100644 --- a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql +++ b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql @@ -1 +1,2 @@ SELECT count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) SETTINGS distributed_group_by_no_merge = 1; +SELECT count(), uniq(dummy) FROM remote('127.0.0.{2,3,4,5}', system.one) SETTINGS distributed_group_by_no_merge = 1; diff --git a/tests/queries/0_stateless/00328_long_case_construction.sql b/tests/queries/0_stateless/00328_long_case_construction.sql index d0fc90d0884..aa653d667f7 100644 --- a/tests/queries/0_stateless/00328_long_case_construction.sql +++ b/tests/queries/0_stateless/00328_long_case_construction.sql @@ -1,4 +1,3 @@ - /* Trivial case */ SELECT CASE WHEN 1 THEN 2 WHEN 3 THEN 4 ELSE 5 END; diff --git a/tests/queries/0_stateless/00561_storage_join.sql b/tests/queries/0_stateless/00561_storage_join.sql index 08f76815702..62ca80d31fe 100644 --- a/tests/queries/0_stateless/00561_storage_join.sql +++ b/tests/queries/0_stateless/00561_storage_join.sql @@ -1,5 +1,3 @@ -SET any_join_distinct_right_table_keys = 1; - drop table IF EXISTS joinbug; CREATE TABLE joinbug ( @@ -21,7 +19,7 @@ CREATE TABLE joinbug_join ( val UInt64, val2 Int32, created UInt64 -) ENGINE = Join(ANY, INNER, id2); +) ENGINE = Join(SEMI, LEFT, id2); insert into joinbug_join (id, id2, val, val2, created) select id, id2, val, val2, created @@ -36,7 +34,7 @@ select id, id2, val, val2, created from ( SELECT toUInt64(arrayJoin(range(50))) AS id2 ) js1 -ANY INNER JOIN joinbug_join using id2; +SEMI LEFT JOIN joinbug_join using id2; DROP TABLE joinbug; DROP TABLE joinbug_join; diff --git a/tests/queries/0_stateless/00564_temporary_table_management.reference b/tests/queries/0_stateless/00564_temporary_table_management.reference index edd17b9ea39..4cfb4230223 100644 --- a/tests/queries/0_stateless/00564_temporary_table_management.reference +++ b/tests/queries/0_stateless/00564_temporary_table_management.reference @@ -1,4 +1,4 @@ 1 -CREATE TEMPORARY TABLE temp_tab (`number` UInt64) ENGINE = Memory +CREATE TEMPORARY TABLE temp_tab\n(\n `number` UInt64\n)\nENGINE = Memory temp_tab 0 diff --git a/tests/queries/0_stateless/00599_create_view_with_subquery.reference b/tests/queries/0_stateless/00599_create_view_with_subquery.reference index 13e0f35b075..d83d2837a18 100644 --- a/tests/queries/0_stateless/00599_create_view_with_subquery.reference +++ b/tests/queries/0_stateless/00599_create_view_with_subquery.reference @@ -1 +1 @@ -CREATE VIEW default.test_view_00599 (`id` UInt64) AS SELECT * FROM default.test_00599 WHERE id = (SELECT 1) +CREATE VIEW default.test_view_00599\n(\n `id` UInt64\n) AS\nSELECT *\nFROM default.test_00599\nWHERE id = \n(\n SELECT 1\n) diff --git a/tests/queries/0_stateless/00604_show_create_database.reference b/tests/queries/0_stateless/00604_show_create_database.reference index 1fe93a5e393..a9ad6abea25 100644 --- a/tests/queries/0_stateless/00604_show_create_database.reference +++ b/tests/queries/0_stateless/00604_show_create_database.reference @@ -1 +1 @@ -CREATE DATABASE test_00604 ENGINE = Ordinary +CREATE DATABASE test_00604\nENGINE = Ordinary diff --git a/tests/queries/0_stateless/00642_cast.reference b/tests/queries/0_stateless/00642_cast.reference index f75503efffe..907861c1784 100644 --- a/tests/queries/0_stateless/00642_cast.reference +++ b/tests/queries/0_stateless/00642_cast.reference @@ -7,7 +7,14 @@ hello hello hello 1970-01-01 00:00:01 -CREATE TABLE default.cast (`x` UInt8, `e` Enum8('hello' = 1, 'world' = 2) DEFAULT CAST(x, 'Enum8(\'hello\' = 1, \'world\' = 2)')) ENGINE = MergeTree ORDER BY e SETTINGS index_granularity = 8192 +CREATE TABLE default.cast +( + `x` UInt8, + `e` Enum8('hello' = 1, 'world' = 2) DEFAULT CAST(x, 'Enum8(\'hello\' = 1, \'world\' = 2)') +) +ENGINE = MergeTree +ORDER BY e +SETTINGS index_granularity = 8192 x UInt8 e Enum8(\'hello\' = 1, \'world\' = 2) DEFAULT CAST(x, \'Enum8(\\\'hello\\\' = 1, \\\'world\\\' = 2)\') 1 hello diff --git a/tests/queries/0_stateless/00643_cast_zookeeper.reference b/tests/queries/0_stateless/00643_cast_zookeeper.reference index 86a8b164844..b79eb07aee3 100644 --- a/tests/queries/0_stateless/00643_cast_zookeeper.reference +++ b/tests/queries/0_stateless/00643_cast_zookeeper.reference @@ -1,4 +1,11 @@ -CREATE TABLE test.cast1 (`x` UInt8, `e` Enum8('hello' = 1, 'world' = 2) DEFAULT CAST(x, 'Enum8(\'hello\' = 1, \'world\' = 2)')) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_cast', 'r1') ORDER BY e SETTINGS index_granularity = 8192 +CREATE TABLE test.cast1 +( + `x` UInt8, + `e` Enum8('hello' = 1, 'world' = 2) DEFAULT CAST(x, 'Enum8(\'hello\' = 1, \'world\' = 2)') +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_cast', 'r1') +ORDER BY e +SETTINGS index_granularity = 8192 x UInt8 e Enum8(\'hello\' = 1, \'world\' = 2) DEFAULT CAST(x, \'Enum8(\\\'hello\\\' = 1, \\\'world\\\' = 2)\') 1 hello diff --git a/tests/queries/0_stateless/00646_url_engine.python b/tests/queries/0_stateless/00646_url_engine.python index 960048dbb8f..494eb12b0ef 100644 --- a/tests/queries/0_stateless/00646_url_engine.python +++ b/tests/queries/0_stateless/00646_url_engine.python @@ -2,6 +2,7 @@ from __future__ import print_function import csv import sys +import time import tempfile import threading import os, urllib @@ -180,7 +181,15 @@ def main(): if __name__ == "__main__": - try: - main() - except: + exception_text = '' + for i in range(1, 5): + try: + main() + break + except Exception as ex: + exception_text = str(ex) + time.sleep(0.1) + + if exception_text: + print("Exception: {}".format(exception_text), file=sys.stderr) os._exit(1) diff --git a/tests/queries/0_stateless/00725_comment_columns.reference b/tests/queries/0_stateless/00725_comment_columns.reference index 7204496753c..86794581daf 100644 --- a/tests/queries/0_stateless/00725_comment_columns.reference +++ b/tests/queries/0_stateless/00725_comment_columns.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.check_query_comment_column (`first_column` UInt8 DEFAULT 1 COMMENT \'comment 1\', `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2\', `third_column` UInt8 ALIAS second_column COMMENT \'comment 3\', `fourth_column` UInt8 COMMENT \'comment 4\', `fifth_column` UInt8) ENGINE = TinyLog +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1\', \n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2\', \n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3\', \n `fourth_column` UInt8 COMMENT \'comment 4\', \n `fifth_column` UInt8\n)\nENGINE = TinyLog first_column UInt8 DEFAULT 1 comment 1 second_column UInt8 MATERIALIZED first_column comment 2 third_column UInt8 ALIAS second_column comment 3 @@ -11,7 +11,7 @@ fifth_column UInt8 │ check_query_comment_column │ fourth_column │ comment 4 │ │ check_query_comment_column │ fifth_column │ │ └────────────────────────────┴───────────────┴───────────┘ -CREATE TABLE default.check_query_comment_column (`first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_1\', `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_1\', `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_1\', `fourth_column` UInt8 COMMENT \'comment 4_1\', `fifth_column` UInt8 COMMENT \'comment 5_1\') ENGINE = TinyLog +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_1\', \n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_1\', \n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_1\', \n `fourth_column` UInt8 COMMENT \'comment 4_1\', \n `fifth_column` UInt8 COMMENT \'comment 5_1\'\n)\nENGINE = TinyLog ┌─table──────────────────────┬─name──────────┬─comment─────┐ │ check_query_comment_column │ first_column │ comment 1_2 │ │ check_query_comment_column │ second_column │ comment 2_2 │ @@ -19,8 +19,8 @@ CREATE TABLE default.check_query_comment_column (`first_column` UInt8 DEFAULT 1 │ check_query_comment_column │ fourth_column │ comment 4_2 │ │ check_query_comment_column │ fifth_column │ comment 5_2 │ └────────────────────────────┴───────────────┴─────────────┘ -CREATE TABLE default.check_query_comment_column (`first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_2\', `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_2\', `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_2\', `fourth_column` UInt8 COMMENT \'comment 4_2\', `fifth_column` UInt8 COMMENT \'comment 5_2\') ENGINE = TinyLog -CREATE TABLE default.check_query_comment_column (`first_column` UInt8 COMMENT \'comment 1\', `second_column` UInt8 COMMENT \'comment 2\', `third_column` UInt8 COMMENT \'comment 3\') ENGINE = MergeTree() PARTITION BY second_column ORDER BY first_column SAMPLE BY first_column SETTINGS index_granularity = 8192 +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_2\', \n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_2\', \n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_2\', \n `fourth_column` UInt8 COMMENT \'comment 4_2\', \n `fifth_column` UInt8 COMMENT \'comment 5_2\'\n)\nENGINE = TinyLog +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1\', \n `second_column` UInt8 COMMENT \'comment 2\', \n `third_column` UInt8 COMMENT \'comment 3\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 first_column UInt8 comment 1 second_column UInt8 comment 2 third_column UInt8 comment 3 @@ -29,8 +29,8 @@ third_column UInt8 comment 3 │ check_query_comment_column │ second_column │ comment 2 │ │ check_query_comment_column │ third_column │ comment 3 │ └────────────────────────────┴───────────────┴───────────┘ -CREATE TABLE default.check_query_comment_column (`first_column` UInt8 COMMENT \'comment 1_2\', `second_column` UInt8 COMMENT \'comment 2_2\', `third_column` UInt8 COMMENT \'comment 3_2\') ENGINE = MergeTree() PARTITION BY second_column ORDER BY first_column SAMPLE BY first_column SETTINGS index_granularity = 8192 -CREATE TABLE default.check_query_comment_column (`first_column` UInt8 COMMENT \'comment 1_3\', `second_column` UInt8 COMMENT \'comment 2_3\', `third_column` UInt8 COMMENT \'comment 3_3\') ENGINE = MergeTree() PARTITION BY second_column ORDER BY first_column SAMPLE BY first_column SETTINGS index_granularity = 8192 +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1_2\', \n `second_column` UInt8 COMMENT \'comment 2_2\', \n `third_column` UInt8 COMMENT \'comment 3_2\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1_3\', \n `second_column` UInt8 COMMENT \'comment 2_3\', \n `third_column` UInt8 COMMENT \'comment 3_3\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 ┌─table──────────────────────┬─name──────────┬─comment─────┐ │ check_query_comment_column │ first_column │ comment 1_3 │ │ check_query_comment_column │ second_column │ comment 2_3 │ diff --git a/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference b/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference index a31b4bd7308..28051d15f65 100644 --- a/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference +++ b/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.ipv4_test (`ipv4_` IPv4) ENGINE = Memory +CREATE TABLE default.ipv4_test\n(\n `ipv4_` IPv4\n)\nENGINE = Memory 0.0.0.0 00 8.8.8.8 08080808 127.0.0.1 7F000001 @@ -10,7 +10,7 @@ CREATE TABLE default.ipv4_test (`ipv4_` IPv4) ENGINE = Memory > 127.0.0.1 255.255.255.255 = 127.0.0.1 127.0.0.1 euqality of IPv4-mapped IPv6 value and IPv4 promoted to IPv6 with function: 1 -CREATE TABLE default.ipv6_test (`ipv6_` IPv6) ENGINE = Memory +CREATE TABLE default.ipv6_test\n(\n `ipv6_` IPv6\n)\nENGINE = Memory :: 00000000000000000000000000000000 :: 00000000000000000000000000000000 ::ffff:8.8.8.8 00000000000000000000FFFF08080808 diff --git a/tests/queries/0_stateless/00751_default_databasename_for_view.reference b/tests/queries/0_stateless/00751_default_databasename_for_view.reference index 2873fcbee3b..5ba1861e3ef 100644 --- a/tests/queries/0_stateless/00751_default_databasename_for_view.reference +++ b/tests/queries/0_stateless/00751_default_databasename_for_view.reference @@ -1,4 +1,29 @@ -CREATE MATERIALIZED VIEW test_00751.t_mv_00751 (`date` Date, `platform` Enum8('a' = 0, 'b' = 1), `app` Enum8('a' = 0, 'b' = 1)) ENGINE = MergeTree ORDER BY date SETTINGS index_granularity = 8192 AS SELECT date, platform, app FROM test_00751.t_00751 WHERE (app = (SELECT min(app) FROM test_00751.u_00751)) AND (platform = (SELECT (SELECT min(platform) FROM test_00751.v_00751))) +CREATE MATERIALIZED VIEW test_00751.t_mv_00751 +( + `date` Date, + `platform` Enum8('a' = 0, 'b' = 1), + `app` Enum8('a' = 0, 'b' = 1) +) +ENGINE = MergeTree +ORDER BY date +SETTINGS index_granularity = 8192 AS +SELECT + date, + platform, + app +FROM test_00751.t_00751 +WHERE (app = +( + SELECT min(app) + FROM test_00751.u_00751 +)) AND (platform = +( + SELECT + ( + SELECT min(platform) + FROM test_00751.v_00751 + ) +)) 2000-01-01 a a 2000-01-02 b b 2000-01-03 a a diff --git a/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference b/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference index 4425f2afeb9..d2705135440 100644 --- a/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference +++ b/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference @@ -1,6 +1,6 @@ -CREATE TABLE default.check_comments (`column_name1` UInt8 DEFAULT 1 COMMENT \'comment\', `column_name2` UInt8 COMMENT \'non default comment\') ENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\') ORDER BY column_name1 SETTINGS index_granularity = 8192 +CREATE TABLE default.check_comments\n(\n `column_name1` UInt8 DEFAULT 1 COMMENT \'comment\', \n `column_name2` UInt8 COMMENT \'non default comment\'\n)\nENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\')\nORDER BY column_name1\nSETTINGS index_granularity = 8192 column_name1 UInt8 DEFAULT 1 comment column_name2 UInt8 non default comment -CREATE TABLE default.check_comments (`column_name1` UInt8 DEFAULT 1 COMMENT \'another comment\', `column_name2` UInt8 COMMENT \'non default comment\') ENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\') ORDER BY column_name1 SETTINGS index_granularity = 8192 +CREATE TABLE default.check_comments\n(\n `column_name1` UInt8 DEFAULT 1 COMMENT \'another comment\', \n `column_name2` UInt8 COMMENT \'non default comment\'\n)\nENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\')\nORDER BY column_name1\nSETTINGS index_granularity = 8192 column_name1 UInt8 DEFAULT 1 another comment column_name2 UInt8 non default comment diff --git a/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference b/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference index 93f15318634..a1493508b61 100644 --- a/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference +++ b/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference @@ -1,5 +1,5 @@ *** Check SHOW CREATE TABLE *** -CREATE TABLE default.alter_column (`x` UInt32, `y` Int32) ENGINE = MergeTree PARTITION BY x ORDER BY x SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_column\n(\n `x` UInt32, \n `y` Int32\n)\nENGINE = MergeTree\nPARTITION BY x\nORDER BY x\nSETTINGS index_granularity = 8192 *** Check parts *** 0 0 10 -10 @@ -52,7 +52,7 @@ CREATE TABLE default.alter_column (`x` UInt32, `y` Int32) ENGINE = MergeTree PAR 8 -8 9 -9 *** Check SHOW CREATE TABLE after ALTER MODIFY *** -CREATE TABLE default.alter_column (`x` UInt32, `y` Int64) ENGINE = MergeTree PARTITION BY x ORDER BY x SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_column\n(\n `x` UInt32, \n `y` Int64\n)\nENGINE = MergeTree\nPARTITION BY x\nORDER BY x\nSETTINGS index_granularity = 8192 *** Check parts after ALTER MODIFY *** 0 0 10 -10 diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by.reference b/tests/queries/0_stateless/00754_alter_modify_order_by.reference index 1bcdae884f8..f0dc413a186 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by.reference +++ b/tests/queries/0_stateless/00754_alter_modify_order_by.reference @@ -9,4 +9,4 @@ 1 2 1 30 1 2 4 90 *** Check SHOW CREATE TABLE *** -CREATE TABLE default.summing (`x` UInt32, `y` UInt32, `z` UInt32, `val` UInt32) ENGINE = SummingMergeTree PRIMARY KEY (x, y) ORDER BY (x, y, -z) SETTINGS index_granularity = 8192 +CREATE TABLE default.summing\n(\n `x` UInt32, \n `y` UInt32, \n `z` UInt32, \n `val` UInt32\n)\nENGINE = SummingMergeTree\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference index ebe30941f3f..938a90a27b4 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference @@ -9,6 +9,6 @@ 1 2 1 30 1 2 4 90 *** Check SHOW CREATE TABLE *** -CREATE TABLE test.summing_r2 (`x` UInt32, `y` UInt32, `z` UInt32, `val` UInt32) ENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\') PRIMARY KEY (x, y) ORDER BY (x, y, -z) SETTINGS index_granularity = 8192 +CREATE TABLE test.summing_r2\n(\n `x` UInt32, \n `y` UInt32, \n `z` UInt32, \n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 *** Check SHOW CREATE TABLE after offline ALTER *** -CREATE TABLE test.summing_r2 (`x` UInt32, `y` UInt32, `z` UInt32, `t` UInt32, `val` UInt32) ENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\') PRIMARY KEY (x, y) ORDER BY (x, y, t * t) SETTINGS index_granularity = 8192 +CREATE TABLE test.summing_r2\n(\n `x` UInt32, \n `y` UInt32, \n `z` UInt32, \n `t` UInt32, \n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, t * t)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference index 6da97ff6091..f778c4f5d90 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference @@ -9,10 +9,10 @@ 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_multiple_more_types (`id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)), `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)), `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)), `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))) ENGINE = MergeTree() ORDER BY tuple() SETTINGS index_granularity = 8192 +CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)), \n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)), \n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)), \n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] 7.1000000000000 xxxxxxxxxxxx [127] ['Henry'] ! 222 !ZSTD -CREATE TABLE default.test_default_delta (`id` UInt64 CODEC(Delta(8)), `data` String CODEC(Delta(1)), `somedate` Date CODEC(Delta(2)), `somenum` Float64 CODEC(Delta(8)), `somestr` FixedString(3) CODEC(Delta(1)), `othernum` Int64 CODEC(Delta(8)), `yetothernum` Float32 CODEC(Delta(4)), `ddd.age` Array(UInt8) CODEC(Delta(1)), `ddd.Name` Array(String) CODEC(Delta(1)), `ddd.OName` Array(String) CODEC(Delta(1)), `ddd.BName` Array(String) CODEC(Delta(1))) ENGINE = MergeTree() ORDER BY tuple() SETTINGS index_granularity = 8192 +CREATE TABLE default.test_default_delta\n(\n `id` UInt64 CODEC(Delta(8)), \n `data` String CODEC(Delta(1)), \n `somedate` Date CODEC(Delta(2)), \n `somenum` Float64 CODEC(Delta(8)), \n `somestr` FixedString(3) CODEC(Delta(1)), \n `othernum` Int64 CODEC(Delta(8)), \n `yetothernum` Float32 CODEC(Delta(4)), \n `ddd.age` Array(UInt8) CODEC(Delta(1)), \n `ddd.Name` Array(String) CODEC(Delta(1)), \n `ddd.OName` Array(String) CODEC(Delta(1)), \n `ddd.BName` Array(String) CODEC(Delta(1))\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference index 322b207bf7d..b33535364e5 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference @@ -1,9 +1,9 @@ -CREATE TABLE default.compression_codec_log (`id` UInt64 CODEC(LZ4), `data` String CODEC(ZSTD(1)), `ddd` Date CODEC(NONE), `somenum` Float64 CODEC(ZSTD(2)), `somestr` FixedString(3) CODEC(LZ4HC(7)), `othernum` Int64 CODEC(Delta(8))) ENGINE = Log() +CREATE TABLE default.compression_codec_log\n(\n `id` UInt64 CODEC(LZ4), \n `data` String CODEC(ZSTD(1)), \n `ddd` Date CODEC(NONE), \n `somenum` Float64 CODEC(ZSTD(2)), \n `somestr` FixedString(3) CODEC(LZ4HC(7)), \n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = Log() 1 hello 2018-12-14 1.1 aaa 5 2 world 2018-12-15 2.2 bbb 6 3 ! 2018-12-16 3.3 ccc 7 2 -CREATE TABLE default.compression_codec_multiple_log (`id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))) ENGINE = Log() +CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), \n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), \n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), \n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = Log() 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 @@ -11,12 +11,12 @@ CREATE TABLE default.compression_codec_multiple_log (`id` UInt64 CODEC(LZ4, ZSTD 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_tiny_log (`id` UInt64 CODEC(LZ4), `data` String CODEC(ZSTD(1)), `ddd` Date CODEC(NONE), `somenum` Float64 CODEC(ZSTD(2)), `somestr` FixedString(3) CODEC(LZ4HC(7)), `othernum` Int64 CODEC(Delta(8))) ENGINE = TinyLog() +CREATE TABLE default.compression_codec_tiny_log\n(\n `id` UInt64 CODEC(LZ4), \n `data` String CODEC(ZSTD(1)), \n `ddd` Date CODEC(NONE), \n `somenum` Float64 CODEC(ZSTD(2)), \n `somestr` FixedString(3) CODEC(LZ4HC(7)), \n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = TinyLog() 1 hello 2018-12-14 1.1 aaa 5 2 world 2018-12-15 2.2 bbb 6 3 ! 2018-12-16 3.3 ccc 7 2 -CREATE TABLE default.compression_codec_multiple_tiny_log (`id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))) ENGINE = TinyLog() +CREATE TABLE default.compression_codec_multiple_tiny_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), \n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), \n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), \n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = TinyLog() 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 diff --git a/tests/queries/0_stateless/00836_indices_alter.reference b/tests/queries/0_stateless/00836_indices_alter.reference index e30c17eb673..6efa25f47b7 100644 --- a/tests/queries/0_stateless/00836_indices_alter.reference +++ b/tests/queries/0_stateless/00836_indices_alter.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 @@ -6,15 +6,15 @@ CREATE TABLE default.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i3 1 2 1 2 1 2 -CREATE TABLE default.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 1 2 1 2 1 2 -CREATE TABLE default.minmax_idx (`u64` UInt64, `i32` Int32) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE default.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 @@ -23,6 +23,6 @@ CREATE TABLE default.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i3 1 2 1 2 1 2 -CREATE TABLE default.minmax_idx2 (`u64` UInt64, `i32` Int32) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx2\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 diff --git a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference index b2c2b41f460..ec9de160fcc 100644 --- a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference @@ -1,5 +1,5 @@ -CREATE TABLE test.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\') ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\') ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 @@ -14,8 +14,8 @@ CREATE TABLE test.minmax_idx_r (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\') ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r (`u64` UInt64, `i32` Int32, INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\') ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 4 1 5 @@ -28,10 +28,10 @@ CREATE TABLE test.minmax_idx_r (`u64` UInt64, `i32` Int32, INDEX idx3 u64 - i32 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx (`u64` UInt64, `i32` Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\') ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r (`u64` UInt64, `i32` Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\') ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\') ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\') ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 4 1 5 @@ -44,14 +44,14 @@ CREATE TABLE test.minmax_idx_r (`u64` UInt64, `i32` Int32, INDEX idx1 u64 * i32 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx2 (`u64` UInt64, `i32` Int32, INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\') ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx2_r (`u64` UInt64, `i32` Int32, INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10, INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\') ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 3 1 2 1 3 -CREATE TABLE test.minmax_idx2 (`u64` UInt64, `i32` Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\') ORDER BY u64 SETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx2_r (`u64` UInt64, `i32` Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\') ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 3 1 2 diff --git a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference index 29bda49a8e5..ee481c88d89 100644 --- a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference +++ b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference @@ -20,7 +20,7 @@ 274972506.6 9175437371954010821 9175437371954010821 -CREATE TABLE test.compression_codec_multiple_more_types_replicated (`id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)), `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)), `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)), `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/compression_codec_multiple_more_types_replicated\', \'1\') ORDER BY tuple() SETTINGS index_granularity = 8192 +CREATE TABLE test.compression_codec_multiple_more_types_replicated\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)), \n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)), \n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)), \n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/compression_codec_multiple_more_types_replicated\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] 7.1000000000000 xxxxxxxxxxxx [127] ['Henry'] ! diff --git a/tests/queries/0_stateless/00916_create_or_replace_view.reference b/tests/queries/0_stateless/00916_create_or_replace_view.reference index 30d14bf1e41..50323e47556 100644 --- a/tests/queries/0_stateless/00916_create_or_replace_view.reference +++ b/tests/queries/0_stateless/00916_create_or_replace_view.reference @@ -1,2 +1,2 @@ -CREATE VIEW default.t (`number` UInt64) AS SELECT number FROM system.numbers -CREATE VIEW default.t (`next_number` UInt64) AS SELECT number + 1 AS next_number FROM system.numbers +CREATE VIEW default.t\n(\n `number` UInt64\n) AS\nSELECT number\nFROM system.numbers +CREATE VIEW default.t\n(\n `next_number` UInt64\n) AS\nSELECT number + 1 AS next_number\nFROM system.numbers diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 32cde7bbfb4..b83cbe17a6e 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -76,6 +76,21 @@ Friday {} "\\n\\u0000" "☺" +--JSONExtractArrayRaw-- +[] +[] +[] +['[]','[]'] +['-100','200','300'] +['1','2','3','4','5','"hello"'] +['1','2','3'] +['4','5','6'] +--JSONExtractKeysAndValuesRaw-- +[] +[] +[('a','"hello"'),('b','[-100,200,300]')] +[('a','"hello"'),('b','[-100,200,300]'),('c','{"d":[121,144]}')] +[('d','[121,144]')] --const/non-const mixed-- a b @@ -93,6 +108,10 @@ v 1 1 0 +--isValidJSON-- +1 +0 +0 --JSONKey-- a b @@ -158,14 +177,6 @@ Friday {} "\\n\\u0000" "☺" ---const/non-const mixed-- -a -b -c -d -e -u -v --JSONExtractArrayRaw-- [] [] @@ -175,3 +186,17 @@ v ['1','2','3','4','5','"hello"'] ['1','2','3'] ['4','5','6'] +--JSONExtractKeysAndValuesRaw-- +[] +[] +[('a','"hello"'),('b','[-100,200,300]')] +[('a','"hello"'),('b','[-100,200,300]'),('c','{"d":[121,144]}')] +[('d','[121,144]')] +--const/non-const mixed-- +a +b +c +d +e +u +v diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index 0db9540377e..7a030d3dab6 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -88,6 +88,22 @@ SELECT JSONExtractRaw('{}'); SELECT JSONExtractRaw('{"abc":"\\n\\u0000"}', 'abc'); SELECT JSONExtractRaw('{"abc":"\\u263a"}', 'abc'); +SELECT '--JSONExtractArrayRaw--'; +SELECT JSONExtractArrayRaw(''); +SELECT JSONExtractArrayRaw('{"a": "hello", "b": "not_array"}'); +SELECT JSONExtractArrayRaw('[]'); +SELECT JSONExtractArrayRaw('[[],[]]'); +SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONExtractArrayRaw('[1,2,3,4,5,"hello"]'); +SELECT JSONExtractArrayRaw(arrayJoin(JSONExtractArrayRaw('[[1,2,3],[4,5,6]]'))); + +SELECT '--JSONExtractKeysAndValuesRaw--'; +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c'); + SELECT '--const/non-const mixed--'; SELECT JSONExtractString('["a", "b", "c", "d", "e"]', idx) FROM (SELECT arrayJoin([1,2,3,4,5]) AS idx); SELECT JSONExtractString(json, 's') FROM (SELECT arrayJoin(['{"s":"u"}', '{"s":"v"}']) AS json); @@ -107,6 +123,11 @@ SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'c'); +SELECT '--isValidJSON--'; +SELECT isValidJSON('{"a": "hello", "b": [-100, 200.0, 300]}'); +SELECT isValidJSON('not a json'); +SELECT isValidJSON('"HX-='); + SELECT '--JSONKey--'; SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', 1); SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', 2); @@ -179,10 +200,6 @@ SELECT JSONExtractRaw('{}'); SELECT JSONExtractRaw('{"abc":"\\n\\u0000"}', 'abc'); SELECT JSONExtractRaw('{"abc":"\\u263a"}', 'abc'); -SELECT '--const/non-const mixed--'; -SELECT JSONExtractString('["a", "b", "c", "d", "e"]', idx) FROM (SELECT arrayJoin([1,2,3,4,5]) AS idx); -SELECT JSONExtractString(json, 's') FROM (SELECT arrayJoin(['{"s":"u"}', '{"s":"v"}']) AS json); - SELECT '--JSONExtractArrayRaw--'; SELECT JSONExtractArrayRaw(''); SELECT JSONExtractArrayRaw('{"a": "hello", "b": "not_array"}'); @@ -191,3 +208,14 @@ SELECT JSONExtractArrayRaw('[[],[]]'); SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); SELECT JSONExtractArrayRaw('[1,2,3,4,5,"hello"]'); SELECT JSONExtractArrayRaw(arrayJoin(JSONExtractArrayRaw('[[1,2,3],[4,5,6]]'))); + +SELECT '--JSONExtractKeysAndValuesRaw--'; +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c'); + +SELECT '--const/non-const mixed--'; +SELECT JSONExtractString('["a", "b", "c", "d", "e"]', idx) FROM (SELECT arrayJoin([1,2,3,4,5]) AS idx); +SELECT JSONExtractString(json, 's') FROM (SELECT arrayJoin(['{"s":"u"}', '{"s":"v"}']) AS json); diff --git a/tests/queries/0_stateless/00933_alter_ttl.reference b/tests/queries/0_stateless/00933_alter_ttl.reference index 44ba49026a7..9b5cec0f773 100644 --- a/tests/queries/0_stateless/00933_alter_ttl.reference +++ b/tests/queries/0_stateless/00933_alter_ttl.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.ttl (`d` Date, `a` Int32) ENGINE = MergeTree PARTITION BY toDayOfMonth(d) ORDER BY a TTL d + toIntervalDay(1) SETTINGS index_granularity = 8192 +CREATE TABLE default.ttl\n(\n `d` Date, \n `a` Int32\n)\nENGINE = MergeTree\nPARTITION BY toDayOfMonth(d)\nORDER BY a\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 2100-10-10 3 2100-10-10 4 d Date diff --git a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference index 986bc6b4a24..629fbf2a4a3 100644 --- a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference @@ -1,3 +1,3 @@ 200 400 -CREATE TABLE test.ttl_repl2 (`d` Date, `x` UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/ttl_repl\', \'2\') PARTITION BY toDayOfMonth(d) ORDER BY x TTL d + toIntervalDay(1) SETTINGS index_granularity = 8192 +CREATE TABLE test.ttl_repl2\n(\n `d` Date, \n `x` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/ttl_repl\', \'2\')\nPARTITION BY toDayOfMonth(d)\nORDER BY x\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00933_ttl_simple.reference b/tests/queries/0_stateless/00933_ttl_simple.reference index e8b0c699aec..102639947a3 100644 --- a/tests/queries/0_stateless/00933_ttl_simple.reference +++ b/tests/queries/0_stateless/00933_ttl_simple.reference @@ -6,11 +6,11 @@ 2000-10-10 00:00:00 0 2100-10-10 00:00:00 3 2100-10-10 2 -CREATE TABLE default.ttl_00933_1 (`b` Int32, `a` Int32 TTL now() - 1000) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple() SETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL now() - 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 0 -CREATE TABLE default.ttl_00933_1 (`b` Int32, `a` Int32 TTL now() + 1000) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple() SETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL now() + 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 1 -CREATE TABLE default.ttl_00933_1 (`b` Int32, `a` Int32 TTL today() - 1) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple() SETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL today() - 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 0 -CREATE TABLE default.ttl_00933_1 (`b` Int32, `a` Int32 TTL today() + 1) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple() SETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL today() + 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 1 diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql index 82549a5d395..6f93ae89a42 100755 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -247,19 +247,19 @@ SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(fixed_string DROP TABLE IF EXISTS bloom_filter_array_lc_null_types_test; DROP TABLE IF EXISTS bloom_filter_array_offsets_lc_str; -CREATE TABLE bloom_filter_array_offsets_lc_str (order_key int, str Array(LowCardinality((String))), INDEX idx str TYPE bloom_filter(1.01) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; +CREATE TABLE bloom_filter_array_offsets_lc_str (order_key int, str Array(LowCardinality((String))), INDEX idx str TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; INSERT INTO bloom_filter_array_offsets_lc_str SELECT number AS i, if(i%2, ['value'], []) FROM system.numbers LIMIT 10000; SELECT count() FROM bloom_filter_array_offsets_lc_str WHERE has(str, 'value'); DROP TABLE IF EXISTS bloom_filter_array_offsets_lc_str; DROP TABLE IF EXISTS bloom_filter_array_offsets_str; -CREATE TABLE bloom_filter_array_offsets_str (order_key int, str Array(String), INDEX idx str TYPE bloom_filter(1.01) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; +CREATE TABLE bloom_filter_array_offsets_str (order_key int, str Array(String), INDEX idx str TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; INSERT INTO bloom_filter_array_offsets_str SELECT number AS i, if(i%2, ['value'], []) FROM system.numbers LIMIT 10000; SELECT count() FROM bloom_filter_array_offsets_str WHERE has(str, 'value'); DROP TABLE IF EXISTS bloom_filter_array_offsets_str; DROP TABLE IF EXISTS bloom_filter_array_offsets_i; -CREATE TABLE bloom_filter_array_offsets_i (order_key int, i Array(int), INDEX idx i TYPE bloom_filter(1.01) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; +CREATE TABLE bloom_filter_array_offsets_i (order_key int, i Array(int), INDEX idx i TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; INSERT INTO bloom_filter_array_offsets_i SELECT number AS i, if(i%2, [99999], []) FROM system.numbers LIMIT 10000; SELECT count() FROM bloom_filter_array_offsets_i WHERE has(i, 99999); DROP TABLE IF EXISTS bloom_filter_array_offsets_i; diff --git a/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py index 9d0fb7e941a..4418c22872a 100755 --- a/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py +++ b/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py @@ -23,7 +23,7 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send('DROP TABLE IF EXISTS test.mt') client1.expect(prompt) client1.send('SET live_view_heartbeat_interval=1') client1.expect(prompt) @@ -31,14 +31,16 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client1.send('CREATE LIVE VIEW test.lv WITH TIMEOUT AS SELECT sum(a) FROM test.mt') client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') + client1.send('WATCH test.lv EVENTS FORMAT CSV') + client1.expect('Progress: 1.00 rows.*\)') client2.send('INSERT INTO test.mt VALUES (1)') - client1.expect('1.*' + end_of_block) + client2.expect(prompt) + client1.expect('Progress: 2.00 rows.*\)') client2.send('INSERT INTO test.mt VALUES (2),(3)') - client1.expect('[23].*' + end_of_block) - client1.expect('Progress: [23]\.00 rows.*\)') + client2.expect(prompt) + client1.expect('Progress: 3.00 rows.*\)') # wait for heartbeat - client1.expect('Progress: [23]\.00 rows.*\)') + client1.expect('Progress: 3.00 rows.*\)') # send Ctrl-C client1.send('\x03', eol='') match = client1.expect('(%s)|([#\$] )' % prompt) diff --git a/tests/queries/0_stateless/00967_insert_into_distributed_different_types.reference b/tests/queries/0_stateless/00967_insert_into_distributed_different_types.reference index e69de29bb2d..573541ac970 100644 --- a/tests/queries/0_stateless/00967_insert_into_distributed_different_types.reference +++ b/tests/queries/0_stateless/00967_insert_into_distributed_different_types.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql b/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql index 6b23c72981a..33f16eb241c 100644 --- a/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql +++ b/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql @@ -4,6 +4,6 @@ DROP TABLE IF EXISTS underlying_00967; CREATE TABLE dist_00967 (key UInt64) Engine=Distributed('test_shard_localhost', currentDatabase(), underlying_00967); -- fails for TinyLog()/MergeTree()/... but not for Memory() CREATE TABLE underlying_00967 (key Nullable(UInt64)) Engine=TinyLog(); -INSERT INTO dist_00967 SELECT toUInt64(number) FROM system.numbers LIMIT 1; -- { serverError 171; } +INSERT INTO dist_00967 SELECT toUInt64(number) FROM system.numbers LIMIT 1; SELECT * FROM dist_00967; diff --git a/tests/queries/0_stateless/00980_merge_alter_settings.reference b/tests/queries/0_stateless/00980_merge_alter_settings.reference index ee3818d25dc..340cf29ce89 100644 --- a/tests/queries/0_stateless/00980_merge_alter_settings.reference +++ b/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -1,6 +1,6 @@ -CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096 -CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 2 -CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 -CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String, \n `Data2` UInt64\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 diff --git a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference index 159102e1ca7..ab006ea6931 100644 --- a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference +++ b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference @@ -1,12 +1,12 @@ -CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192 -CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 4 4 4 4 6 6 -CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 -CREATE TABLE default.replicated_table_for_alter2 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\') ORDER BY id SETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 -CREATE TABLE default.replicated_table_for_alter2 (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\') ORDER BY id SETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 +CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String, \n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 +CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64, \n `Data` String, \n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 diff --git a/tests/queries/0_stateless/00990_hasToken.sh b/tests/queries/0_stateless/00990_hasToken.sh index 4ccb77b8ecc..4ef62bc69c0 100755 --- a/tests/queries/0_stateless/00990_hasToken.sh +++ b/tests/queries/0_stateless/00990_hasToken.sh @@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # We should have correct env vars from shell_config.sh to run this test -python $CURDIR/00990_hasToken.python | ${CLICKHOUSE_CLIENT} -nm +python $CURDIR/00990_hasToken.python | ${CLICKHOUSE_CLIENT} --max_query_size 1048576 -nm diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 2d458ba7e28..01127a3ef7b 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -72,28 +72,28 @@ timeout $TIMEOUT bash -c thread2 2> /dev/null & timeout $TIMEOUT bash -c thread3 2> /dev/null & timeout $TIMEOUT bash -c thread4 2> /dev/null & timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & timeout $TIMEOUT bash -c thread1 2> /dev/null & timeout $TIMEOUT bash -c thread2 2> /dev/null & timeout $TIMEOUT bash -c thread3 2> /dev/null & timeout $TIMEOUT bash -c thread4 2> /dev/null & timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & timeout $TIMEOUT bash -c thread1 2> /dev/null & timeout $TIMEOUT bash -c thread2 2> /dev/null & timeout $TIMEOUT bash -c thread3 2> /dev/null & timeout $TIMEOUT bash -c thread4 2> /dev/null & timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & timeout $TIMEOUT bash -c thread1 2> /dev/null & timeout $TIMEOUT bash -c thread2 2> /dev/null & timeout $TIMEOUT bash -c thread3 2> /dev/null & timeout $TIMEOUT bash -c thread4 2> /dev/null & timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & wait diff --git a/tests/queries/0_stateless/00998_constraints_all_tables.reference b/tests/queries/0_stateless/00998_constraints_all_tables.reference index 730df555af3..3de251daa71 100644 --- a/tests/queries/0_stateless/00998_constraints_all_tables.reference +++ b/tests/queries/0_stateless/00998_constraints_all_tables.reference @@ -10,5 +10,5 @@ 0 0 3 -CREATE TABLE default.constrained (`URL` String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log -CREATE TABLE default.constrained2 (`URL` String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log +CREATE TABLE default.constrained\n(\n `URL` String, \n CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = \'yandex.ru\', \n CONSTRAINT is_utf8 CHECK isValidUTF8(URL)\n)\nENGINE = Log +CREATE TABLE default.constrained2\n(\n `URL` String, \n CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = \'yandex.ru\', \n CONSTRAINT is_utf8 CHECK isValidUTF8(URL)\n)\nENGINE = Log diff --git a/tests/queries/0_stateless/00998_constraints_all_tables.sql b/tests/queries/0_stateless/00998_constraints_all_tables.sql index 66b93fca97b..e47b7eaf83c 100644 --- a/tests/queries/0_stateless/00998_constraints_all_tables.sql +++ b/tests/queries/0_stateless/00998_constraints_all_tables.sql @@ -45,8 +45,8 @@ DROP TABLE constrained; DROP TABLE IF EXISTS constrained2; CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log; CREATE TABLE constrained2 AS constrained; -SHOW CREATE TABLE constrained FORMAT TSVRaw; -SHOW CREATE TABLE constrained2 FORMAT TSVRaw; +SHOW CREATE TABLE constrained; +SHOW CREATE TABLE constrained2; INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } INSERT INTO constrained2 VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } DROP TABLE constrained; diff --git a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.reference b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.reference new file mode 100755 index 00000000000..678f9a34e6f --- /dev/null +++ b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.reference @@ -0,0 +1 @@ +Test OK diff --git a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh new file mode 100755 index 00000000000..8bf21d3cb02 --- /dev/null +++ b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +export CURR_DATABASE="test_lazy_01014_concurrent_${CLICKHOUSE_DATABASE}" + + +function recreate_lazy_func1() +{ + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; + "; + + while true; do + $CLICKHOUSE_CLIENT -q " + DETACH TABLE $CURR_DATABASE.log; + "; + + $CLICKHOUSE_CLIENT -q " + ATTACH TABLE $CURR_DATABASE.log; + "; + done +} + +function recreate_lazy_func2() +{ + while true; do + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; + "; + + $CLICKHOUSE_CLIENT -q " + DROP TABLE $CURR_DATABASE.tlog; + "; + done +} + +function recreate_lazy_func3() +{ + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; + "; + + while true; do + $CLICKHOUSE_CLIENT -q " + ATTACH TABLE $CURR_DATABASE.slog; + "; + + $CLICKHOUSE_CLIENT -q " + DETACH TABLE $CURR_DATABASE.slog; + "; + done +} + +function recreate_lazy_func4() +{ + while true; do + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; + "; + + $CLICKHOUSE_CLIENT -q " + DROP TABLE $CURR_DATABASE.tlog2; + "; + done +} + +function show_tables_func() +{ + while true; do + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE database = '$CURR_DATABASE' FORMAT Null"; + done +} + + +export -f recreate_lazy_func1; +export -f recreate_lazy_func2; +export -f recreate_lazy_func3; +export -f recreate_lazy_func4; +export -f show_tables_func; + + +${CLICKHOUSE_CLIENT} -n -q " + DROP DATABASE IF EXISTS $CURR_DATABASE; + CREATE DATABASE $CURR_DATABASE ENGINE = Lazy(1); +" + + +TIMEOUT=30 + +timeout $TIMEOUT bash -c recreate_lazy_func1 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func2 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func3 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func4 2> /dev/null & +timeout $TIMEOUT bash -c show_tables_func 2> /dev/null & + +wait +sleep 1 + +${CLICKHOUSE_CLIENT} -n -q " + DROP TABLE IF EXISTS $CURR_DATABASE.log; + DROP TABLE IF EXISTS $CURR_DATABASE.slog; + DROP TABLE IF EXISTS $CURR_DATABASE.tlog; + DROP TABLE IF EXISTS $CURR_DATABASE.tlog2; +" +# DROP DATABASE $CURR_DATABASE; -- This fails for some reason + +echo "Test OK" diff --git a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.reference b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.reference index b462a5a7baa..6bf25043399 100644 --- a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.reference +++ b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.reference @@ -2,3 +2,8 @@ OK OK OK OK +OK +OK +OK +OK +OK diff --git a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh index ac66dbc352a..68cb5e0e760 100755 --- a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh +++ b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh @@ -11,6 +11,22 @@ T1=table_1017_merge ${CLICKHOUSE_CLIENT} -n -q " DROP TABLE IF EXISTS $R1; DROP TABLE IF EXISTS $R2; + DROP TABLE IF EXISTS $T1; + + DROP TABLE IF EXISTS lookup_table; + DROP TABLE IF EXISTS table_for_dict; + DROP DICTIONARY IF EXISTS dict1; + + CREATE TABLE table_for_dict (y UInt64, y_new UInt32) ENGINE = Log; + INSERT INTO table_for_dict VALUES (3, 3003),(4,4004); + + CREATE DICTIONARY dict1( y UInt64 DEFAULT 0, y_new UInt32 DEFAULT 0 ) PRIMARY KEY y + SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB '${CLICKHOUSE_DATABASE}')) + LIFETIME(MIN 1 MAX 10) + LAYOUT(FLAT()); + + CREATE TABLE lookup_table (y UInt32, y_new UInt32) ENGINE = Join(ANY, LEFT, y); + INSERT INTO lookup_table VALUES(1,1001),(2,1002); CREATE TABLE $R1 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1017', 'r1') ORDER BY x; CREATE TABLE $R2 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1017', 'r2') ORDER BY x; @@ -27,6 +43,12 @@ ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 DELETE WHERE ignore(rand())" 2>&1 ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 UPDATE y = y + rand() % 1 WHERE not ignore()" 2>&1 \ | fgrep -q "must use only deterministic functions" && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 UPDATE y = x + arrayCount(x -> (x + y) % 2, range(y)) WHERE not ignore()" 2>&1 > /dev/null \ +&& echo 'OK' || echo 'FAIL' + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 UPDATE y = x + arrayCount(x -> (rand() + x) % 2, range(y)) WHERE not ignore()" 2>&1 \ +| fgrep -q "must use only deterministic functions" && echo 'OK' || echo 'FAIL' + # For regular tables we do not enforce deterministic functions ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $T1 DELETE WHERE rand() = 0" 2>&1 > /dev/null \ @@ -35,9 +57,21 @@ ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $T1 DELETE WHERE rand() = 0" 2>&1 > /d ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $T1 UPDATE y = y + rand() % 1 WHERE not ignore()" 2>&1 > /dev/null \ && echo 'OK' || echo 'FAIL' +# hm... it looks like joinGet condidered determenistic +${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 UPDATE y = joinGet('${CLICKHOUSE_DATABASE}.lookup_table', 'y_new', y) WHERE x=1" 2>&1 \ +| echo 'OK' || echo 'FAIL' + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 DELETE WHERE dictHas('${CLICKHOUSE_DATABASE}.dict1', toUInt64(x))" 2>&1 \ +| fgrep -q "must use only deterministic functions" && echo 'OK' || echo 'FAIL' + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE $R1 DELETE WHERE dictHas('${CLICKHOUSE_DATABASE}.dict1', toUInt64(x))" --allow_nondeterministic_mutations=1 2>&1 \ +&& echo 'OK' || echo 'FAIL' ${CLICKHOUSE_CLIENT} -n -q " DROP TABLE IF EXISTS $R2; DROP TABLE IF EXISTS $R1; DROP TABLE IF EXISTS $T1; + DROP TABLE IF EXISTS lookup_table; + DROP TABLE IF EXISTS table_for_dict; + DROP DICTIONARY IF EXISTS dict1; " diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference index 327c02a4b8a..ad16e8ae7f2 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference @@ -1,5 +1,5 @@ =DICTIONARY in Ordinary DB -CREATE DICTIONARY ordinary_db.dict1 (`key_column` UInt64 DEFAULT 0, `second_column` UInt8 DEFAULT 1, `third_column` String DEFAULT \'qqq\') PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict\')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) +CREATE DICTIONARY ordinary_db.dict1\n(\n `key_column` UInt64 DEFAULT 0, \n `second_column` UInt8 DEFAULT 1, \n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) dict1 1 ordinary_db dict1 diff --git a/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.reference b/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.reference index 87dc6a5b6bf..4a22b3a52cf 100644 --- a/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.reference +++ b/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.reference @@ -9,6 +9,7 @@ dict1 dict2 dict3 +dict4 table_for_dict dict1 dict2 diff --git a/tests/queries/0_stateless/01033_quota_dcl.reference b/tests/queries/0_stateless/01033_quota_dcl.reference index 7f92f992dd5..7bd2d2923d2 100644 --- a/tests/queries/0_stateless/01033_quota_dcl.reference +++ b/tests/queries/0_stateless/01033_quota_dcl.reference @@ -1,2 +1,2 @@ default -CREATE QUOTA default KEYED BY \'user name\' FOR INTERVAL 1 HOUR TRACKING TO default, readonly +CREATE QUOTA default KEYED BY \'user name\' FOR INTERVAL 1 HOUR TRACKING ONLY TO default, readonly diff --git a/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference b/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference index bba4944f4a8..c2d7d849fae 100644 --- a/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference +++ b/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference @@ -1 +1 @@ -CREATE TABLE default.BannerDict (`BannerID` UInt64, `CompaignID` UInt64) ENGINE = ODBC(\'DSN=pgconn;Database=postgres\', \'somedb\', \'bannerdict\') +CREATE TABLE default.BannerDict\n(\n `BannerID` UInt64, \n `CompaignID` UInt64\n)\nENGINE = ODBC(\'DSN=pgconn;Database=postgres\', \'somedb\', \'bannerdict\') diff --git a/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql b/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql index ffc33ce6949..dbec319ab76 100644 --- a/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql +++ b/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql @@ -2,8 +2,11 @@ SET distributed_directory_monitor_batch_inserts=1; SET distributed_directory_monitor_sleep_time_ms=10; SET distributed_directory_monitor_max_sleep_time_ms=100; -CREATE TABLE test (key UInt64) ENGINE=TinyLog(); -CREATE TABLE dist_test AS test Engine=Distributed(test_cluster_two_shards, currentDatabase(), test, key); -INSERT INTO dist_test SELECT toUInt64(number) FROM numbers(2); -SYSTEM FLUSH DISTRIBUTED dist_test; -SELECT * FROM dist_test; +DROP TABLE IF EXISTS test_01040; +DROP TABLE IF EXISTS dist_test_01040; + +CREATE TABLE test_01040 (key UInt64) ENGINE=TinyLog(); +CREATE TABLE dist_test_01040 AS test_01040 Engine=Distributed(test_cluster_two_shards, currentDatabase(), test_01040, key); +INSERT INTO dist_test_01040 SELECT toUInt64(number) FROM numbers(2); +SYSTEM FLUSH DISTRIBUTED dist_test_01040; +SELECT * FROM dist_test_01040; diff --git a/tests/queries/0_stateless/01048_exists_query.sql b/tests/queries/0_stateless/01048_exists_query.sql index 9a4c0558b60..700b4f5983d 100644 --- a/tests/queries/0_stateless/01048_exists_query.sql +++ b/tests/queries/0_stateless/01048_exists_query.sql @@ -32,7 +32,7 @@ EXISTS TABLE db_01048.t_01048; -- Dictionaries are tables as well. But not all t EXISTS DICTIONARY db_01048.t_01048; -- But dictionary-tables cannot be dropped as usual tables. -DROP TABLE db_01048.t_01048; -- { serverError 60 } +DROP TABLE db_01048.t_01048; -- { serverError 520 } DROP DICTIONARY db_01048.t_01048; EXISTS db_01048.t_01048; EXISTS TABLE db_01048.t_01048; diff --git a/tests/queries/0_stateless/01055_compact_parts_1.reference b/tests/queries/0_stateless/01055_compact_parts_1.reference index 7c9dd4a0ef9..b99f336d3b0 100644 --- a/tests/queries/0_stateless/01055_compact_parts_1.reference +++ b/tests/queries/0_stateless/01055_compact_parts_1.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.mt_compact (`a` Int32, `s` String) ENGINE = MergeTree PARTITION BY a ORDER BY a SETTINGS index_granularity_bytes = 0, index_granularity = 8192 -CREATE TABLE default.mt_compact (`a` Int32, `s` String) ENGINE = MergeTree PARTITION BY a ORDER BY a SETTINGS index_granularity_bytes = 0, min_rows_for_wide_part = 0, index_granularity = 8192, parts_to_delay_insert = 300 +CREATE TABLE default.mt_compact\n(\n `a` Int32, \n `s` String\n)\nENGINE = MergeTree\nPARTITION BY a\nORDER BY a\nSETTINGS index_granularity_bytes = 0, index_granularity = 8192 +CREATE TABLE default.mt_compact\n(\n `a` Int32, \n `s` String\n)\nENGINE = MergeTree\nPARTITION BY a\nORDER BY a\nSETTINGS index_granularity_bytes = 0, min_rows_for_wide_part = 0, index_granularity = 8192, parts_to_delay_insert = 300 diff --git a/tests/queries/0_stateless/01056_create_table_as.sql b/tests/queries/0_stateless/01056_create_table_as.sql index 868e1f082dd..f95df9b7906 100644 --- a/tests/queries/0_stateless/01056_create_table_as.sql +++ b/tests/queries/0_stateless/01056_create_table_as.sql @@ -15,6 +15,7 @@ CREATE TABLE t3 AS v; -- { serverError 80; } DROP TABLE v; -- dictionary +DROP DICTIONARY IF EXISTS dict; DROP DATABASE if exists test_01056_dict_data; CREATE DATABASE test_01056_dict_data; CREATE TABLE test_01056_dict_data.dict_data (key Int, value UInt16) Engine=Memory(); diff --git a/tests/queries/0_stateless/01060_avro.reference b/tests/queries/0_stateless/01060_avro.reference index a21e7a3a101..0550967a224 100644 --- a/tests/queries/0_stateless/01060_avro.reference +++ b/tests/queries/0_stateless/01060_avro.reference @@ -19,6 +19,10 @@ = references "a1","c1" "a2","c2" += nested +1,"b1",2.2,2.3,"c3" +2.3,"b1",1,"c3" +not found = compression 1000 1000 diff --git a/tests/queries/0_stateless/01060_avro.sh b/tests/queries/0_stateless/01060_avro.sh index 15e97abfa52..a64b2884731 100755 --- a/tests/queries/0_stateless/01060_avro.sh +++ b/tests/queries/0_stateless/01060_avro.sh @@ -27,6 +27,11 @@ cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --out echo = references cat $DATA_DIR/references.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "a String, c String" -q 'select * from table' +echo = nested +cat $DATA_DIR/nested.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64, "b.a" String, "b.b" Double, "b.c" Double, c String' -q 'select * from table' +cat $DATA_DIR/nested.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S '"b.c" Double, "b.a" String, a Int64, c String' -q 'select * from table' +cat $DATA_DIR/nested.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S '"b" Double' -q 'select * from table' 2>&1 | grep -i 'not found' -o + echo = compression cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' cat $DATA_DIR/simple.deflate.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' @@ -68,4 +73,4 @@ ${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(0) format Avro ${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(1000) format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S4" -q 'select count() from table' # type supported via conversion -${CLICKHOUSE_LOCAL} -q "select toInt16(123) as a format Avro" | wc -c \ No newline at end of file +${CLICKHOUSE_LOCAL} -q "select toInt16(123) as a format Avro" | wc -c | tr -d ' ' \ No newline at end of file diff --git a/tests/queries/0_stateless/01062_alter_on_mutataion.reference b/tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper.reference similarity index 100% rename from tests/queries/0_stateless/01062_alter_on_mutataion.reference rename to tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper.reference diff --git a/tests/queries/0_stateless/01062_alter_on_mutataion.sql b/tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper.sql similarity index 100% rename from tests/queries/0_stateless/01062_alter_on_mutataion.sql rename to tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper.sql diff --git a/tests/queries/0_stateless/01068_parens.sql b/tests/queries/0_stateless/01068_parens.sql index 7cb4f097b15..42948760594 100644 --- a/tests/queries/0_stateless/01068_parens.sql +++ b/tests/queries/0_stateless/01068_parens.sql @@ -1 +1,2 @@ -((((((((((((((SELECT((((((((((((((((((((((((((((((((1)))))))))))))))))))))))))))))))))))))))))))))) +SET max_parser_depth = 10000; +((((((((((((((SELECT((((((((((((((((((((((((((((((((1)))))))))))))))))))))))))))))))))))))))))))))); diff --git a/tests/queries/0_stateless/01069_database_memory.reference b/tests/queries/0_stateless/01069_database_memory.reference index 393c85070b9..e7486d57276 100644 --- a/tests/queries/0_stateless/01069_database_memory.reference +++ b/tests/queries/0_stateless/01069_database_memory.reference @@ -1,8 +1,8 @@ -CREATE DATABASE memory_01069 ENGINE = Memory() +CREATE DATABASE memory_01069\nENGINE = Memory() 1 2 3 4 3 4 -CREATE TABLE memory_01069.file (`n` UInt8) ENGINE = File(\'CSV\') +CREATE TABLE memory_01069.file\n(\n `n` UInt8\n)\nENGINE = File(\'CSV\') diff --git a/tests/queries/0_stateless/01070_alter_with_ttl.reference b/tests/queries/0_stateless/01070_alter_with_ttl.reference index 8b2bd9d1389..de7833472a1 100644 --- a/tests/queries/0_stateless/01070_alter_with_ttl.reference +++ b/tests/queries/0_stateless/01070_alter_with_ttl.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.alter_ttl (`i` Int32, `s` String TTL toDate(\'2020-01-01\')) ENGINE = MergeTree ORDER BY i TTL toDate(\'2020-05-05\') SETTINGS index_granularity = 8192 -CREATE TABLE default.alter_ttl (`d` Date, `s` String TTL d + toIntervalDay(1)) ENGINE = MergeTree ORDER BY d TTL d + toIntervalMonth(1) SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_ttl\n(\n `i` Int32, \n `s` String TTL toDate(\'2020-01-01\')\n)\nENGINE = MergeTree\nORDER BY i\nTTL toDate(\'2020-05-05\')\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_ttl\n(\n `d` Date, \n `s` String TTL d + toIntervalDay(1)\n)\nENGINE = MergeTree\nORDER BY d\nTTL d + toIntervalMonth(1)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.reference b/tests/queries/0_stateless/01075_allowed_client_hosts.reference index 0082653059c..73f54c6027a 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.reference +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.reference @@ -8,10 +8,10 @@ CREATE USER test_user_01075 HOST LOCAL, IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765 CREATE USER test_user_01075 HOST LOCAL CREATE USER test_user_01075 HOST NONE CREATE USER test_user_01075 HOST LIKE \'@.somesite.com\' -CREATE USER test_user_01075 HOST NAME REGEXP \'.*.anothersite.com\' -CREATE USER test_user_01075 HOST NAME REGEXP \'.*.anothersite.com\', \'.*.anothersite.org\' -CREATE USER test_user_01075 HOST NAME REGEXP \'.*.anothersite2.com\', \'.*.anothersite2.org\' -CREATE USER test_user_01075 HOST NAME REGEXP \'.*.anothersite3.com\', \'.*.anothersite3.org\' +CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite.com\' +CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite.com\', \'.*.anothersite.org\' +CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite2.com\', \'.*.anothersite2.org\' +CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite3.com\', \'.*.anothersite3.org\' CREATE USER `test_user_01075_x@localhost` HOST LOCAL CREATE USER test_user_01075_x CREATE USER `test_user_01075_x@192.168.23.15` HOST LIKE \'192.168.23.15\' diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.sql b/tests/queries/0_stateless/01075_allowed_client_hosts.sql index e0b1c0f9905..2960a93f0f2 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.sql +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.sql @@ -30,16 +30,16 @@ SHOW CREATE USER test_user_01075; ALTER USER test_user_01075 HOST LIKE '@.somesite.com'; SHOW CREATE USER test_user_01075; -ALTER USER test_user_01075 HOST NAME REGEXP '.*\.anothersite\.com'; +ALTER USER test_user_01075 HOST REGEXP '.*\.anothersite\.com'; SHOW CREATE USER test_user_01075; -ALTER USER test_user_01075 HOST NAME REGEXP '.*\.anothersite\.com', '.*\.anothersite\.org'; +ALTER USER test_user_01075 HOST REGEXP '.*\.anothersite\.com', '.*\.anothersite\.org'; SHOW CREATE USER test_user_01075; -ALTER USER test_user_01075 HOST NAME REGEXP '.*\.anothersite2\.com', NAME REGEXP '.*\.anothersite2\.org'; +ALTER USER test_user_01075 HOST REGEXP '.*\.anothersite2\.com', REGEXP '.*\.anothersite2\.org'; SHOW CREATE USER test_user_01075; -ALTER USER test_user_01075 HOST NAME REGEXP '.*\.anothersite3\.com' HOST NAME REGEXP '.*\.anothersite3\.org'; +ALTER USER test_user_01075 HOST REGEXP '.*\.anothersite3\.com' HOST REGEXP '.*\.anothersite3\.org'; SHOW CREATE USER test_user_01075; DROP USER test_user_01075; diff --git a/tests/queries/0_stateless/01079_alter_default_zookeeper.reference b/tests/queries/0_stateless/01079_alter_default_zookeeper.reference index 35ba20aff3e..62d26bc9b4b 100644 --- a/tests/queries/0_stateless/01079_alter_default_zookeeper.reference +++ b/tests/queries/0_stateless/01079_alter_default_zookeeper.reference @@ -1,11 +1,11 @@ -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` String DEFAULT \'10\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` String DEFAULT \'10\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 1000 -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` UInt64 DEFAULT \'10\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` UInt64 DEFAULT 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt64 DEFAULT \'10\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt64 DEFAULT 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 1000 -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` UInt64 DEFAULT 100) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` UInt16 DEFAULT 100) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt64 DEFAULT 100\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt16 DEFAULT 100\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 10000 -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` UInt8 DEFAULT 10) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` UInt8 DEFAULT 10, `better_column` UInt8 DEFAULT \'1\') ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default (`date` Date, `key` UInt64, `value` UInt8 DEFAULT 10, `better_column` UInt8 DEFAULT \'1\', `other_date` String DEFAULT 1) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt8 DEFAULT 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt8 DEFAULT 10, \n `better_column` UInt8 DEFAULT \'1\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt8 DEFAULT 10, \n `better_column` UInt8 DEFAULT \'1\', \n `other_date` String DEFAULT 1\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference index 198f79cf9a4..ea3fbec34a8 100644 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference @@ -1,6 +1,6 @@ Wrong column name. -CREATE TABLE default.table_for_bad_alters (`key` UInt64, `value1` UInt8, `value2` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_bad_alters (`key` UInt64, `value1` UInt8, `value2` UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64, \n `value1` UInt8, \n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64, \n `value1` UInt8, \n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 syntax error at begin of string. 7 Hello diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference index 5b376a0654f..2007eda0f07 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference @@ -1,11 +1,11 @@ -CREATE TABLE test_01083.file (`n` Int8) ENGINE = File(\'TSVWithNamesAndTypes\') -CREATE TABLE test_01083.buffer (`n` Int8) ENGINE = Buffer(\'test_01083\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) -CREATE TABLE test_01083.merge (`n` Int8) ENGINE = Merge(\'test_01083\', \'distributed\') +CREATE TABLE test_01083.file\n(\n `n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') +CREATE TABLE test_01083.buffer\n(\n `n` Int8\n)\nENGINE = Buffer(\'test_01083\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) +CREATE TABLE test_01083.merge\n(\n `n` Int8\n)\nENGINE = Merge(\'test_01083\', \'distributed\') CREATE TABLE test_01083.merge_tf AS merge(\'test_01083\', \'.*\') -CREATE TABLE test_01083.distributed (`n` Int8) ENGINE = Distributed(\'test_shard_localhost\', \'test_01083\', \'file\') +CREATE TABLE test_01083.distributed\n(\n `n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'test_01083\', \'file\') CREATE TABLE test_01083.distributed_tf AS cluster(\'test_shard_localhost\', \'test_01083\', \'buffer\') -CREATE TABLE test_01083.url (`n` UInt64, `col` String) ENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+test_01083.merge+format+CSV\', \'CSV\') +CREATE TABLE test_01083.url\n(\n `n` UInt64, \n `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+test_01083.merge+format+CSV\', \'CSV\') CREATE TABLE test_01083.rich_syntax AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'test_01083\', \'view\'))) -CREATE VIEW test_01083.view (`n` Int64) AS SELECT toInt64(n) AS n FROM (SELECT toString(n) AS n FROM test_01083.merge WHERE _table != \'qwerty\' ORDER BY _table ASC) UNION ALL SELECT * FROM test_01083.file -CREATE DICTIONARY test_01083.dict (`n` UInt64, `col` String DEFAULT \'42\') PRIMARY KEY n SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\' DB \'test_01083\')) LIFETIME(MIN 0 MAX 1) LAYOUT(CACHE(SIZE_IN_CELLS 1)) +CREATE VIEW test_01083.view\n(\n `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM test_01083.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM test_01083.file +CREATE DICTIONARY test_01083.dict\n(\n `n` UInt64, \n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\' DB \'test_01083\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) 16 diff --git a/tests/queries/0_stateless/01086_odbc_roundtrip.sh b/tests/queries/0_stateless/01086_odbc_roundtrip.sh new file mode 100755 index 00000000000..71ea517f4dd --- /dev/null +++ b/tests/queries/0_stateless/01086_odbc_roundtrip.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + + +for i in $(seq 1 10); do + ${CLICKHOUSE_CLIENT} -q "select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (ANSI)}','system','tables'))" 2>/dev/null && break + sleep 0.1 +done + +${CLICKHOUSE_CLIENT} --query "select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (Unicode)}','system','tables'))" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS test_01086" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE test_01086" + + +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_01086.t (x UInt8, y Float32, z String) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_01086.t VALUES (1,0.1,'a я'),(2,0.2,'b ą'),(3,0.3,'c d')" + +${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (ANSI)}','test_01086','t') ORDER BY x" + +${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (Unicode)}','test_01086','t') ORDER BY x" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE test_01086;" diff --git a/tests/queries/0_stateless/01086_odbc_roundtrip.sql b/tests/queries/0_stateless/01086_odbc_roundtrip.sql deleted file mode 100644 index 2c31711d895..00000000000 --- a/tests/queries/0_stateless/01086_odbc_roundtrip.sql +++ /dev/null @@ -1,14 +0,0 @@ -select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (ANSI)}','system','tables')); -select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (Unicode)}','system','tables')); - -DROP DATABASE IF EXISTS test_01086; -CREATE DATABASE test_01086; -USE test_01086; - -CREATE TABLE t (x UInt8, y Float32, z String) ENGINE = Memory; -INSERT INTO t VALUES (1,0.1,'a я'),(2,0.2,'b ą'),(3,0.3,'c d'); - -select * from odbc('DSN={ClickHouse DSN (ANSI)}','test_01086','t') ORDER BY x; -select * from odbc('DSN={ClickHouse DSN (Unicode)}','test_01086','t') ORDER BY x; - -DROP DATABASE test_01086; diff --git a/tests/queries/0_stateless/01087_table_function_generate.reference b/tests/queries/0_stateless/01087_table_function_generate.reference index 68238faee48..d7cc6b0a933 100644 --- a/tests/queries/0_stateless/01087_table_function_generate.reference +++ b/tests/queries/0_stateless/01087_table_function_generate.reference @@ -1,14 +1,14 @@ UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 -2804162938822577320 -2776833771540858 3467776823 1163715250 23903 13655 137 -41 -7885388429666205427 -1363628932535403038 484159052 -308788249 56810 -22227 51 -41 -4357435422797280898 1355609803008819271 4126129912 -852056475 64304 -11401 139 86 -5935810273536892891 -804738887697332962 3109335413 -80126721 258 12889 18 88 -368066018677693974 -4927165984347126295 1015254922 2026080544 44305 21973 16 0 -8124171311239967992 -1179703908046100129 1720727300 -138469036 61343 10573 252 -32 -15657812979985370729 -5733276247123822513 3254757884 -500590428 45913 19153 105 -102 -18371568619324220532 -6793779541583578394 1686821450 -455892108 49050 -28603 248 80 -821735343441964030 3148260644406230976 256251035 -885069056 58858 -29361 58 61 -9558594037060121162 -2907172753635797124 4276198376 1947296644 26801 -13531 204 -66 +2804162938822577320 -2776833771540858 3467776823 1163715250 31161 -2916 220 -117 +7885388429666205427 -1363628932535403038 484159052 -308788249 43346 13638 143 -105 +4357435422797280898 1355609803008819271 4126129912 -852056475 34184 9166 49 33 +5935810273536892891 -804738887697332962 3109335413 -80126721 47877 -31421 186 -77 +368066018677693974 -4927165984347126295 1015254922 2026080544 46037 -29626 240 108 +8124171311239967992 -1179703908046100129 1720727300 -138469036 33028 -12819 138 16 +15657812979985370729 -5733276247123822513 3254757884 -500590428 3829 30527 3 -81 +18371568619324220532 -6793779541583578394 1686821450 -455892108 43475 2284 252 -90 +821735343441964030 3148260644406230976 256251035 -885069056 11643 11455 176 90 +9558594037060121162 -2907172753635797124 4276198376 1947296644 45922 26632 97 43 - Enum8(\'hello\' = 1, \'world\' = 5) hello @@ -47,16 +47,16 @@ h o - Date DateTime DateTime(\'Europe/Moscow\') -2106-02-07 2050-12-17 02:46:35 2096-02-16 22:18:22 -2106-02-07 2013-10-17 23:35:26 1976-01-24 12:52:48 -2039-08-16 1974-11-17 23:22:46 1980-03-04 21:02:50 -1997-04-11 1972-09-18 23:44:08 2040-07-10 14:46:42 -2103-11-03 2044-11-23 20:57:12 1970-10-09 02:30:14 -2066-11-19 2029-12-10 03:13:55 2106-01-30 21:52:44 -2064-08-14 2016-07-14 11:33:45 2096-12-12 00:40:50 -2046-09-13 2085-07-10 18:51:14 2096-01-15 16:31:33 -2008-03-16 2047-05-16 23:28:36 2103-02-11 16:44:39 -2000-07-07 2105-07-19 19:29:06 1980-01-02 05:18:22 +2077-09-17 1970-10-09 02:30:14 2074-08-12 11:31:27 +2005-11-19 2106-01-30 21:52:44 2097-05-25 07:54:35 +2007-02-24 2096-12-12 00:40:50 1988-08-10 11:16:31 +2019-06-30 2096-01-15 16:31:33 2063-10-20 08:48:17 +2039-01-16 2103-02-11 16:44:39 2036-10-09 04:29:10 +1994-11-03 1980-01-02 05:18:22 2055-12-23 12:33:52 +2083-08-20 2079-06-11 16:29:02 2000-12-05 17:46:24 +2030-06-25 2100-03-01 18:50:22 1993-03-25 01:19:12 +2087-03-16 2034-08-25 19:46:33 2045-12-10 16:47:40 +2006-04-30 2069-09-30 16:07:48 2084-08-26 03:33:12 - DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 1978-06-07 23:50:57.320 2013-08-28 10:21:54.010758 1991-08-25 16:23:26.140215 @@ -225,14 +225,14 @@ RL,{Xs\\tw [114] -84125.1554 ('2023-06-06 06:55:06.492','bf9ab359-ef9f-ad11-7e6c-160368b1e5ea') [124] -114719.5228 ('2010-11-11 22:57:23.722','c1046ffb-3415-cc3a-509a-e0005856d7d7') - -[] 1900051923 { -189530.5846 h -5.6279699579452485e47 ('1984-12-06','2028-08-17 06:05:01','2036-04-02 23:52:28.468','4b3d498c-dd44-95c1-5b75-921504ec5d8d') F743 -[-102,-118] 392272782 Eb -14818.0200 o -2.664492247169164e59 ('2082-12-26','2052-09-09 06:50:50','2088-04-21 05:07:08.245','aeb9c26e-0ee7-2b8e-802b-2a96319b8e60') CBF4 -[-71] 775049089 \N -158115.1178 w 4.1323844687113747e-305 ('2106-02-07','2090-07-31 16:45:26','2076-07-10 09:11:06.385','57c69bc6-dddd-0975-e932-a7b5173a1304') EB1D -[-28,100] 3675466147 { -146685.1749 h 3.6676044396877755e142 ('2017-10-25','2100-02-28 18:07:18','2055-10-14 06:36:20.056','14949dae-dfa8-a124-af83-887348b2f609') 6D88 -[-23] 2514120753 (`u, -119659.6174 w 1.3231258347475906e34 ('2106-02-07','2074-08-10 06:25:12','1976-12-04 18:31:55.745','86a9b3c1-4593-4d56-7762-3aa1dd22cbbf') AD43 -[11,-36] 3308237300 \N 171205.1896 \N 5.634708707075817e195 ('1974-10-31','1993-12-24 09:38:45','2038-07-15 05:22:51.805','63d999b8-8cca-e237-c4a4-4dd7d0096f65') 609E -[39] 1614362420 `4A8P 157144.0630 o -1.1843143253872814e-255 ('2106-02-07','2072-09-28 18:27:27','2073-07-10 12:19:58.146','6483f5c0-8733-364c-4fa0-9948d32e8903') A886 -[48,-120] 3848918261 1 $CURDIR/tmp_msgpac_test_all_types.msgpk; + +cat $CURDIR/tmp_msgpac_test_all_types.msgpk | $CLICKHOUSE_CLIENT --query="INSERT INTO msgpack FORMAT MsgPack"; + +rm $CURDIR/tmp_msgpac_test_all_types.msgpk + +$CLICKHOUSE_CLIENT --query="SELECT * FROM msgpack"; + +$CLICKHOUSE_CLIENT --query="DROP TABLE msgpack"; + + +$CLICKHOUSE_CLIENT --query="CREATE TABLE msgpack (array1 Array(Array(UInt32)), array2 Array(Array(Array(String)))) ENGINE = Memory"; + +$CLICKHOUSE_CLIENT --query="INSERT INTO msgpack VALUES ([[1,2,3], [1001, 2002], [3167]], [[['one'], ['two']], [['three']],[['four'], ['five']]])"; + +$CLICKHOUSE_CLIENT --query="SELECT * FROM msgpack FORMAT MsgPack" > $CURDIR/tmp_msgpack_test_nested_arrays.msgpk; + +cat $CURDIR/tmp_msgpack_test_nested_arrays.msgpk | $CLICKHOUSE_CLIENT --query="INSERT INTO msgpack FORMAT MsgPack"; +rm $CURDIR/tmp_msgpack_test_nested_arrays.msgpk; + +$CLICKHOUSE_CLIENT --query="SELECT * FROM msgpack"; + +$CLICKHOUSE_CLIENT --query="DROP TABLE msgpack"; + + +$CLICKHOUSE_CLIENT --query="CREATE TABLE msgpack (array Array(UInt8)) ENGINE = Memory"; + +$CLICKHOUSE_CLIENT --query="INSERT INTO msgpack VALUES ([0, 1, 2, 3, 42, 253, 254, 255]), ([255, 254, 253, 42, 3, 2, 1, 0])"; + +$CLICKHOUSE_CLIENT --query="SELECT * FROM msgpack FORMAT MsgPack" > $CURDIR/tmp_msgpack_type_conversion.msgpk; + +$CLICKHOUSE_CLIENT --query="DROP TABLE msgpack"; + +$CLICKHOUSE_CLIENT --query="CREATE TABLE msgpack (array Array(Int64)) ENGINE = Memory"; + +cat $CURDIR/tmp_msgpack_type_conversion.msgpk | $CLICKHOUSE_CLIENT --query="INSERT INTO msgpack FORMAT MsgPack"; +rm $CURDIR/tmp_msgpack_type_conversion.msgpk; + +$CLICKHOUSE_CLIENT --query="SELECT * FROM msgpack"; + +$CLICKHOUSE_CLIENT --query="DROP TABLE msgpack"; + +$CLICKHOUSE_CLIENT --query="CREATE TABLE msgpack (date FixedString(10)) ENGINE = Memory"; + +$CLICKHOUSE_CLIENT --query="INSERT INTO msgpack VALUES ('2020-01-01'), ('2020-01-02'), ('2020-01-02')"; + +$CLICKHOUSE_CLIENT --query="SELECT * FROM msgpack"; + +$CLICKHOUSE_CLIENT --query="DROP TABLE msgpack"; + diff --git a/tests/queries/0_stateless/01098_temporary_and_external_tables.sh b/tests/queries/0_stateless/01098_temporary_and_external_tables.sh index f8b9862c1c1..c984f363c31 100755 --- a/tests/queries/0_stateless/01098_temporary_and_external_tables.sh +++ b/tests/queries/0_stateless/01098_temporary_and_external_tables.sh @@ -5,12 +5,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) url="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/?session_id=test_01098" -${CLICKHOUSE_CURL} -sSk "$url" --data "CREATE TEMPORARY TABLE tmp_table AS SELECT number AS n FROM numbers(42)" > /dev/null; +${CLICKHOUSE_CURL} -m 30 -sSk "$url" --data "CREATE TEMPORARY TABLE tmp_table AS SELECT number AS n FROM numbers(42)" > /dev/null; name_expr="'\`' || database || '\`.\`' || name || '\`'" -full_tmp_name=`echo "SELECT $name_expr FROM system.tables WHERE database='_temporary_and_external_tables' AND create_table_query LIKE '%tmp_table%'" | ${CLICKHOUSE_CURL} -sSgk $url -d @-` +full_tmp_name=`echo "SELECT $name_expr FROM system.tables WHERE database='_temporary_and_external_tables' AND create_table_query LIKE '%tmp_table%'" | ${CLICKHOUSE_CURL} -m 30 -sSgk $url -d @-` -echo "SELECT * FROM $full_tmp_name" | ${CLICKHOUSE_CURL} -sSgk $url -d @- | grep -F "Code: 291" > /dev/null && echo "OK" +echo "SELECT * FROM $full_tmp_name" | ${CLICKHOUSE_CURL} -m 60 -sSgk $url -d @- | grep -F "Code: 291" > /dev/null && echo "OK" -echo -ne '0\n1\n' | ${CLICKHOUSE_CURL} -sSkF 'file=@-' "$url&file_format=CSV&file_types=UInt64&query=SELECT+sum((number+GLOBAL+IN+(SELECT+number+AS+n+FROM+remote('127.0.0.2',+numbers(5))+WHERE+n+GLOBAL+IN+(SELECT+*+FROM+tmp_table)+AND+n+GLOBAL+NOT+IN+(SELECT+*+FROM+file)+))+AS+res),+sum(number*res)+FROM+remote('127.0.0.2',+numbers(10))"; +echo -ne '0\n1\n' | ${CLICKHOUSE_CURL} -m 30 -sSkF 'file=@-' "$url&file_format=CSV&file_types=UInt64&query=SELECT+sum((number+GLOBAL+IN+(SELECT+number+AS+n+FROM+remote('127.0.0.2',+numbers(5))+WHERE+n+GLOBAL+IN+(SELECT+*+FROM+tmp_table)+AND+n+GLOBAL+NOT+IN+(SELECT+*+FROM+file)+))+AS+res),+sum(number*res)+FROM+remote('127.0.0.2',+numbers(10))"; diff --git a/tests/queries/0_stateless/01101_literal_column_clash.reference b/tests/queries/0_stateless/01101_literal_column_clash.reference new file mode 100644 index 00000000000..47458541f79 --- /dev/null +++ b/tests/queries/0_stateless/01101_literal_column_clash.reference @@ -0,0 +1,14 @@ +1 +7 3 +7 0 +7 1 +xyzabc 2 +1 3 +1 2 0 0 +1 0 0 3 +\N 1 2 \N 0 +\N 1 0 \N 3 +1 +1 +1 hello\0\0\0 +1 hello\0\0\0 diff --git a/tests/queries/0_stateless/01101_literal_column_clash.sql b/tests/queries/0_stateless/01101_literal_column_clash.sql new file mode 100644 index 00000000000..2c665d3833b --- /dev/null +++ b/tests/queries/0_stateless/01101_literal_column_clash.sql @@ -0,0 +1,23 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/9810 +select cast(1 as String) +from (select 1 as iid) as t1 +join (select '1' as sid) as t2 on t2.sid = cast(t1.iid as String); + +-- even simpler cases +select cast(7 as String), * from (select 3 "'String'"); +select cast(7 as String), * from (select number "'String'" FROM numbers(2)); +SELECT concat('xyz', 'abc'), * FROM (SELECT 2 AS "'xyz'"); +with 3 as "1" select 1, "1"; + +-- https://github.com/ClickHouse/ClickHouse/issues/9953 +select 1, * from (select 2 x) a left join (select 1, 3 y) b on y = x; +select 1, * from (select 2 x, 1) a right join (select 3 y) b on y = x; +select null, isConstant(null), * from (select 2 x) a left join (select null, 3 y) b on y = x; +select null, isConstant(null), * from (select 2 x, null) a right join (select 3 y) b on y = x; + +-- other cases with joins and constants + +select cast(1, 'UInt8') from (select arrayJoin([1, 2]) as a) t1 left join (select 1 as b) t2 on b = ignore('UInt8'); + +select isConstant('UInt8'), toFixedString('hello', toUInt8(substring('UInt8', 5, 1))) from (select arrayJoin([1, 2]) as a) t1 left join (select 1 as b) t2 on b = ignore('UInt8'); + diff --git a/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference b/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference index a0518e78891..852abeea187 100644 --- a/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference +++ b/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference @@ -1,3 +1,3 @@ World -CREATE DICTIONARY db_for_dict.dict_with_hashed_layout (`key1` UInt64, `value` String) PRIMARY KEY key1 SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' DB \'db_for_dict\')) LIFETIME(MIN 1 MAX 10) LAYOUT(HASHED) +CREATE DICTIONARY db_for_dict.dict_with_hashed_layout\n(\n `key1` UInt64, \n `value` String\n)\nPRIMARY KEY key1\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' DB \'db_for_dict\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(HASHED) Hello diff --git a/tests/queries/0_stateless/01114_alter_modify_compact_parts.reference b/tests/queries/0_stateless/01114_alter_modify_compact_parts.reference new file mode 100644 index 00000000000..4ec38dfb475 --- /dev/null +++ b/tests/queries/0_stateless/01114_alter_modify_compact_parts.reference @@ -0,0 +1 @@ +999000 diff --git a/tests/queries/0_stateless/01114_alter_modify_compact_parts.sql b/tests/queries/0_stateless/01114_alter_modify_compact_parts.sql new file mode 100644 index 00000000000..a5aa12548e7 --- /dev/null +++ b/tests/queries/0_stateless/01114_alter_modify_compact_parts.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS mt_compact; + +CREATE TABLE mt_compact (d Date, id UInt32, s String) + ENGINE = MergeTree ORDER BY id PARTITION BY d + SETTINGS min_bytes_for_wide_part = 10000000, index_granularity = 128; + +INSERT INTO mt_compact SELECT toDate('2020-01-05'), number, toString(number) FROM numbers(1000); +INSERT INTO mt_compact SELECT toDate('2020-01-06'), number, toString(number) FROM numbers(1000); +ALTER TABLE mt_compact MODIFY COLUMN s UInt64; +SELECT sum(s) from mt_compact; + +DROP TABLE IF EXISTS mt_compact; diff --git a/tests/queries/0_stateless/01114_clear_column_compact_parts.reference b/tests/queries/0_stateless/01114_clear_column_compact_parts.reference new file mode 100644 index 00000000000..e8c9b37ff3a --- /dev/null +++ b/tests/queries/0_stateless/01114_clear_column_compact_parts.reference @@ -0,0 +1,4 @@ +1 0 +2 3 +1 0 +2 0 diff --git a/tests/queries/0_stateless/01114_clear_column_compact_parts.sql b/tests/queries/0_stateless/01114_clear_column_compact_parts.sql new file mode 100644 index 00000000000..bdfed06ea9a --- /dev/null +++ b/tests/queries/0_stateless/01114_clear_column_compact_parts.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS clear_column; + +CREATE TABLE clear_column(x UInt32, y UInt32) ENGINE MergeTree ORDER BY x PARTITION by x; +INSERT INTO clear_column VALUES (1, 1), (2, 3); + +ALTER TABLE clear_column CLEAR COLUMN y IN PARTITION 1; +SELECT * FROM clear_column ORDER BY x; +ALTER TABLE clear_column CLEAR COLUMN y IN PARTITION 2; +SELECT * FROM clear_column ORDER BY x; + +DROP TABLE clear_column; diff --git a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.reference b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.reference new file mode 100644 index 00000000000..083edaac248 --- /dev/null +++ b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.reference @@ -0,0 +1,3 @@ +2 +2 +2 diff --git a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql new file mode 100644 index 00000000000..404922f36bb --- /dev/null +++ b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS minmax_compact; + +CREATE TABLE minmax_compact +( + u64 UInt64, + i64 Int64, + i32 Int32 +) ENGINE = MergeTree() +PARTITION BY i32 +ORDER BY u64 +SETTINGS index_granularity = 2, min_rows_for_wide_part = 1000000; + +INSERT INTO minmax_compact VALUES (0, 2, 1), (1, 1, 1), (2, 1, 1), (3, 1, 1), (4, 1, 1), (5, 2, 1), (6, 1, 2), (7, 1, 2), (8, 1, 2), (9, 1, 2); + +SET mutations_sync = 1; +ALTER TABLE minmax_compact ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1; + +ALTER TABLE minmax_compact MATERIALIZE INDEX idx IN PARTITION 1; +set max_rows_to_read = 8; +SELECT count() FROM minmax_compact WHERE i64 = 2; + +ALTER TABLE minmax_compact MATERIALIZE INDEX idx IN PARTITION 2; +set max_rows_to_read = 6; +SELECT count() FROM minmax_compact WHERE i64 = 2; + +ALTER TABLE minmax_compact CLEAR INDEX idx IN PARTITION 1; +ALTER TABLE minmax_compact CLEAR INDEX idx IN PARTITION 2; + +SELECT count() FROM minmax_compact WHERE i64 = 2; -- { serverError 158 } + +set max_rows_to_read = 10; +SELECT count() FROM minmax_compact WHERE i64 = 2; diff --git a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.reference b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql new file mode 100644 index 00000000000..af88c5af53a --- /dev/null +++ b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql @@ -0,0 +1 @@ +CREATE DATABASE conv_main ENGINE = MySQL('127.0.0.1:3456', conv_main, 'metrika', 'password'); -- { serverError 501 } diff --git a/tests/queries/0_stateless/01115_join_with_dictionary.reference b/tests/queries/0_stateless/01115_join_with_dictionary.reference new file mode 100644 index 00000000000..f909a3d61f5 --- /dev/null +++ b/tests/queries/0_stateless/01115_join_with_dictionary.reference @@ -0,0 +1,103 @@ +flat: left on +0 0 0 0 0 +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 0 0 0 +flat: left +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 0 0 +flat: any left +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 0 0 +flat: semi left +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +flat: anti left +4 0 0 +flat: inner +0 0 0 0 +1 1 1 1 +flat: inner on +0 0 0 0 0 +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +hashed: left on +0 0 0 0 0 +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 \N \N \N \N +hashed: left +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 \N \N \N +hashed: any left +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 \N \N \N +hashed: semi left +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +hashed: anti left +4 \N \N \N +hashed: inner +0 0 0 0 +1 1 1 1 +hashed: inner on +0 0 0 0 0 +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +complex_cache (smoke) +0 \N \N \N \N +1 \N \N \N \N +2 \N \N \N \N +3 \N \N \N \N +4 \N \N \N \N +not optimized (smoke) +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +- +0 0 0 0 0 +1 1 1 1 1 +\N 2 2 2 2 +\N 3 3 3 3 +- +2 2 2 2 +3 3 3 3 +4 \N \N \N +5 \N \N \N +\N 0 0 0 +\N 1 1 1 +- +0 0 0 0 +1 1 1 1 +- +0 0 0 0 +1 1 1 1 +3 3 3 3 +2 2 2 2 +- +0 0 0 0 +1 1 1 1 +- +3 3 3 3 +2 2 2 2 diff --git a/tests/queries/0_stateless/01115_join_with_dictionary.sql b/tests/queries/0_stateless/01115_join_with_dictionary.sql new file mode 100644 index 00000000000..65704f2b3eb --- /dev/null +++ b/tests/queries/0_stateless/01115_join_with_dictionary.sql @@ -0,0 +1,90 @@ +SET send_logs_level = 'none'; + +DROP DATABASE IF EXISTS db_01115; +CREATE DATABASE db_01115 Engine = Ordinary; + +USE db_01115; + +DROP DICTIONARY IF EXISTS dict_flat; +DROP DICTIONARY IF EXISTS dict_hashed; +DROP DICTIONARY IF EXISTS dict_complex_cache; + +CREATE TABLE t1 (key UInt64, a UInt8, b String, c Float64) ENGINE = MergeTree() ORDER BY key; +INSERT INTO t1 SELECT number, number, toString(number), number from numbers(4); + +CREATE DICTIONARY dict_flat (key UInt64 DEFAULT 0, a UInt8 DEFAULT 42, b String DEFAULT 'x', c Float64 DEFAULT 42.0) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 't1' PASSWORD '' DB 'db_01115')) +LIFETIME(MIN 1 MAX 10) +LAYOUT(FLAT()); + +CREATE DICTIONARY db_01115.dict_hashed (key UInt64 DEFAULT 0, a UInt8 DEFAULT 42, b String DEFAULT 'x', c Float64 DEFAULT 42.0) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 't1' DB 'db_01115')) +LIFETIME(MIN 1 MAX 10) +LAYOUT(HASHED()); + +CREATE DICTIONARY dict_complex_cache (key UInt64 DEFAULT 0, a UInt8 DEFAULT 42, b String DEFAULT 'x', c Float64 DEFAULT 42.0) +PRIMARY KEY key, b +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 't1' DB 'db_01115')) +LIFETIME(MIN 1 MAX 10) +LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 1)); + +SET join_use_nulls = 0; + +SELECT 'flat: left on'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 LEFT JOIN dict_flat d ON s1.key = d.key ORDER BY s1.key; +SELECT 'flat: left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 LEFT JOIN dict_flat d USING(key) ORDER BY key; +SELECT 'flat: any left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 ANY LEFT JOIN dict_flat d USING(key) ORDER BY key; +SELECT 'flat: semi left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 SEMI JOIN dict_flat d USING(key) ORDER BY key; +SELECT 'flat: anti left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 ANTI JOIN dict_flat d USING(key) ORDER BY key; +SELECT 'flat: inner'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 JOIN dict_flat d USING(key); +SELECT 'flat: inner on'; +SELECT * FROM (SELECT number AS k FROM numbers(100)) s1 JOIN dict_flat d ON k = key ORDER BY k; + +SET join_use_nulls = 1; + +SELECT 'hashed: left on'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 LEFT JOIN dict_hashed d ON s1.key = d.key ORDER BY s1.key; +SELECT 'hashed: left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 LEFT JOIN dict_hashed d USING(key) ORDER BY key; +SELECT 'hashed: any left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 ANY LEFT JOIN dict_hashed d USING(key) ORDER BY key; +SELECT 'hashed: semi left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 SEMI JOIN dict_hashed d USING(key) ORDER BY key; +SELECT 'hashed: anti left'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 ANTI JOIN dict_hashed d USING(key) ORDER BY key; +SELECT 'hashed: inner'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 JOIN dict_hashed d USING(key); +SELECT 'hashed: inner on'; +SELECT * FROM (SELECT number AS k FROM numbers(100)) s1 JOIN dict_hashed d ON k = key ORDER BY k; + +SELECT 'complex_cache (smoke)'; +SELECT * FROM (SELECT number AS key FROM numbers(5)) s1 LEFT JOIN dict_complex_cache d ON s1.key = d.key ORDER BY s1.key; + +SELECT 'not optimized (smoke)'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 RIGHT JOIN dict_flat d USING(key) ORDER BY key; +SELECT '-'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 RIGHT JOIN dict_flat d ON s1.key = d.key ORDER BY d.key; +SELECT '-'; +SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s1 FULL JOIN dict_flat d USING(key) ORDER BY s1.key, d.key; +SELECT '-'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 ANY INNER JOIN dict_flat d USING(key); +SELECT '-'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 ANY RIGHT JOIN dict_flat d USING(key); +SELECT '-'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 SEMI RIGHT JOIN dict_flat d USING(key); +SELECT '-'; +SELECT * FROM (SELECT number AS key FROM numbers(2)) s1 ANTI RIGHT JOIN dict_flat d USING(key); + +DROP DICTIONARY dict_flat; +DROP DICTIONARY dict_hashed; +DROP DICTIONARY dict_complex_cache; + +DROP TABLE t1; +DROP DATABASE IF EXISTS db_01115; diff --git a/tests/queries/0_stateless/01115_prewhere_array_join.reference b/tests/queries/0_stateless/01115_prewhere_array_join.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01115_prewhere_array_join.sql b/tests/queries/0_stateless/01115_prewhere_array_join.sql new file mode 100644 index 00000000000..e614bdf402b --- /dev/null +++ b/tests/queries/0_stateless/01115_prewhere_array_join.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS prewhere; + +CREATE TABLE prewhere (light UInt8, heavy String) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO prewhere SELECT 0, randomPrintableASCII(10000) FROM numbers(10000); +SELECT arrayJoin([light]) != 0 AS cond, length(heavy) FROM prewhere WHERE light != 0 AND cond != 0; + +DROP TABLE prewhere; diff --git a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference new file mode 100644 index 00000000000..1055a67ea5b --- /dev/null +++ b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference @@ -0,0 +1,3 @@ +v1 o1 ['s2','s1'] +v1 o2 ['s4'] +v2 o3 ['s5','s3'] diff --git a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql new file mode 100644 index 00000000000..8a94b6ddd24 --- /dev/null +++ b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql @@ -0,0 +1,18 @@ +CREATE TEMPORARY TABLE sessions (date DateTime, visitorId String, sessionId String); +CREATE TEMPORARY TABLE orders (date DateTime, visitorId String, orderId String); + +INSERT INTO sessions VALUES ('2018-01-01 00:00:00', 'v1', 's1'), ('2018-01-02 00:00:00', 'v1', 's2'), ('2018-01-03 00:00:00', 'v2', 's3'), ('2018-01-04 00:00:00', 'v1', 's4'), ('2018-01-05 00:00:00', 'v2', 's5'), ('2018-01-06 00:00:00', 'v3', 's6'); +INSERT INTO orders VALUES ('2018-01-03 00:00:00', 'v1', 'o1'), ('2018-01-05 00:00:00', 'v1', 'o2'), ('2018-01-06 00:00:00', 'v2', 'o3'); + +SELECT + visitorId, + orderId, + groupUniqArray(sessionId) +FROM sessions +ASOF INNER JOIN orders ON (sessions.visitorId = orders.visitorId) AND (sessions.date <= orders.date) +GROUP BY + visitorId, + orderId +ORDER BY + visitorId ASC, + orderId ASC; diff --git a/tests/queries/0_stateless/01116_cross_count_asterisks.reference b/tests/queries/0_stateless/01116_cross_count_asterisks.reference new file mode 100644 index 00000000000..8347b144a35 --- /dev/null +++ b/tests/queries/0_stateless/01116_cross_count_asterisks.reference @@ -0,0 +1,4 @@ +2 +1 +2 +1 diff --git a/tests/queries/0_stateless/01116_cross_count_asterisks.sql b/tests/queries/0_stateless/01116_cross_count_asterisks.sql new file mode 100644 index 00000000000..1fb8b0b0e66 --- /dev/null +++ b/tests/queries/0_stateless/01116_cross_count_asterisks.sql @@ -0,0 +1,29 @@ +SET multiple_joins_rewriter_version = 2; + +SELECT count(*) +FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 +WHERE (n1.number = n2.number) AND (n2.number = n3.number); + +SELECT count(*) c FROM ( + SELECT count(*), count(*) as c + FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 + WHERE (n1.number = n2.number) AND (n2.number = n3.number) + AND (SELECT count(*) FROM numbers(1)) = 1 +) +WHERE (SELECT count(*) FROM numbers(2)) = 2 +HAVING c IN(SELECT count(*) c FROM numbers(1)); + +SET multiple_joins_rewriter_version = 1; + +SELECT count(*) +FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 +WHERE (n1.number = n2.number) AND (n2.number = n3.number); + +SELECT count(*) c FROM ( + SELECT count(*), count(*) as c + FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 + WHERE (n1.number = n2.number) AND (n2.number = n3.number) + AND (SELECT count(*) FROM numbers(1)) = 1 +) +WHERE (SELECT count(*) FROM numbers(2)) = 2 +HAVING c IN(SELECT count(*) c FROM numbers(1)); diff --git a/tests/queries/0_stateless/01117_chain_finalize_bug.reference b/tests/queries/0_stateless/01117_chain_finalize_bug.reference new file mode 100644 index 00000000000..5704c399b5c --- /dev/null +++ b/tests/queries/0_stateless/01117_chain_finalize_bug.reference @@ -0,0 +1,9 @@ +1 0 +1 1 +2 0 +2 1 +1 0 +1 1 +2 0 +2 1 +200000 diff --git a/tests/queries/0_stateless/01117_chain_finalize_bug.sql b/tests/queries/0_stateless/01117_chain_finalize_bug.sql new file mode 100644 index 00000000000..f79f82b8d4d --- /dev/null +++ b/tests/queries/0_stateless/01117_chain_finalize_bug.sql @@ -0,0 +1,24 @@ +SELECT arrayJoin(arrayMap(i -> (i + 1), range(2))) AS index, number +FROM numbers(2) +GROUP BY number +ORDER BY index, number; + +SET max_bytes_before_external_group_by = 1; + +SELECT arrayJoin(arrayMap(i -> (i + 1), range(2))) AS index, number +FROM numbers(2) +GROUP BY number +ORDER BY index, number; + +SET group_by_two_level_threshold = 2; + +SELECT count() FROM +( + SELECT + arrayJoin(arrayMap(i -> (i + 1), range(2))) AS index, + number + FROM numbers_mt(100000) + GROUP BY number + ORDER BY index ASC + SETTINGS max_block_size = 100000, max_threads = 2 +); diff --git a/tests/queries/0_stateless/01117_comma_and_others_join_mix.reference b/tests/queries/0_stateless/01117_comma_and_others_join_mix.reference new file mode 100644 index 00000000000..e50321284f4 --- /dev/null +++ b/tests/queries/0_stateless/01117_comma_and_others_join_mix.reference @@ -0,0 +1,23 @@ +0 0 0 +0 0 1 +0 0 2 +0 0 3 +1 1 0 +1 1 1 +1 1 2 +1 1 3 +- +0 0 0 +0 1 0 +1 0 1 +1 1 1 +2 0 \N +2 1 \N +- +0 0 0 +0 1 1 +0 2 2 +1 0 0 +1 1 1 +1 2 2 +\N \N 3 diff --git a/tests/queries/0_stateless/01117_comma_and_others_join_mix.sql b/tests/queries/0_stateless/01117_comma_and_others_join_mix.sql new file mode 100644 index 00000000000..5fe297956b8 --- /dev/null +++ b/tests/queries/0_stateless/01117_comma_and_others_join_mix.sql @@ -0,0 +1,20 @@ +SET join_use_nulls = 1; + +SELECT * +FROM numbers(2) AS n1 +JOIN numbers(3) AS n2 ON n1.number = n2.number, numbers(4) AS n3 +ORDER BY n1.number, n2.number, n3.number; + +SELECT '-'; + +SELECT * +FROM numbers(3) AS n1, numbers(2) AS n2 +LEFT JOIN numbers(2) AS n3 ON n1.number = n3.number +ORDER BY n1.number, n2.number, n3.number; + +SELECT '-'; + +SELECT * +FROM numbers(2) AS n1, numbers(3) AS n2 +RIGHT JOIN numbers(4) AS n3 ON n2.number = n3.number +ORDER BY n1.number, n2.number, n3.number; diff --git a/tests/queries/0_stateless/01117_greatest_least_case.reference b/tests/queries/0_stateless/01117_greatest_least_case.reference new file mode 100644 index 00000000000..4bbcfcf5682 --- /dev/null +++ b/tests/queries/0_stateless/01117_greatest_least_case.reference @@ -0,0 +1,2 @@ +2 +-1 diff --git a/tests/queries/0_stateless/01117_greatest_least_case.sql b/tests/queries/0_stateless/01117_greatest_least_case.sql new file mode 100644 index 00000000000..21bfd240f5a --- /dev/null +++ b/tests/queries/0_stateless/01117_greatest_least_case.sql @@ -0,0 +1,2 @@ +SELECT GREATEST(1, 2); +SELECT LEAST(1, -1); diff --git a/tests/queries/0_stateless/01118_is_constant.reference b/tests/queries/0_stateless/01118_is_constant.reference new file mode 100644 index 00000000000..aba2b912a08 --- /dev/null +++ b/tests/queries/0_stateless/01118_is_constant.reference @@ -0,0 +1,9 @@ +1 +1 +0 +1 +1 +--- +0 +0 +--- diff --git a/tests/queries/0_stateless/01118_is_constant.sql b/tests/queries/0_stateless/01118_is_constant.sql new file mode 100644 index 00000000000..5cbff986dd2 --- /dev/null +++ b/tests/queries/0_stateless/01118_is_constant.sql @@ -0,0 +1,10 @@ +select isConstant(1); +select isConstant([1]); +select isConstant(arrayJoin([1])); +SELECT isConstant((SELECT 1)); +SELECT isConstant(x) FROM (SELECT 1 x); +SELECT '---'; +SELECT isConstant(x) FROM (SELECT 1 x UNION ALL SELECT 2); +SELECT '---'; +select isConstant(); -- { serverError 42 } +select isConstant(1, 2); -- { serverError 42 } diff --git a/tests/queries/0_stateless/01120_join_constants.reference b/tests/queries/0_stateless/01120_join_constants.reference new file mode 100644 index 00000000000..a16427fbdf7 --- /dev/null +++ b/tests/queries/0_stateless/01120_join_constants.reference @@ -0,0 +1,2 @@ +1 hello 1 world world 1 +2 hello 0 world 1 diff --git a/tests/queries/0_stateless/01120_join_constants.sql b/tests/queries/0_stateless/01120_join_constants.sql new file mode 100644 index 00000000000..443559c3ea1 --- /dev/null +++ b/tests/queries/0_stateless/01120_join_constants.sql @@ -0,0 +1,17 @@ +SELECT + t1.*, + t2.*, + 'world', + isConstant('world') +FROM +( + SELECT + arrayJoin([1, 2]) AS k, + 'hello' +) AS t1 +LEFT JOIN +( + SELECT + arrayJoin([1, 3]) AS k, + 'world' +) AS t2 ON t1.k = t2.k; diff --git a/tests/queries/0_stateless/01121_remote_scalar_subquery.reference b/tests/queries/0_stateless/01121_remote_scalar_subquery.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/01121_remote_scalar_subquery.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/bugs/remote_scalar_subquery.sql b/tests/queries/0_stateless/01121_remote_scalar_subquery.sql similarity index 100% rename from tests/queries/bugs/remote_scalar_subquery.sql rename to tests/queries/0_stateless/01121_remote_scalar_subquery.sql diff --git a/tests/queries/0_stateless/01122_totals_rollup_having_block_header.reference b/tests/queries/0_stateless/01122_totals_rollup_having_block_header.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/bugs/totals_rollup_having_block_header.sql b/tests/queries/0_stateless/01122_totals_rollup_having_block_header.sql similarity index 75% rename from tests/queries/bugs/totals_rollup_having_block_header.sql rename to tests/queries/0_stateless/01122_totals_rollup_having_block_header.sql index 4f7f9692fd0..4f4f3355912 100644 --- a/tests/queries/bugs/totals_rollup_having_block_header.sql +++ b/tests/queries/0_stateless/01122_totals_rollup_having_block_header.sql @@ -1,5 +1,3 @@ --- triggers assertion in debug build - DROP TABLE IF EXISTS test.rollup_having; CREATE TABLE test.rollup_having ( a Nullable(String), @@ -10,7 +8,7 @@ INSERT INTO test.rollup_having VALUES (NULL, NULL); INSERT INTO test.rollup_having VALUES ('a', NULL); INSERT INTO test.rollup_having VALUES ('a', 'b'); -SELECT a, b, count(*) FROM test.rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL; -SELECT a, b, count(*) FROM test.rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL and b IS NOT NULL; +SELECT a, b, count(*) FROM test.rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL; -- { serverError 48 } +SELECT a, b, count(*) FROM test.rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL and b IS NOT NULL; -- { serverError 48 } DROP TABLE test.rollup_having; diff --git a/tests/queries/0_stateless/01123_parse_date_time_best_effort_even_more.reference b/tests/queries/0_stateless/01123_parse_date_time_best_effort_even_more.reference new file mode 100644 index 00000000000..558ba34abcd --- /dev/null +++ b/tests/queries/0_stateless/01123_parse_date_time_best_effort_even_more.reference @@ -0,0 +1,2 @@ +2018-08-18 07:22:16 +2018-08-16 07:22:16 diff --git a/tests/queries/0_stateless/01123_parse_date_time_best_effort_even_more.sql b/tests/queries/0_stateless/01123_parse_date_time_best_effort_even_more.sql new file mode 100644 index 00000000000..a4f6f173402 --- /dev/null +++ b/tests/queries/0_stateless/01123_parse_date_time_best_effort_even_more.sql @@ -0,0 +1,2 @@ +SELECT toTimeZone(parseDateTimeBestEffort('Thu, 18 Aug 2018 07:22:16 GMT'), 'UTC'); +SELECT toTimeZone(parseDateTimeBestEffort('Tue, 16 Aug 2018 07:22:16 GMT'), 'UTC'); diff --git a/tests/queries/0_stateless/01124_view_bad_types.reference b/tests/queries/0_stateless/01124_view_bad_types.reference new file mode 100644 index 00000000000..af98bcd6397 --- /dev/null +++ b/tests/queries/0_stateless/01124_view_bad_types.reference @@ -0,0 +1,10 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 diff --git a/tests/queries/bugs/view_bad_types.sql b/tests/queries/0_stateless/01124_view_bad_types.sql similarity index 84% rename from tests/queries/bugs/view_bad_types.sql rename to tests/queries/0_stateless/01124_view_bad_types.sql index 38daabfd6b8..81fc53930c1 100644 --- a/tests/queries/bugs/view_bad_types.sql +++ b/tests/queries/0_stateless/01124_view_bad_types.sql @@ -5,7 +5,7 @@ INSERT INTO test.table SELECT * FROM system.numbers LIMIT 10; DROP TABLE IF EXISTS test.view; CREATE VIEW test.view (x UInt64) AS SELECT * FROM test.table; -SELECT x, any(x) FROM test.view GROUP BY x; +SELECT x, any(x) FROM test.view GROUP BY x ORDER BY x; DROP TABLE test.view; DROP TABLE test.table; diff --git a/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.reference b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.reference new file mode 100644 index 00000000000..1a9e5685a6a --- /dev/null +++ b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.reference @@ -0,0 +1,3 @@ +1 2019-01-05 2020-01-10 1 +date_table +somedict diff --git a/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.sql b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.sql new file mode 100644 index 00000000000..3f87235bdf4 --- /dev/null +++ b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.sql @@ -0,0 +1,34 @@ +DROP DATABASE IF EXISTS database_for_dict; + +CREATE DATABASE database_for_dict; + +use database_for_dict; + +CREATE TABLE date_table +( + id UInt32, + val String, + start Date, + end Date +) Engine = Memory(); + +INSERT INTO date_table VALUES(1, '1', toDate('2019-01-05'), toDate('2020-01-10')); + +CREATE DICTIONARY somedict +( + id UInt32, + val String, + start Date, + end Date +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'date_table' DB 'database_for_dict')) +LAYOUT(RANGE_HASHED()) +RANGE (MIN start MAX end) +LIFETIME(MIN 300 MAX 360); + +SELECT * from somedict; + +SHOW TABLES; + +DROP DATABASE IF EXISTS database_for_dict; diff --git a/tests/queries/0_stateless/01125_generate_random_qoega.reference b/tests/queries/0_stateless/01125_generate_random_qoega.reference new file mode 100644 index 00000000000..1cb416a722b --- /dev/null +++ b/tests/queries/0_stateless/01125_generate_random_qoega.reference @@ -0,0 +1 @@ +100 4456446406473339606 diff --git a/tests/queries/0_stateless/01125_generate_random_qoega.sql b/tests/queries/0_stateless/01125_generate_random_qoega.sql new file mode 100644 index 00000000000..7fb586ad2b5 --- /dev/null +++ b/tests/queries/0_stateless/01125_generate_random_qoega.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS mass_table_117; +CREATE TABLE mass_table_117 (`dt` Date, `site_id` Int32, `site_key` String) ENGINE = MergeTree(dt, (site_id, site_key, dt), 8192); +INSERT INTO mass_table_117 SELECT * FROM generateRandom('`dt` Date,`site_id` Int32,`site_key` String', 1, 10, 2) LIMIT 100; +SELECT count(), sum(cityHash64(*)) FROM mass_table_117; +DROP TABLE mass_table_117; diff --git a/tests/queries/0_stateless/01126_month_partitioning_consistent_code.reference b/tests/queries/0_stateless/01126_month_partitioning_consistent_code.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01126_month_partitioning_consistent_code.sql b/tests/queries/0_stateless/01126_month_partitioning_consistent_code.sql new file mode 100644 index 00000000000..c9bfbbe5111 --- /dev/null +++ b/tests/queries/0_stateless/01126_month_partitioning_consistent_code.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS mt; +CREATE TABLE mt (d Date, x UInt8) ENGINE = MergeTree(d, x, 8192); +INSERT INTO mt VALUES (52392, 1), (62677, 2); +DROP TABLE mt; diff --git a/tests/queries/0_stateless/01127_month_partitioning_consistency_select.reference b/tests/queries/0_stateless/01127_month_partitioning_consistency_select.reference new file mode 100644 index 00000000000..1b08e7f2d6f --- /dev/null +++ b/tests/queries/0_stateless/01127_month_partitioning_consistency_select.reference @@ -0,0 +1,4 @@ +Q1 2106-02-07 Hello +Q2 0000-00-00 World +Q1 2106-02-07 Hello +Q2 0000-00-00 World diff --git a/tests/queries/0_stateless/01127_month_partitioning_consistency_select.sql b/tests/queries/0_stateless/01127_month_partitioning_consistency_select.sql new file mode 100644 index 00000000000..59edd0c37b8 --- /dev/null +++ b/tests/queries/0_stateless/01127_month_partitioning_consistency_select.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS mt; +CREATE TABLE mt (d Date, x String) ENGINE = MergeTree(d, x, 8192); +INSERT INTO mt VALUES ('2106-02-07', 'Hello'), ('1970-01-01', 'World'); + +SELECT 'Q1', * FROM mt WHERE d = '2106-02-07'; +SELECT 'Q2', * FROM mt WHERE d = '1970-01-01'; + +DETACH TABLE mt; +ATTACH TABLE mt; + +SELECT 'Q1', * FROM mt WHERE d = '2106-02-07'; +SELECT 'Q2', * FROM mt WHERE d = '1970-01-01'; + +DROP TABLE mt; diff --git a/tests/queries/0_stateless/01128_generate_random_nested.reference b/tests/queries/0_stateless/01128_generate_random_nested.reference new file mode 100644 index 00000000000..d9d2b251702 --- /dev/null +++ b/tests/queries/0_stateless/01128_generate_random_nested.reference @@ -0,0 +1,2 @@ +100 12366141706519416319 +109 2990700419202507835 diff --git a/tests/queries/0_stateless/01128_generate_random_nested.sql b/tests/queries/0_stateless/01128_generate_random_nested.sql new file mode 100644 index 00000000000..2af52e69893 --- /dev/null +++ b/tests/queries/0_stateless/01128_generate_random_nested.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS mass_table_312; +CREATE TABLE mass_table_312 (d Date DEFAULT '2000-01-01', x UInt64, n Nested(a String, b String)) ENGINE = MergeTree(d, x, 1); +INSERT INTO mass_table_312 SELECT * FROM generateRandom('`d` Date,`x` UInt64,`n.a` Array(String),`n.b` Array(String)', 1, 10, 2) LIMIT 100; + +SELECT count(), sum(cityHash64(*)) FROM mass_table_312; +SELECT count(), sum(cityHash64(*)) FROM mass_table_312 ARRAY JOIN n; + +DROP TABLE mass_table_312; diff --git a/tests/queries/0_stateless/01134_max_rows_to_group_by.reference b/tests/queries/0_stateless/01134_max_rows_to_group_by.reference new file mode 100644 index 00000000000..caaf3394975 --- /dev/null +++ b/tests/queries/0_stateless/01134_max_rows_to_group_by.reference @@ -0,0 +1,33 @@ +test2 0 +test2 1 +test2 2 +test2 3 +test2 4 +test2 5 +test2 6 +test2 7 +test2 8 +test2 9 +test2 10 +test3 0 +test3 1 +test3 2 +test3 3 +test3 4 +test3 5 +test3 6 +test3 7 +test3 8 +test3 9 +test3 10 +test5 0 +test5 1 +test5 2 +test5 3 +test5 4 +test5 5 +test5 6 +test5 7 +test5 8 +test5 9 +test5 10 diff --git a/tests/queries/0_stateless/01134_max_rows_to_group_by.sql b/tests/queries/0_stateless/01134_max_rows_to_group_by.sql new file mode 100644 index 00000000000..bfbc499e1c3 --- /dev/null +++ b/tests/queries/0_stateless/01134_max_rows_to_group_by.sql @@ -0,0 +1,17 @@ +SET max_block_size = 1; +SET max_rows_to_group_by = 10; +SET group_by_overflow_mode = 'throw'; + +SELECT 'test1', number FROM system.numbers GROUP BY number; -- { serverError 158 } + +SET group_by_overflow_mode = 'break'; +SELECT 'test2', number FROM system.numbers GROUP BY number ORDER BY number; + +SET max_rows_to_read = 500; +SELECT 'test3', number FROM system.numbers GROUP BY number ORDER BY number; + +SET group_by_overflow_mode = 'any'; +SELECT 'test4', number FROM numbers(1000) GROUP BY number ORDER BY number; -- { serverError 158 } + +SET max_rows_to_read = 1000; +SELECT 'test5', number FROM numbers(1000) GROUP BY number ORDER BY number; diff --git a/tests/queries/0_stateless/01196_max_parser_depth.reference b/tests/queries/0_stateless/01196_max_parser_depth.reference new file mode 100644 index 00000000000..a72c1b18aa2 --- /dev/null +++ b/tests/queries/0_stateless/01196_max_parser_depth.reference @@ -0,0 +1,3 @@ +Code: 306 +Code: 306 +Code: 306 diff --git a/tests/queries/0_stateless/01196_max_parser_depth.sh b/tests/queries/0_stateless/01196_max_parser_depth.sh new file mode 100755 index 00000000000..471c1c22ecb --- /dev/null +++ b/tests/queries/0_stateless/01196_max_parser_depth.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +{ printf "select "; for x in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for x in {1..1000}; do printf ")"; done; } | $CLICKHOUSE_CLIENT 2>&1 | grep -o -F 'Code: 306' +{ printf "select "; for x in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for x in {1..1000}; do printf ")"; done; } | $CLICKHOUSE_LOCAL 2>&1 | grep -o -F 'Code: 306' +{ printf "select "; for x in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for x in {1..1000}; do printf ")"; done; } | $CLICKHOUSE_CURL --data-binary @- -vsS "$CLICKHOUSE_URL" 2>&1 | grep -o -F 'Code: 306' diff --git a/tests/queries/0_stateless/01197_summing_enum.reference b/tests/queries/0_stateless/01197_summing_enum.reference new file mode 100644 index 00000000000..aa6a25fb0cf --- /dev/null +++ b/tests/queries/0_stateless/01197_summing_enum.reference @@ -0,0 +1 @@ + 2000 hello diff --git a/tests/queries/0_stateless/01197_summing_enum.sql b/tests/queries/0_stateless/01197_summing_enum.sql new file mode 100644 index 00000000000..c76f43aca4e --- /dev/null +++ b/tests/queries/0_stateless/01197_summing_enum.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS summing; + +CREATE TABLE summing (k String, x UInt64, e Enum('hello' = 1, 'world' = 2)) ENGINE = SummingMergeTree ORDER BY k; +INSERT INTO summing SELECT '', 1, e FROM generateRandom('e Enum(\'hello\' = 1, \'world\' = 2)', 1) LIMIT 1000; +INSERT INTO summing SELECT '', 1, e FROM generateRandom('e Enum(\'hello\' = 1, \'world\' = 2)', 1) LIMIT 1000; + +OPTIMIZE TABLE summing; +SELECT k, x, e FROM summing; + +DROP TABLE summing; \ No newline at end of file diff --git a/tests/queries/0_stateless/01198_plus_inf.reference b/tests/queries/0_stateless/01198_plus_inf.reference new file mode 100644 index 00000000000..f726b8429b6 --- /dev/null +++ b/tests/queries/0_stateless/01198_plus_inf.reference @@ -0,0 +1,3 @@ +inf +-inf +inf diff --git a/tests/queries/0_stateless/01198_plus_inf.sql b/tests/queries/0_stateless/01198_plus_inf.sql new file mode 100644 index 00000000000..e06faa2fd21 --- /dev/null +++ b/tests/queries/0_stateless/01198_plus_inf.sql @@ -0,0 +1,3 @@ +SELECT DISTINCT toFloat64(arrayJoin(['+inf', '+Inf', '+INF', '+infinity', '+Infinity'])); +SELECT DISTINCT toFloat64(arrayJoin(['-inf', '-Inf', '-INF', '-infinity', '-Infinity'])); +SELECT DISTINCT toFloat64(arrayJoin(['inf', 'Inf', 'INF', 'infinity', 'Infinity'])); diff --git a/tests/queries/0_stateless/01199_url_functions_path_without_schema_yiurule.reference b/tests/queries/0_stateless/01199_url_functions_path_without_schema_yiurule.reference new file mode 100644 index 00000000000..9d75f9c90df --- /dev/null +++ b/tests/queries/0_stateless/01199_url_functions_path_without_schema_yiurule.reference @@ -0,0 +1,2 @@ +/a/b/c +/?query=hello world+foo+bar diff --git a/tests/queries/0_stateless/01199_url_functions_path_without_schema_yiurule.sql b/tests/queries/0_stateless/01199_url_functions_path_without_schema_yiurule.sql new file mode 100644 index 00000000000..14b0f4fd8d5 --- /dev/null +++ b/tests/queries/0_stateless/01199_url_functions_path_without_schema_yiurule.sql @@ -0,0 +1,2 @@ +SELECT path('www.example.com:443/a/b/c') AS Path; +SELECT decodeURLComponent(materialize(pathFull('www.example.com/?query=hello%20world+foo%2Bbar'))) AS Path; diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference index a2c0e0d7d77..e2d6007c57f 100644 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference @@ -1,6 +1,6 @@ 1 -CREATE TABLE default.table_for_rename_replicated (`date` Date, `key` UInt64, `value1` String, `value2` String, `value3` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\') PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_rename_replicated (`date` Date, `key` UInt64, `renamed_value1` String, `value2` String, `value3` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\') PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date, \n `key` UInt64, \n `renamed_value1` String, \n `value2` String, \n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 1 date key renamed_value1 value2 value3 2019-10-02 1 1 1 1 diff --git a/tests/queries/0_stateless/01213_alter_rename_nested.reference b/tests/queries/0_stateless/01213_alter_rename_nested.reference index 8b7aaaa3d5a..2641df46aeb 100644 --- a/tests/queries/0_stateless/01213_alter_rename_nested.reference +++ b/tests/queries/0_stateless/01213_alter_rename_nested.reference @@ -1,10 +1,10 @@ [8,9,10] ['a','b','c'] -CREATE TABLE default.table_for_rename_nested (`date` Date, `key` UInt64, `n.x` Array(UInt32), `n.y` Array(String), `value1` Array(Array(LowCardinality(String)))) ENGINE = MergeTree() PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_rename_nested (`date` Date, `key` UInt64, `n.renamed_x` Array(UInt32), `n.renamed_y` Array(String), `value1` Array(Array(LowCardinality(String)))) ENGINE = MergeTree() PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.x` Array(UInt32), \n `n.y` Array(String), \n `value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.renamed_x` Array(UInt32), \n `n.renamed_y` Array(String), \n `value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 7 [8,9,10] 7 ['a','b','c'] [['7']] -CREATE TABLE default.table_for_rename_nested (`date` Date, `key` UInt64, `n.renamed_x` Array(UInt32), `n.renamed_y` Array(String), `renamed_value1` Array(Array(LowCardinality(String)))) ENGINE = MergeTree() PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.renamed_x` Array(UInt32), \n `n.renamed_y` Array(String), \n `renamed_value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 date key n.renamed_x n.renamed_y renamed_value1 2019-10-01 7 [8,9,10] ['a','b','c'] [['7']] diff --git a/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference b/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference index 06f136d6dbc..251e664b522 100644 --- a/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference +++ b/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference @@ -1,17 +1,17 @@ date key value1 value2 2019-10-02 1 1 Hello 1 -CREATE TABLE default.table_rename_with_default (`date` Date, `key` UInt64, `value1` String, `value2` String DEFAULT concat(\'Hello \', value1), `value3` String ALIAS concat(\'Word \', value1)) ENGINE = MergeTree() PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_default\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String DEFAULT concat(\'Hello \', value1), \n `value3` String ALIAS concat(\'Word \', value1)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 date key renamed_value1 value2 2019-10-02 1 1 Hello 1 -CREATE TABLE default.table_rename_with_default (`date` Date, `key` UInt64, `renamed_value1` String, `value2` String DEFAULT concat(\'Hello \', renamed_value1), `value3` String ALIAS concat(\'Word \', renamed_value1)) ENGINE = MergeTree() PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_default\n(\n `date` Date, \n `key` UInt64, \n `renamed_value1` String, \n `value2` String DEFAULT concat(\'Hello \', renamed_value1), \n `value3` String ALIAS concat(\'Word \', renamed_value1)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 Hello 1 Word 1 date1 date2 value1 value2 2019-10-02 2018-10-02 1 1 -CREATE TABLE default.table_rename_with_ttl (`date1` Date, `date2` Date, `value1` String, `value2` String TTL date1 + toIntervalMonth(10000)) ENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\') ORDER BY tuple() TTL date2 + toIntervalMonth(10000) SETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_ttl\n(\n `date1` Date, \n `date2` Date, \n `value1` String, \n `value2` String TTL date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 renamed_date1 date2 value1 value2 2019-10-02 2018-10-02 1 1 -CREATE TABLE default.table_rename_with_ttl (`renamed_date1` Date, `date2` Date, `value1` String, `value2` String TTL renamed_date1 + toIntervalMonth(10000)) ENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\') ORDER BY tuple() TTL date2 + toIntervalMonth(10000) SETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date, \n `date2` Date, \n `value1` String, \n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 renamed_date1 renamed_date2 value1 value2 2019-10-02 2018-10-02 1 1 -CREATE TABLE default.table_rename_with_ttl (`renamed_date1` Date, `renamed_date2` Date, `value1` String, `value2` String TTL renamed_date1 + toIntervalMonth(10000)) ENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\') ORDER BY tuple() TTL renamed_date2 + toIntervalMonth(10000) SETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date, \n `renamed_date2` Date, \n `value1` String, \n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL renamed_date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01213_alter_table_rename_nested.reference b/tests/queries/0_stateless/01213_alter_table_rename_nested.reference index 51647dc2e7b..8e6d93dbcce 100644 --- a/tests/queries/0_stateless/01213_alter_table_rename_nested.reference +++ b/tests/queries/0_stateless/01213_alter_table_rename_nested.reference @@ -1,6 +1,6 @@ [8,9,10] ['a','b','c'] -CREATE TABLE default.table_for_rename_nested (`date` Date, `key` UInt64, `n.x` Array(UInt32), `n.y` Array(String), `value1` String) ENGINE = MergeTree() PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_rename_nested (`date` Date, `key` UInt64, `n.renamed_x` Array(UInt32), `n.renamed_y` Array(String), `value1` String) ENGINE = MergeTree() PARTITION BY date ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.x` Array(UInt32), \n `n.y` Array(String), \n `value1` String\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.renamed_x` Array(UInt32), \n `n.renamed_y` Array(String), \n `value1` String\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 7 [8,9,10] 7 ['a','b','c'] diff --git a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference index 4ade9cd9c5d..eac8957dd47 100644 --- a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference +++ b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference @@ -6,3 +6,5 @@ optimize_skip_unused_shards optimize_skip_unused_shards lack of WHERE 0 1 +0 +1 diff --git a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql index 5b45bea9046..67ba1cda870 100644 --- a/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql +++ b/tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql @@ -12,6 +12,7 @@ SELECT DISTINCT id FROM dist_01213 WHERE id = 1 SETTINGS distributed_group_by_no SELECT 'optimize_skip_unused_shards'; SELECT DISTINCT id FROM dist_01213 WHERE id = 1 SETTINGS optimize_skip_unused_shards=1; -- check that querying all shards is ok +-- (there will be duplicates, since the INSERT was done via local table) SELECT 'optimize_skip_unused_shards lack of WHERE'; SELECT DISTINCT id FROM dist_01213 SETTINGS optimize_skip_unused_shards=1; diff --git a/tests/queries/0_stateless/01223_dist_on_dist.reference b/tests/queries/0_stateless/01223_dist_on_dist.reference index 4a5dd8f316c..aca2f070db4 100644 --- a/tests/queries/0_stateless/01223_dist_on_dist.reference +++ b/tests/queries/0_stateless/01223_dist_on_dist.reference @@ -99,3 +99,5 @@ merge() distributed_group_by_no_merge 33 33 +GLOBAL IN +1 diff --git a/tests/queries/0_stateless/01223_dist_on_dist.sql b/tests/queries/0_stateless/01223_dist_on_dist.sql index 1b9175f622e..65a240fd48b 100644 --- a/tests/queries/0_stateless/01223_dist_on_dist.sql +++ b/tests/queries/0_stateless/01223_dist_on_dist.sql @@ -82,6 +82,10 @@ select count() from merge_dist_01223; select 'distributed_group_by_no_merge'; select count() from merge_dist_01223 settings distributed_group_by_no_merge=1; +-- global in +select 'GLOBAL IN'; +select distinct * from dist_01223 where key global in (select toInt32(1)); + drop table merge_dist_01223; drop table dist_01223; drop table dist_layer_01223; diff --git a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference new file mode 100644 index 00000000000..524fbdd26fc --- /dev/null +++ b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference @@ -0,0 +1,24 @@ +NOT_LOADED +NOT_LOADED +CREATE DICTIONARY dict_db_01224.dict +( + `key` UInt64 DEFAULT 0, + `val` UInt64 DEFAULT 10 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict_data' PASSWORD '' DB 'dict_db_01224')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT()) +NOT_LOADED +CREATE TABLE dict_db_01224_dictionary.`dict_db_01224.dict` +( + `key` UInt64, + `val` UInt64 +) +ENGINE = Dictionary(`dict_db_01224.dict`) +NOT_LOADED +Dictionary 1 CREATE DICTIONARY dict_db_01224.dict (`key` UInt64 DEFAULT 0, `val` UInt64 DEFAULT 10) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'dict_data\' PASSWORD \'\' DB \'dict_db_01224\')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +NOT_LOADED +key UInt64 +val UInt64 +NOT_LOADED diff --git a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql new file mode 100644 index 00000000000..cf8b2a471c4 --- /dev/null +++ b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql @@ -0,0 +1,38 @@ +DROP DATABASE IF EXISTS dict_db_01224; +DROP DATABASE IF EXISTS dict_db_01224_dictionary; +CREATE DATABASE dict_db_01224; +CREATE DATABASE dict_db_01224_dictionary Engine=Dictionary; + +CREATE TABLE dict_db_01224.dict_data (key UInt64, val UInt64) Engine=Memory(); +CREATE DICTIONARY dict_db_01224.dict +( + key UInt64 DEFAULT 0, + val UInt64 DEFAULT 10 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict_data' PASSWORD '' DB 'dict_db_01224')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT()); + +SELECT status FROM system.dictionaries WHERE database = 'dict_db_01224' AND name = 'dict'; + +SELECT * FROM system.tables FORMAT Null; +SELECT status FROM system.dictionaries WHERE database = 'dict_db_01224' AND name = 'dict'; + +SHOW CREATE TABLE dict_db_01224.dict FORMAT TSVRaw; +SELECT status FROM system.dictionaries WHERE database = 'dict_db_01224' AND name = 'dict'; + +SHOW CREATE TABLE dict_db_01224_dictionary.`dict_db_01224.dict` FORMAT TSVRaw; +SELECT status FROM system.dictionaries WHERE database = 'dict_db_01224' AND name = 'dict'; + +SELECT engine, metadata_path LIKE '%/metadata/dict\_db\_01224/dict.sql', create_table_query FROM system.tables WHERE database = 'dict_db_01224' AND name = 'dict'; +SELECT status FROM system.dictionaries WHERE database = 'dict_db_01224' AND name = 'dict'; + +SELECT name, type FROM system.columns WHERE database = 'dict_db_01224' AND table = 'dict'; +SELECT status FROM system.dictionaries WHERE database = 'dict_db_01224' AND name = 'dict'; + +DROP DICTIONARY dict_db_01224.dict; +SELECT status FROM system.dictionaries WHERE database = 'dict_db_01224' AND name = 'dict'; + +DROP DATABASE dict_db_01224; +DROP DATABASE dict_db_01224_dictionary; diff --git a/tests/queries/0_stateless/01225_drop_dictionary_as_table.reference b/tests/queries/0_stateless/01225_drop_dictionary_as_table.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01225_drop_dictionary_as_table.sql b/tests/queries/0_stateless/01225_drop_dictionary_as_table.sql new file mode 100644 index 00000000000..866f2dff56b --- /dev/null +++ b/tests/queries/0_stateless/01225_drop_dictionary_as_table.sql @@ -0,0 +1,20 @@ +DROP DATABASE IF EXISTS dict_db_01225; +CREATE DATABASE dict_db_01225; + +CREATE TABLE dict_db_01225.dict_data (key UInt64, val UInt64) Engine=Memory(); +CREATE DICTIONARY dict_db_01225.dict +( + key UInt64 DEFAULT 0, + val UInt64 DEFAULT 10 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict_data' PASSWORD '' DB 'dict_db_01225')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT()); + +SYSTEM RELOAD DICTIONARY dict_db_01225.dict; + +DROP TABLE dict_db_01225.dict; -- { serverError 520; } +DROP DICTIONARY dict_db_01225.dict; + +DROP DATABASE dict_db_01225; diff --git a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.reference b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.reference new file mode 100644 index 00000000000..14ddc093143 --- /dev/null +++ b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.reference @@ -0,0 +1,6 @@ +CREATE TABLE dict_db_01225_dictionary.`dict_db_01225.dict` +( + `key` UInt64, + `val` UInt64 +) +ENGINE = Dictionary(`dict_db_01225.dict`) diff --git a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql new file mode 100644 index 00000000000..a494511ebd8 --- /dev/null +++ b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql @@ -0,0 +1,21 @@ +DROP DATABASE IF EXISTS dict_db_01225; +DROP DATABASE IF EXISTS dict_db_01225_dictionary; +CREATE DATABASE dict_db_01225; +CREATE DATABASE dict_db_01225_dictionary Engine=Dictionary; + +CREATE TABLE dict_db_01225.dict_data (key UInt64, val UInt64) Engine=Memory(); +CREATE DICTIONARY dict_db_01225.dict +( + key UInt64 DEFAULT 0, + val UInt64 DEFAULT 10 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict_data' PASSWORD '' DB 'dict_db_01225')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT()); + +SHOW CREATE TABLE dict_db_01225_dictionary.`dict_db_01225.dict` FORMAT TSVRaw; +SHOW CREATE TABLE dict_db_01225_dictionary.`dict_db_01225.no_such_dict`; -- { serverError 487; } + +DROP DATABASE dict_db_01225; +DROP DATABASE dict_db_01225_dictionary; diff --git a/tests/queries/0_stateless/01226_dist_on_dist_global_in.reference b/tests/queries/0_stateless/01226_dist_on_dist_global_in.reference new file mode 100644 index 00000000000..3d8d7fb770d --- /dev/null +++ b/tests/queries/0_stateless/01226_dist_on_dist_global_in.reference @@ -0,0 +1,6 @@ +GLOBAL IN +0 +0 +0 +0 +GLOBAL NOT IN diff --git a/tests/queries/0_stateless/01226_dist_on_dist_global_in.sql b/tests/queries/0_stateless/01226_dist_on_dist_global_in.sql new file mode 100644 index 00000000000..588ea9c1048 --- /dev/null +++ b/tests/queries/0_stateless/01226_dist_on_dist_global_in.sql @@ -0,0 +1,10 @@ +SELECT 'GLOBAL IN'; +select * from remote('localhost', system.one) where dummy global in (0); +select * from remote('localhost', system.one) where toUInt64(dummy) global in numbers(1); +select * from remote('localhost', system.one) where dummy global in system.one; +select * from remote('localhost', system.one) where dummy global in (select 0); +SELECT 'GLOBAL NOT IN'; +select * from remote('localhost', system.one) where dummy global not in (0); +select * from remote('localhost', system.one) where toUInt64(dummy) global not in numbers(1); +select * from remote('localhost', system.one) where dummy global not in system.one; +select * from remote('localhost', system.one) where dummy global not in (select 0); diff --git a/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.reference b/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.reference new file mode 100644 index 00000000000..083edaac248 --- /dev/null +++ b/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.reference @@ -0,0 +1,3 @@ +2 +2 +2 diff --git a/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.sql b/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.sql new file mode 100644 index 00000000000..a063e417e3a --- /dev/null +++ b/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.sql @@ -0,0 +1,6 @@ +-- Test from the issue https://github.com/ClickHouse/ClickHouse/issues/2610 +drop table if exists data_01227; +create table data_01227 (key Int) Engine=MergeTree() order by key; +insert into data_01227 select * from numbers(10); +select * from remote('127.1', currentDatabase(), data_01227) prewhere key global in (select key from data_01227 prewhere key = 2); +select * from cluster('test_cluster_two_shards', currentDatabase(), data_01227) prewhere key global in (select key from data_01227 prewhere key = 2); diff --git a/tests/queries/0_stateless/01231_log_queries_min_type.reference b/tests/queries/0_stateless/01231_log_queries_min_type.reference new file mode 100644 index 00000000000..a358d022033 --- /dev/null +++ b/tests/queries/0_stateless/01231_log_queries_min_type.reference @@ -0,0 +1,5 @@ +01231_log_queries_min_type/QUERY_START +2 +01231_log_queries_min_type/EXCEPTION_BEFORE_START +2 +3 diff --git a/tests/queries/0_stateless/01231_log_queries_min_type.sql b/tests/queries/0_stateless/01231_log_queries_min_type.sql new file mode 100644 index 00000000000..f2229c94a8a --- /dev/null +++ b/tests/queries/0_stateless/01231_log_queries_min_type.sql @@ -0,0 +1,15 @@ +set log_queries=1; + +select '01231_log_queries_min_type/QUERY_START'; +system flush logs; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; + +set log_queries_min_type='EXCEPTION_BEFORE_START'; +select '01231_log_queries_min_type/EXCEPTION_BEFORE_START'; +system flush logs; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; + +set log_queries_min_type='EXCEPTION_WHILE_PROCESSING'; +select '01231_log_queries_min_type/', max(number) from system.numbers limit 1e6 settings max_rows_to_read='100K'; -- { serverError 158; } +system flush logs; +select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; diff --git a/tests/queries/0_stateless/01231_markdown_format.reference b/tests/queries/0_stateless/01231_markdown_format.reference new file mode 100644 index 00000000000..57ad6c51de8 --- /dev/null +++ b/tests/queries/0_stateless/01231_markdown_format.reference @@ -0,0 +1,5 @@ +|id|name|array| +|:-:|:-:|:-:| +|1|name1|[1,2,3]| +|2|name2|[4,5,6]| +|3|name3|[7,8,9]| diff --git a/tests/queries/0_stateless/01231_markdown_format.sql b/tests/queries/0_stateless/01231_markdown_format.sql new file mode 100644 index 00000000000..693664be1ab --- /dev/null +++ b/tests/queries/0_stateless/01231_markdown_format.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS makrdown; +CREATE TABLE markdown (id UInt32, name String, array Array(Int8)) ENGINE = Memory; +INSERT INTO markdown VALUES (1, 'name1', [1,2,3]), (2, 'name2', [4,5,6]), (3, 'name3', [7,8,9]); + +SELECT * FROM markdown FORMAT Markdown; +DROP TABLE IF EXISTS markdown diff --git a/tests/queries/0_stateless/01231_operator_null_in.reference b/tests/queries/0_stateless/01231_operator_null_in.reference new file mode 100644 index 00000000000..b76f42e9af4 --- /dev/null +++ b/tests/queries/0_stateless/01231_operator_null_in.reference @@ -0,0 +1,80 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01231_operator_null_in.sql b/tests/queries/0_stateless/01231_operator_null_in.sql new file mode 100644 index 00000000000..ddebaf23900 --- /dev/null +++ b/tests/queries/0_stateless/01231_operator_null_in.sql @@ -0,0 +1,143 @@ +DROP TABLE IF EXISTS null_in; +CREATE TABLE null_in (dt DateTime, idx int, i Nullable(int), s Nullable(String)) ENGINE = MergeTree() PARTITION BY dt ORDER BY idx; + +INSERT INTO null_in VALUES (1, 1, 1, '1') (2, 2, NULL, NULL) (3, 3, 3, '3') (4, 4, NULL, NULL) (5, 5, 5, '5'); + +SELECT count() == 2 FROM null_in WHERE i in (1, 3, NULL); +SELECT count() == 2 FROM null_in WHERE i in range(4); +SELECT count() == 2 FROM null_in WHERE s in ('1', '3', NULL); +SELECT count() == 2 FROM null_in WHERE i global in (1, 3, NULL); +SELECT count() == 2 FROM null_in WHERE i global in range(4); +SELECT count() == 2 FROM null_in WHERE s global in ('1', '3', NULL); + +SELECT count() == 1 FROM null_in WHERE i not in (1, 3, NULL); +SELECT count() == 1 FROM null_in WHERE i not in range(4); +SELECT count() == 1 FROM null_in WHERE s not in ('1', '3', NULL); +SELECT count() == 1 FROM null_in WHERE i global not in (1, 3, NULL); +SELECT count() == 1 FROM null_in WHERE i global not in range(4); +SELECT count() == 1 FROM null_in WHERE s global not in ('1', '3', NULL); + +SET transform_null_in = 1; + +SELECT count() == 4 FROM null_in WHERE i in (1, 3, NULL); +SELECT count() == 2 FROM null_in WHERE i in range(4); +SELECT count() == 4 FROM null_in WHERE s in ('1', '3', NULL); +SELECT count() == 4 FROM null_in WHERE i global in (1, 3, NULL); +SELECT count() == 2 FROM null_in WHERE i global in range(4); +SELECT count() == 4 FROM null_in WHERE s global in ('1', '3', NULL); + +SELECT count() == 1 FROM null_in WHERE i not in (1, 3, NULL); +SELECT count() == 3 FROM null_in WHERE i not in range(4); +SELECT count() == 1 FROM null_in WHERE s not in ('1', '3', NULL); +SELECT count() == 1 FROM null_in WHERE i global not in (1, 3, NULL); +SELECT count() == 3 FROM null_in WHERE i global not in range(4); +SELECT count() == 1 FROM null_in WHERE s global not in ('1', '3', NULL); + +SELECT count() == 3 FROM null_in WHERE i not in (1, 3); +SELECT count() == 3 FROM null_in WHERE i not in range(4); +SELECT count() == 3 FROM null_in WHERE s not in ('1', '3'); +SELECT count() == 3 FROM null_in WHERE i global not in (1, 3); +SELECT count() == 3 FROM null_in WHERE i global not in range(4); +SELECT count() == 3 FROM null_in WHERE s global not in ('1', '3'); + +DROP TABLE IF EXISTS test_set; +CREATE TABLE test_set (i Nullable(int)) ENGINE = Set(); +INSERT INTO test_set VALUES (1), (NULL); + +SET transform_null_in = 0; + +SELECT count() == 1 FROM null_in WHERE i in test_set; +SELECT count() == 2 FROM null_in WHERE i not in test_set; +SELECT count() == 1 FROM null_in WHERE i global in test_set; +SELECT count() == 2 FROM null_in WHERE i global not in test_set; + +SET transform_null_in = 1; + +SELECT count() == 3 FROM null_in WHERE i in test_set; +SELECT count() == 2 FROM null_in WHERE i not in test_set; +SELECT count() == 3 FROM null_in WHERE i global in test_set; +SELECT count() == 2 FROM null_in WHERE i global not in test_set; + +-- Create with transform_null_in +CREATE TABLE test_set2 (i Nullable(int)) ENGINE = Set(); +INSERT INTO test_set2 VALUES (1), (NULL); + +SET transform_null_in = 0; + +SELECT count() == 1 FROM null_in WHERE i in test_set2; +SELECT count() == 2 FROM null_in WHERE i not in test_set2; +SELECT count() == 1 FROM null_in WHERE i global in test_set2; +SELECT count() == 2 FROM null_in WHERE i global not in test_set2; + +SET transform_null_in = 1; + +SELECT count() == 3 FROM null_in WHERE i in test_set2; +SELECT count() == 2 FROM null_in WHERE i not in test_set2; +SELECT count() == 3 FROM null_in WHERE i global in test_set2; +SELECT count() == 2 FROM null_in WHERE i global not in test_set2; + +DROP TABLE IF EXISTS test_set; +DROP TABLE IF EXISTS null_in; + + +DROP TABLE IF EXISTS null_in_subquery; +CREATE TABLE null_in_subquery (dt DateTime, idx int, i Nullable(UInt64)) ENGINE = MergeTree() PARTITION BY dt ORDER BY idx; +INSERT INTO null_in_subquery SELECT number % 3, number, number FROM system.numbers LIMIT 99999; + +SELECT count() == 33333 FROM null_in_subquery WHERE i in (SELECT i FROM null_in_subquery WHERE dt = 0); +SELECT count() == 66666 FROM null_in_subquery WHERE i not in (SELECT i FROM null_in_subquery WHERE dt = 1); +SELECT count() == 33333 FROM null_in_subquery WHERE i global in (SELECT i FROM null_in_subquery WHERE dt = 2); +SELECT count() == 66666 FROM null_in_subquery WHERE i global not in (SELECT i FROM null_in_subquery WHERE dt = 0); + +-- For index column +SELECT count() == 33333 FROM null_in_subquery WHERE idx in (SELECT idx FROM null_in_subquery WHERE dt = 0); +SELECT count() == 66666 FROM null_in_subquery WHERE idx not in (SELECT idx FROM null_in_subquery WHERE dt = 1); +SELECT count() == 33333 FROM null_in_subquery WHERE idx global in (SELECT idx FROM null_in_subquery WHERE dt = 2); +SELECT count() == 66666 FROM null_in_subquery WHERE idx global not in (SELECT idx FROM null_in_subquery WHERE dt = 0); + +INSERT INTO null_in_subquery VALUES (0, 123456780, NULL); +INSERT INTO null_in_subquery VALUES (1, 123456781, NULL); + +SELECT count() == 33335 FROM null_in_subquery WHERE i in (SELECT i FROM null_in_subquery WHERE dt = 0); +SELECT count() == 66666 FROM null_in_subquery WHERE i not in (SELECT i FROM null_in_subquery WHERE dt = 1); +SELECT count() == 33333 FROM null_in_subquery WHERE i in (SELECT i FROM null_in_subquery WHERE dt = 2); +SELECT count() == 66668 FROM null_in_subquery WHERE i not in (SELECT i FROM null_in_subquery WHERE dt = 2); +SELECT count() == 33335 FROM null_in_subquery WHERE i global in (SELECT i FROM null_in_subquery WHERE dt = 0); +SELECT count() == 66666 FROM null_in_subquery WHERE i global not in (SELECT i FROM null_in_subquery WHERE dt = 1); +SELECT count() == 33333 FROM null_in_subquery WHERE i global in (SELECT i FROM null_in_subquery WHERE dt = 2); +SELECT count() == 66668 FROM null_in_subquery WHERE i global not in (SELECT i FROM null_in_subquery WHERE dt = 2); + +DROP TABLE IF EXISTS null_in_subquery; + + +DROP TABLE IF EXISTS null_in_tuple; +CREATE TABLE null_in_tuple (dt DateTime, idx int, t Tuple(Nullable(UInt64), Nullable(String))) ENGINE = MergeTree() PARTITION BY dt ORDER BY idx; +INSERT INTO null_in_tuple VALUES (1, 1, (1, '1')) (2, 2, (2, NULL)) (3, 3, (NULL, '3')) (4, 4, (NULL, NULL)) + +SET transform_null_in = 0; + +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1')] FROM null_in_tuple WHERE t in ((1, '1'), (NULL, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, '3'), (NULL, NULL)] FROM null_in_tuple WHERE t not in ((1, '1'), (NULL, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1')] FROM null_in_tuple WHERE t global in ((1, '1'), (NULL, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, '3'), (NULL, NULL)] FROM null_in_tuple WHERE t global not in ((1, '1'), (NULL, NULL)); + +SET transform_null_in = 1; + +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1'), (NULL, NULL)] FROM null_in_tuple WHERE t in ((1, '1'), (NULL, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, '3')] FROM null_in_tuple WHERE t not in ((1, '1'), (NULL, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1'), (NULL, NULL)] FROM null_in_tuple WHERE t global in ((1, '1'), (NULL, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, '3')] FROM null_in_tuple WHERE t global not in ((1, '1'), (NULL, NULL)); + +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1')] FROM null_in_tuple WHERE t in ((1, '1'), (1, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1')] FROM null_in_tuple WHERE t in ((1, '1'), (NULL, '1')); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1'), (2, NULL)] FROM null_in_tuple WHERE t in ((1, '1'), (NULL, '1'), (2, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1'), (NULL, '3')] FROM null_in_tuple WHERE t in ((1, '1'), (1, NULL), (NULL, '3')); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(1, '1'), (2, NULL), (NULL, '3'), (NULL, NULL)] FROM null_in_tuple WHERE t in ((1, '1'), (1, NULL), (2, NULL), (NULL, '3'), (NULL, NULL)); + +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, '3'), (NULL, NULL)] FROM null_in_tuple WHERE t not in ((1, '1'), (1, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, '3'), (NULL, NULL)] FROM null_in_tuple WHERE t not in ((1, '1'), (NULL, '1')); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(NULL, '3'), (NULL, NULL)] FROM null_in_tuple WHERE t not in ((1, '1'), (NULL, '1'), (2, NULL)); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, NULL)] FROM null_in_tuple WHERE t not in ((1, '1'), (1, NULL), (NULL, '3')); +SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [] FROM null_in_tuple WHERE t not in ((1, '1'), (1, NULL), (2, NULL), (NULL, '3'), (NULL, NULL)); + +DROP TABLE IF EXISTS null_in_tuple; diff --git a/tests/queries/0_stateless/01232_extremes.reference b/tests/queries/0_stateless/01232_extremes.reference new file mode 100644 index 00000000000..d5b66dcbd4b --- /dev/null +++ b/tests/queries/0_stateless/01232_extremes.reference @@ -0,0 +1,110 @@ +0 +1 + +0 +1 +- + +- +- +0 +1 +0 +1 + +0 +1 +- + +- +- +0 +1 +0 +1 + +0 +1 +- + +- +- +0 +1 +0 +1 + +0 +1 +- + +- +- +0 +1 + +0 +1 +------ + +------ +------ +0 +0 +0 +1 +1 +2 + +0 +2 +- + +- +- +0 +0 +0 +1 +1 +2 + +0 +2 +- + +- +- +0 +0 +0 +1 +1 +2 + +0 +2 +------ + +------ +------ +0 +0 +1 +1 +2 + +0 +2 +- + +- +- +0 +0 +1 +1 +2 + +0 +2 diff --git a/tests/queries/0_stateless/01232_extremes.sql b/tests/queries/0_stateless/01232_extremes.sql new file mode 100644 index 00000000000..9379dc1cd38 --- /dev/null +++ b/tests/queries/0_stateless/01232_extremes.sql @@ -0,0 +1,55 @@ +set send_logs_level = 'error'; +set extremes = 1; +-- set experimental_use_processors=0; + +select * from remote('127.0.0.1', numbers(2)); +select '-'; +select * from remote('127.0.0.{1,1}', numbers(2)); +select '-'; +select * from remote('127.0.0.{1,2}', numbers(2)); +select '-'; +select * from remote('127.0.0.{2,2}', numbers(2)); +select '-'; +select * from remote('127.0.0.2', numbers(2)); +select '------'; + +select * from (select * from numbers(2) union all select * from numbers(3) union all select * from numbers(1)) order by number; +select '-'; +select * from (select * from numbers(1) union all select * from numbers(2) union all select * from numbers(3)) order by number; +select '-'; +select * from (select * from numbers(3) union all select * from numbers(1) union all select * from numbers(2)) order by number; +select '------'; + +create database if not exists shard_0; +create database if not exists shard_1; + +drop table if exists shard_0.num_01232; +drop table if exists shard_0.num2_01232; +drop table if exists shard_1.num_01232; +drop table if exists shard_1.num2_01232; +drop table if exists distr; +drop table if exists distr2; + +create table shard_0.num_01232 (number UInt64) engine = MergeTree order by number; +create table shard_1.num_01232 (number UInt64) engine = MergeTree order by number; +insert into shard_0.num_01232 select number from numbers(2); +insert into shard_1.num_01232 select number from numbers(3); +create table distr (number UInt64) engine = Distributed(test_cluster_two_shards_different_databases, '', num_01232); + +create table shard_0.num2_01232 (number UInt64) engine = MergeTree order by number; +create table shard_1.num2_01232 (number UInt64) engine = MergeTree order by number; +insert into shard_0.num2_01232 select number from numbers(3); +insert into shard_1.num2_01232 select number from numbers(2); +create table distr2 (number UInt64) engine = Distributed(test_cluster_two_shards_different_databases, '', num2_01232); + +select * from distr order by number; +select '-'; +select * from distr2 order by number; + +drop table if exists shard_0.num_01232; +drop table if exists shard_0.num2_01232; +drop table if exists shard_1.num_01232; +drop table if exists shard_1.num2_01232; +drop table if exists distr; +drop table if exists distr2; + diff --git a/tests/queries/0_stateless/01232_preparing_sets_race_condition.reference b/tests/queries/0_stateless/01232_preparing_sets_race_condition.reference index 2fc36ed5c97..45a4fb75db8 100644 --- a/tests/queries/0_stateless/01232_preparing_sets_race_condition.reference +++ b/tests/queries/0_stateless/01232_preparing_sets_race_condition.reference @@ -1,10 +1 @@ 8 -8 -8 -8 -8 -8 -8 -8 -8 -8 diff --git a/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh b/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh index 5f7b76c0e99..25a8cdb12ea 100755 --- a/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh +++ b/tests/queries/0_stateless/01232_preparing_sets_race_condition.sh @@ -18,7 +18,7 @@ echo " insert into tableB select number, number % 100000, addDays(toDate('2020-01-01'), number % 90) from numbers(50000000); " | $CLICKHOUSE_CLIENT -n -for i in {1..10}; do echo " +for i in {1..1}; do echo " SELECT tableName FROM ( diff --git a/tests/queries/0_stateless/01234_to_string_monotonic.reference b/tests/queries/0_stateless/01234_to_string_monotonic.reference new file mode 100644 index 00000000000..75404a347a4 --- /dev/null +++ b/tests/queries/0_stateless/01234_to_string_monotonic.reference @@ -0,0 +1,2 @@ +1234 +1234 diff --git a/tests/queries/0_stateless/01234_to_string_monotonic.sql b/tests/queries/0_stateless/01234_to_string_monotonic.sql new file mode 100644 index 00000000000..87324fdda27 --- /dev/null +++ b/tests/queries/0_stateless/01234_to_string_monotonic.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS test1; +DROP TABLE IF EXISTS test2; + +CREATE TABLE test1 (s String) ENGINE = MergeTree ORDER BY s SETTINGS index_granularity = 1; +CREATE TABLE test2 (s LowCardinality(String)) ENGINE = MergeTree ORDER BY s SETTINGS index_granularity = 1; + +INSERT INTO test1 SELECT toString(number) FROM numbers(10000); +INSERT INTO test2 SELECT toString(number) FROM numbers(10000); + +SELECT s FROM test1 WHERE toString(s) = '1234' SETTINGS max_rows_to_read = 2; +SELECT s FROM test2 WHERE toString(s) = '1234' SETTINGS max_rows_to_read = 2; + +DROP TABLE test1; +DROP TABLE test2; diff --git a/tests/queries/0_stateless/01235_live_view_over_distributed.reference b/tests/queries/0_stateless/01235_live_view_over_distributed.reference new file mode 100644 index 00000000000..00fc99d96ba --- /dev/null +++ b/tests/queries/0_stateless/01235_live_view_over_distributed.reference @@ -0,0 +1,4 @@ +2020-01-01 +2020-01-01 +2020-01-02 +2020-01-02 diff --git a/tests/queries/0_stateless/01235_live_view_over_distributed.sql b/tests/queries/0_stateless/01235_live_view_over_distributed.sql new file mode 100644 index 00000000000..f3950c16002 --- /dev/null +++ b/tests/queries/0_stateless/01235_live_view_over_distributed.sql @@ -0,0 +1,19 @@ +SET allow_experimental_live_view = 1; + +DROP TABLE IF EXISTS lv; +DROP TABLE IF EXISTS visits; +DROP TABLE IF EXISTS visits_layer; + +CREATE TABLE visits(StartDate Date) ENGINE MergeTree ORDER BY(StartDate); +CREATE TABLE visits_layer(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); + +CREATE LIVE VIEW lv AS SELECT * FROM visits_layer ORDER BY StartDate; + +INSERT INTO visits_layer (StartDate) VALUES ('2020-01-01'); +INSERT INTO visits_layer (StartDate) VALUES ('2020-01-02'); + +SELECT * FROM lv; + +DROP TABLE visits; +DROP TABLE visits_layer; + diff --git a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.reference b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.reference new file mode 100644 index 00000000000..dfb4d0552f5 --- /dev/null +++ b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.reference @@ -0,0 +1,8 @@ +2020-01-01 +2020-01-01 +2020-01-02 +2020-01-02 +2020-01-01 +2020-01-01 +2020-01-02 +2020-01-02 diff --git a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql new file mode 100644 index 00000000000..4408880ec5f --- /dev/null +++ b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql @@ -0,0 +1,21 @@ +SET allow_experimental_live_view = 1; + +DROP TABLE IF EXISTS lv; +DROP TABLE IF EXISTS visits; +DROP TABLE IF EXISTS visits_layer; + +CREATE TABLE visits(StartDate Date) ENGINE MergeTree ORDER BY(StartDate); +CREATE TABLE visits_layer(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); + +CREATE LIVE VIEW lv AS SELECT * FROM visits_layer ORDER BY StartDate; + +CREATE TABLE visits_layer_lv(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'lv', rand()); + +INSERT INTO visits_layer (StartDate) VALUES ('2020-01-01'); +INSERT INTO visits_layer (StartDate) VALUES ('2020-01-02'); + +SELECT * FROM visits_layer_lv; + +DROP TABLE visits; +DROP TABLE visits_layer; + diff --git a/tests/queries/0_stateless/01236_graphite_mt.reference b/tests/queries/0_stateless/01236_graphite_mt.reference new file mode 100644 index 00000000000..a30d2495265 --- /dev/null +++ b/tests/queries/0_stateless/01236_graphite_mt.reference @@ -0,0 +1,344 @@ +1 max_1 9 1 0 +1 max_1 19 1 10 +1 max_1 29 1 20 +1 max_1 39 1 30 +1 max_1 49 1 40 +1 max_1 59 1 50 +1 max_1 69 1 60 +1 max_1 79 1 70 +1 max_1 89 1 80 +1 max_1 99 1 90 +1 max_1 109 1 100 +1 max_1 119 1 110 +1 max_1 129 1 120 +1 max_1 139 1 130 +1 max_1 149 1 140 +1 max_1 159 1 150 +1 max_1 169 1 160 +1 max_1 179 1 170 +1 max_1 189 1 180 +1 max_1 199 1 190 +1 max_1 209 1 200 +1 max_1 219 1 210 +1 max_1 229 1 220 +1 max_1 239 1 230 +1 max_1 249 1 240 +1 max_1 259 1 250 +1 max_1 269 1 260 +1 max_1 279 1 270 +1 max_1 289 1 280 +1 max_1 299 1 290 +1 max_1 39 1 0 +1 max_1 139 1 40 +1 max_1 239 1 140 +1 max_1 339 1 240 +1 max_1 439 1 340 +1 max_1 539 1 440 +1 max_1 639 1 540 +1 max_1 739 1 640 +1 max_1 839 1 740 +1 max_1 939 1 840 +1 max_1 1039 1 940 +1 max_1 1139 1 1040 +1 max_1 1199 1 1140 +1 max_2 9 1 0 +1 max_2 19 1 10 +1 max_2 29 1 20 +1 max_2 39 1 30 +1 max_2 49 1 40 +1 max_2 59 1 50 +1 max_2 69 1 60 +1 max_2 79 1 70 +1 max_2 89 1 80 +1 max_2 99 1 90 +1 max_2 109 1 100 +1 max_2 119 1 110 +1 max_2 129 1 120 +1 max_2 139 1 130 +1 max_2 149 1 140 +1 max_2 159 1 150 +1 max_2 169 1 160 +1 max_2 179 1 170 +1 max_2 189 1 180 +1 max_2 199 1 190 +1 max_2 209 1 200 +1 max_2 219 1 210 +1 max_2 229 1 220 +1 max_2 239 1 230 +1 max_2 249 1 240 +1 max_2 259 1 250 +1 max_2 269 1 260 +1 max_2 279 1 270 +1 max_2 289 1 280 +1 max_2 299 1 290 +1 max_2 39 1 0 +1 max_2 139 1 40 +1 max_2 239 1 140 +1 max_2 339 1 240 +1 max_2 439 1 340 +1 max_2 539 1 440 +1 max_2 639 1 540 +1 max_2 739 1 640 +1 max_2 839 1 740 +1 max_2 939 1 840 +1 max_2 1039 1 940 +1 max_2 1139 1 1040 +1 max_2 1199 1 1140 +1 sum_1 45 1 0 +1 sum_1 145 1 10 +1 sum_1 245 1 20 +1 sum_1 345 1 30 +1 sum_1 445 1 40 +1 sum_1 545 1 50 +1 sum_1 645 1 60 +1 sum_1 745 1 70 +1 sum_1 845 1 80 +1 sum_1 945 1 90 +1 sum_1 1045 1 100 +1 sum_1 1145 1 110 +1 sum_1 1245 1 120 +1 sum_1 1345 1 130 +1 sum_1 1445 1 140 +1 sum_1 1545 1 150 +1 sum_1 1645 1 160 +1 sum_1 1745 1 170 +1 sum_1 1845 1 180 +1 sum_1 1945 1 190 +1 sum_1 2045 1 200 +1 sum_1 2145 1 210 +1 sum_1 2245 1 220 +1 sum_1 2345 1 230 +1 sum_1 2445 1 240 +1 sum_1 2545 1 250 +1 sum_1 2645 1 260 +1 sum_1 2745 1 270 +1 sum_1 2845 1 280 +1 sum_1 2945 1 290 +1 sum_1 780 1 0 +1 sum_1 8950 1 40 +1 sum_1 18950 1 140 +1 sum_1 28950 1 240 +1 sum_1 38950 1 340 +1 sum_1 48950 1 440 +1 sum_1 58950 1 540 +1 sum_1 68950 1 640 +1 sum_1 78950 1 740 +1 sum_1 88950 1 840 +1 sum_1 98950 1 940 +1 sum_1 108950 1 1040 +1 sum_1 70170 1 1140 +1 sum_2 45 1 0 +1 sum_2 145 1 10 +1 sum_2 245 1 20 +1 sum_2 345 1 30 +1 sum_2 445 1 40 +1 sum_2 545 1 50 +1 sum_2 645 1 60 +1 sum_2 745 1 70 +1 sum_2 845 1 80 +1 sum_2 945 1 90 +1 sum_2 1045 1 100 +1 sum_2 1145 1 110 +1 sum_2 1245 1 120 +1 sum_2 1345 1 130 +1 sum_2 1445 1 140 +1 sum_2 1545 1 150 +1 sum_2 1645 1 160 +1 sum_2 1745 1 170 +1 sum_2 1845 1 180 +1 sum_2 1945 1 190 +1 sum_2 2045 1 200 +1 sum_2 2145 1 210 +1 sum_2 2245 1 220 +1 sum_2 2345 1 230 +1 sum_2 2445 1 240 +1 sum_2 2545 1 250 +1 sum_2 2645 1 260 +1 sum_2 2745 1 270 +1 sum_2 2845 1 280 +1 sum_2 2945 1 290 +1 sum_2 780 1 0 +1 sum_2 8950 1 40 +1 sum_2 18950 1 140 +1 sum_2 28950 1 240 +1 sum_2 38950 1 340 +1 sum_2 48950 1 440 +1 sum_2 58950 1 540 +1 sum_2 68950 1 640 +1 sum_2 78950 1 740 +1 sum_2 88950 1 840 +1 sum_2 98950 1 940 +1 sum_2 108950 1 1040 +1 sum_2 70170 1 1140 +2 max_1 9 1 0 +2 max_1 19 1 10 +2 max_1 29 1 20 +2 max_1 39 1 30 +2 max_1 49 1 40 +2 max_1 59 1 50 +2 max_1 69 1 60 +2 max_1 79 1 70 +2 max_1 89 1 80 +2 max_1 99 1 90 +2 max_1 109 1 100 +2 max_1 119 1 110 +2 max_1 129 1 120 +2 max_1 139 1 130 +2 max_1 149 1 140 +2 max_1 159 1 150 +2 max_1 169 1 160 +2 max_1 179 1 170 +2 max_1 189 1 180 +2 max_1 199 1 190 +2 max_1 209 1 200 +2 max_1 219 1 210 +2 max_1 229 1 220 +2 max_1 239 1 230 +2 max_1 249 1 240 +2 max_1 259 1 250 +2 max_1 269 1 260 +2 max_1 279 1 270 +2 max_1 289 1 280 +2 max_1 299 1 290 +2 max_1 39 1 0 +2 max_1 139 1 40 +2 max_1 239 1 140 +2 max_1 339 1 240 +2 max_1 439 1 340 +2 max_1 539 1 440 +2 max_1 639 1 540 +2 max_1 739 1 640 +2 max_1 839 1 740 +2 max_1 939 1 840 +2 max_1 1039 1 940 +2 max_1 1139 1 1040 +2 max_1 1199 1 1140 +2 max_2 9 1 0 +2 max_2 19 1 10 +2 max_2 29 1 20 +2 max_2 39 1 30 +2 max_2 49 1 40 +2 max_2 59 1 50 +2 max_2 69 1 60 +2 max_2 79 1 70 +2 max_2 89 1 80 +2 max_2 99 1 90 +2 max_2 109 1 100 +2 max_2 119 1 110 +2 max_2 129 1 120 +2 max_2 139 1 130 +2 max_2 149 1 140 +2 max_2 159 1 150 +2 max_2 169 1 160 +2 max_2 179 1 170 +2 max_2 189 1 180 +2 max_2 199 1 190 +2 max_2 209 1 200 +2 max_2 219 1 210 +2 max_2 229 1 220 +2 max_2 239 1 230 +2 max_2 249 1 240 +2 max_2 259 1 250 +2 max_2 269 1 260 +2 max_2 279 1 270 +2 max_2 289 1 280 +2 max_2 299 1 290 +2 max_2 39 1 0 +2 max_2 139 1 40 +2 max_2 239 1 140 +2 max_2 339 1 240 +2 max_2 439 1 340 +2 max_2 539 1 440 +2 max_2 639 1 540 +2 max_2 739 1 640 +2 max_2 839 1 740 +2 max_2 939 1 840 +2 max_2 1039 1 940 +2 max_2 1139 1 1040 +2 max_2 1199 1 1140 +2 sum_1 45 1 0 +2 sum_1 145 1 10 +2 sum_1 245 1 20 +2 sum_1 345 1 30 +2 sum_1 445 1 40 +2 sum_1 545 1 50 +2 sum_1 645 1 60 +2 sum_1 745 1 70 +2 sum_1 845 1 80 +2 sum_1 945 1 90 +2 sum_1 1045 1 100 +2 sum_1 1145 1 110 +2 sum_1 1245 1 120 +2 sum_1 1345 1 130 +2 sum_1 1445 1 140 +2 sum_1 1545 1 150 +2 sum_1 1645 1 160 +2 sum_1 1745 1 170 +2 sum_1 1845 1 180 +2 sum_1 1945 1 190 +2 sum_1 2045 1 200 +2 sum_1 2145 1 210 +2 sum_1 2245 1 220 +2 sum_1 2345 1 230 +2 sum_1 2445 1 240 +2 sum_1 2545 1 250 +2 sum_1 2645 1 260 +2 sum_1 2745 1 270 +2 sum_1 2845 1 280 +2 sum_1 2945 1 290 +2 sum_1 780 1 0 +2 sum_1 8950 1 40 +2 sum_1 18950 1 140 +2 sum_1 28950 1 240 +2 sum_1 38950 1 340 +2 sum_1 48950 1 440 +2 sum_1 58950 1 540 +2 sum_1 68950 1 640 +2 sum_1 78950 1 740 +2 sum_1 88950 1 840 +2 sum_1 98950 1 940 +2 sum_1 108950 1 1040 +2 sum_1 70170 1 1140 +2 sum_2 45 1 0 +2 sum_2 145 1 10 +2 sum_2 245 1 20 +2 sum_2 345 1 30 +2 sum_2 445 1 40 +2 sum_2 545 1 50 +2 sum_2 645 1 60 +2 sum_2 745 1 70 +2 sum_2 845 1 80 +2 sum_2 945 1 90 +2 sum_2 1045 1 100 +2 sum_2 1145 1 110 +2 sum_2 1245 1 120 +2 sum_2 1345 1 130 +2 sum_2 1445 1 140 +2 sum_2 1545 1 150 +2 sum_2 1645 1 160 +2 sum_2 1745 1 170 +2 sum_2 1845 1 180 +2 sum_2 1945 1 190 +2 sum_2 2045 1 200 +2 sum_2 2145 1 210 +2 sum_2 2245 1 220 +2 sum_2 2345 1 230 +2 sum_2 2445 1 240 +2 sum_2 2545 1 250 +2 sum_2 2645 1 260 +2 sum_2 2745 1 270 +2 sum_2 2845 1 280 +2 sum_2 2945 1 290 +2 sum_2 780 1 0 +2 sum_2 8950 1 40 +2 sum_2 18950 1 140 +2 sum_2 28950 1 240 +2 sum_2 38950 1 340 +2 sum_2 48950 1 440 +2 sum_2 58950 1 540 +2 sum_2 68950 1 640 +2 sum_2 78950 1 740 +2 sum_2 88950 1 840 +2 sum_2 98950 1 940 +2 sum_2 108950 1 1040 +2 sum_2 70170 1 1140 diff --git a/tests/queries/0_stateless/01236_graphite_mt.sql b/tests/queries/0_stateless/01236_graphite_mt.sql new file mode 100644 index 00000000000..cee9b8c9fde --- /dev/null +++ b/tests/queries/0_stateless/01236_graphite_mt.sql @@ -0,0 +1,26 @@ +drop table if exists test_graphite; +create table test_graphite (key UInt32, Path String, Time DateTime, Value Float64, Version UInt32, col UInt64) engine = GraphiteMergeTree('graphite_rollup') order by key settings index_granularity=10; + +insert into test_graphite +select 1, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 2, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 1, 'sum_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 2, 'sum_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 1, 'max_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 2, 'max_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 1, 'max_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 2, 'max_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300); + +insert into test_graphite +select 1, 'sum_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 2, 'sum_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 1, 'sum_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 2, 'sum_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 1, 'max_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 2, 'max_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 1, 'max_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 2, 'max_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200); + +optimize table test_graphite; + +select key, Path, Value, Version, col from test_graphite order by key, Path, Time desc; diff --git a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.reference b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.reference new file mode 100644 index 00000000000..00fc99d96ba --- /dev/null +++ b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.reference @@ -0,0 +1,4 @@ +2020-01-01 +2020-01-01 +2020-01-02 +2020-01-02 diff --git a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql new file mode 100644 index 00000000000..dc57e001122 --- /dev/null +++ b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql @@ -0,0 +1,19 @@ +SET allow_experimental_live_view = 1; + +DROP TABLE IF EXISTS lv; +DROP TABLE IF EXISTS visits; +DROP TABLE IF EXISTS visits_layer; + +CREATE TABLE visits(StartDate Date) ENGINE MergeTree ORDER BY(StartDate); +CREATE TABLE visits_layer(StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); + +CREATE LIVE VIEW lv AS SELECT foo.x FROM (SELECT StartDate AS x FROM visits_layer) AS foo ORDER BY foo.x; + +INSERT INTO visits_layer (StartDate) VALUES ('2020-01-01'); +INSERT INTO visits_layer (StartDate) VALUES ('2020-01-02'); + +SELECT * FROM lv; + +DROP TABLE visits; +DROP TABLE visits_layer; + diff --git a/tests/queries/0_stateless/01240_join_get_or_null.reference b/tests/queries/0_stateless/01240_join_get_or_null.reference new file mode 100644 index 00000000000..96e34d5a44c --- /dev/null +++ b/tests/queries/0_stateless/01240_join_get_or_null.reference @@ -0,0 +1,2 @@ +\N +\N diff --git a/tests/queries/0_stateless/01240_join_get_or_null.sql b/tests/queries/0_stateless/01240_join_get_or_null.sql new file mode 100644 index 00000000000..48fd8228b55 --- /dev/null +++ b/tests/queries/0_stateless/01240_join_get_or_null.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS join_test; + +CREATE TABLE join_test (id UInt16, num UInt16) engine = Join(ANY, LEFT, id); +SELECT joinGetOrNull('join_test', 'num', 500); +DROP TABLE join_test; + +CREATE TABLE join_test (id UInt16, num Nullable(UInt16)) engine = Join(ANY, LEFT, id); +SELECT joinGetOrNull('join_test', 'num', 500); +DROP TABLE join_test; + +CREATE TABLE join_test (id UInt16, num Array(UInt16)) engine = Join(ANY, LEFT, id); +SELECT joinGetOrNull('join_test', 'num', 500); -- { serverError 43 } +DROP TABLE join_test; diff --git a/tests/queries/0_stateless/01245_distributed_group_by_no_merge_with-extremes_and_totals.reference b/tests/queries/0_stateless/01245_distributed_group_by_no_merge_with-extremes_and_totals.reference new file mode 100644 index 00000000000..3e6590d769c --- /dev/null +++ b/tests/queries/0_stateless/01245_distributed_group_by_no_merge_with-extremes_and_totals.reference @@ -0,0 +1,289 @@ +20 +20 + +20 +distributed_group_by_no_merge = 0, extremes = 0 +10 +- +10 +- +10 + +10 +- +10 + +10 +------ +10 +- +10 +- +20 +- +20 +------ +10 + +10 +- +10 + +10 +- +20 + +20 +- +20 + +20 +distributed_group_by_no_merge = 1, extremes = 0 +10 +- +10 +- +10 + +10 +- +10 + +10 +------ +10 +- +10 +- +20 +- +20 +------ +10 + +10 +- +10 + +10 +- +20 + +20 +- +20 + +20 +distributed_group_by_no_merge = 0, extremes = 1 +10 + +10 +10 +- + +- +- +10 + +10 +10 +- + +- +- +10 + +10 + +10 +10 +- + +- +- +10 + +10 + +10 +10 +------ + +------ +------ +10 + +10 +10 +- + +- +- +10 + +10 +10 +- + +- +- +20 + +20 +20 +- + +- +- +20 + +20 +20 +------ + +------ +------ +10 + +10 + +10 +10 +- + +- +- +10 + +10 + +10 +10 +- + +- +- +20 + +20 + +20 +20 +- + +- +- +20 + +20 + +20 +20 +distributed_group_by_no_merge = 1, extremes = 1 + +distributed_group_by_no_merge = 1, extremes = 1 +distributed_group_by_no_merge = 1, extremes = 1 +10 + +10 +10 +- + +- +- +10 + +10 +10 +- + +- +- +10 + +10 + +10 +10 +- + +- +- +10 + +10 + +10 +10 +------ + +------ +------ +10 + +10 +10 +- + +- +- +10 + +10 +10 +- + +- +- +20 + +20 +20 +- + +- +- +20 + +20 +20 +------ + +------ +------ +10 + +10 + +10 +10 +- + +- +- +10 + +10 + +10 +10 +- + +- +- +20 + +20 + +20 +20 +- + +- +- +20 + +20 + +20 +20 diff --git a/tests/queries/0_stateless/01245_distributed_group_by_no_merge_with-extremes_and_totals.sql b/tests/queries/0_stateless/01245_distributed_group_by_no_merge_with-extremes_and_totals.sql new file mode 100644 index 00000000000..030330e4cae --- /dev/null +++ b/tests/queries/0_stateless/01245_distributed_group_by_no_merge_with-extremes_and_totals.sql @@ -0,0 +1,106 @@ +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.{1,2}', system.numbers) LIMIT 5 SETTINGS distributed_group_by_no_merge = 1); +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.{1,2}', system.numbers) LIMIT 5 SETTINGS distributed_group_by_no_merge = 1) with totals; + +SELECT 'distributed_group_by_no_merge = 0, extremes = 0'; +SET distributed_group_by_no_merge = 0, extremes = 0; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5) with totals; +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5) with totals; +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5)) with totals); +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5))) with totals; + +SELECT 'distributed_group_by_no_merge = 1, extremes = 0'; +SET distributed_group_by_no_merge = 1, extremes = 0; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5) with totals; +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5) with totals; +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5)) with totals); +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5))) with totals; + +SELECT 'distributed_group_by_no_merge = 0, extremes = 1'; +SET distributed_group_by_no_merge = 0, extremes = 1; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5) with totals; +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5) with totals; +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5)) with totals); +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5))) with totals; + +SELECT 'distributed_group_by_no_merge = 1, extremes = 1'; +SET distributed_group_by_no_merge = 1, extremes = 1; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5); +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.1', system.numbers) LIMIT 5) with totals; +SELECT '-'; +SELECT sum(number) FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 5) with totals; +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5)) with totals); +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5)) with totals); +SELECT '------'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.1', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.2', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{1,2}', numbers(5))) with totals; +SELECT '-'; +SELECT sum(s) FROM (SELECT sum(number) as s FROM remote('127.0.0.{2,3}', numbers(5))) with totals; diff --git a/tests/queries/0_stateless/01245_limit_infinite_sources.reference b/tests/queries/0_stateless/01245_limit_infinite_sources.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01245_limit_infinite_sources.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01245_limit_infinite_sources.sql b/tests/queries/0_stateless/01245_limit_infinite_sources.sql new file mode 100644 index 00000000000..803a2d14c39 --- /dev/null +++ b/tests/queries/0_stateless/01245_limit_infinite_sources.sql @@ -0,0 +1,11 @@ +SELECT number +FROM +( + SELECT zero AS number + FROM remote('127.0.0.2', system.zeros) + UNION ALL + SELECT number + sleep(0.5) + FROM system.numbers +) +WHERE number = 1 +LIMIT 1 diff --git a/tests/queries/0_stateless/01246_buffer_flush.reference b/tests/queries/0_stateless/01246_buffer_flush.reference new file mode 100644 index 00000000000..a877e94b919 --- /dev/null +++ b/tests/queries/0_stateless/01246_buffer_flush.reference @@ -0,0 +1,10 @@ +min +0 +5 +max +5 +10 +direct +20 +drop +30 diff --git a/tests/queries/0_stateless/01246_buffer_flush.sql b/tests/queries/0_stateless/01246_buffer_flush.sql new file mode 100644 index 00000000000..efe0adf703a --- /dev/null +++ b/tests/queries/0_stateless/01246_buffer_flush.sql @@ -0,0 +1,44 @@ +drop table if exists data_01256; +drop table if exists buffer_01256; + +create table data_01256 as system.numbers Engine=Memory(); + +select 'min'; +create table buffer_01256 as system.numbers Engine=Buffer(currentDatabase(), data_01256, 1, + 2, 100, /* time */ + 4, 100, /* rows */ + 1, 1e6 /* bytes */ +); +insert into buffer_01256 select * from system.numbers limit 5; +select count() from data_01256; +-- sleep 2 (min time) + 1 (round up) + bias (1) = 4 +select sleepEachRow(2) from numbers(2) FORMAT Null; +select count() from data_01256; +drop table buffer_01256; + +select 'max'; +create table buffer_01256 as system.numbers Engine=Buffer(currentDatabase(), data_01256, 1, + 100, 2, /* time */ + 0, 100, /* rows */ + 0, 1e6 /* bytes */ +); +insert into buffer_01256 select * from system.numbers limit 5; +select count() from data_01256; +-- sleep 2 (min time) + 1 (round up) + bias (1) = 4 +select sleepEachRow(2) from numbers(2) FORMAT Null; +select count() from data_01256; +drop table buffer_01256; + +select 'direct'; +create table buffer_01256 as system.numbers Engine=Buffer(currentDatabase(), data_01256, 1, + 100, 100, /* time */ + 0, 9, /* rows */ + 0, 1e6 /* bytes */ +); +insert into buffer_01256 select * from system.numbers limit 10; +select count() from data_01256; + +select 'drop'; +insert into buffer_01256 select * from system.numbers limit 10; +drop table if exists buffer_01256; +select count() from data_01256; diff --git a/tests/queries/0_stateless/01246_least_greatest_generic.reference b/tests/queries/0_stateless/01246_least_greatest_generic.reference new file mode 100644 index 00000000000..24c2233eed2 --- /dev/null +++ b/tests/queries/0_stateless/01246_least_greatest_generic.reference @@ -0,0 +1,22 @@ +hello +world + +z +hello +world +1 +\N +\N +nan +inf +-0 +123 +-1 +4294967295 +['world'] +[[[]]] +[[[],[]]] +[] +[NULL] +[0] +[NULL] diff --git a/tests/queries/0_stateless/01246_least_greatest_generic.sql b/tests/queries/0_stateless/01246_least_greatest_generic.sql new file mode 100644 index 00000000000..f0dceabfcb5 --- /dev/null +++ b/tests/queries/0_stateless/01246_least_greatest_generic.sql @@ -0,0 +1,36 @@ +SELECT least('hello', 'world'); +SELECT greatest('hello', 'world'); +SELECT least('hello', 'world', ''); +SELECT greatest('hello', 'world', 'z'); + +SELECT least('hello'); +SELECT greatest('world'); + +SELECT least(1, inf, nan); +SELECT least(1, inf, nan, NULL); +SELECT greatest(1, inf, nan, NULL); +SELECT greatest(1, inf, nan); +SELECT greatest(1, inf); + +SELECT least(0., -0.); +SELECT least(toNullable(123), 456); + +-- This can be improved +SELECT LEAST(-1, 18446744073709551615); -- { serverError 43 } +SELECT LEAST(-1., 18446744073709551615); -- { serverError 43 } + +SELECT LEAST(-1., 18446744073709551615.); +SELECT greatest(-1, 1, 4294967295); + +SELECT greatest([], ['hello'], ['world']); + +SELECT least([[[], []]], [[[]]], [[[]], [[]]]); +SELECT greatest([[[], []]], [[[]]], [[[]], [[]]]); + +SELECT least([], [NULL]); +SELECT greatest([], [NULL]); + +SELECT LEAST([NULL], [0]); +SELECT GREATEST([NULL], [0]); + +SELECT Greatest(); -- { serverError 42 } diff --git a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.reference b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.reference new file mode 100644 index 00000000000..b8cf0042ed3 --- /dev/null +++ b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.reference @@ -0,0 +1,69 @@ +- +0 +1 +0 +1 +optimize_skip_unused_shards +0 +1 +0 +1 +GROUP BY number +1 0 +1 1 +1 0 +1 1 +GROUP BY number distributed_group_by_no_merge +1 0 +1 1 +1 0 +1 1 +GROUP BY number, 1 +1 0 +1 1 +1 0 +1 1 +GROUP BY 1 +4 0 +GROUP BY number ORDER BY number DESC +2 1 +2 0 +GROUP BY toString(number) +1 0 +1 1 +1 0 +1 1 +GROUP BY number%2 +2 0 +2 1 +countDistinct +2 +countDistinct GROUP BY number +1 +1 +1 +1 +DISTINCT +0 +1 +0 +1 +HAVING +LIMIT +2 0 +2 1 +LIMIT BY +2 0 +2 1 +GROUP BY (Distributed-over-Distributed) +4 0 +4 1 +GROUP BY (Distributed-over-Distributed) distributed_group_by_no_merge +1 0 +1 1 +1 0 +1 1 +1 0 +1 1 +1 0 +1 1 diff --git a/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql new file mode 100644 index 00000000000..56345e23094 --- /dev/null +++ b/tests/queries/0_stateless/01247_distributed_group_by_no_merge_GROUP_BY_injective_sharding_key.sql @@ -0,0 +1,62 @@ +drop table if exists dist_01247; +drop table if exists data_01247; + +create table data_01247 as system.numbers engine=Memory(); +insert into data_01247 select * from system.numbers limit 2; +create table dist_01247 as data_01247 engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01247, number); +-- since data is not inserted via distributed it will have duplicates +-- (and this is how we ensure that this optimization will work) + +set max_distributed_connections=1; + +select '-'; +select * from dist_01247; + +select 'optimize_skip_unused_shards'; +set optimize_skip_unused_shards=1; +select * from dist_01247; + +select 'GROUP BY number'; +select count(), * from dist_01247 group by number; +select 'GROUP BY number distributed_group_by_no_merge'; +select count(), * from dist_01247 group by number settings distributed_group_by_no_merge=1; + +-- dumb, but should work, since "GROUP BY 1" optimized out +select 'GROUP BY number, 1'; +select count(), * from dist_01247 group by number, 1; +select 'GROUP BY 1'; +select count(), min(number) from dist_01247 group by 1; + +select 'GROUP BY number ORDER BY number DESC'; +select count(), * from dist_01247 group by number order by number desc; + +select 'GROUP BY toString(number)'; +select count(), * from dist_01247 group by toString(number); + +select 'GROUP BY number%2'; +select count(), any(number) from dist_01247 group by number%2; + +select 'countDistinct'; +select count(DISTINCT number) from dist_01247; + +select 'countDistinct GROUP BY number'; +select count(DISTINCT number) from dist_01247 group by number; + +select 'DISTINCT'; +select DISTINCT number from dist_01247; + +select 'HAVING'; +select count() cnt, * from dist_01247 group by number having cnt < 0; + +select 'LIMIT'; +select count(), * from dist_01247 group by number limit 1; +select count(), * from dist_01247 group by number limit 1 offset 1; + +select 'LIMIT BY'; +select count(), * from dist_01247 group by number limit 0 by number; +select count(), * from dist_01247 group by number limit 1 by number; + +select 'GROUP BY (Distributed-over-Distributed)'; +select count(), * from cluster(test_cluster_two_shards, currentDatabase(), dist_01247) group by number; +select 'GROUP BY (Distributed-over-Distributed) distributed_group_by_no_merge'; +select count(), * from cluster(test_cluster_two_shards, currentDatabase(), dist_01247) group by number settings distributed_group_by_no_merge=1; diff --git a/tests/queries/0_stateless/01247_least_greatest_filimonov.reference b/tests/queries/0_stateless/01247_least_greatest_filimonov.reference new file mode 100644 index 00000000000..5b3b2abada8 --- /dev/null +++ b/tests/queries/0_stateless/01247_least_greatest_filimonov.reference @@ -0,0 +1,3 @@ +2 +767 +C diff --git a/tests/queries/0_stateless/01247_least_greatest_filimonov.sql b/tests/queries/0_stateless/01247_least_greatest_filimonov.sql new file mode 100644 index 00000000000..b845d65dcb9 --- /dev/null +++ b/tests/queries/0_stateless/01247_least_greatest_filimonov.sql @@ -0,0 +1,3 @@ +SELECT GREATEST(2,0); +SELECT GREATEST(34.0,3.0,5.0,767.0); +SELECT GREATEST('B','A','C'); diff --git a/tests/queries/0_stateless/01248_least_greatest_mixed_const.reference b/tests/queries/0_stateless/01248_least_greatest_mixed_const.reference new file mode 100644 index 00000000000..bbdc93bd5ee --- /dev/null +++ b/tests/queries/0_stateless/01248_least_greatest_mixed_const.reference @@ -0,0 +1,10 @@ +0 6 +1 6 +2 6 +3 6 +4 6 +4 6 +4 6 +4 7 +4 8 +4 9 diff --git a/tests/queries/0_stateless/01248_least_greatest_mixed_const.sql b/tests/queries/0_stateless/01248_least_greatest_mixed_const.sql new file mode 100644 index 00000000000..3fcf20623d6 --- /dev/null +++ b/tests/queries/0_stateless/01248_least_greatest_mixed_const.sql @@ -0,0 +1 @@ +SELECT least(4, number, 6), greatest(4, number, 6) FROM numbers(10); diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql new file mode 100644 index 00000000000..987380dcc73 --- /dev/null +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql @@ -0,0 +1,3 @@ +CREATE TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(0, 1) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; -- { serverError 42 } +CREATE TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(-0.1) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; -- { serverError 36 } +CREATE TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(1.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; -- { serverError 36 } diff --git a/tests/queries/0_stateless/01250_fixed_string_comparison.reference b/tests/queries/0_stateless/01250_fixed_string_comparison.reference new file mode 100644 index 00000000000..19754573478 --- /dev/null +++ b/tests/queries/0_stateless/01250_fixed_string_comparison.reference @@ -0,0 +1,110 @@ +Row 1: +────── +equals(b, b): 1 +greater(b, b): 0 +less(b, b): 0 +equals(b, c): 0 +greater(b, c): 0 +less(b, c): 1 +equals(b, d): 0 +greater(b, d): 0 +less(b, d): 1 +equals(b, bf): 1 +greater(b, bf): 0 +less(b, bf): 0 +equals(b, cf): 0 +greater(b, cf): 0 +less(b, cf): 1 +equals(b, df): 0 +greater(b, df): 0 +less(b, df): 1 +equals(c, b): 0 +greater(c, b): 1 +less(c, b): 0 +equals(c, c): 1 +greater(c, c): 0 +less(c, c): 0 +equals(c, d): 0 +greater(c, d): 0 +less(c, d): 1 +equals(c, bf): 0 +greater(c, bf): 1 +less(c, bf): 0 +equals(c, cf): 1 +greater(c, cf): 0 +less(c, cf): 0 +equals(c, df): 0 +greater(c, df): 0 +less(c, df): 1 +equals(d, b): 0 +greater(d, b): 1 +less(d, b): 0 +equals(d, c): 0 +greater(d, c): 1 +less(d, c): 0 +equals(d, d): 1 +greater(d, d): 0 +less(d, d): 0 +equals(d, bf): 0 +greater(d, bf): 1 +less(d, bf): 0 +equals(d, cf): 0 +greater(d, cf): 1 +less(d, cf): 0 +equals(d, df): 1 +greater(d, df): 0 +less(d, df): 0 +equals(bf, b): 1 +greater(bf, b): 0 +less(bf, b): 0 +equals(bf, c): 0 +greater(bf, c): 0 +less(bf, c): 1 +equals(bf, d): 0 +greater(bf, d): 0 +less(bf, d): 1 +equals(bf, bf): 1 +greater(bf, bf): 0 +less(bf, bf): 0 +equals(bf, cf): 0 +greater(bf, cf): 0 +less(bf, cf): 1 +equals(bf, df): 0 +greater(bf, df): 0 +less(bf, df): 1 +equals(cf, b): 0 +greater(cf, b): 1 +less(cf, b): 0 +equals(cf, c): 1 +greater(cf, c): 0 +less(cf, c): 0 +equals(cf, d): 0 +greater(cf, d): 0 +less(cf, d): 1 +equals(cf, bf): 0 +greater(cf, bf): 1 +less(cf, bf): 0 +equals(cf, cf): 1 +greater(cf, cf): 0 +less(cf, cf): 0 +equals(cf, df): 0 +greater(cf, df): 0 +less(cf, df): 1 +equals(df, b): 0 +greater(df, b): 1 +less(df, b): 0 +equals(df, c): 0 +greater(df, c): 1 +less(df, c): 0 +equals(df, d): 1 +greater(df, d): 0 +less(df, d): 0 +equals(df, bf): 0 +greater(df, bf): 1 +less(df, bf): 0 +equals(df, cf): 0 +greater(df, cf): 1 +less(df, cf): 0 +equals(df, df): 1 +greater(df, df): 0 +less(df, df): 0 diff --git a/tests/queries/0_stateless/01250_fixed_string_comparison.sql b/tests/queries/0_stateless/01250_fixed_string_comparison.sql new file mode 100644 index 00000000000..8481b3572bb --- /dev/null +++ b/tests/queries/0_stateless/01250_fixed_string_comparison.sql @@ -0,0 +1,45 @@ +WITH 'abb' AS b, 'abc' AS c, 'abd' AS d, toFixedString(b, 5) AS bf, toFixedString(c, 5) AS cf, toFixedString(d, 5) AS df +SELECT + b = b, b > b, b < b, + b = c, b > c, b < c, + b = d, b > d, b < d, + b = bf, b > bf, b < bf, + b = cf, b > cf, b < cf, + b = df, b > df, b < df, + + c = b, c > b, c < b, + c = c, c > c, c < c, + c = d, c > d, c < d, + c = bf, c > bf, c < bf, + c = cf, c > cf, c < cf, + c = df, c > df, c < df, + + d = b, d > b, d < b, + d = c, d > c, d < c, + d = d, d > d, d < d, + d = bf, d > bf, d < bf, + d = cf, d > cf, d < cf, + d = df, d > df, d < df, + + bf = b, bf > b, bf < b, + bf = c, bf > c, bf < c, + bf = d, bf > d, bf < d, + bf = bf, bf > bf, bf < bf, + bf = cf, bf > cf, bf < cf, + bf = df, bf > df, bf < df, + + cf = b, cf > b, cf < b, + cf = c, cf > c, cf < c, + cf = d, cf > d, cf < d, + cf = bf, cf > bf, cf < bf, + cf = cf, cf > cf, cf < cf, + cf = df, cf > df, cf < df, + + df = b, df > b, df < b, + df = c, df > c, df < c, + df = d, df > d, df < d, + df = bf, df > bf, df < bf, + df = cf, df > cf, df < cf, + df = df, df > df, df < df + +FORMAT Vertical; diff --git a/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.reference b/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.reference new file mode 100644 index 00000000000..757d2858524 --- /dev/null +++ b/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.reference @@ -0,0 +1,39 @@ +1 +1 +1 +1 +1 +0 +0 +0 +0 +[11,22] +[22,11] +[11,22] +[22,11] +1 +1 +1 +1 +1 +0 +0 +0 +0 +[11,22] +[22,11] +[11,22] +[22,11] +1 +1 +1 +1 +1 +255 +255 +0 +255 +[11,22] +[22,11] +[11,22] +[22,11] diff --git a/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql b/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql new file mode 100644 index 00000000000..decf65dc8cf --- /dev/null +++ b/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql @@ -0,0 +1,97 @@ +DROP DATABASE IF EXISTS database_for_dict; +CREATE DATABASE database_for_dict Engine = Ordinary; + +DROP TABLE IF EXISTS database_for_dict.dict_source; +CREATE TABLE database_for_dict.dict_source (id UInt64, parent_id UInt64, value String) ENGINE = Memory; +INSERT INTO database_for_dict.dict_source VALUES (1, 0, 'hello'), (2, 1, 'world'), (3, 2, 'upyachka'), (11, 22, 'a'), (22, 11, 'b'); + +DROP DICTIONARY IF EXISTS database_for_dict.dictionary_with_hierarchy; + +CREATE DICTIONARY database_for_dict.dictionary_with_hierarchy +( + id UInt64, parent_id UInt64 HIERARCHICAL, value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'dict_source')) +LAYOUT(HASHED()) +LIFETIME(MIN 1 MAX 1); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(2), toUInt64(1)); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(11)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(11)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(22))); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(11))); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(111)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(111)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(222))); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(111))); + +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(11)); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(22)); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(11))); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22))); + + +DROP DICTIONARY IF EXISTS database_for_dict.dictionary_with_hierarchy; + +CREATE DICTIONARY database_for_dict.dictionary_with_hierarchy +( + id UInt64, parent_id UInt64 HIERARCHICAL, value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'dict_source')) +LAYOUT(FLAT()) +LIFETIME(MIN 1 MAX 1); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(2), toUInt64(1)); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(11)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(11)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(22))); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(11))); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(111)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(111)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(222))); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(111))); + +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(11)); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(22)); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(11))); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22))); + + +DROP DICTIONARY IF EXISTS database_for_dict.dictionary_with_hierarchy; + +CREATE DICTIONARY database_for_dict.dictionary_with_hierarchy +( + id UInt64, parent_id UInt64 HIERARCHICAL, value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'dict_source')) +LAYOUT(CACHE(SIZE_IN_CELLS 10)) +LIFETIME(MIN 1 MAX 1); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(2), toUInt64(1)); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(11)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(11)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(22))); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(11))); + +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(111)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(111)); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(222))); +SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(111))); + +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(11)); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(22)); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(11))); +SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22))); + + +DROP DICTIONARY database_for_dict.dictionary_with_hierarchy; +DROP TABLE database_for_dict.dict_source; +DROP DATABASE database_for_dict; diff --git a/tests/queries/0_stateless/01251_string_comparison.reference b/tests/queries/0_stateless/01251_string_comparison.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01251_string_comparison.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01251_string_comparison.sql b/tests/queries/0_stateless/01251_string_comparison.sql new file mode 100644 index 00000000000..b2923ba4320 --- /dev/null +++ b/tests/queries/0_stateless/01251_string_comparison.sql @@ -0,0 +1 @@ +SELECT isConstant('a' = 'b'); diff --git a/tests/queries/0_stateless/01252_weird_time_zone.reference b/tests/queries/0_stateless/01252_weird_time_zone.reference new file mode 100644 index 00000000000..f2968d4efa6 --- /dev/null +++ b/tests/queries/0_stateless/01252_weird_time_zone.reference @@ -0,0 +1,7 @@ +2020-01-02 03:04:05 2020-01-02 00:00:00 3 +2020-01-02 03:04:05 2020-01-02 00:00:00 3 +2020-01-02 03:04:05 2020-01-02 00:00:00 3 +2020-01-02 03:04:05 2020-01-02 00:00:00 3 +2020-01-02 03:04:05 2020-01-02 00:00:00 3 +2020-01-02 03:04:05 2020-01-02 00:00:00 3 +2020-01-02 03:04:05 2020-01-02 00:00:00 3 diff --git a/tests/queries/0_stateless/01252_weird_time_zone.sql b/tests/queries/0_stateless/01252_weird_time_zone.sql new file mode 100644 index 00000000000..68ea903a797 --- /dev/null +++ b/tests/queries/0_stateless/01252_weird_time_zone.sql @@ -0,0 +1,15 @@ +SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kiritimati') AS x, toStartOfDay(x), toHour(x); +SELECT toDateTime('2020-01-02 03:04:05', 'Africa/El_Aaiun') AS x, toStartOfDay(x), toHour(x); +SELECT toDateTime('2020-01-02 03:04:05', 'Asia/Pyongyang') AS x, toStartOfDay(x), toHour(x); +SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kwajalein') AS x, toStartOfDay(x), toHour(x); +SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Apia') AS x, toStartOfDay(x), toHour(x); +SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Enderbury') AS x, toStartOfDay(x), toHour(x); +SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Fakaofo') AS x, toStartOfDay(x), toHour(x); + +SELECT toHour(toDateTime(rand(), 'Pacific/Kiritimati') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT toHour(toDateTime(rand(), 'Africa/El_Aaiun') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT toHour(toDateTime(rand(), 'Asia/Pyongyang') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT toHour(toDateTime(rand(), 'Pacific/Kwajalein') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT toHour(toDateTime(rand(), 'Pacific/Apia') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT toHour(toDateTime(rand(), 'Pacific/Enderbury') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT toHour(toDateTime(rand(), 'Pacific/Fakaofo') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; diff --git a/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.reference b/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.reference new file mode 100644 index 00000000000..72c3262a309 --- /dev/null +++ b/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.reference @@ -0,0 +1 @@ +Hello 1 diff --git a/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.sql b/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.sql new file mode 100644 index 00000000000..9659db7973d --- /dev/null +++ b/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS test_table; +DROP TABLE IF EXISTS test_table_sharded; + +create table + test_table_sharded( + date Date, + text String, + hash UInt64 + ) +engine=MergeTree(date, (hash, date), 8192); + +create table test_table as test_table_sharded +engine=Distributed(test_cluster_two_shards, currentDatabase(), test_table_sharded, hash); + +SET distributed_product_mode = 'local'; +SET insert_distributed_sync = 1; + +INSERT INTO test_table VALUES ('2020-04-20', 'Hello', 123); + +SELECT + text, + uniqExactIf(hash, hash IN ( + SELECT DISTINCT + hash + FROM test_table AS t1 + )) as counter +FROM test_table AS t2 +GROUP BY text +ORDER BY counter, text; + +DROP TABLE test_table; +DROP TABLE test_table_sharded; diff --git a/tests/queries/0_stateless/01254_array_of_unnamed_tuples.reference b/tests/queries/0_stateless/01254_array_of_unnamed_tuples.reference new file mode 100644 index 00000000000..0342c7b7310 --- /dev/null +++ b/tests/queries/0_stateless/01254_array_of_unnamed_tuples.reference @@ -0,0 +1,10 @@ +[(nan,1.1460459347849696e-169)] `xbguF 1493493065813843889 +[(-2.6774132404843463e217,-1.5941466176583548e-32)] ?/;UTko 8163231169061670909 +[(4.559039863342969e-218,1.1023812988249403e186)] k3Igp@ 9512519566588292358 +[(-6.156499824044965e254,2.125276757267567e176)] 12763761121429990320 +[] TlL 10781278062399783511 +[(-1.6511853645079817e-21,-1.5094564365588905e303)] UeS}D 1158449958889177529 +[(-5.109297304033652e229,-8.565674764550042e219),(-2.163260216301827e-75,-3.771562357185976e285)] 0Z3|h 12502841092151487477 +[] -] 2092132020040612180 +[(-2.9588901009588613e-146,-6.4241843242744556e268)] &b!M-e;7 10616749141511339887 +[(3.084905744030789e-98,-4.973771413288743e177),(-1.8198487259356486e114,1.2449864950522508e251)] Dj7peUH{T 5992776369613298329 diff --git a/tests/queries/0_stateless/01254_array_of_unnamed_tuples.sql b/tests/queries/0_stateless/01254_array_of_unnamed_tuples.sql new file mode 100644 index 00000000000..3660d66620c --- /dev/null +++ b/tests/queries/0_stateless/01254_array_of_unnamed_tuples.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS mass_table_457; +CREATE TABLE mass_table_457 (key Array(Tuple(Float64, Float64)), name String, value UInt64) ENGINE = Memory; +INSERT INTO mass_table_457 SELECT * FROM generateRandom('`key` Array(Tuple(Float64, Float64)),`name` String,`value` UInt64', 1, 10, 2) LIMIT 10; +SELECT * FROM mass_table_457; +DROP TABLE mass_table_457; diff --git a/tests/queries/0_stateless/01254_dict_create_without_db.reference b/tests/queries/0_stateless/01254_dict_create_without_db.reference new file mode 100644 index 00000000000..2f2d638a294 --- /dev/null +++ b/tests/queries/0_stateless/01254_dict_create_without_db.reference @@ -0,0 +1,4 @@ +0 NOT_LOADED +0 LOADED +10 +1 LOADED diff --git a/tests/queries/0_stateless/01254_dict_create_without_db.sql b/tests/queries/0_stateless/01254_dict_create_without_db.sql new file mode 100644 index 00000000000..e7c7702bb21 --- /dev/null +++ b/tests/queries/0_stateless/01254_dict_create_without_db.sql @@ -0,0 +1,23 @@ +DROP DATABASE IF EXISTS dict_db_01254; +CREATE DATABASE dict_db_01254; +USE dict_db_01254; + +CREATE TABLE dict_data (key UInt64, val UInt64) Engine=Memory(); +CREATE DICTIONARY dict +( + key UInt64 DEFAULT 0, + val UInt64 DEFAULT 10 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict_data' PASSWORD '' DB 'dict_db_01254')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT()); + +SELECT query_count, status FROM system.dictionaries WHERE database = 'dict_db_01254' AND name = 'dict'; +SYSTEM RELOAD DICTIONARY dict_db_01254.dict; +SELECT query_count, status FROM system.dictionaries WHERE database = 'dict_db_01254' AND name = 'dict'; +SELECT dictGetUInt64('dict_db_01254.dict', 'val', toUInt64(0)); +SELECT query_count, status FROM system.dictionaries WHERE database = 'dict_db_01254' AND name = 'dict'; + +USE system; +DROP DATABASE dict_db_01254; diff --git a/tests/queries/0_stateless/01254_dict_load_after_detach_attach.reference b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.reference new file mode 100644 index 00000000000..2f2d638a294 --- /dev/null +++ b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.reference @@ -0,0 +1,4 @@ +0 NOT_LOADED +0 LOADED +10 +1 LOADED diff --git a/tests/queries/0_stateless/01254_dict_load_after_detach_attach.sql b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.sql new file mode 100644 index 00000000000..5a5f694d28f --- /dev/null +++ b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.sql @@ -0,0 +1,24 @@ +DROP DATABASE IF EXISTS dict_db_01254; +CREATE DATABASE dict_db_01254; + +CREATE TABLE dict_db_01254.dict_data (key UInt64, val UInt64) Engine=Memory(); +CREATE DICTIONARY dict_db_01254.dict +( + key UInt64 DEFAULT 0, + val UInt64 DEFAULT 10 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict_data' PASSWORD '' DB 'dict_db_01254')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT()); + +DETACH DATABASE dict_db_01254; +ATTACH DATABASE dict_db_01254; + +SELECT query_count, status FROM system.dictionaries WHERE database = 'dict_db_01254' AND name = 'dict'; +SYSTEM RELOAD DICTIONARY dict_db_01254.dict; +SELECT query_count, status FROM system.dictionaries WHERE database = 'dict_db_01254' AND name = 'dict'; +SELECT dictGetUInt64('dict_db_01254.dict', 'val', toUInt64(0)); +SELECT query_count, status FROM system.dictionaries WHERE database = 'dict_db_01254' AND name = 'dict'; + +DROP DATABASE dict_db_01254; diff --git a/tests/queries/0_stateless/01255_geo_types_livace.reference b/tests/queries/0_stateless/01255_geo_types_livace.reference new file mode 100644 index 00000000000..3982d6fb18a --- /dev/null +++ b/tests/queries/0_stateless/01255_geo_types_livace.reference @@ -0,0 +1,2 @@ +[(123,456),(789,234)] [(567,890)] +[] [(11,22),(33,44),(55,66)] diff --git a/tests/queries/0_stateless/01255_geo_types_livace.sql b/tests/queries/0_stateless/01255_geo_types_livace.sql new file mode 100644 index 00000000000..0838f0fa219 --- /dev/null +++ b/tests/queries/0_stateless/01255_geo_types_livace.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS tutorial; +create table tutorial ( inner_poly Array(Tuple(Int32, Int32)), outer_poly Array(Tuple(Int32, Int32)) ) engine = Log(); + +SELECT * FROM tutorial; + +INSERT INTO tutorial VALUES ([(123, 456), (789, 234)], [(567, 890)]), ([], [(11, 22), (33, 44), (55, 66)]); +SELECT * FROM tutorial; + +DROP TABLE tutorial; diff --git a/tests/queries/0_stateless/01256_misspell_layout_name_podshumok.reference b/tests/queries/0_stateless/01256_misspell_layout_name_podshumok.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01256_misspell_layout_name_podshumok.sql b/tests/queries/0_stateless/01256_misspell_layout_name_podshumok.sql new file mode 100644 index 00000000000..a41402a12e4 --- /dev/null +++ b/tests/queries/0_stateless/01256_misspell_layout_name_podshumok.sql @@ -0,0 +1,9 @@ +CREATE DICTIONARY testip +( + `network` String, + `test_field` String +) +PRIMARY KEY network +SOURCE(FILE(PATH '/tmp/test.csv' FORMAT CSVWithNames)) +LIFETIME(MIN 0 MAX 300) +LAYOUT(IPTRIE()); -- { serverError 137 } diff --git a/tests/queries/0_stateless/01256_negative_generate_random.reference b/tests/queries/0_stateless/01256_negative_generate_random.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01256_negative_generate_random.sql b/tests/queries/0_stateless/01256_negative_generate_random.sql new file mode 100644 index 00000000000..11c169edd75 --- /dev/null +++ b/tests/queries/0_stateless/01256_negative_generate_random.sql @@ -0,0 +1,5 @@ +SELECT * FROM generateRandom('i8', 1, 10, 10); -- { serverError 62 } +SELECT * FROM generateRandom; -- { serverError 60 } +SELECT * FROM generateRandom(); -- { serverError 42 } +SELECT * FROM generateRandom('i8 UInt8', 1, 10, 10, 10, 10); -- { serverError 42 } +SELECT * FROM generateRandom('', 1, 10, 10); -- { serverError 62 } diff --git a/tests/queries/0_stateless/01257_dictionary_mismatch_types.reference b/tests/queries/0_stateless/01257_dictionary_mismatch_types.reference new file mode 100644 index 00000000000..358922133d7 --- /dev/null +++ b/tests/queries/0_stateless/01257_dictionary_mismatch_types.reference @@ -0,0 +1,31 @@ +0 300 0000-00-00 00:00:00 N1 +0 300 0000-00-00 00:00:00 N1 +0 300 0000-00-00 00:00:00 N1 +0 300 0000-00-00 00:00:00 N1 +0 300 0000-00-00 00:00:00 N1 +0 300 0000-00-00 00:00:00 N1 +0 200 0000-00-00 00:00:00 C2 +0 0 0000-00-00 00:00:00 C3 +0 0 0000-00-00 00:00:00 C3 +0 0 0000-00-00 00:00:00 N1 +0 0 0000-00-00 00:00:00 N1 +0 0 0000-00-00 00:00:00 N1 +0 0 0000-00-00 00:00:00 N1 +0 0 0000-00-00 00:00:00 N1 +0 210 2020-02-16 05:22:04 N1 +0 210 2020-02-16 05:22:04 N1 +0 270 2020-02-16 05:22:04 N1 +0 270 2020-02-16 05:22:04 N1 +0 380 0000-00-00 00:00:00 N1 +0 380 0000-00-00 00:00:00 N1 +0 280 0000-00-00 00:00:00 C2 +0 0 0000-00-00 00:00:00 N1 +0 190 2020-02-16 05:22:05 N1 +0 160 2020-02-14 05:22:13 N1 +0 230 2020-02-14 05:22:13 N1 +0 130 2020-02-14 05:22:14 N1 +0 300 0000-00-00 00:00:00 N1 +0 300 0000-00-00 00:00:00 N1 +0 0 0000-00-00 00:00:00 C2 +0 0 0000-00-00 00:00:00 N1 +0 0 0000-00-00 00:00:00 N1 diff --git a/tests/queries/0_stateless/01257_dictionary_mismatch_types.sql b/tests/queries/0_stateless/01257_dictionary_mismatch_types.sql new file mode 100644 index 00000000000..575dd4b09d0 --- /dev/null +++ b/tests/queries/0_stateless/01257_dictionary_mismatch_types.sql @@ -0,0 +1,110 @@ +DROP DATABASE IF EXISTS test_dict_db; +CREATE DATABASE test_dict_db; + +DROP TABLE IF EXISTS test_dict_db.table1; +DROP DICTIONARY IF EXISTS test_dict_db.table1_dict; + +CREATE TABLE test_dict_db.table1 +( + `col1` String, + `col2` Int16, + `col3` String, + `col4` Int32, + `col5` String, + `col6` Nullable(Float64), + `col7` Nullable(Float64), + `col8` Nullable(DateTime), + `col9` Nullable(String), + `col10` Nullable(String), + `col11` Nullable(String), + `col12` Nullable(String), + `col13` Nullable(Int32), + `col14` Nullable(DateTime), + `col15` Nullable(DateTime), + `col16` Nullable(DateTime), + `col17` Nullable(DateTime), + `col18` Nullable(DateTime), + `col19` Nullable(DateTime), + `col20` Nullable(String) +) +ENGINE = MergeTree +ORDER BY (col1, col2, col3, col4, col5); + +INSERT INTO test_dict_db.table1 VALUES ('id1',1,'20200127-1',701,'20200127-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',701,'20200127-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',702,'20200127-1-02',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',703,'20200127-1-03',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',704,'20200127-1-04',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',705,'20200127-1-05',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200202-1',711,'20200202-1-01',0,200,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 11:07:57',NULL,NULL,NULL,'2020-02-03 11:09:23',NULL,NULL),('id1',1,'20200202-2',712,'20200202-2-01',0,0,NULL,'C3','bye','R40',NULL,1,'2020-02-03 14:13:10',NULL,'2020-02-03 16:11:31','2020-02-07 05:32:05','2020-02-07 11:18:15','2020-02-07 11:18:16','123455'),('id1',1,'20200202-2',713,'20200202-2-02',0,0,NULL,'C3','bye','R40',NULL,1,'2020-02-03 14:13:10',NULL,'2020-02-03 16:11:31','2020-02-07 05:32:05','2020-02-07 11:18:15','2020-02-07 11:18:16','123455'),('id1',2,'20200128-1',701,'20200128-1-01',0,0,NULL,'N1','Hi','N40',NULL,2,'2020-02-03 17:07:27',NULL,'2020-02-05 13:33:55','2020-02-13 05:32:04',NULL,NULL,'A123755'),('id1',2,'20200131-1',701,'20200131-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 13:07:17',NULL,'2020-02-04 13:47:55','2020-02-12 05:32:04',NULL,NULL,'A123485'),('id1',2,'20200201-1',701,'20200201-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 21:07:37',NULL,'2020-02-05 13:40:51','2020-02-13 05:32:04',NULL,NULL,'A123455'),('id1',2,'20200202-1',711,'20200202-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 02:06:54',NULL,'2020-02-04 13:36:45','2020-02-12 05:32:04',NULL,NULL,'A123459'),('id1',2,'20200202-1',712,'20200202-1-02',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 02:06:54',NULL,'2020-02-04 13:36:45','2020-02-12 05:32:04',NULL,NULL,'A123429'),('id2',1,'20200131-1',401,'20200131-1-01',0,210,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',402,'20200131-1-02',0,210,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',403,'20200131-1-03',0,270,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',404,'20200131-1-04',0,270,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',405,'20200131-1-05',0,380,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-11 16:52:58','2020-02-15 05:32:04',NULL,NULL,'6892144935823'),('id2',1,'20200131-1',406,'20200131-1-06',0,380,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-11 16:52:58','2020-02-15 05:32:04',NULL,NULL,'6892144935823'),('id2',1,'20200131-1',407,'20200131-1-07',0,280,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 10:11:00',NULL,NULL,NULL,'2020-02-04 11:01:21',NULL,NULL),('id2',1,'20200131-1',408,'20200131-1-08',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:04',NULL,NULL,'454545'),('id2',1,'20200201-1',401,'20200201-1-01',0,190,'2020-02-16 05:22:05','N1','Hi','N40',NULL,1,'2020-02-03 12:06:17',NULL,'2020-02-05 17:30:30','2020-02-09 05:32:03',NULL,NULL,'90071'),('id2',1,'20200201-1',402,'20200201-1-01',0,160,'2020-02-14 05:22:13','N1','Hi','N40',NULL,1,'2020-02-03 06:21:05',NULL,'2020-02-03 17:42:35','2020-02-07 05:32:04',NULL,NULL,'96575'),('id2',1,'20200201-1',403,'20200201-1-02',0,230,'2020-02-14 05:22:13','N1','Hi','N40',NULL,1,'2020-02-03 06:21:05',NULL,'2020-02-03 17:42:35','2020-02-07 05:32:04',NULL,NULL,'96575'),('id2',1,'20200202-1',404,'20200202-1-01',0,130,'2020-02-14 05:22:14','N1','Hi','N40',NULL,1,'2020-02-03 14:00:39',NULL,'2020-02-03 17:42:45','2020-02-07 05:32:04',NULL,NULL,'96850'),('id3',1,'20200130-1',391,'20200130-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:26:46',NULL,'2020-02-05 15:33:01','2020-02-08 05:32:05',NULL,NULL,'27243'),('id3',1,'20200130-1',392,'20200130-1-02',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:26:46',NULL,'2020-02-10 16:16:11','2020-02-13 05:32:06',NULL,NULL,'92512'),('id3',1,'20200131-1',393,'20200131-1-01',0,0,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 10:24:38',NULL,NULL,NULL,'2020-02-05 14:04:40',NULL,NULL),('id3',1,'20200131-2',391,'20200131-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:22:08',NULL,'2020-02-06 14:27:06','2020-02-09 05:32:04',NULL,NULL,'46433'),('id3',1,'20200131-2',392,'20200131-1-02',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:22:08',NULL,'2020-02-06 14:27:06','2020-02-09 05:32:02',NULL,NULL,'46433'); + +CREATE DICTIONARY test_dict_db.table1_dict +( + col1 String, + col2 Int16, + col3 String, + col4 Int32, + col5 String, + col6 Float64, + col7 Float64, + col8 DateTime, + col9 String, + col10 String, + col11 String, + col12 String, + col13 Int32, + col14 DateTime, + col15 DateTime, + col16 DateTime, + col17 DateTime, + col18 DateTime, + col19 DateTime, + col20 String +) +PRIMARY KEY col1,col2,col3,col4,col5 +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 DB test_dict_db TABLE table1 USER 'default')) +LIFETIME(MIN 0 MAX 0) LAYOUT(COMPLEX_KEY_HASHED()); + +SELECT + dictGet('test_dict_db.table1_dict', 'col6', (col1, col2, col3, col4, col5)), + dictGet('test_dict_db.table1_dict', 'col7', (col1, col2, col3, col4, col5)), + dictGet('test_dict_db.table1_dict', 'col8', (col1, col2, col3, col4, col5)), + dictGet('test_dict_db.table1_dict', 'col9', (col1, col2, col3, col4, col5)) +FROM test_dict_db.table1 +WHERE dictHas('test_dict_db.table1_dict', (col1, col2, col3, col4, col5)); -- { serverError 349 } + +DROP TABLE test_dict_db.table1; +CREATE TABLE test_dict_db.table1 +( + `col1` String, + `col2` Int16, + `col3` String, + `col4` Int32, + `col5` String, + `col6` Float64, + `col7` Float64, + `col8` DateTime, + `col9` String, + `col10` String, + `col11` String, + `col12` String, + `col13` Int32, + `col14` DateTime, + `col15` DateTime, + `col16` DateTime, + `col17` DateTime, + `col18` DateTime, + `col19` DateTime, + `col20` String +) +ENGINE = MergeTree +ORDER BY (col1, col2, col3, col4, col5); + +SET input_format_null_as_default = 1; +INSERT INTO test_dict_db.table1 VALUES ('id1',1,'20200127-1',701,'20200127-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',701,'20200127-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',702,'20200127-1-02',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',703,'20200127-1-03',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',704,'20200127-1-04',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',705,'20200127-1-05',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200202-1',711,'20200202-1-01',0,200,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 11:07:57',NULL,NULL,NULL,'2020-02-03 11:09:23',NULL,NULL),('id1',1,'20200202-2',712,'20200202-2-01',0,0,NULL,'C3','bye','R40',NULL,1,'2020-02-03 14:13:10',NULL,'2020-02-03 16:11:31','2020-02-07 05:32:05','2020-02-07 11:18:15','2020-02-07 11:18:16','123455'),('id1',1,'20200202-2',713,'20200202-2-02',0,0,NULL,'C3','bye','R40',NULL,1,'2020-02-03 14:13:10',NULL,'2020-02-03 16:11:31','2020-02-07 05:32:05','2020-02-07 11:18:15','2020-02-07 11:18:16','123455'),('id1',2,'20200128-1',701,'20200128-1-01',0,0,NULL,'N1','Hi','N40',NULL,2,'2020-02-03 17:07:27',NULL,'2020-02-05 13:33:55','2020-02-13 05:32:04',NULL,NULL,'A123755'),('id1',2,'20200131-1',701,'20200131-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 13:07:17',NULL,'2020-02-04 13:47:55','2020-02-12 05:32:04',NULL,NULL,'A123485'),('id1',2,'20200201-1',701,'20200201-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 21:07:37',NULL,'2020-02-05 13:40:51','2020-02-13 05:32:04',NULL,NULL,'A123455'),('id1',2,'20200202-1',711,'20200202-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 02:06:54',NULL,'2020-02-04 13:36:45','2020-02-12 05:32:04',NULL,NULL,'A123459'),('id1',2,'20200202-1',712,'20200202-1-02',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 02:06:54',NULL,'2020-02-04 13:36:45','2020-02-12 05:32:04',NULL,NULL,'A123429'),('id2',1,'20200131-1',401,'20200131-1-01',0,210,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',402,'20200131-1-02',0,210,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',403,'20200131-1-03',0,270,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',404,'20200131-1-04',0,270,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',405,'20200131-1-05',0,380,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-11 16:52:58','2020-02-15 05:32:04',NULL,NULL,'6892144935823'),('id2',1,'20200131-1',406,'20200131-1-06',0,380,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-11 16:52:58','2020-02-15 05:32:04',NULL,NULL,'6892144935823'),('id2',1,'20200131-1',407,'20200131-1-07',0,280,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 10:11:00',NULL,NULL,NULL,'2020-02-04 11:01:21',NULL,NULL),('id2',1,'20200131-1',408,'20200131-1-08',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:04',NULL,NULL,'454545'),('id2',1,'20200201-1',401,'20200201-1-01',0,190,'2020-02-16 05:22:05','N1','Hi','N40',NULL,1,'2020-02-03 12:06:17',NULL,'2020-02-05 17:30:30','2020-02-09 05:32:03',NULL,NULL,'90071'),('id2',1,'20200201-1',402,'20200201-1-01',0,160,'2020-02-14 05:22:13','N1','Hi','N40',NULL,1,'2020-02-03 06:21:05',NULL,'2020-02-03 17:42:35','2020-02-07 05:32:04',NULL,NULL,'96575'),('id2',1,'20200201-1',403,'20200201-1-02',0,230,'2020-02-14 05:22:13','N1','Hi','N40',NULL,1,'2020-02-03 06:21:05',NULL,'2020-02-03 17:42:35','2020-02-07 05:32:04',NULL,NULL,'96575'),('id2',1,'20200202-1',404,'20200202-1-01',0,130,'2020-02-14 05:22:14','N1','Hi','N40',NULL,1,'2020-02-03 14:00:39',NULL,'2020-02-03 17:42:45','2020-02-07 05:32:04',NULL,NULL,'96850'),('id3',1,'20200130-1',391,'20200130-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:26:46',NULL,'2020-02-05 15:33:01','2020-02-08 05:32:05',NULL,NULL,'27243'),('id3',1,'20200130-1',392,'20200130-1-02',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:26:46',NULL,'2020-02-10 16:16:11','2020-02-13 05:32:06',NULL,NULL,'92512'),('id3',1,'20200131-1',393,'20200131-1-01',0,0,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 10:24:38',NULL,NULL,NULL,'2020-02-05 14:04:40',NULL,NULL),('id3',1,'20200131-2',391,'20200131-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:22:08',NULL,'2020-02-06 14:27:06','2020-02-09 05:32:04',NULL,NULL,'46433'),('id3',1,'20200131-2',392,'20200131-1-02',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:22:08',NULL,'2020-02-06 14:27:06','2020-02-09 05:32:02',NULL,NULL,'46433'); + +SYSTEM RELOAD DICTIONARIES; + +SELECT + dictGet('test_dict_db.table1_dict', 'col6', (col1, col2, col3, col4, col5)), + dictGet('test_dict_db.table1_dict', 'col7', (col1, col2, col3, col4, col5)), + dictGet('test_dict_db.table1_dict', 'col8', (col1, col2, col3, col4, col5)), + dictGet('test_dict_db.table1_dict', 'col9', (col1, col2, col3, col4, col5)) +FROM test_dict_db.table1 +WHERE dictHas('test_dict_db.table1_dict', (col1, col2, col3, col4, col5)); + +DROP DATABASE IF EXISTS test_dict_db; diff --git a/tests/queries/0_stateless/01258_bom_tsv.reference b/tests/queries/0_stateless/01258_bom_tsv.reference new file mode 100644 index 00000000000..6622044ee6e --- /dev/null +++ b/tests/queries/0_stateless/01258_bom_tsv.reference @@ -0,0 +1,2 @@ +1 2 3 +4 5 6 diff --git a/tests/queries/0_stateless/01258_bom_tsv.sh b/tests/queries/0_stateless/01258_bom_tsv.sh new file mode 100755 index 00000000000..5668b00f069 --- /dev/null +++ b/tests/queries/0_stateless/01258_bom_tsv.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +# BOM can be parsed if TSV format has first column that cannot contain arbitrary binary data (such as integer) +# In contrast, BOM cannot be parsed if the first column in String as it can contain arbitrary binary data. + +echo 'DROP TABLE IF EXISTS bom' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- +echo 'CREATE TABLE bom (a UInt8, b UInt8, c UInt8) ENGINE = Memory' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- +echo -ne '1\t2\t3\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+bom+FORMAT+TSV" --data-binary @- +echo -ne '\xEF\xBB\xBF4\t5\t6\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+bom+FORMAT+TSV" --data-binary @- +echo 'SELECT * FROM bom ORDER BY a' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- +echo 'DROP TABLE bom' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- diff --git a/tests/queries/0_stateless/01258_wrong_cast_filimonov.reference b/tests/queries/0_stateless/01258_wrong_cast_filimonov.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01258_wrong_cast_filimonov.sql b/tests/queries/0_stateless/01258_wrong_cast_filimonov.sql new file mode 100644 index 00000000000..5bcc159b384 --- /dev/null +++ b/tests/queries/0_stateless/01258_wrong_cast_filimonov.sql @@ -0,0 +1 @@ +create table x( id UInt64, t AggregateFunction(argMax, Enum8('' = -1, 'Male' = 1, 'Female' = 2), UInt64) DEFAULT arrayReduce('argMaxState', ['cast(-1, \'Enum8(\'\' = -1, \'Male\' = 1, \'Female\' = 2)'], [toUInt64(0)]) ) Engine=MergeTree ORDER BY id; -- { serverError 70 } diff --git a/tests/queries/0_stateless/data_avro/generate_avro.sh b/tests/queries/0_stateless/data_avro/generate_avro.sh index 6ec26efc049..b6ec75ad4dd 100755 --- a/tests/queries/0_stateless/data_avro/generate_avro.sh +++ b/tests/queries/0_stateless/data_avro/generate_avro.sh @@ -8,6 +8,7 @@ avro-tools fromjson --schema-file complex.avsc complex.json > complex.avro avro-tools fromjson --schema-file logical_types.avsc logical_types.json > logical_types.avro avro-tools fromjson --schema-file empty.avsc empty.json > empty.avro avro-tools fromjson --schema-file references.avsc references.json > references.avro +avro-tools fromjson --schema-file nested.avsc nested.json > nested.avro #compression avro-tools fromjson --codec null --schema-file simple.avsc simple.json > simple.null.avro diff --git a/tests/queries/0_stateless/data_avro/nested.avro b/tests/queries/0_stateless/data_avro/nested.avro new file mode 100644 index 00000000000..1415c45d328 Binary files /dev/null and b/tests/queries/0_stateless/data_avro/nested.avro differ diff --git a/tests/queries/0_stateless/data_avro/nested.avsc b/tests/queries/0_stateless/data_avro/nested.avsc new file mode 100644 index 00000000000..966dc6defb3 --- /dev/null +++ b/tests/queries/0_stateless/data_avro/nested.avsc @@ -0,0 +1,17 @@ +{ + "type": "record", + "name": "main", + "fields": [ + {"name": "a", "type": "long"}, + {"name": "b", "type": { + "type": "record", + "name": "sub1", + "fields": [ + {"name": "a", "type": "string"}, + {"name": "b", "type": "double"}, + {"name": "c", "type": "double"} + ] + }}, + {"name": "c", "type": "string"} + ] + } \ No newline at end of file diff --git a/tests/queries/0_stateless/data_avro/nested.json b/tests/queries/0_stateless/data_avro/nested.json new file mode 100644 index 00000000000..63a7bc40e4b --- /dev/null +++ b/tests/queries/0_stateless/data_avro/nested.json @@ -0,0 +1 @@ +{"a":1, "b": {"a":"b1", "b": 2.2, "c": 2.3}, "c": "c3"} diff --git a/tests/queries/1_stateful/00065_loyalty_with_storage_join.sql b/tests/queries/1_stateful/00065_loyalty_with_storage_join.sql index d3e73faa7be..15a2a75cf58 100644 --- a/tests/queries/1_stateful/00065_loyalty_with_storage_join.sql +++ b/tests/queries/1_stateful/00065_loyalty_with_storage_join.sql @@ -1,16 +1,14 @@ -SET any_join_distinct_right_table_keys = 1; - USE test; DROP TABLE IF EXISTS join; -CREATE TABLE join (UserID UInt64, loyalty Int8) ENGINE = Join(ANY, INNER, UserID); +CREATE TABLE join (UserID UInt64, loyalty Int8) ENGINE = Join(SEMI, LEFT, UserID); INSERT INTO join SELECT UserID, toInt8(if((sum(SearchEngineID = 2) AS yandex) > (sum(SearchEngineID = 3) AS google), - yandex / (yandex + google), - -google / (yandex + google)) * 10) AS loyalty + yandex / (yandex + google), + -google / (yandex + google)) * 10) AS loyalty FROM hits WHERE (SearchEngineID = 2) OR (SearchEngineID = 3) GROUP BY UserID @@ -19,17 +17,17 @@ HAVING (yandex + google) > 10; SELECT loyalty, count() -FROM hits ANY INNER JOIN join USING UserID +FROM hits SEMI LEFT JOIN join USING UserID GROUP BY loyalty ORDER BY loyalty ASC; DETACH TABLE join; -ATTACH TABLE join (UserID UInt64, loyalty Int8) ENGINE = Join(ANY, INNER, UserID); +ATTACH TABLE join (UserID UInt64, loyalty Int8) ENGINE = Join(SEMI, LEFT, UserID); SELECT loyalty, count() -FROM hits ANY INNER JOIN join USING UserID +FROM hits SEMI LEFT JOIN join USING UserID GROUP BY loyalty ORDER BY loyalty ASC; diff --git a/tests/queries/1_stateful/00093_prewhere_array_join.reference b/tests/queries/1_stateful/00093_prewhere_array_join.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/1_stateful/00093_prewhere_array_join.sql b/tests/queries/1_stateful/00093_prewhere_array_join.sql new file mode 100644 index 00000000000..a1263144bb1 --- /dev/null +++ b/tests/queries/1_stateful/00093_prewhere_array_join.sql @@ -0,0 +1,9 @@ +SELECT arrayJoin([SearchEngineID]) AS search_engine, URL FROM test.hits WHERE SearchEngineID != 0 AND search_engine != 0 FORMAT Null; + +SELECT + arrayJoin([0]) AS browser, + arrayJoin([SearchEngineID]) AS search_engine, + URL +FROM test.hits +WHERE 1 AND (SearchEngineID != 0) AND (browser != 0) AND (search_engine != 0) +FORMAT Null; diff --git a/tests/queries/1_stateful/00094_order_by_array_join_limit.reference b/tests/queries/1_stateful/00094_order_by_array_join_limit.reference new file mode 100644 index 00000000000..a8be94b5470 --- /dev/null +++ b/tests/queries/1_stateful/00094_order_by_array_join_limit.reference @@ -0,0 +1,8 @@ +[''] +[''] + + +[] +[] +[''] +[''] diff --git a/tests/queries/1_stateful/00094_order_by_array_join_limit.sql b/tests/queries/1_stateful/00094_order_by_array_join_limit.sql new file mode 100644 index 00000000000..1c2d9f0164e --- /dev/null +++ b/tests/queries/1_stateful/00094_order_by_array_join_limit.sql @@ -0,0 +1,9 @@ +SELECT `ParsedParams.Key2` AS x +FROM test.hits +ARRAY JOIN ParsedParams AS PP +ORDER BY x ASC +LIMIT 2; + +SELECT arrayJoin(`ParsedParams.Key2`) AS x FROM test.hits ORDER BY x ASC LIMIT 2; +WITH arrayJoin(`ParsedParams.Key2`) AS pp SELECT ParsedParams.Key2 AS x FROM test.hits ORDER BY x ASC LIMIT 2; +WITH arrayJoin(`ParsedParams.Key2`) AS pp SELECT ParsedParams.Key2 AS x FROM test.hits WHERE NOT ignore(pp) ORDER BY x ASC LIMIT 2; diff --git a/tests/queries/bugs/00938_client_suggestions.sh b/tests/queries/bugs/00938_client_suggestions.sh deleted file mode 100755 index b4bd9e4480d..00000000000 --- a/tests/queries/bugs/00938_client_suggestions.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -for i in {1..100}; do $CLICKHOUSE_CLIENT --always_load_suggestion_data --query="SELECT 1 FORMAT Null"; done diff --git a/tests/queries/bugs/01224_dist_on_dist_global_in.reference b/tests/queries/bugs/01224_dist_on_dist_global_in.reference deleted file mode 100644 index 7f75aa873cb..00000000000 --- a/tests/queries/bugs/01224_dist_on_dist_global_in.reference +++ /dev/null @@ -1,4 +0,0 @@ -GLOBAL IN distributed_group_by_no_merge -1 -GLOBAL IN -1 diff --git a/tests/queries/bugs/01224_dist_on_dist_global_in.sql b/tests/queries/bugs/01224_dist_on_dist_global_in.sql deleted file mode 100644 index e363fef2d2b..00000000000 --- a/tests/queries/bugs/01224_dist_on_dist_global_in.sql +++ /dev/null @@ -1,18 +0,0 @@ -create table if not exists data_01224 (key Int) Engine=Memory(); -create table if not exists dist_layer_01224 as data_01224 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01224); -create table if not exists dist_01224 as data_01224 Engine=Distributed(test_cluster_two_shards, currentDatabase(), dist_layer_01224); - -select * from dist_01224; -insert into data_01224 select * from numbers(3); - --- "Table expression is undefined, Method: ExpressionAnalyzer::interpretSubquery" -select 'GLOBAL IN distributed_group_by_no_merge'; -select distinct * from dist_01224 where key global in (1) settings distributed_group_by_no_merge=1; - --- requires #9923 -select 'GLOBAL IN'; -select distinct * from dist_01224 where key global in (1); - -drop table dist_01224; -drop table dist_layer_01224; -drop table data_01224; diff --git a/tests/server-test.xml b/tests/server-test.xml index c2356ec1ba0..7f792479065 100644 --- a/tests/server-test.xml +++ b/tests/server-test.xml @@ -31,7 +31,7 @@ true - + true true sslv2,sslv3 diff --git a/tests/integration/test_authentication/configs/users.d/access_management.xml b/tests/users.d/access_management.xml similarity index 100% rename from tests/integration/test_authentication/configs/users.d/access_management.xml rename to tests/users.d/access_management.xml diff --git a/utils/check-style/check-include b/utils/check-style/check-include index 9cdcbbaea7b..211172979bd 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -59,8 +59,8 @@ inc="-I. \ -I./contrib/lz4/lib \ -I./contrib/hyperscan/src \ -I./contrib/simdjson/include \ --I./dbms \ --I${BUILD_DIR}/dbms" +-I./src \ +-I${BUILD_DIR}/src" if [ -z $1 ]; then cd ${ROOT_DIR=${CUR_DIR}../..} diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 08b0e81c123..2a2e9dab42d 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -56,3 +56,6 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | while # Broken XML files (requires libxml2-utils) find $ROOT_PATH/{src,base,programs,utils} -name '*.xml' | xargs xmllint --noout --nonet + +# Machine translation to Russian is strictly prohibited +find $ROOT_PATH/docs/ru -name '*.md' | xargs grep -l -F 'machine_translated: true' diff --git a/utils/check-style/check-ungrouped-includes.sh b/utils/check-style/check-ungrouped-includes.sh index c394c8a04c3..f1f124fc13e 100755 --- a/utils/check-style/check-ungrouped-includes.sh +++ b/utils/check-style/check-ungrouped-includes.sh @@ -3,6 +3,6 @@ ROOT_PATH=$(git rev-parse --show-toplevel) # Find files with includes not grouped together by first component of path -find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | while read file; do +find $ROOT_PATH/src -name '*.h' -or -name '*.cpp' | while read file; do [[ $(grep -oP '^#include <\w+' $file | uniq -c | wc -l) > $(grep -oP '^#include <\w+' $file | sort | uniq -c | wc -l) ]] && echo $file && grep -oP '^#include <\w+' $file | uniq -c; done diff --git a/utils/clickhouse-docker b/utils/clickhouse-docker new file mode 100755 index 00000000000..6f2d1197c0a --- /dev/null +++ b/utils/clickhouse-docker @@ -0,0 +1,57 @@ +#!/bin/bash + +if [ $# -lt 1 ] +then +cat << HELP + +clickhouse-docker -- open clickhouse-client of desired version in docker container (automatically removed after you exit bash shell). + +EXAMPLE: + - start latest version: + clickhouse-docker latest + + - start version 20.1: + clickhouse-docker 20.1 + + - list avaliable versions: + clickhouse-docker list +HELP +exit +fi + +param="$1" + +if [ "${param}" = "list" ] +then + # https://stackoverflow.com/a/39454426/1555175 + wget -q https://registry.hub.docker.com/v1/repositories/yandex/clickhouse-server/tags -O - | sed -e 's/[][]//g' -e 's/"//g' -e 's/ //g' | tr '}' '\n' | awk -F: '{print $3}' +else + docker pull yandex/clickhouse-server:${param} + tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX) # older version require /nonexistent folder to exist to run clickhouse client :D + chmod 777 ${tmp_dir} + set -e + containerid=`docker run -v${tmp_dir}:/nonexistent -d yandex/clickhouse-server:${param}` + set +e + while : + do + # that trick with init-file allows to start clickhouse client inside bash shell (nice if you need exit to bash, check smth, and get back to clickhouse-client) + docker exec -it ${containerid} bash -c 'bash --init-file <(echo "clickhouse client -m")' + + printf "\n\nYou exited the session. What next?\n" + echo " [Q]uit and remove container." + echo " [R]estart clickhouse and run clickhouse-client in shell again." + echo "You can also hit Ctrl+C to exit and keep container running." + + while : + do + read -p "Quit or restart [Q/R]?" choice + case "$choice" in + q|Q|exit ) break 2;; + r|R|restart ) echo "Restarting container ..."; docker restart ${containerid} > /dev/null; break 1;; + * ) echo "I don't understand. Please type Q or R" ;; + esac + done + done + docker rm -f ${containerid} > /dev/null + rm -rf ${tmp_dir} +fi diff --git a/utils/github/__main__.py b/utils/github/__main__.py index e05d27c03d6..401908298eb 100644 --- a/utils/github/__main__.py +++ b/utils/github/__main__.py @@ -129,6 +129,7 @@ if bad_commits and not args.login: # TODO: check backports. if need_backporting: re_vlabel = re.compile(r'^v\d+\.\d+$') + re_vlabel_backported = re.compile(r'^v\d+\.\d+-backported$') re_vlabel_conflicts = re.compile(r'^v\d+\.\d+-conflicts$') print('\nPull-requests need to be backported:') @@ -146,8 +147,8 @@ if need_backporting: # FIXME: compatibility logic - check for a manually set label, that indicates status 'backported'. # FIXME: O(n²) - no need to iterate all labels for every `stable` for label in github.get_labels(pull_request): - if re_vlabel.match(label['name']): - if f'v{stable[0]}' == label['name']: + if re_vlabel.match(label['name']) or re_vlabel_backported.match(label['name']): + if f'v{stable[0]}' == label['name'] or f'v{stable[0]}-backported' == label['name']: backport_labeled.add(stable[0]) if re_vlabel_conflicts.match(label['name']): if f'v{stable[0]}-conflicts' == label['name']: diff --git a/utils/junit_to_html/junit-noframes.xsl b/utils/junit_to_html/junit-noframes.xsl new file mode 100644 index 00000000000..01f09ced557 --- /dev/null +++ b/utils/junit_to_html/junit-noframes.xsl @@ -0,0 +1,390 @@ + + + + + + + + Test Results + + + + + + + + +
+ + + + + + + + + + + + + + + + + +

+ + + + + + + + + + +
+

+ Back to top + + +

Summary

+ + + + + + + + + + + + + + + + + Failure + Error + + + + + + + + +
TestsFailuresErrorsSuccess rateTime
+ + + + + + + +
+ + + + +
+ Note: failures are anticipated and checked for with assertions while errors are unanticipated. +
+ + + + + +

Test Results

+
+ + + + Name + Tests + Errors + Failures + Time(s) + + + + + + Name + Tests + Errors + Failures + Time(s) + Time Stamp + Host + + + + + + Name + Status + Type + Time(s) + + + + + + + + + Failure + Error + + + + + + + + + + + + + + + + + + + + + Error + Failure + TableRowColor + + + + + + Failure + + + + Error + + + + Success + + + + + + + + + + + + +

+ + + + + + + + + +

+ + + + + + + + + + N/A + + + + + + +

+ at line + + + , column + + + + + + + + + + + + + + + 32 + + + + + + + + + + + + +
+ + + + + + +
+ + + + + + + + + + + + + + + + + + diff --git a/utils/junit_to_html/junit_to_html b/utils/junit_to_html/junit_to_html new file mode 100755 index 00000000000..cf50e7df00a --- /dev/null +++ b/utils/junit_to_html/junit_to_html @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import lxml.etree as etree +import json +import argparse + +def export_testcases_json(report, path): + with open(os.path.join(path, "cases.jer"), "w") as testcases_file: + for testsuite in report.getroot(): + for testcase in testsuite: + row = {} + row["hostname"] = testsuite.get("hostname") + row["suite"] = testsuite.get("name") + row["suite_duration"] = testsuite.get("time") + row["timestamp"] = testsuite.get("timestamp") + row["testname"] = testcase.get("name") + row["classname"] = testcase.get("classname") + row["file"] = testcase.get("file") + row["line"] = testcase.get("line") + row["duration"] = testcase.get("time") + for el in testcase: + if el.tag == "system-err": + row["stderr"] = el.text + else: + row["stderr"] = "" + + if el.tag == "system-out": + row["stdout"] = el.text + else: + row["stdout"] = "" + + json.dump(row, testcases_file) + testcases_file.write("\n") + +def export_testsuites_json(report, path): + with open(os.path.join(path, "suites.jer"), "w") as testsuites_file: + for testsuite in report.getroot(): + row = {} + row["suite"] = testsuite.get("name") + row["errors"] = testsuite.get("errors") + row["failures"] = testsuite.get("failures") + row["hostname"] = testsuite.get("hostname") + row["skipped"] = testsuite.get("skipped") + row["duration"] = testsuite.get("time") + row["timestamp"] = testsuite.get("timestamp") + json.dump(row, testsuites_file) + testsuites_file.write("\n") + + +def _convert_junit_to_html(junit_path, result_path, export_cases, export_suites): + with open(os.path.join(os.path.dirname(__file__), "junit-noframes.xsl")) as xslt_file: + junit_to_html_xslt = etree.parse(xslt_file) + if not os.path.exists(result_path): + os.makedirs(result_path) + + with open(junit_path) as junit_file: + junit_xml = etree.parse(junit_file) + + if export_suites: + export_testsuites_json(junit_xml, result_path) + if export_cases: + export_testcases_json(junit_xml, result_path) + transform = etree.XSLT(junit_to_html_xslt) + html = etree.tostring(transform(junit_xml), encoding="utf-8") + + with open(os.path.join(result_path, "result.html"), "w") as html_file: + html_file.write(html) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Convert JUnit XML.') + parser.add_argument('junit', help='path to junit.xml report') + parser.add_argument('result_dir', nargs='?', help='directory for result files. Default to junit.xml directory') + parser.add_argument('--export-cases', help='Export JSONEachRow result for testcases to upload in CI', action='store_true') + parser.add_argument('--export-suites', help='Export JSONEachRow result for testsuites to upload in CI', action='store_true') + + args = parser.parse_args() + + junit_path = args.junit + if args.result_dir: + result_path = args.result_dir + else: + result_path = os.path.dirname(junit_path) + print "junit_path: {}, result_path: {}, export cases:{}, export suites: {}".format(junit_path, result_path, args.export_cases, args.export_suites) + _convert_junit_to_html(junit_path, result_path, args.export_cases, args.export_suites) diff --git a/utils/release/push_packages b/utils/release/push_packages index 4a0548bb4ca..68d72bb39fe 100755 --- a/utils/release/push_packages +++ b/utils/release/push_packages @@ -152,7 +152,7 @@ def transfer_packages_dupload(ssh_key, path, repo_user, repo_url, incoming_direc } with DebRelease(config, repo_user, ssh_key): logging.info("Duploading") - subprocess.check_call("dupload --nomail --to {repo} {path}".format(repo=repo_short_name, path=path), shell=True) + subprocess.check_call("dupload -f --nomail --to {repo} {path}".format(repo=repo_short_name, path=path), shell=True) logging.info("Dupload finished") diff --git a/utils/simple-backport/.gitignore b/utils/simple-backport/.gitignore new file mode 100644 index 00000000000..72e8ffc0db8 --- /dev/null +++ b/utils/simple-backport/.gitignore @@ -0,0 +1 @@ +* diff --git a/utils/simple-backport/README.md b/utils/simple-backport/README.md new file mode 100644 index 00000000000..c5a625ca0d1 --- /dev/null +++ b/utils/simple-backport/README.md @@ -0,0 +1,107 @@ +# Упрощённый скрипт для бекпортирования + +Это упрощённый скрипт для бекпортирования. Он определяет, какие пулреквесты ещё не бекпортировали из мастера в указанную ветку. Запускать скрипт нужно из папки, где он лежит, указав ему название ветки. Он предполагает, что ваш апстримный remote называется origin. +``` +cd my-clickhouse-repo/utils/simple-backport +git fetch origin +time GITHUB_TOKEN= ./backport.sh 20.1 +``` + +Скрипт выведет примитивный отчёт: +``` +$ time GITHUB_TOKEN= ~/backport.sh 20.3 +144 PRs differ between 20.3 and master. +backport https://github.com/ClickHouse/ClickHouse/pull/10135 +backport https://github.com/ClickHouse/ClickHouse/pull/10121 +... +backport https://github.com/ClickHouse/ClickHouse/pull/9808 +backport https://github.com/ClickHouse/ClickHouse/pull/9410 + +real 0m1.213s +user 0m1.065s +sys 0m0.311s +``` + +Также в рабочей папке сгенерируется отчёт `<ваша-ветка>-report.tsv`: + +``` +$ cat 20.3-report.tsv +skip 10153 https://github.com/ClickHouse/ClickHouse/pull/10153 pr10153.json +skip 10147 https://github.com/ClickHouse/ClickHouse/pull/10147 pr10147.json +no-backport 10138 https://github.com/ClickHouse/ClickHouse/pull/10138 pr10138.json +backport 10135 https://github.com/ClickHouse/ClickHouse/pull/10135 pr10135.json +skip 10134 https://github.com/ClickHouse/ClickHouse/pull/10134 pr10134.json +... +``` + +Можно кликать по ссылкам прям из консоли, а можно ещё проще: + +``` +$ cat <ветка>-report.tsv | grep ^backport | cut -f3 +$ cat <ветка>-report.tsv | grep ^backport | cut -f3 | xargs -n1 xdg-open +``` + +Такая команда откроет в браузере все пулреквесты, которые надо бекпортировать. Есть и другие статусы, посмотрите какие: + +``` +$ cat 20.1-report.tsv | cut -f1 | sort | uniq -c | sort -rn + 446 skip + 38 done + 25 conflict + 18 backport + 10 no-backport +``` + +### Как разметить пулреквест? +По умолчанию бекпортируются все пулреквесты, у которых в описании указана +категория чейнжлога Bug fix. Если этого недостаточно, используйте теги: +* v20.1-no-backport -- в ветку 20.1 бекпортировать не нужно. +* pr-no-backport -- ни в какие ветки бекпортировать не нужно. +* v20.1-conflicts -- при бекпорте в 20.1 произошёл конфликт. Такие пулреквесты + скрипт пропускает, к ним можно потом вернуться. +* pr-must-backport -- нужно бекпортировать в поддерживаемые ветки. +* v20.1-must-backport -- нужно бекпортировать в 20.1. + +### Я бекпортировал, почему скрипт не видит? +* Сообщение коммита должно содержать текст backport/cherry-pick #12345, или + иметь вид стандартного гитхабовского мерж-коммита для ПР #12345. +* Коммит должен быть достижим по `git log --first-parent my-branch`. Возможно, + в ветке сделали pull с merge, от чего некоторые коммиты из ветки становятся +недоступны по `--first-parent`. + +В качестве обхода, добавьте в ветку пустой коммит с текстом вроде "backport +#12345 -- real backport commit is ". + +### Я поправил пулреквест, почему скрипт не видит? +В процессе работы скрипт кеширует данные о пулреквестах в текущей папке, чтобы +экономить квоту гитхаба. Удалите закешированные файлы, например, для всех +реквестов, которые не помечены как пропущенные: +``` +$ cat <ваша-ветка>-report.tsv | grep -v "^skip" | cut -f4 +$ cat <ваша-ветка>-report.tsv | grep -v "^skip" | cut -f4 | xargs rm +``` + +## Как сформировать change log +В этой же папке запустите: +``` +$ time GITHUB_TOKEN=... ./changelog.sh v20.3.4.10-stable v20.3.5.21-stable +9 PRs added between v20.3.4.10-stable and v20.3.5.21-stable. +### ClickHouse release v20.3.5.21-stable FIXME as compared to v20.3.4.10-stable + +#### Bug Fix + +* Fix 'Different expressions with the same alias' error when query has PREWHERE + and WHERE on distributed table and `SET distributed_product_mode = 'local'`. +[#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem +Zuikov](https://github.com/4ertus2)). +... +``` + +Скрипт выведет changelog на экран, а также сохранит его в `./changelog.md`. +Скопируйте этот текст в большой changelog, проверьте и поправьте версию и дату +релиза, вычитайте сообщения. Если сообщения неправильные, обязательно исправьте +их на гитхабе -- это поможет при последующей генерации changelog для других +версий, содержащих этот пулреквест. Чтобы скрипт подтянул изменения с гитхаба, +удалите соответствующие файлы `./pr12345.json`. Если вы часто видите +неправильно оформленные пулреквесты, это повод подумать об улучшении проверки +Description check в CI. diff --git a/utils/simple-backport/backport.sh b/utils/simple-backport/backport.sh new file mode 100755 index 00000000000..7d5b12d6f7f --- /dev/null +++ b/utils/simple-backport/backport.sh @@ -0,0 +1,108 @@ +#!/bin/bash +set -e + +branch="$1" +merge_base=$(git merge-base origin/master "origin/$branch") +master_git_cmd=(git log "$merge_base..origin/master" --first-parent) +branch_git_cmd=(git log "$merge_base..origin/$branch" --first-parent) + +# Make lists of PRs that were merged into each branch. Use first parent here, or else +# we'll get weird things like seeing older master that was merged into a PR branch +# that was then merged into master. +"${master_git_cmd[@]}" > master-log.txt +"${branch_git_cmd[@]}" > "$branch-log.txt" + +# Check for diamond merges. +"${master_git_cmd[@]}" --oneline --grep "Merge branch '" | grep '' +diamonds_in_master=$? + +"${branch_git_cmd[@]}" --oneline --grep "Merge branch '" | grep '' +diamonds_in_branch=$? + +if [ "$diamonds_in_master" -eq 0 ] || [ "$diamonds_in_branch" -eq 0 ] +then + # DO NOT ADD automated handling of diamond merges to this script. + # It is an unsustainable way to work with git, and it MUST be visible. + echo Warning: suspected diamond merges above. + echo Some commits will be missed, review these manually. +fi + +# NOTE keep in sync with ./changelog.sh. +# Search for PR numbers in commit messages. First variant is normal merge, and second +# variant is squashed. Next are some backport message variants. +find_prs=(sed -n "s/^.*Merge pull request #\([[:digit:]]\+\).*$/\1/p; + s/^.*(#\([[:digit:]]\+\))$/\1/p; + s/^.*back[- ]*port[ed of]*#\([[:digit:]]\+\).*$/\1/Ip; + s/^.*cherry[- ]*pick[ed of]*#\([[:digit:]]\+\).*$/\1/Ip") + +"${find_prs[@]}" master-log.txt | sort -rn > master-prs.txt +"${find_prs[@]}" "$branch-log.txt" | sort -rn > "$branch-prs.txt" + +# Find all master PRs that are not in branch by calculating differences of two PR lists. +grep -f "$branch-prs.txt" -F -x -v master-prs.txt > "$branch-diff-prs.txt" + +echo "$(wc -l < "$branch-diff-prs".txt) PRs differ between $branch and master." + +function github_download() +{ + local url=${1} + local file=${2} + if ! [ -f "$file" ] + then + if ! curl -H "Authorization: token $GITHUB_TOKEN" \ + -sSf "$url" \ + > "$file" + then + >&2 echo "Failed to download '$url' to '$file'. Contents: '$(cat "$file")'." + rm "$file" + return 1 + fi + sleep 0.1 + fi +} + +rm "$branch-report.tsv" &> /dev/null ||: +for pr in $(cat "$branch-diff-prs.txt") +do + # Download PR info from github. + file="pr$pr.json" + github_download "https://api.github.com/repos/ClickHouse/ClickHouse/pulls/$pr" "$file" || continue + + if ! [ "$pr" == "$(jq -r .number "$file")" ] + then + >&2 echo "Got wrong data for PR #$pr (please check and remove '$file')." + continue + fi + + action="skip" + + # First, check the changelog category. We port all bugfixes. + if jq -r .body "$file" | grep -i "^- bug[ -]*fix" > /dev/null + then + action="backport" + fi + + # Next, check the tag. They might override the decision. Checks are ordered by priority. + labels="$(jq -r .labels[].name "$file")" + if echo "$labels" | grep -x "pr-must-backport\|v$branch-must-backport" > /dev/null; then action="backport"; fi + if echo "$labels" | grep -x "v$branch-conflicts" > /dev/null; then action="conflict"; fi + if echo "$labels" | grep -x "pr-no-backport\|v$branch-no-backport" > /dev/null; then action="no-backport"; fi + # FIXME Ignore "backported" labels for now. If we can't find the backport commit, + # this means that the changelog script also won't be able to. An alternative + # way to mark PR as backported is to add an empty commit with text like + # "backported #12345", so that it can be found between tags and put in proper + # place in changelog. + #if echo "$labels" | grep -x "v$branch\|v$branch-backported" > /dev/null; then action="done"; fi + + # Find merge commit SHA for convenience + merge_sha="$(jq -r .merge_commit_sha "$file")" + + url="https://github.com/ClickHouse/ClickHouse/pull/$pr" + printf "%s\t%s\t%s\t%s\t%s\n" "$action" "$pr" "$url" "$file" "$merge_sha" >> "$branch-report.tsv" + if [ "$action" == "backport" ] + then + printf "%s\t%s\t%s\n" "$action" "$url" "$merge_sha" + fi +done + +echo "Done." diff --git a/utils/simple-backport/changelog.sh b/utils/simple-backport/changelog.sh new file mode 100755 index 00000000000..75a54a50b92 --- /dev/null +++ b/utils/simple-backport/changelog.sh @@ -0,0 +1,86 @@ +#!/bin/bash +set -e + +from="$1" +to="$2" +log_command=(git log "$from..$to" --first-parent) + +"${log_command[@]}" > "changelog-log.txt" + +# Check for diamond merges. +if "${log_command[@]}" --oneline --grep "Merge branch '" | grep '' +then + # DO NOT ADD automated handling of diamond merges to this script. + # It is an unsustainable way to work with git, and it MUST be visible. + echo Warning: suspected diamond merges above. + echo Some commits will be missed, review these manually. +fi + +# NOTE keep in sync with ./backport.sh. +# Search for PR numbers in commit messages. First variant is normal merge, and second +# variant is squashed. Next are some backport message variants. +find_prs=(sed -n "s/^.*Merge pull request #\([[:digit:]]\+\).*$/\1/p; + s/^.*(#\([[:digit:]]\+\))$/\1/p; + s/^.*back[- ]*port[ed of]*#\([[:digit:]]\+\).*$/\1/Ip; + s/^.*cherry[- ]*pick[ed of]*#\([[:digit:]]\+\).*$/\1/Ip") + +"${find_prs[@]}" "changelog-log.txt" | sort -rn | uniq > "changelog-prs.txt" + +echo "$(wc -l < "changelog-prs.txt") PRs added between $from and $to." + +function github_download() +{ + local url=${1} + local file=${2} + if ! [ -f "$file" ] + then + if ! curl -H "Authorization: token $GITHUB_TOKEN" \ + -sSf "$url" \ + > "$file" + then + >&2 echo "Failed to download '$url' to '$file'. Contents: '$(cat "$file")'." + rm "$file" + return 1 + fi + sleep 0.1 + fi +} + +rm changelog-prs-filtered.txt &> /dev/null ||: +for pr in $(cat "changelog-prs.txt") +do + # Download PR info from github. + file="pr$pr.json" + github_download "https://api.github.com/repos/ClickHouse/ClickHouse/pulls/$pr" "$file" || continue + + if ! [ "$pr" == "$(jq -r .number "$file")" ] + then + >&2 echo "Got wrong data for PR #$pr (please check and remove '$file')." + continue + fi + + # Filter out PRs by bots. + user_login=$(jq -r .user.login "$file") + if echo "$user_login" | grep "\[bot\]$" > /dev/null + then + continue + fi + + # Download author info from github. + user_id=$(jq -r .user.id "$file") + user_file="user$user_id.json" + github_download "$(jq -r .user.url "$file")" "$user_file" || continue + + if ! [ "$user_id" == "$(jq -r .id "$user_file")" ] + then + >&2 echo "Got wrong data for user #$user_id (please check and remove '$user_file')." + continue + fi + + echo "$pr" >> changelog-prs-filtered.txt +done + +echo "### ClickHouse release $to FIXME as compared to $from +" > changelog.md +./format-changelog.py changelog-prs-filtered.txt >> changelog.md +cat changelog.md diff --git a/utils/simple-backport/format-changelog.py b/utils/simple-backport/format-changelog.py new file mode 100755 index 00000000000..356ed48b6fd --- /dev/null +++ b/utils/simple-backport/format-changelog.py @@ -0,0 +1,114 @@ +#!/usr/bin/python3 + +import os +import sys +import itertools +import argparse +import json +import collections +import re + +parser = argparse.ArgumentParser(description='Format changelog for given PRs.') +parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs='?', default=sys.stdin, help='File with PR numbers, one per line.') +args = parser.parse_args() + +# This function mirrors the PR description checks in ClickhousePullRequestTrigger. +# Returns False if the PR should not be mentioned changelog. +def parse_one_pull_request(item): + description = item['body'] + # Don't skip empty lines because they delimit parts of description + lines = [line for line in map(lambda x: x.strip(), description.split('\n') if description else [])] + lines = [re.sub(r'\s+', ' ', l) for l in lines] + + category = '' + entry = '' + + if lines: + i = 0 + while i < len(lines): + if re.match(r'(?i).*category.*:$', lines[i]): + i += 1 + if i >= len(lines): + break + # Can have one empty line between header and the category itself. Filter it out. + if not lines[i]: + i += 1 + if i >= len(lines): + break + category = re.sub(r'^[-*\s]*', '', lines[i]) + i += 1 + elif re.match(r'(?i)^\**\s*(Short description|Change\s*log entry)', lines[i]): + i += 1 + # Can have one empty line between header and the entry itself. Filter it out. + if i < len(lines) and not lines[i]: + i += 1 + # All following lines until empty one are the changelog entry. + entry_lines = [] + while i < len(lines) and lines[i]: + entry_lines.append(lines[i]) + i += 1 + entry = ' '.join(entry_lines) + else: + i += 1 + + if not category: + # Shouldn't happen, because description check in CI should catch such PRs. + # Fall through, so that it shows up in output and the user can fix it. + category = "NO CL CATEGORY" + + # Filter out the PR categories that are not for changelog. + if re.match(r'(?i)doc|((non|in|not|un)[-\s]*significant)', category): + return False + + if not entry: + # Shouldn't happen, because description check in CI should catch such PRs. + category = "NO CL ENTRY" + entry = "NO CL ENTRY: '" + item['title'] + "'" + + entry = entry.strip() + if entry[-1] != '.': + entry += '.' + + item['entry'] = entry + item['category'] = category + + return True + + +category_to_pr = collections.defaultdict(lambda: []) +users = {} +for line in args.file: + pr = json.loads(open(f'pr{line.strip()}.json').read()) + assert(pr['number']) + if not parse_one_pull_request(pr): + continue + + assert(pr['category']) + category_to_pr[pr['category']].append(pr) + user_id = pr['user']['id'] + users[user_id] = json.loads(open(f'user{user_id}.json').read()) + +def print_category(category): + print("#### " + category) + print() + for pr in category_to_pr[category]: + user = users[pr["user"]["id"]] + user_name = user["name"] if user["name"] else user["login"] + + # Substitute issue links + pr["entry"] = re.sub(r'([^[])#([0-9]{4,})', r'\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)', pr["entry"]) + + print(f'* {pr["entry"]} [#{pr["number"]}]({pr["html_url"]}) ([{user_name}]({user["html_url"]})).') + + print() + +# Print categories in preferred order +categories_preferred_order = ['Backward Incompatible Change', 'New Feature', 'Bug Fix', 'Improvement', 'Performance Improvement', 'Build/Testing/Packaging Improvement', 'Other'] +for category in categories_preferred_order: + if category in category_to_pr: + print_category(category) + category_to_pr.pop(category) + +# Print the rest of the categories +for category in category_to_pr: + print_category(category) diff --git a/utils/upload_test_results/README.md b/utils/upload_test_results/README.md new file mode 100644 index 00000000000..e6b361081a2 --- /dev/null +++ b/utils/upload_test_results/README.md @@ -0,0 +1,34 @@ +## Tool to upload results to CI ClickHouse + +Currently allows to upload results from `junit_to_html` tool to ClickHouse CI + +``` +usage: upload_test_results [-h] --sha SHA --pr PR --file FILE --type + {suites,cases} [--user USER] --password PASSWORD + [--ca-cert CA_CERT] [--host HOST] [--db DB] + +Upload test result to CI ClickHouse. + +optional arguments: + -h, --help show this help message and exit + --sha SHA sha of current commit + --pr PR pr of current commit. 0 for master + --file FILE file to upload + --type {suites,cases} + Export type + --user USER user name + --password PASSWORD password + --ca-cert CA_CERT CA certificate path + --host HOST CI ClickHouse host + --db DB CI ClickHouse database name +``` + +$ ./upload_test_results --sha "cf7eaee3301d4634acdacbfa308ddbe0cc6a061d" --pr "0" --file xyz/cases.jer --type cases --password $PASSWD + +CI checks has single commit sha and pr identifier. +While uploading your local results for testing purposes try to use correct sha and pr. + +CA Certificate for ClickHouse CI can be obtained from Yandex.Cloud where CI database is hosted +``` bash +wget "https://storage.yandexcloud.net/cloud-certs/CA.pem" -O YandexInternalRootCA.crt +``` \ No newline at end of file diff --git a/utils/upload_test_results/upload_test_results b/utils/upload_test_results/upload_test_results new file mode 100755 index 00000000000..058a73d8081 --- /dev/null +++ b/utils/upload_test_results/upload_test_results @@ -0,0 +1,127 @@ +#!/usr/bin/env python +import requests +import argparse + +# CREATE TABLE test_suites +# ( +# sha String, +# pr UInt16, +# suite String, +# errors UInt16, +# failures UInt16, +# hostname String, +# skipped UInt16, +# duration Double, +# timestamp DateTime +# ) ENGINE = MergeTree ORDER BY tuple(timestamp, suite); + +QUERY_SUITES="INSERT INTO test_suites "\ + "SELECT '{sha}' AS sha, "\ + "{pr} AS pr, "\ + "suite, "\ + "errors, "\ + "failures, "\ + "hostname, "\ + "skipped, "\ + "duration, "\ + "timestamp "\ + "FROM input('"\ + "suite String, "\ + "errors UInt16, "\ + "failures UInt16, "\ + "hostname String, "\ + "skipped UInt16, "\ + "duration Double, "\ + "timestamp DateTime"\ + "') FORMAT JSONEachRow" + +# CREATE TABLE test_cases +# ( +# sha String, +# pr UInt16, +# hostname String, +# suite String, +# timestamp DateTime, +# testname String, +# classname String, +# file String, +# line UInt16, +# duration Double, +# suite_duration Double, +# stderr String, +# stdout String +# ) ENGINE = MergeTree ORDER BY tuple(timestamp, testname); + +QUERY_CASES="INSERT INTO test_cases "\ + "SELECT '{sha}' AS sha, "\ + "{pr} AS pr, "\ + "hostname, "\ + "suite, "\ + "timestamp, "\ + "testname, "\ + "classname, "\ + "file, "\ + "line, "\ + "duration, "\ + "suite_duration, "\ + "stderr,"\ + "stdout "\ + "FROM input('"\ + "hostname String, "\ + "suite String, "\ + "timestamp DateTime, "\ + "testname String, "\ + "classname String, "\ + "file String, "\ + "line UInt16, "\ + "duration Double, "\ + "suite_duration Double, "\ + "stderr String, "\ + "stdout String"\ + "') FORMAT JSONEachRow" + + +def upload_request(sha, pr, file, q_type, user, password, ca_cert, host, db): + with open(file) as upload_f: + query = QUERY_SUITES if q_type=="suites" else QUERY_CASES + query = query.format(sha=sha, pr=pr) + url = 'https://{host}:8443/?database={db}&query={query}&date_time_input_format=best_effort'.format( + host=host, + db=db, + query=query + ) + data=upload_f + auth = { + 'X-ClickHouse-User': user, + 'X-ClickHouse-Key': password, + } + + print query; + + res = requests.post( + url, + data=data, + headers=auth, + verify=ca_cert) + res.raise_for_status() + return res.text + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Upload test result to CI ClickHouse.') + parser.add_argument('--sha', help='sha of current commit', type=str, required=True) + parser.add_argument('--pr', help='pr of current commit. 0 for master', type=int, required=True) + parser.add_argument('--file', help='file to upload', required=True) + parser.add_argument('--type', help='Export type', choices=['suites', 'cases'] , required=True) + parser.add_argument('--user', help='user name', type=str, default="clickhouse-ci") + parser.add_argument('--password', help='password', type=str, required=True) + parser.add_argument('--ca-cert', help='CA certificate path', type=str, default="/usr/local/share/ca-certificates/YandexInternalRootCA.crt") + parser.add_argument('--host', help='CI ClickHouse host', type=str, default="c1a-ity5agjmuhyu6nu9.mdb.yandexcloud.net") + parser.add_argument('--db', help='CI ClickHouse database name', type=str, default="clickhouse-ci") + + args = parser.parse_args() + + print(upload_request(args.sha, args.pr, args.file, args.type, args.user, args.password, args.ca_cert, args.host, args.db)) + + + diff --git a/website/benchmark.html b/website/benchmark.html deleted file mode 100644 index 35ffe340711..00000000000 --- a/website/benchmark.html +++ /dev/null @@ -1,2186 +0,0 @@ - - - - - Performance comparison of analytical DBMS - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - -

Performance comparison of analytical DBMS

-
- -
- -

Most results are for single server setup. The server is: two socket Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz; 128 GiB RAM; md RAID-5 on 8 6TB SATA HDD; ext4.

- -

See as well hardware benchmark results page.

- -

Some additional results (marked as x2, x3, x6) are for clustered setup for comparison. These results are contributed from independent teams and hardware specification may differ.

- -

Disclaimer: some results are significantly outdated. Results for MonetDB was obtained at 2013 for version v11.15.11 (Feb2013-SP3). Results for MemSQL was obtained at 2015 for version 3.2. Results for Vertica are from 2015 for version 7.1.1.

- -
- -
- -
- -

Relative query processing time (lower is better):

-
- -
-

Full results:

- -
- -
- - - - - diff --git a/website/benchmark/benchmark.js b/website/benchmark/benchmark.js new file mode 100644 index 00000000000..ce60ca66941 --- /dev/null +++ b/website/benchmark/benchmark.js @@ -0,0 +1,411 @@ +var data_sizes = + [ + {id: "10000000", name: "10 mln."}, + {id: "100000000", name: "100 mln."}, + {id: "1000000000", name: "1 bn."} + ]; + + +var systems = []; +var systems_uniq = {}; +for (r in results) { + if (systems_uniq[results[r].system]) + continue; + systems_uniq[results[r].system] = 1; + systems.push(results[r].system); +} + +var runs = ["first (cold cache)", "second", "third"]; +var current_runs = ['0', '1']; + +try { + var state = JSON.parse(decodeURIComponent(window.location.hash.substring(1))); + current_data_size = state[0]; + current_systems = state[1]; + current_runs = state[2]; +} catch (e) { +} + +function update_hash() { + window.location.hash = JSON.stringify([current_data_size, current_systems, current_runs]); +} + + +function generate_selectors(elem) { + var html = ''; + if (current_data_size) { + html += ''; + html += '
Compare'; + + var available_results = results; + + if (current_data_size) { + available_results = results.filter(function (run) { + return run.data_size == current_data_size; + }); + } + var available_systems_for_current_data_size = available_results.map(function (run) { + return run.system; + }); + + for (var i = 0; i < systems.length; i++) { + var selected = current_systems.indexOf(systems[i]) != -1; + var available = available_systems_for_current_data_size.indexOf(systems[i]) != -1; + + html += ''; + } + + html += '
Dataset size'; + + for (var i = 0; i < data_sizes.length; i++) { + html += ''; + } + } + + html += '
Run'; + + for (var i = 0; i < runs.length; i++) { + html += ''; + } + + html += '
'; + + elem.html(html); + + $('#systems_selector button:not(.disabled)').click(function (event) { + var target = $(event.target || event.srcElement); + + if (target.hasClass("active") && current_systems.length == 1) { + return; + } + + target.toggleClass("active"); + + current_systems = $.map($('#systems_selector button'), function (elem) { + return $(elem).hasClass("active") ? $(elem).html() : null + }).filter(function (x) { + return x; + }); + + update_hash(); + generate_selectors(elem); + generate_comparison_table(); + generate_diagram(); + }); + + if (current_data_size) { + $('#data_size_selector button').click(function (event) { + var target = $(event.target || event.srcElement); + + current_data_size = target.attr("data-size-id"); + + update_hash(); + generate_selectors(elem); + generate_comparison_table(); + generate_diagram(); + }); + } + + $('#runs_selector button').click(function (event) { + var target = $(event.target || event.srcElement); + + if (target.hasClass("active") && current_runs.length == 1) { + return; + } + + target.toggleClass("active"); + + current_runs = $.map($('#runs_selector button'), function (elem) { + return $(elem).hasClass("active") ? $(elem).attr("data-run-id") : null + }).filter(function (x) { + return x; + }); + + update_hash(); + generate_selectors(elem); + generate_comparison_table(); + generate_diagram(); + }); +} + +function format_number_cell(value, ratio) { + var html = ""; + + var redness = (ratio - 1) / ratio; + var blackness = ratio < 10 ? 0 : ((ratio - 10) / ratio / 2); + + var color = !value ? "#FFF" : + ratio == 1 ? + ("rgba(0, 255, 0, 1)") : + ("rgba(" + ~~(255 * (1 - blackness)) + ", 0, 0, " + redness + ")"); + + html += ""; + html += value ? + (ratio == 1 ? "" : ("×" + ratio.toFixed(2))) + " (" + value.toFixed(3) + " s.)" : + "—"; + html += ""; + + return html; +} + +/* Ratio of execution time to best execution time: + * system index -> run index -> query index -> ratio. + */ +var ratios = []; + + +function generate_comparison_table() { + ratios = []; + + var filtered_results = results; + if (current_data_size) { + filtered_results = filtered_results.filter(function (x) { + return x.data_size == current_data_size; + }); + } + filtered_results = filtered_results.filter(function (x) { + return current_systems.indexOf(x.system) != -1; + }); + + var html = ""; + + html += ""; + html += ""; + html += ""; + html += ""; + for (var j = 0; j < filtered_results.length; j++) { + html += ""; + } + html += ""; + + for (var i = 0; i < queries.length; i++) { + html += ""; + html += ""; + + html += ""; + + // Max and min execution time per system, for each of three runs + var minimums = [0, 0, 0], maximums = [0, 0, 0]; + + for (var j = 0; j < filtered_results.length; j++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + var k = current_runs[current_run_idx]; + var value = filtered_results[j].result[i][k]; + + if (value && (!minimums[k] || value < minimums[k])) { + minimums[k] = value; + + // Ignore below 10ms + if (minimums[k] < 0.01) { + minimums[k] = 0.01; + } + } + + if (value > maximums[k]) { + maximums[k] = value; + } + } + } + + for (var j = 0; j < filtered_results.length; j++) { + if (!ratios[j]) { + ratios[j] = []; + } + + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + var k = current_runs[current_run_idx]; + var value = filtered_results[j].result[i][k]; + + var ratio = value / minimums[k]; + + ratios[j][k] = ratios[j][k] || []; + + if (ratio && ratio <= 1) { + ratio = 1; + } + + ratios[j][k].push(ratio); + + html += format_number_cell(value, ratio); + } + } + html += ""; + } + + if (current_systems.length) { + html += ""; + html += ""; + html += ""; + + for (var j = 0; j < filtered_results.length; j++) { + for (var k = 0; k < current_runs.length; k++) { + html += ""; + } + } + + html += ""; + html += ""; + + for (var j = 0; j < filtered_results.length; j++) { + html += ""; + } + + html += ""; + } + + html += "
Query" + filtered_results[j].system + + (filtered_results[j].version ? " (" + filtered_results[j].version + ")" : "") + "
" + queries[i].query + "
Geometric mean of ratios
"; + + $('#comparison_table').html(html); + + for (var i = 0; i < queries.length; i++) { + $('#query_checkbox' + i).click(function () { + calculate_totals(); + generate_diagram(); + }); + } + $('#query_checkbox_toggler').click(function () { + for (var i = 0; i < queries.length; i++) { + var item = $('#query_checkbox' + i); + item.prop("checked", !item.prop("checked")); + } + }); + + calculate_totals(); +} + + +function calculate_totals() { + if (!current_systems.length) return; + var filtered_results = results; + if (current_data_size) { + filtered_results = filtered_results.filter(function (x) { + return x.data_size == current_data_size; + }); + } + + filtered_results = filtered_results.filter(function (x) { + return current_systems.indexOf(x.system) != -1; + }); + + var total_ratios = []; + + for (var j = 0; j < filtered_results.length; j++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + var k = current_runs[current_run_idx]; + + var current_ratios = ratios[j][k].filter( + function (x, i) { + return x && $("#query_checkbox" + i).is(':checked'); + } + ); + + var ratio = Math.pow( + current_ratios.reduce( + function (acc, cur) { + return acc * cur; + }, + 1), + 1 / current_ratios.length); + + total_ratios[j] = total_ratios[j] || 1; + total_ratios[j] *= ratio; + + $("#totals" + j + "_" + k).attr("data-ratio", ratio).html("x" + ratio.toFixed(2)); + } + } + + for (var j = 0; j < filtered_results.length; j++) { + var total_ratio = Math.pow(total_ratios[j], 1 / current_runs.length); + $("#absolute_totals" + j).attr("data-ratio", total_ratio).html("x" + total_ratio.toFixed(2)); + } +} + + +function generate_diagram() { + var html = ""; + var filtered_results = results; + if (current_data_size) { + filtered_results = filtered_results.filter(function (x) { + return x.data_size == current_data_size && current_systems.indexOf(x.system) != -1; + }); + } + filtered_results = filtered_results.filter(function (x) { + return current_systems.indexOf(x.system) != -1; + }); + + var max_ratio = 1; + var min_ratio = 0; + + var max_total_ratio = 1; + var min_total_ratio = 0; + + for (var j = 0; j < filtered_results.length; j++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + var k = current_runs[current_run_idx]; + var ratio = +$("#totals" + j + "_" + k).attr("data-ratio"); + + if (ratio > max_ratio) { + max_ratio = ratio; + } + + if (!min_ratio || ratio < min_ratio) { + min_ratio = ratio; + } + } + + var total_ratio = +$("#absolute_totals" + j).attr("data-ratio"); + + if (total_ratio > max_total_ratio) { + max_total_ratio = total_ratio; + } + + if (!min_total_ratio || total_ratio < min_total_ratio) { + min_total_ratio = total_ratio; + } + } + + html += ""; + + for (var j = 0; j < filtered_results.length; j++) { + var total_ratio = +$("#absolute_totals" + j).attr("data-ratio"); + + html += ""; + html += ""; + + html += ""; + + html += ""; + html += ""; + } + + html += "
" + filtered_results[j].system + "" + + (filtered_results[j].version ? "
(" + filtered_results[j].version.replace(/ /g, ' ') + ")" : "") + "		"; + + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + var k = current_runs[current_run_idx]; + + var ratio = +$("#totals" + j + "_" + k).attr("data-ratio"); + var percents = (ratio * 100 / max_ratio).toFixed(2); + + if (!ratio) { + ratio = +$("#absolute_totals" + j).attr("data-ratio"); + percents = (ratio * 100 / max_total_ratio).toFixed(2); + } + + html += '
 
'; + + } + + html += "		" + (total_ratio / min_total_ratio).toFixed(2) + "
"; + + $('#diagram').html(html); +} + +generate_selectors($('#selectors')); +generate_comparison_table(); +generate_diagram(); diff --git a/website/benchmark/dbms/index.html b/website/benchmark/dbms/index.html new file mode 100644 index 00000000000..16bf92cd052 --- /dev/null +++ b/website/benchmark/dbms/index.html @@ -0,0 +1,60 @@ +{% extends 'templates/base.html' %} + +{% set title = 'Performance comparison of database management systems' %} +{% set extra_js = [ + 'queries.js?' + rev_short, + 'results.js?' + rev_short, + '../benchmark.js?' + rev_short] +%} +{% set url = 'https://clickhouse.tech/benchmark/dbms/' %} +{% set no_footer = True %} + +{% block content %} +
+ +
+
+ + ClickHouse + +

Performance comparison of analytical DBMS

+
+
+ +
+
+
+ +
+
+

Relative query processing time (lower is better)

+
+
+
+ +
+
+

Full results

+
+
+
+ +
+
+

Comments

+

Most results are for single server setup with the following configuration: two socket Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz; 128 GiB RAM; md RAID-5 on 8 6TB SATA HDD; ext4.

+ +

Some additional results (marked as x2, x3, x6) are for clustered setup for comparison. These results are contributed from independent teams and hardware specification may differ.

+ +

+ Disclaimer! Some results are significantly outdated: +

    +
  • Results for MonetDB were obtained in 2013 for version v11.15.11 (Feb2013-SP3)
    • +
  • Results for MemSQL were obtained in 2015 for version 3.2.
    • +
  • Results for Vertica were obtained in 2015 for version 7.1.1.
    • +
+

+

See also: performance comparison of ClickHouse on various hardware.

+
+
+{% endblock %} diff --git a/website/benchmark/dbms/queries.js b/website/benchmark/dbms/queries.js new file mode 100644 index 00000000000..c92353ab0f2 --- /dev/null +++ b/website/benchmark/dbms/queries.js @@ -0,0 +1,179 @@ +var current_data_size = 1000000000; + +var current_systems = ["ClickHouse", "Vertica", "Greenplum"]; + +var queries = + [ + { + "query": "SELECT count() FROM hits", + "comment": "", + }, + { + "query": "SELECT count() FROM hits WHERE AdvEngineID != 0", + "comment": "", + }, + { + "query": "SELECT sum(AdvEngineID), count(), avg(ResolutionWidth) FROM hits", + "comment": "", + }, + { + "query": "SELECT sum(UserID) FROM hits", + "comment": "", + }, + { + "query": "SELECT uniq(UserID) FROM hits", + "comment": "", + }, + { + "query": "SELECT uniq(SearchPhrase) FROM hits", + "comment": "", + }, + { + "query": "SELECT min(EventDate), max(EventDate) FROM hits", + "comment": "", + }, + { + "query": "SELECT AdvEngineID, count() FROM hits WHERE AdvEngineID != 0 GROUP BY AdvEngineID ORDER BY count() DESC", + "comment": "", + }, + { + "query": "SELECT RegionID, uniq(UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT RegionID, sum(AdvEngineID), count() AS c, avg(ResolutionWidth), uniq(UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT MobilePhoneModel, uniq(UserID) AS u FROM hits WHERE MobilePhoneModel != '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT MobilePhone, MobilePhoneModel, uniq(UserID) AS u FROM hits WHERE MobilePhoneModel != '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, count() AS c FROM hits WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, uniq(UserID) AS u FROM hits WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchEngineID, SearchPhrase, count() AS c FROM hits WHERE SearchPhrase != '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, count() FROM hits GROUP BY UserID ORDER BY count() DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, SearchPhrase, count() FROM hits GROUP BY UserID, SearchPhrase ORDER BY count() DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, SearchPhrase, count() FROM hits GROUP BY UserID, SearchPhrase LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, toMinute(EventTime) AS m, SearchPhrase, count() FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY count() DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID FROM hits WHERE UserID = 12345678901234567890", + "comment": "", + }, + { + "query": "SELECT count() FROM hits WHERE URL LIKE '%metrika%'", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, any(URL), count() AS c FROM hits WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, any(URL), any(Title), count() AS c, uniq(UserID) FROM hits WHERE Title LIKE '%Яндекс%' AND URL NOT LIKE '%.yandex.%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT * FROM hits WHERE URL LIKE '%metrika%' ORDER BY EventTime LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase FROM hits WHERE SearchPhrase != '' ORDER BY EventTime LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase FROM hits WHERE SearchPhrase != '' ORDER BY SearchPhrase LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase FROM hits WHERE SearchPhrase != '' ORDER BY EventTime, SearchPhrase LIMIT 10", + "comment": "", + }, + { + "query": "SELECT CounterID, avg(length(URL)) AS l, count() AS c FROM hits WHERE URL != '' GROUP BY CounterID HAVING c > 100000 ORDER BY l DESC LIMIT 25", + "comment": "", + }, + { + "query": "SELECT domainWithoutWWW(Referer) AS key, avg(length(Referer)) AS l, count() AS c, any(Referer) FROM hits WHERE Referer != '' GROUP BY key HAVING c > 100000 ORDER BY l DESC LIMIT 25", + "comment": "", + }, + { + "query": "SELECT sum(ResolutionWidth), sum(ResolutionWidth + 1), sum(ResolutionWidth + 2), sum(ResolutionWidth + 3), sum(ResolutionWidth + 4), sum(ResolutionWidth + 5), sum(ResolutionWidth + 6), sum(ResolutionWidth + 7), sum(ResolutionWidth + 8), sum(ResolutionWidth + 9), sum(ResolutionWidth + 10), sum(ResolutionWidth + 11), sum(ResolutionWidth + 12), sum(ResolutionWidth + 13), sum(ResolutionWidth + 14), sum(ResolutionWidth + 15), sum(ResolutionWidth + 16), sum(ResolutionWidth + 17), sum(ResolutionWidth + 18), sum(ResolutionWidth + 19), sum(ResolutionWidth + 20), sum(ResolutionWidth + 21), sum(ResolutionWidth + 22), sum(ResolutionWidth + 23), sum(ResolutionWidth + 24), sum(ResolutionWidth + 25), sum(ResolutionWidth + 26), sum(ResolutionWidth + 27), sum(ResolutionWidth + 28), sum(ResolutionWidth + 29), sum(ResolutionWidth + 30), sum(ResolutionWidth + 31), sum(ResolutionWidth + 32), sum(ResolutionWidth + 33), sum(ResolutionWidth + 34), sum(ResolutionWidth + 35), sum(ResolutionWidth + 36), sum(ResolutionWidth + 37), sum(ResolutionWidth + 38), sum(ResolutionWidth + 39), sum(ResolutionWidth + 40), sum(ResolutionWidth + 41), sum(ResolutionWidth + 42), sum(ResolutionWidth + 43), sum(ResolutionWidth + 44), sum(ResolutionWidth + 45), sum(ResolutionWidth + 46), sum(ResolutionWidth + 47), sum(ResolutionWidth + 48), sum(ResolutionWidth + 49), sum(ResolutionWidth + 50), sum(ResolutionWidth + 51), sum(ResolutionWidth + 52), sum(ResolutionWidth + 53), sum(ResolutionWidth + 54), sum(ResolutionWidth + 55), sum(ResolutionWidth + 56), sum(ResolutionWidth + 57), sum(ResolutionWidth + 58), sum(ResolutionWidth + 59), sum(ResolutionWidth + 60), sum(ResolutionWidth + 61), sum(ResolutionWidth + 62), sum(ResolutionWidth + 63), sum(ResolutionWidth + 64), sum(ResolutionWidth + 65), sum(ResolutionWidth + 66), sum(ResolutionWidth + 67), sum(ResolutionWidth + 68), sum(ResolutionWidth + 69), sum(ResolutionWidth + 70), sum(ResolutionWidth + 71), sum(ResolutionWidth + 72), sum(ResolutionWidth + 73), sum(ResolutionWidth + 74), sum(ResolutionWidth + 75), sum(ResolutionWidth + 76), sum(ResolutionWidth + 77), sum(ResolutionWidth + 78), sum(ResolutionWidth + 79), sum(ResolutionWidth + 80), sum(ResolutionWidth + 81), sum(ResolutionWidth + 82), sum(ResolutionWidth + 83), sum(ResolutionWidth + 84), sum(ResolutionWidth + 85), sum(ResolutionWidth + 86), sum(ResolutionWidth + 87), sum(ResolutionWidth + 88), sum(ResolutionWidth + 89) FROM hits", + "comment": "", + }, + { + "query": "SELECT SearchEngineID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT URL, count() AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT 1, URL, count() AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM hits GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT URL, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT Title, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT URL, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000", + "comment": "", + }, + { + "query": "SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000", + "comment": "", + }, + { + "query": "SELECT URLHash, EventDate, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://yandex.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100", + "comment": "", + }, + { + "query": "SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://yandex.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000;", + "comment": "", + }, + { + "query": "SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute;", + "comment": "", + } + ] diff --git a/website/benchmark/dbms/results/001_clickhouse_19_1_6.json b/website/benchmark/dbms/results/001_clickhouse_19_1_6.json new file mode 100644 index 00000000000..7b3f68d0b16 --- /dev/null +++ b/website/benchmark/dbms/results/001_clickhouse_19_1_6.json @@ -0,0 +1,163 @@ +[ + { + "system": "ClickHouse", + "version": "19.1.6", + "data_size": 10000000, + "time": "2016-06-01 03:00:00", + "comments": "FORMAT Null", + "result": + [ + [0.092, 0.012, 0.006], + [0.087, 0.006, 0.006], + [0.098, 0.010, 0.011], + [0.134, 0.016, 0.010], + [0.176, 0.037, 0.040], + [0.233, 0.057, 0.058], + [0.066, 0.006, 0.007], + [0.071, 0.006, 0.014], + [0.259, 0.131, 0.128], + [0.266, 0.094, 0.078], + [0.217, 0.033, 0.032], + [0.211, 0.040, 0.034], + [0.249, 0.102, 0.102], + [0.343, 0.122, 0.107], + [0.302, 0.116, 0.122], + [0.209, 0.090, 0.083], + [0.415, 0.222, 0.221], + [0.287, 0.112, 0.115], + [0.562, 0.323, 0.351], + [0.127, 0.009, 0.009], + [0.410, 0.068, 0.073], + [0.494, 0.108, 0.083], + [0.911, 0.197, 0.193], + [2.426, 0.303, 0.286], + [0.311, 0.041, 0.046], + [0.203, 0.037, 0.040], + [0.262, 0.042, 0.052], + [0.365, 0.067, 0.066], + [0.735, 0.158, 0.177], + [0.549, 0.344, 0.521], + [0.357, 0.098, 0.103], + [0.509, 0.136, 0.130], + [0.775, 0.531, 0.532], + [0.641, 0.342, 0.320], + [0.600, 0.351, 0.352], + [0.295, 0.122, 0.134], + [0.291, 0.118, 0.122], + [0.203, 0.053, 0.057], + [0.234, 0.041, 0.037], + [0.502, 0.223, 0.234], + [0.275, 0.016, 0.018], + [0.222, 0.014, 0.017], + [0.182, 0.016, 0.014] + ] + }, + + { + "system": "ClickHouse", + "version": "19.1.6", + "data_size": 100000000, + "time": "2016-06-01 03:00:00", + "comments": "FORMAT Null", + "result": + [ + [0.133, 0.016, 0.016], + [0.086, 0.012, 0.037], + [0.254, 0.041, 0.037], + [0.478, 0.047, 0.045], + [0.655, 0.104, 0.105], + [0.581, 0.228, 0.297], + [0.105, 0.048, 0.022], + [0.076, 0.012, 0.015], + [0.892, 0.509, 0.487], + [1.077, 0.542, 0.589], + [0.529, 0.171, 0.167], + [0.541, 0.188, 0.182], + [0.975, 0.659, 0.603], + [1.479, 0.812, 0.770], + [1.082, 0.734, 0.778], + [0.952, 0.728, 0.729], + [2.212, 1.756, 1.952], + [1.230, 1.033, 0.907], + [4.613, 4.030, 3.955], + [0.477, 0.074, 0.037], + [1.648, 0.532, 0.555], + [2.002, 0.645, 0.651], + [3.676, 1.479, 1.487], + [7.792, 0.759, 0.788], + [1.139, 0.239, 0.215], + [0.522, 0.220, 0.215], + [1.018, 0.230, 0.238], + [1.768, 0.602, 0.603], + [1.818, 0.899, 0.916], + [3.301, 3.174, 3.283], + [1.313, 0.639, 0.631], + [2.136, 0.938, 0.975], + [5.894, 5.412, 5.467], + [3.359, 3.053, 3.061], + [3.355, 2.999, 3.004], + [1.224, 1.103, 1.063], + [0.185, 0.086, 0.092], + [0.138, 0.038, 0.038], + [0.155, 0.031, 0.031], + [0.390, 0.183, 0.162], + [0.203, 0.013, 0.013], + [0.178, 0.011, 0.011], + [0.128, 0.007, 0.007] + ] + }, + + { + "system": "ClickHouse", + "version": "19.1.6", + "data_size": 1000000000, + "time": "2016-06-01 03:00:00", + "comments": "FORMAT Null", + "result": + [ + [0.747, 0.090, 0.075], + [0.197, 0.074, 0.061], + [1.370, 0.419, 0.334], + [3.911, 0.390, 0.365], + [4.096, 0.636, 0.638], + [4.739, 1.626, 1.542], + [0.270, 0.172, 0.188], + [0.184, 0.067, 0.061], + [6.182, 3.016, 2.913], + [7.068, 3.329, 3.409], + [4.332, 1.128, 1.296], + [4.698, 1.351, 1.328], + [7.721, 5.721, 5.802], + [11.683, 7.687, 7.629], + [9.144, 6.987, 6.974], + [8.069, 6.115, 6.386], + [22.657, 20.622, 20.842], + [11.727, 10.574, 10.708], + [52.193, 51.836, 53.738], + [4.179, 0.378, 0.279], + [24.400, 4.712, 4.591], + [29.045, 5.533, 5.502], + [56.733, 13.958, 13.791], + [86.314, 8.349, 7.448], + [11.360, 1.762, 1.781], + [4.890, 1.568, 1.508], + [11.667, 1.962, 1.889], + [24.491, 5.554, 5.556], + [28.096, 8.528, 8.481], + [24.084, 25.500, 26.527], + [10.820, 5.387, 5.406], + [25.187, 8.879, 8.852], + [70.218, 67.707, 68.059], + [42.018, 36.039, 36.391], + [43.128, 35.813, 36.154], + [9.646, 8.490, 8.418], + [0.639, 0.250, 0.289], + [0.350, 0.064, 0.066], + [0.443, 0.106, 0.100], + [0.923, 0.461, 0.460], + [0.479, 0.030, 0.029], + [0.372, 0.025, 0.023], + [0.224, 0.012, 0.013] + ] + } +] diff --git a/website/benchmark/dbms/results/002_vertica_7_1_1.json b/website/benchmark/dbms/results/002_vertica_7_1_1.json new file mode 100644 index 00000000000..2b0c340747b --- /dev/null +++ b/website/benchmark/dbms/results/002_vertica_7_1_1.json @@ -0,0 +1,163 @@ +[ + { + "system": "Vertica", + "version": "7.1.1", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [0.015899, 0.012807, 0.012184], + [0.035133, 0.021397, 0.018946], + [0.060938, 0.034013, 0.037085], + [0.051492, 0.021746, 0.022661], + [0.150695, 0.144041, 0.143313], + [0.412724, 0.276505, 0.27558], + [0.039736, 0.021215, 0.025029], + [0.049819, 0.045784, 0.049303], + [0.248834, 0.222963, 0.217386], + [0.43366, 0.437333, 0.4883], + [0.106483, 0.094236, 0.092362], + [0.122424, 0.10004, 0.100646], + [0.952346, 0.963712, 0.994094], + [0.64299, 0.647605, 0.644699], + [0.606994, 0.552117, 0.563657], + [0.221643, 0.205149, 0.216158], + [0.789877, 0.848421, 0.869198], + [0.439601, 0.438257, 0.424207], + [1.51968, 1.351311, 1.495538], + [0.038791, 0.03504, 0.050796], + [0.847444, 0.412624, 0.413898], + [0.491446, 0.474577, 0.484147], + [1.693912, 1.166251, 1.236441], + [1.905181, 1.257361, 1.437238], + [0.36171, 0.263338, 0.28958], + [0.46795, 0.515716, 0.443451], + [0.28467, 0.248823, 0.251787], + [2.326452, 2.152684, 2.073438], + [2.953462, 2.324174, 2.845123], + [0.631466, 0.822281, 0.873889], + [0.446697, 0.320824, 0.304922], + [0.489312, 0.417575, 0.440902], + [2.178985, 2.094694, 2.164159], + [1.375479, 1.331646, 1.360907], + [1.347677, 1.354772, 1.344533], + [0.429637, 0.43564, 0.436279], + [3.297413, 3.177341, 3.194704], + [0.835327, 0.743157, 0.746247], + [0.248143, 0.20795, 0.218004], + [1.495476, 1.322633, 1.374602], + [0.187092, 0.12099, 0.127517], + [0.148605, 0.109589, 0.107272], + [0.148173, 0.12809, 0.133435] + ] + }, + + { + "system": "Vertica", + "version": "7.1.1", + "data_size": 100000000, + "time": "", + "comments": "", + "result": + [ + [0.044914, 0.033471, 0.029564], + [0.124784, 0.044458, 0.04813], + [0.253575, 0.167392, 0.166981], + [0.267045, 0.060595, 0.059291], + [0.901021, 0.881088, 1.032132], + [1.284296, 0.991411, 1.011576], + [0.149862, 0.068061, 0.067217], + [0.148711, 0.083089, 0.078208], + [1.847624, 1.836724, 1.892968], + [4.278373, 4.313527, 4.564084], + [0.358281, 0.266623, 0.288294], + [0.42072, 0.38109, 0.371086], + [5.294134, 4.280055, 4.179055], + [8.849077, 8.744801, 8.750143], + [4.469753, 4.203493, 4.319043], + [1.542069, 1.506466, 1.605814], + [8.230461, 8.129543, 8.521089], + [5.063301, 5.177715, 4.989504], + [21.097494, 21.113905, 20.863796], + [0.238975, 0.163524, 0.162261], + [3.681673, 3.664944, 3.738555], + [3.996372, 3.875857, 3.897112], + [5.086255, 5.00034, 5.019747], + [5.105649, 4.51027, 4.780023], + [6.028593, 6.027804, 5.998026], + [4.324348, 4.357931, 4.451637], + [6.011405, 6.01204, 6.006612], + [13.744667, 14.174568, 14.053413], + [9.533647, 9.626582, 9.551671], + [2.652615, 2.689042, 2.617271], + [1.881931, 1.808578, 1.80198], + [3.926782, 3.813766, 3.806481], + [19.214651, 19.52602, 19.544008], + [27.55101, 27.641466, 28.128856], + [27.687275, 27.525594, 27.338331], + [4.989802, 5.243158, 4.861738], + [2.471475, 2.239634, 2.270449], + [0.814507, 0.733751, 0.760994], + [0.280513, 0.230994, 0.231817], + [1.479762, 1.293106, 1.277735], + [0.163974, 0.12314, 0.127756], + [0.148318, 0.112932, 0.107095], + [0.069868, 0.071664, 0.065618] + ] + }, + + { + "system": "Vertica", + "version": "7.1.1", + "data_size": 1000000000, + "time": "", + "comments": "", + "result": + [ + [0.09516, 0.094485, 0.092024], + [0.816135, 0.446083, 0.453634], + [2.11051, 1.773246, 1.791243], + [2.120462, 1.033154, 1.020443], + [8.149946, 7.740829, 7.659704], + [26.119083, 25.611287, 25.675649], + [1.035186, 0.818121, 0.822225], + [0.816869, 0.506568, 0.51574], + [13.108386, 13.20647, 13.556902], + [40.935852, 39.508237, 39.034314], + [3.183196, 2.862235, 2.86959], + [3.967054, 3.658079, 3.557328], + [30.73868, 30.722098, 31.301074], + [76.889072, 76.027064, 77.056729], + [32.033544, 31.866097, 32.772241], + [11.350141, 11.177469, 11.064054], + [74.059265, 73.061888, 73.041769], + [52.895306, 52.588376, 52.671085], + [186.891072, 184.998125, 185.913578], + [2.208113, 1.871682, 1.87381], + [21.705589, 21.595201, 21.415026], + [21.080841, 20.856408, 20.696303], + [45.916174, 45.351723, 45.433121], + [47.051243, 35.723046, 35.694351], + [53.209629, 53.406901, 53.069656], + [52.737858, 52.784361, 52.617806], + [53.430247, 53.206678, 53.309617], + [372.194119, 371.545597, 370.507236], + [76.594315, 76.464039, 76.319749], + [24.274602, 24.263616, 24.198579], + [14.400851, 13.927733, 13.747829], + [30.679117, 28.09498, 27.203538], + [210.606242, 214.108745, 214.521569], + [289.044749, 291.983512, 289.419234], + [289.46641, 290.132895, 289.4922], + [43.7288, 43.325352, 43.19419], + [5.028182, 4.798986, 4.728277], + [1.145844, 1.035948, 1.017571], + [0.398542, 0.3224, 0.324956], + [5.497337, 5.271507, 5.329618], + [0.210421, 0.162334, 0.161671], + [0.201661, 0.140586, 0.135319], + [0.178466, 0.162246, 0.159834] + ] + } +] diff --git a/website/benchmark/dbms/results/003_vertica_7_0_0_x3.json b/website/benchmark/dbms/results/003_vertica_7_0_0_x3.json new file mode 100644 index 00000000000..8f5ddaf9e73 --- /dev/null +++ b/website/benchmark/dbms/results/003_vertica_7_0_0_x3.json @@ -0,0 +1,109 @@ +[ + { + "system": "Vertica (x3)", + "version": "7.0.0-0", + "data_size": 1000000000, + "time": "", + "comments": "", + "result": + [ + [3.328, 0.397, 0.382], + [2.889, 0.24, 0.219], + [5.667, 0.781, 0.79], + [5.163, 0.58, 0.545], + [10.743, 7.414, 7.554], + [44.021, 43.629, 42.651], + [3.045, 0.416, 0.419], + [2.987, 0.366, 0.387], + [16.039, 13.626, 13.646], + [23.93, 21.818, 21.376], + [7.084, 2.683, 2.763], + [9.39, 3.176, 3.085], + [41.674, 42.039, 42.239], + [138.331, 136.452, 137.238], + [68.416, 67.551, 67.478], + [12.463, 10.125, 9.974], + [188.023, 186.817, 186.116], + [172.149, 170.75, 171.178], + [489.181, 488.154, 489.749], + [5.466, 0.916, 0.891], + [24.91, 18.009, 17.985], + [25.49, 18.525, 18.803], + [39.856, 17.993, 18.436], + [128.041, 9.876, 9.599], + [31.162, 30.831, 30.708], + [33.871, 32.901, 33.198], + [31.26, 30.795, 30.982], + [97.13, 93.233, 93.352], + [123.941, 123.625, 123.509], + [10.032, 9.418, 9.431], + [19.05, 9.184, 8.907], + [22.433, 19.726, 19.764], + [171.142, 162.149, 162.224], + [286.116, 283.672, 282.848], + [281.98, 278.234, 280.236], + [30.897, 30.486, 30.338], + [11.785, 11.42, 11.262], + [2.188, 1.739, 1.782], + [1.754, 1.582, 1.852], + [25.553, 24.89, 24.753], + [0.853, 0.324, 0.304], + [0.656, 0.368, 0.332], + [0.436, 0.356, 0.31] + ] + }, + + { + "system": "Vertica (x6)", + "version": "7.0.0-0", + "data_size": 1000000000, + "time": "", + "comments": "", + "result": + [ + [2.203, 0.392, 0.421], + [2.201, 0.336, 0.36], + [3.669, 0.704, 0.624], + [3.512, 0.516, 0.501], + [7.482, 5.696, 5.812], + [43.298, 48.75, 42.419], + [2.285, 0.411, 0.396], + [2.111, 0.454, 0.447], + [12.751, 10.454, 10.447], + [16.751, 15.247, 16.011], + [4.789, 2.06, 2.023], + [6.015, 2.207, 2.273], + [42.854, 41.299, 42.517], + [50.338, 48.5, 48.569], + [27.033, 25.38, 25.336], + [8.766, 6.73, 6.584], + [63.644, 64.514, 63.864], + [56.751, 56.018, 56.263], + [151.68, 149.595, 150.832], + [3.249, 0.701, 0.639], + [25.006, 18.019, 17.994], + [27.427, 18.74, 18.759], + [23.331, 9.915, 10.314], + [92.277, 6.708, 6.496], + [16.708, 15.827, 16.021], + [18.68, 18.99, 18.492], + [16.39, 16, 16.023], + [50.455, 48.204, 49.54], + [64.871, 64.565, 63.996], + [5.638, 5.103, 5.086], + [14.41, 6.785, 6.714], + [22.602, 14.089, 13.952], + [106.629, 105.692, 103.34], + [301.935, 293.388, 295.326], + [299.916, 290.717, 297.424], + [21.476, 21.447, 21.519], + [7.853, 7.158, 7.353], + [2.087, 1.573, 1.543], + [2.536, 1.788, 1.912], + [11.355, 10.46, 9.62], + [1.008, 0.401, 0.434], + [1.358, 0.373, 0.422], + [0.841, 0.438, 0.442] + ] + } +] diff --git a/website/benchmark/dbms/results/004_infinidb_3_6_23.json b/website/benchmark/dbms/results/004_infinidb_3_6_23.json new file mode 100644 index 00000000000..0a446564af8 --- /dev/null +++ b/website/benchmark/dbms/results/004_infinidb_3_6_23.json @@ -0,0 +1,109 @@ +[ + { + "system": "InfiniDB", + "version": "Enterprise 3.6.23", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [1.15, 0.17, 0.16], + [0.31, 0.17, 0.16], + [0.47, 0.31, 0.30], + [null, null, null], + [0.97, 0.87, 0.92], + [6.14, 6.56, 5.43], + [0.35, 0.25, 0.26], + [0.22, 0.17, 0.16], + [1.19, 0.94, 1.02], + [1.34, 1.37, 1.33], + [0.51, 0.39, 0.39], + [0.45, 0.40, 0.38], + [8.06, 8.02, 8.02], + [8.43, 8.83, 8.89], + [8.21, 8.31, 8.42], + [1.73, 1.76, 1.78], + [18.95, 17.76, 19.98], + [12.59, 13.64, 12.24], + [37.04, 35.00, 36.76], + [0.25, 0.14, 0.13], + [2.28, 0.81, 0.86], + [0.61, 0.63, 0.57], + [3.02, 1.15, 1.17], + [12.44, 1.19, 1.19], + [8.92, 8.83, 9.07], + [8.15, 8.11, 8.11], + [10.39, 10.18, 10.33], + [5.70, 5.70, 5.82], + [13.77, 15.06, 13.88], + [8.57, 9.29, 8.58], + [2.03, 2.02, 2.00], + [3.18, 3.31, 3.26], + [20.23, 19.45, 20.16], + [183.55, 156.42, 124.94], + [160.14, 164.08, 162.15], + [3.49, 1.67, 1.71], + [23.03, 21.05, 21.21], + [3.14, 1.70, 1.65], + [1.64, 1.27, 1.23], + [82.86, 72.81, 77.55], + [0.32, 0.18, 0.18], + [0.28, 0.18, 0.19], + [3.43, 1.61, 1.53] + ] + }, + + { + "system": "InfiniDB", + "version": "Enterprise 3.6.23", + "data_size": 100000000, + "time": "", + "comments": "", + "result": + [ + [2.07, 0.34, 0.35], + [0.76, 0.3, 0.31], + [1.45, 1.23, 1.24], + [null, null, null], + [4.18, 3.89, 3.85], + [26.32, 28.07, 23.96], + [1.36, 1.04, 1.03], + [0.56, 0.32, 0.3], + [5.14, 4.54, 4.51], + [7.83, 8.18, 8.0], + [1.96, 1.4, 1.45], + [1.75, 1.52, 1.46], + [23.72, 23.01, 23.87], + [30.74, 30.86, 28.36], + [25.55, 24.76, 24.41], + [11.66, 11.59, 11.67], + [80.45, 85.49, 116.21], + [52.27, 50.76, 48.3], + [null, null, null], + [4.31, 0.24, 0.16], + [130.37, 7.24, 7.78], + [66.62, 10.19, 10.2], + [32.34, 19.66, 19.59], + [288.38, 58.86, 7.35], + [57.88, 57.95, 57.82], + [47.32, 52.59, 47.03], + [73.32, 65.1, 73.43], + [50.6, 51.5, 50.93], + [89.16, 85.75, 87.26], + [61.97, 60.49, 62.13], + [10.3, 10.4, 10.31], + [21.11, 20.86, 20.99], + [157.67, 151.81, 153.5], + [null, null, null], + [null, null, null], + [11.86, 11.08, 11.13], + [12.35, 12.49, 12.36], + [3.11, 3.12, 3.14], + [1.03, 0.89, 0.9], + [34.01, 45.75, 50.3], + [0.21, 0.23, 0.24], + [0.23, 0.21, 0.23], + [0.14, 0.15, 0.17] + ] + } +] diff --git a/website/benchmark/dbms/results/005_monetdb.json b/website/benchmark/dbms/results/005_monetdb.json new file mode 100644 index 00000000000..caa335babc9 --- /dev/null +++ b/website/benchmark/dbms/results/005_monetdb.json @@ -0,0 +1,110 @@ +[ + { + "system": "MonetDB", + "version": "", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [0.003851, 0.003389, 0.003633], + [0.400058, 0.045117, 0.007999], + [0.207544, 0.022219, 0.033007], + [null, null, null], + [0.873236, 0.61427, 0.564001], + [3.1, 2.8, 2.9], + [0.191616, 0.004704, 0.008579], + [0.056658, 0.049444, 0.035463], + [5.8, 5.8, 6.4], + [5.8, 8.4, 8.5], + [0.458164, 0.319166, 0.413914], + [0.344021, 0.314183, 0.348057], + [6.1, 13.3, 6.1], + [5.8, 5.6, 5.5], + [54.9, 70.0, 58.4], + [0.886465, 0.711899, 0.7329], + [5.8, 5.9, 5.9], + [5.8, 5.9, 6.0], + [20.2, 19.7, 21.0], + [0.331309, 0.000618, 0.085817], + [2.0, 0.056549, 0.09292], + [0.053756, 0.050622, 0.054916], + [2.2, 0.069379, 0.066628], + [13.0, 0.511351, 0.484708], + [0.137787, 0.030937, 0.030501], + [0.039574, 0.027741, 0.025684], + [0.046094, 0.043038, 0.039139], + [4.1, 2.7, 2.7], + [null, null, null], + [1.1, 0.940847, 0.921118], + [17.7, 18.2, 17.5], + [2.0, 0.236466, 0.236223], + [1.3, 1.3, 1.3], + [138.0, 140.0, 173.0], + [null, null, null], + [null, null, null], + [169.0, 175.0, null], + [261.0, 257.0, 268.0], + [3.1, 0.513628, 0.464017], + [262.0, 252.0, 257.0], + [0.524173, 0.066989, 0.123604], + [0.305343, 0.030736, 0.048725], + [2.3, 1.9, 1.9] + ] + }, + + { + "system": "MonetDB", + "version": "", + "data_size": 100000000, + "time": "", + "comments": "", + "result": + [ + [0.028886, 0.003581, 0.003829], + [2.7, 0.139637, 0.060264], + [1.1, 0.146556, 0.146063], + [null, null, null], + [7.5, 5.6, 5.4], + [15.5, 12.7, 11.5], + [1.0, 0.038566, 0.036837], + [0.687604, 0.026271, 0.030261], + [195.0, 214.0, null], + [254.0, 267.0, 265.0], + [22.9, 20.0, 20.8], + [20.4, 20.8, 23.3], + [97.0, 115.0, null], + [111.0, 84.0, 58.8], + [656.0, null, null], + [6.8, 6.9, 6.7], + [264.0, 234.0, 208.0], + [222.0, 279.0, null], + [1173.0, null, null], + [3.9, 0.000787, 0.032566], + [27.4, 1.7, 2.1], + [4.0, 2.6, 2.6], + [null, null, null], + [209.0, 155, 143], + [10.5, 5.7, 40.9], + [4.8, 4.7, 4.7], + [85.0, 4.7, 5.7], + [52.4, 30.3, 29.1], + [null, null, null], + [null, null, null], + [12.7, 11.2, 11.4], + [392.0, 370.0, null], + [3.2, 1.8, 1.8], + [41.9, 37.0, 40.7], + [null, null, null], + [null, null, null], + [null, null, null], + [6.4, 2.3, 2.1], + [3.6, 1.0, 1.1], + [1.5, 1.2, 1.3], + [7.2, 4.6, 4.9], + [2.8, 0.050916, 0.04148], + [3.0, 0.045935, 0.047277], + [2.5, 0.049119, 0.04828] + ] + } +] diff --git a/website/benchmark/dbms/results/006_infobright_4_0_7.json b/website/benchmark/dbms/results/006_infobright_4_0_7.json new file mode 100644 index 00000000000..25cf680f9d9 --- /dev/null +++ b/website/benchmark/dbms/results/006_infobright_4_0_7.json @@ -0,0 +1,55 @@ +[ + { + "system": "Infobright", + "version": "CE 4.0.7", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [0.00, 0.00, 0.00], + [0.40, 0.38, 0.39], + [0.10, 0.00, 0.00], + [null, null, null], + [2.83, 1.91, 1.95], + [9.16, 1.65, 1.70], + [0.03, 0.00, 0.00], + [0.46, 0.40, 0.41], + [4.13, 2.97, 3.43], + [5.12, 4.46, 4.15], + [1.98, 1.24, 1.36], + [1.58, 1.26, 1.27], + [13.37, 30.81, 29.76], + [32.59, 30.22, 13.00], + [12.93, 11.39, 30.46], + [2.98, 3.05, 2.96], + [9.90, 20.86, 25.17], + [10.38, 10.19, 10.28], + [162.43, 164.35, 169.28], + [1, 0, 22, 0, 24], + [47.80, 4.40, 4.47], + [4.83, 1.15, 1.14], + [43.82, 2.14, 2.16], + [6.14, 4.39, 4.35], + [0.41, 0.41, 0.41], + [0.80, 0.80, 0.80], + [0.41, 0.41, 0.40], + [10.39, 10.12, 9.88], + [139.25, 79.33, 78.92], + [145.8, 146.2, 144.97], + [3.34, 2.02, 2.06], + [3.68, 3.05, 3.02], + [14.76, 14.82, 14.76], + [77.49, 91.4, 90.9], + [75.37, 83.55, 63.55], + [10.80, 10.52, 10.67], + [23.77, 9.47, 9.30], + [50.21, 3.37, 3.36], + [1.26, 0.77, 0.76], + [98.25, 104.6, 94.29], + [2.04, 0.47, 0.54], + [1.36, 0.42, 0.41], + [4.58, 4.01, 3.98] + ] + } +] diff --git a/website/benchmark/dbms/results/007_hive_0_11.json b/website/benchmark/dbms/results/007_hive_0_11.json new file mode 100644 index 00000000000..a025fe9d884 --- /dev/null +++ b/website/benchmark/dbms/results/007_hive_0_11.json @@ -0,0 +1,109 @@ +[ + { + "system": "Hive", + "version": "0.11, ORC File", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [47.388, 44.55, 43.513], + [25.332, 22.592, 22.629], + [27.558, 23.861, 24.986], + [26.148, 23.564, 23.508], + [35.237, 31.445, 32.552], + [34.063, 29.607, 29.268], + [25.999, 22.443, 22.559], + [38.784, 37.082, 37.652], + [49.973, 47.282, 46.027], + [54.759, 50.301, 51.858], + [42.793, 39.001, 38.998], + [42.858, 38.928, 40.035], + [55.967, 53.253, 53.053], + [58.068, 54.393, 53.189], + [58.359, 53.181, 54.164], + [63.096, 58.614, 60.153], + [73.175, 70.386, 69.204], + [35.511, 31.512, 31.482], + [109.132, 107.333, 106.376], + [17.948, 14.47, 14.154], + [27.452, 24.527, 24.674], + [41.792, 40.17, 40.052], + [45.079, 42.12, 43.438], + [50.847, 46.004, 45.95], + [31.007, 26.473, 26.277], + [30.985, 27.724, 27.357], + [32.747, 28.329, 27.439], + [62.932, 57.159, 59.233], + [63.563, 63.375, 63.307], + [74.663, 67.206, 68.586], + [58.017, 52.364, 53.155], + [62.907, 60.202, 59.653], + [127.206, 124.701, 123.291], + [89.931, 87.6, 87.325], + [98.879, 89.299, 90.377], + [63.792, 61.127, 61.517], + [44.325, 39.995, 39.979], + [43.852, 40.178, 40.131], + [44.493, 40.17, 40.171], + [36.108, 36.293, 36.241], + [43.025, 39.168, 40.042], + [42.914, 40.129, 39.135], + [33.91, 34.161, 34.191] + ] + }, + + { + "system": "Hive", + "version": "0.11, ORC File", + "data_size": 100000000, + "time": "", + "comments": "", + "result": + [ + [110.676, 105.13, 107.358], + [55.195, 36.435, 32.201], + [39.991, 35.143, 35.085], + [44.465, 34.131, 34.032], + [110.69, 105.953, 107.343], + [68.119, 64.831, 64.269], + [37.809, 33.021, 33.13], + [53.788, 51.261, 48.653], + [87.479, 85.062, 85.039], + [106.577, 102.879, 101.705], + [60.4, 53.498, 53.516], + [61.275, 53.698, 53.577], + [87.924, 82.999, 82.867], + [94.281, 86.991, 87.084], + [91.05, 87.267, 87.731], + [132.697, 132.306, 130.91], + [141.357, 147.059, 140.75], + [60.884, 57.376, 57.367], + [237.554, 234.361, 234.271], + [34.019, 21.834, 21.08], + [41.195, 36.443, 35.979], + [60.385, 54.888, 56.541], + [67.257, 58.995, 59.828], + [87.697, 88.521, 89.324], + [53.796, 50.592, 50.118], + [68.786, 63.993, 62.886], + [60.715, 56.14, 55.303], + [112.58, 107.297, 106.493], + [115.068, 110.622, 109.541], + [136.36, 133.102, 135.896], + [113.348, 100.032, 99.905], + [124.002, 117.366, 109.524], + [301.77, 324.867, 294.034], + [233.937, 272.053, 238.167], + [241.283, 228.198, 246.999], + [120.684, 118.948, 118.18], + [70.292, 55.211, 55.076], + [63.4, 52.093, 52.895], + [67.483, 53.704, 54.814], + [60.588, 52.321, 53.356], + [62.644, 51.812, 53.23], + [69.068, 53.234, 52.853], + [46.67, 46.041, 45.95] + ] + } +] diff --git a/website/benchmark/dbms/results/008_mysql_5_5.json b/website/benchmark/dbms/results/008_mysql_5_5.json new file mode 100644 index 00000000000..c132ee3b2bb --- /dev/null +++ b/website/benchmark/dbms/results/008_mysql_5_5.json @@ -0,0 +1,109 @@ +[ + { + "system": "MySQL", + "version": "5.5.32, MyISAM", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [0.01, 0.01, 0.01], + [21.55, 18.91, 18.28], + [22.71, 19.86, 20.53], + [21.3, 18.93, 19.25], + [26.77, 25.74, 25.65], + [29.14, 26.92, 26.53], + [7.47, 7.38, 7.23], + [20.56, 18.84, 18.41], + [27.53, 25.14, 24.45], + [30.08, 26.07, 26.75], + [22.93, 19.82, 20.23], + [21.79, 19.75, 19.64], + [51.3, 42.27, 46.45], + [43.75, 42.38, 42.36], + [43.76, 48.66, 46.8], + [106.76, 106.53, 105.32], + [172.51, 181.59, 177.7], + [189.92, 166.09, 172.03], + [185.61, 181.38, 206.92], + [20.3, 19.24, 18.49], + [21.43, 20.03, 19.99], + [21.88, 20.06, 20.3], + [25.51, 20.72, 20.58], + [24.14, 20.04, 19.29], + [21.65, 19.11, 19.0], + [22.44, 20.35, 20.02], + [21.41, 19.96, 19.91], + [27.15, 26.28, 25.32], + [135.45, 134.31, 133.12], + [107.44, 106.26, 106.16], + [40.47, 36.67, 37.07], + [39.2, 36.86, 37.22], + [134.5, 130.75, 133.55], + [1057.45, 1075.29, 928.38], + [867.64, 1023.33, 1063.4], + [111.01, 109.86, 109.34], + [1160.03, 23.32, 23.08], + [1109.2, 14.24, 14.38], + [1086.92, 7.29, 6.63], + [31.74, 31.79, null], + [1074.29, 5.51, 5.43], + [1069.16, 5.6, 5.3], + [652.84, 4.84, 4.57] + ] + }, + + { + "system": "MySQL", + "version": "5.5.32, MyISAM", + "data_size": 100000000, + "time": "", + "comments": "", + "result": + [ + [0.01, 0.01, 0.01], + [220.39, 234.32, 305.28], + [220.45, 198.31, 198.37], + [207.6, 190.59, 188.35], + [275.96, 250.84, 246.93], + [292.17, 254.14, 251.06], + [75.51, 76.11, 74.98], + [203.94, 184.14, 180.82], + [287.28, 252.52, 249.48], + [299.44, 282.02, 271.33], + [218.71, 197.51, 195.94], + [220.1, 197.17, 199.88], + [929.45, 869.74, 739.53], + [null, null, null], + [1196.42, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [447.72, 199.09, 185.82], + [582.73, 196.73, 195.8], + [582.54, 205.89, 199.15], + [568.75, 217.15, 209.65], + [562.72, 206.77, 203.19], + [602.47, 186.8, 186.62], + [565.26, 199.44, 199.24], + [657.78, 202.53, 196.95], + [675.84, 250.11, 248.9], + [null, null, null], + [1061.89, 1054.6, null], + [993.89, 918.67, null], + [604.48, 553.33, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [873.29, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [0.67, 0.0, 0.0] + ] + } +] diff --git a/website/benchmark/dbms/results/009_memsql_3_2.json b/website/benchmark/dbms/results/009_memsql_3_2.json new file mode 100644 index 00000000000..7195f904778 --- /dev/null +++ b/website/benchmark/dbms/results/009_memsql_3_2.json @@ -0,0 +1,55 @@ +[ + { + "system": "MemSQL", + "version": "3.2, column store", + "data_size": 10000000, + "time": "2015-04-05", + "comments": "", + "result": + [ + [0.01, 0.01, 0.01], + [0.29, 0.26, 0.25], + [0.48, 0.45, 0.39], + [0.75, 0.63, 0.62], + [1.03, 0.97, 0.89], + [2.76, 2.65, 2.57], + [0.36, 0.32, 0.32], + [0.29, 0.24, 0.24], + [1.71, 1.51, 1.47], + [1.91, 1.69, 1.67], + [0.83, 0.66, 0.65], + [0.88, 0.70, 0.70], + [2.55, 2.59, 2.32], + [null, null, null], + [4.36, 2.34, 2.39], + [1.23, 1.09, 1.09], + [3.26, 3.18, 2.81], + [2.76, 2.58, 2.58], + [5.23, 4.74, 4.45], + [null, null, null], + [5.12, 4.62, 4.81], + [5.43, 4.91, 4.90], + [7.32, 6.18, 6.14], + [22.61, 17.85, 17.89], + [1.04, 0.77, 0.75], + [0.93, 0.77, 0.90], + [1.04, 0.75, 0.76], + [2.84, 2.41, 2.31], + [18.64, 18.19, 18.38], + [5.78, 5.68, 5.67], + [2.24, 1.90, 1.85], + [2.65, 2.22, 2.22], + [8.82, 8.32, 8.01], + [11.30, 10.93, 11.21], + [11.22, 10.73, 10.72], + [1.60, 1.46, 1.45], + [1.86, 1.75, 1.83], + [1.16, 1.10, 1.11], + [0.54, 0.44, 0.44], + [3.79, 3.59, 3.58], + [null, null, null], + [null, null, null], + [0.37, 0.35, 0.35] + ] + } +] diff --git a/website/benchmark/dbms/results/010_greenplum_4_3_9.json b/website/benchmark/dbms/results/010_greenplum_4_3_9.json new file mode 100644 index 00000000000..c2c43b22de6 --- /dev/null +++ b/website/benchmark/dbms/results/010_greenplum_4_3_9.json @@ -0,0 +1,161 @@ +[ + { + "system": "Greenplum", + "version": "4.3.9.1", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [0.77, 0.77, 0.92], + [0.44, 0.27, 0.46], + [0.95, 0.90, 0.89], + [0.74, 0.91, 0.83], + [1.43, 1.47, 1.44], + [1.74, 1.63, 1.51], + [0.77, 0.83, 0.88], + [1.86, 1.95, 1.86], + [2.09, 1.92, 1.92], + [2.33, 2.41, 2.46], + [0.75, 0.78, 0.77], + [0.71, 0.70, 0.75], + [2.11, 2.22, 2.50], + [2.65, 2.47, 2.59], + [2.11, 2.40, 2.46], + [1.59, 1.66, 1.14], + [1.75, 1.96, 1.71], + [1.29, 1.30, 1.02], + [1.99, 2.25, 2.29], + [0.65, 0.53, 0.49], + [1.23, 0.98, 1.01], + [1.85, 1.81, 1.91], + [1.85, 1.24, 1.17], + [8.21, 3.48, 2.71], + [0.77, 0.49, 0.75], + [0.68, 0.68, 0.88], + [0.76, 0.46, 0.78], + [2.12, 2.07, 2.36], + [3.08, 2.86, 3.12], + [11.50, 11.90, 10.32], + [1.89, 1.85, 1.86], + [1.99, 1.96, 2.02], + [5.24, 4.93, 5.03], + [3.24, 3.40, 3.27], + [3.01, 2.64, 2.65], + [3.14, 2.80, 2.73], + [1.47, 1.14, 1.24], + [1.79, 1.05, 1.13], + [1.32, 1.11, 1.14], + [2.20, 1.51, 1.48], + [1.42, 1.01, 1.02], + [2.17, 1.74, 2.23], + [2.17, 1.70, 1.94] + ] + }, + { + "system": "Greenplum", + "version": "4.3.9.1", + "data_size": 100000000, + "time": "", + "comments": "", + "result": + [ + [4.11, 4.32, 4.36], + [2.56, 2.53, 2.80], + [4.98, 5.14, 5.45], + [6.64, 5.99, 5.69], + [5.86, 5.72, 5.98], + [6.71, 6.76, 7.36], + [4.25, 4.39, 4.62], + [4.53, 4.23, 4.14], + [9.12, 9.23, 9.18], + [14.77, 15.38, 14.37], + [3.22, 2.87, 3.89], + [4.41, 4.29, 4.89], + [6.78, 7.19, 6.36], + [15.04, 16.93, 16.14], + [7.08, 6.21, 6.96], + [6.83, 6.17, 5.02], + [8.90, 9.81, 9.29], + [6.84, 5.31, 7.67], + [15.33, 16.31, 15.42], + [3.18, 4.29, 3.59], + [8.47, 8.73, 9.91], + [10.40, 8.38, 9.67], + [10.68, 8.16, 7.90], + [50.37, 32.46, 30.45], + [5.21, 3.64, 4.13], + [5.47, 5.68, 5.91], + [4.33, 3.20, 3.83], + [10.76, 11.14, 11.22], + [18.45, 17.26, 16.28], + [96.58, 97.29, 92.61], + [7.23, 7.36, 7.57], + [10.35, 8.20, 9.87], + [38.32, 37.57, 40.99], + [16.27, 15.24, 16.74], + [15.90, 15.33, 16.27], + [13.44, 13.84, 13.24], + [1.92, 1.16, 1.10], + [1.88, 1.16, 1.10], + [1.87, 1.07, 0.98], + [2.71, 1.22, 1.22], + [1.71, 0.97, 1.07], + [2.44, 1.78, 1.68], + [2.19, 1.72, 2.24] + ] + }, + { + "system": "Greenplum", + "version": "4.3.9.1", + "data_size": 1000000000, + "time": "", + "comments": "", + "result": + [ + [29.01, 32.82, 30.09], + [20.93, 22.20, 20.63], + [34.41, 35.26, 36.59], + [44.01, 41.56, 41.36], + [36.87, 35.96, 39.32], + [46.44, 50.24, 45.50], + [29.22, 31.75, 30.19], + [20.58, 20.76, 24.18], + [56.29, 56.67, 57.36], + [79.13, 81.78, 78.60], + [32.11, 33.21, 29.69], + [33.69, 29.92, 30.92], + [44.85, 42.52, 40.64], + [63.16, 63.16, 64.79], + [47.16, 43.26, 42.45], + [41.04, 43.67, 41.76], + [63.45, 64.64, 60.74], + [48.56, 51.07, 48.81], + [79.89, 81.48, 81.27], + [25.73, 29.27, 31.48], + [72.20, 75.93, 71.44], + [74.50, 73.46, 74.82], + [81.19, 80.76, 78.67], + [339.14, 296.80, 296.95], + [28.51, 31.36, 28.67], + [36.49, 36.47, 38.96], + [31.26, 29.18, 31.65], + [80.03, 83.33, 80.84], + [73.36, 73.27, 73.57], + [961.62, 935.00, 944.02], + [51.55, 50.44, 45.14], + [73.03, 72.86, 70.11], + [29.24, 28.79, 29.78], + [37.51, 39.70, 39.66], + [53.86, 53.37, 53.77], + [84.54, 84.86, 85.62], + [7.01, 1.93, 2.16], + [5.39, 1.55, 1.69], + [6.85, 1.43, 1.57], + [13.18, 3.17, 3.08], + [5.97, 1.30, 1.47], + [5.69, 2.12, 1.96], + [4.11, 2.27, 2.43] + ] + } +] diff --git a/website/benchmark/dbms/results/011_greenplum_4_3_9_x2.json b/website/benchmark/dbms/results/011_greenplum_4_3_9_x2.json new file mode 100644 index 00000000000..269f44d8dff --- /dev/null +++ b/website/benchmark/dbms/results/011_greenplum_4_3_9_x2.json @@ -0,0 +1,161 @@ +[ + { + "system": "Greenplum(x2)", + "version": "4.3.9.1", + "data_size": 100000000, + "time": "", + "comments": "", + "result": + [ + [2.11, 1.61, 1.60], + [1.30, 1.30, 1.25], + [2.19, 2.12, 2.10], + [2.48, 2.53, 2.50], + [2.96, 3.05, 3.12], + [4.02, 4.02, 3.98], + [1.99, 1.79, 1.81], + [4.26, 3.77, 3.80], + [7.83, 7.19, 6.67], + [11.48, 11.72, 11.62], + [2.12, 2.06, 2.02], + [2.18, 2.05, 2.03], + [5.23, 5.16, 5.29], + [7.01, 6.84, 6.91], + [5.11, 5.31, 5.25], + [3.47, 3.20, 3.24], + [4.19, 4.18, 4.19], + [3.25, 3.16, 3.25], + [6.72, 6.65, 6.84], + [1.70, 1.57, 1.55], + [3.84, 3.58, 3.46], + [6.80, 5.48, 5.31], + [10.50, 3.71, 3.86], + [31.39, 13.54, 14.30], + [3.00, 1.76, 1.70], + [2.13, 2.01, 2.11], + [2.69, 1.72, 1.67], + [6.26, 5.90, 5.58], + [16.77, 16.00, 15.89], + [45.96, 46.69, 47.78], + [4.61, 4.15, 4.22], + [7.08, 5.49, 5.64], + [28.43, 26.11, 28.32], + [19.05, 19.68, 19.23], + [19.23, 19.01, 20.48], + [7.71, 7.69, 7.61], + [2.50, 1.74, 1.74], + [2.15, 1.60, 1.77], + [2.21, 1.67, 1.97], + [2.88, 1.90, 1.82], + [2.25, 1.81, 1.84], + [3.36, 3.13, 3.18], + [3.16, 3.21, 2.90] + ] + }, + { + "system": "Greenplum(x2)", + "version": "4.3.9.1", + "data_size": 10000000, + "time": "", + "comments": "", + "result": + [ + [0.37, 0.17, 0.22], + [0.25, 0.13, 0.12], + [0.39, 0.22, 0.23], + [0.47, 0.26, 0.26], + [1.30, 1.49, 1.44], + [1.87, 1.85, 2.06], + [0.32, 0.19, 0.19], + [2.85, 3.32, 3.06], + [2.60, 2.44, 2.57], + [2.79, 2.46, 2.79], + [0.80, 0.57, 0.59], + [0.83, 0.60, 0.61], + [3.47, 3.21, 3.43], + [1.54, 1.33, 1.40], + [3.29, 3.11, 3.45], + [1.35, 1.45, 1.49], + [1.65, 1.54, 1.65], + [0.92, 0.81, 0.86], + [1.67, 1.25, 1.28], + [0.33, 0.19, 0.17], + [0.59, 0.40, 0.41], + [2.07, 2.22, 1.96], + [1.70, 0.95, 0.87], + [6.83, 1.74, 1.80], + [0.46, 0.20, 0.19], + [0.44, 0.28, 0.28], + [0.50, 0.19, 0.20], + [2.06, 2.07, 2.15], + [3.02, 2.94, 2.80], + [5.42, 5.62, 5.37], + [2.10, 1.95, 2.05], + [2.31, 2.16, 2.16], + [3.74, 3.59, 3.67], + [3.89, 3.99, 3.93], + [3.03, 2.89, 3.10], + [3.56, 3.41, 3.19], + [2.12, 1.96, 1.96], + [2.03, 1.87, 1.86], + [2.11, 1.83, 1.76], + [2.53, 2.10, 1.96], + [2.31, 1.68, 1.87], + [3.26, 3.22, 3.27], + [3.19, 3.23, 3.11] + ] + }, + { + "system": "Greenplum(x2)", + "version": "4.3.9.1", + "data_size": 1000000000, + "time": "", + "comments": "", + "result": + [ + [17.70, 12.63, 12.16], + [9.14, 9.22, 9.15], + [17.13, 17.56, 16.67], + [21.25, 20.67, 21.22], + [17.16, 16.49, 17.86], + [25.70, 25.60, 25.78], + [14.77, 15.00, 14.73], + [11.80, 12.09, 12.05], + [33.57, 33.21, 33.01], + [61.65, 59.21, 60.16], + [13.69, 13.74, 13.77], + [14.48, 14.19, 14.28], + [25.59, 26.22, 26.35], + [30.75, 31.32, 31.37], + [27.70, 28.49, 28.18], + [20.28, 20.50, 20.28], + [27.80, 27.13, 29.00], + [23.66, 14.42, 24.30], + [59.58, 58.06, 58.91], + [12.86, 13.18, 13.26], + [36.04, 32.46, 32.59], + [45.28, 34.80, 34.56], + [97.71, 34.57, 33.62], + [215.97, 121.61, 120.47], + [24.44, 13.65, 13.62], + [17.15, 17.01, 17.12], + [23.84, 13.32, 13.46], + [40.83, 39.39, 38.71], + [155.70, 155.18, 158.97], + [451.18, 448.88, 449.55], + [27.37, 25.11, 25.06], + [56.58, 42.46, 43.33], + [29.77, 29.24, 29.11], + [36.94, 31.05, 29.40], + [104.88, 102.26, 101.88], + [41.06, 41.52, 41.15], + [6.91, 2.29, 2.37], + [5.35, 2.00, 2.02], + [6.49, 1.98, 1.88], + [12.55, 3.30, 3.44], + [6.18, 1.95, 2.06], + [6.12, 3.02, 3.43], + [5.21, 3.66, 3.58] + ] + } +] diff --git a/website/benchmark/hardware/index.html b/website/benchmark/hardware/index.html new file mode 100644 index 00000000000..386865b3a0f --- /dev/null +++ b/website/benchmark/hardware/index.html @@ -0,0 +1,70 @@ +{% extends 'templates/base.html' %} + +{% set title = 'Performance comparison of ClickHouse on various hardware' %} +{% set extra_js = [ + 'queries.js?' + rev_short, + 'results.js?' + rev_short, + '../benchmark.js?' + rev_short] +%} +{% set url = 'https://clickhouse.tech/benchmark/hardware/' %} +{% set no_footer = True %} + +{% block content %} +
+ +
+
+ + ClickHouse + +

{{ title }}

+
+
+ +
+
+
+ +
+
+

Relative query processing time (lower is better)

+
+
+
+ +
+
+

Full results

+
+
+
+ +
+
+

Comments

+

Submit your own results: https://clickhouse.tech/docs/en/operations/performance_test/

+

+Results for Lenovo B580 Laptop are from Ragıp Ünal. 16GB RAM 1600 GHz, 240GB SSD, Intel(R) Core(TM) i5-3210M CPU @ 2.50GHz (2 Core / 4 HT)
+Results for Time4vps.eu are from Ragıp Ünal.
+Results for Dell PowerEdge R640, R641 (in Hetzner) are from Dmirty Titov.
+Results for Dell PowerEdge R730 are from Amos Bird.
+Results for Dell R530 are from Yuriy Zolkin.
+Results for Xeon 2176G are from Sergey Golod.
+Results for Azure DS3v2 are from Boris Granveaud.
+Results for AWS are from Wolf Kreuzerkrieg.
+Results for Huawei Taishan are from Peng Gao in sina.com.
+Results for Selectel and AMD EPYC 7402P are from Andrey Dudin.
+Results for ProLiant are from Denis Ustinov.
+Results for AMD EPYC 7502P 128GiB are from Kostiantyn Velychkovskyi.
+Results for AMD EPYC 7502P 512GiB are from Sergey Zakharov.
+Results for Pinebook Pro are from Aleksey R. @kITerE.
+Results for AMD Ryzen are from Alexey Milovidov. Firefox was running in background.
+Results for Azure E32s are from Piotr Maśko.
+Results for MacBook Pro are from Denis Glazachev. MacOS Catalina Version 10.15.4 (19E266). For "drop caches", the "Free Up RAM" in CleanMyMac is used.
+Results for AMD EPYC 7702 are from Peng Gao in sina.com.
+Results for Intel NUC are from Alexander Zaitsev, Altinity.
+Xeon Gold 6230 server is using 4 x SAMSUNG datacenter class SSD in RAID-10.
+ Results for Yandex Managed ClickHouse for "cold cache" are biased and should not be compared, because cache was not flushed for every next query.

+
+
+{% endblock %} diff --git a/website/benchmark/hardware/queries.js b/website/benchmark/hardware/queries.js new file mode 100644 index 00000000000..9527da84abf --- /dev/null +++ b/website/benchmark/hardware/queries.js @@ -0,0 +1,182 @@ +var current_data_size = 0; + +var current_systems = [ +'Xeon Gold 6230, 2 sockets, 40 threads', +'Dell PowerEdge R640 DX292 2x Xeon SP Gold 16-Core 2.10GHz, 196 GB RAM, 2x SSD 960 GB RAID-1', +'E5-2650 v2 @ 2.60GHz, 2 sockets, 16 threads, 8xHDD RAID-5']; + +var queries = + [ + { + "query": "SELECT count() FROM hits", + "comment": "", + }, + { + "query": "SELECT count() FROM hits WHERE AdvEngineID != 0", + "comment": "", + }, + { + "query": "SELECT sum(AdvEngineID), count(), avg(ResolutionWidth) FROM hits", + "comment": "", + }, + { + "query": "SELECT sum(UserID) FROM hits", + "comment": "", + }, + { + "query": "SELECT uniq(UserID) FROM hits", + "comment": "", + }, + { + "query": "SELECT uniq(SearchPhrase) FROM hits", + "comment": "", + }, + { + "query": "SELECT min(EventDate), max(EventDate) FROM hits", + "comment": "", + }, + { + "query": "SELECT AdvEngineID, count() FROM hits WHERE AdvEngineID != 0 GROUP BY AdvEngineID ORDER BY count() DESC", + "comment": "", + }, + { + "query": "SELECT RegionID, uniq(UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT RegionID, sum(AdvEngineID), count() AS c, avg(ResolutionWidth), uniq(UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT MobilePhoneModel, uniq(UserID) AS u FROM hits WHERE MobilePhoneModel != '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT MobilePhone, MobilePhoneModel, uniq(UserID) AS u FROM hits WHERE MobilePhoneModel != '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, count() AS c FROM hits WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, uniq(UserID) AS u FROM hits WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchEngineID, SearchPhrase, count() AS c FROM hits WHERE SearchPhrase != '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, count() FROM hits GROUP BY UserID ORDER BY count() DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, SearchPhrase, count() FROM hits GROUP BY UserID, SearchPhrase ORDER BY count() DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, SearchPhrase, count() FROM hits GROUP BY UserID, SearchPhrase LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID, toMinute(EventTime) AS m, SearchPhrase, count() FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY count() DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT UserID FROM hits WHERE UserID = 12345678901234567890", + "comment": "", + }, + { + "query": "SELECT count() FROM hits WHERE URL LIKE '%metrika%'", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, any(URL), count() AS c FROM hits WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase, any(URL), any(Title), count() AS c, uniq(UserID) FROM hits WHERE Title LIKE '%Яндекс%' AND URL NOT LIKE '%.yandex.%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT * FROM hits WHERE URL LIKE '%metrika%' ORDER BY EventTime LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase FROM hits WHERE SearchPhrase != '' ORDER BY EventTime LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase FROM hits WHERE SearchPhrase != '' ORDER BY SearchPhrase LIMIT 10", + "comment": "", + }, + { + "query": "SELECT SearchPhrase FROM hits WHERE SearchPhrase != '' ORDER BY EventTime, SearchPhrase LIMIT 10", + "comment": "", + }, + { + "query": "SELECT CounterID, avg(length(URL)) AS l, count() AS c FROM hits WHERE URL != '' GROUP BY CounterID HAVING c > 100000 ORDER BY l DESC LIMIT 25", + "comment": "", + }, + { + "query": "SELECT domainWithoutWWW(Referer) AS key, avg(length(Referer)) AS l, count() AS c, any(Referer) FROM hits WHERE Referer != '' GROUP BY key HAVING c > 100000 ORDER BY l DESC LIMIT 25", + "comment": "", + }, + { + "query": "SELECT sum(ResolutionWidth), sum(ResolutionWidth + 1), sum(ResolutionWidth + 2), sum(ResolutionWidth + 3), sum(ResolutionWidth + 4), sum(ResolutionWidth + 5), sum(ResolutionWidth + 6), sum(ResolutionWidth + 7), sum(ResolutionWidth + 8), sum(ResolutionWidth + 9), sum(ResolutionWidth + 10), sum(ResolutionWidth + 11), sum(ResolutionWidth + 12), sum(ResolutionWidth + 13), sum(ResolutionWidth + 14), sum(ResolutionWidth + 15), sum(ResolutionWidth + 16), sum(ResolutionWidth + 17), sum(ResolutionWidth + 18), sum(ResolutionWidth + 19), sum(ResolutionWidth + 20), sum(ResolutionWidth + 21), sum(ResolutionWidth + 22), sum(ResolutionWidth + 23), sum(ResolutionWidth + 24), sum(ResolutionWidth + 25), sum(ResolutionWidth + 26), sum(ResolutionWidth + 27), sum(ResolutionWidth + 28), sum(ResolutionWidth + 29), sum(ResolutionWidth + 30), sum(ResolutionWidth + 31), sum(ResolutionWidth + 32), sum(ResolutionWidth + 33), sum(ResolutionWidth + 34), sum(ResolutionWidth + 35), sum(ResolutionWidth + 36), sum(ResolutionWidth + 37), sum(ResolutionWidth + 38), sum(ResolutionWidth + 39), sum(ResolutionWidth + 40), sum(ResolutionWidth + 41), sum(ResolutionWidth + 42), sum(ResolutionWidth + 43), sum(ResolutionWidth + 44), sum(ResolutionWidth + 45), sum(ResolutionWidth + 46), sum(ResolutionWidth + 47), sum(ResolutionWidth + 48), sum(ResolutionWidth + 49), sum(ResolutionWidth + 50), sum(ResolutionWidth + 51), sum(ResolutionWidth + 52), sum(ResolutionWidth + 53), sum(ResolutionWidth + 54), sum(ResolutionWidth + 55), sum(ResolutionWidth + 56), sum(ResolutionWidth + 57), sum(ResolutionWidth + 58), sum(ResolutionWidth + 59), sum(ResolutionWidth + 60), sum(ResolutionWidth + 61), sum(ResolutionWidth + 62), sum(ResolutionWidth + 63), sum(ResolutionWidth + 64), sum(ResolutionWidth + 65), sum(ResolutionWidth + 66), sum(ResolutionWidth + 67), sum(ResolutionWidth + 68), sum(ResolutionWidth + 69), sum(ResolutionWidth + 70), sum(ResolutionWidth + 71), sum(ResolutionWidth + 72), sum(ResolutionWidth + 73), sum(ResolutionWidth + 74), sum(ResolutionWidth + 75), sum(ResolutionWidth + 76), sum(ResolutionWidth + 77), sum(ResolutionWidth + 78), sum(ResolutionWidth + 79), sum(ResolutionWidth + 80), sum(ResolutionWidth + 81), sum(ResolutionWidth + 82), sum(ResolutionWidth + 83), sum(ResolutionWidth + 84), sum(ResolutionWidth + 85), sum(ResolutionWidth + 86), sum(ResolutionWidth + 87), sum(ResolutionWidth + 88), sum(ResolutionWidth + 89) FROM hits", + "comment": "", + }, + { + "query": "SELECT SearchEngineID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT URL, count() AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT 1, URL, count() AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM hits GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT URL, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT Title, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10", + "comment": "", + }, + { + "query": "SELECT URL, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000", + "comment": "", + }, + { + "query": "SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000", + "comment": "", + }, + { + "query": "SELECT URLHash, EventDate, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://yandex.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100", + "comment": "", + }, + { + "query": "SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://yandex.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000;", + "comment": "", + }, + { + "query": "SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM hits WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute;", + "comment": "", + } + ]; diff --git a/website/benchmark/hardware/results/001_xeon_gold_6230.json b/website/benchmark/hardware/results/001_xeon_gold_6230.json new file mode 100644 index 00000000000..d43b5707967 --- /dev/null +++ b/website/benchmark/hardware/results/001_xeon_gold_6230.json @@ -0,0 +1,52 @@ +[ + { + "system": "Xeon Gold 6230, 2 sockets, 40 threads", + "time": "2020-01-01 00:00:00", + "result": + [ + [0.009, 0.002, 0.001], + [0.028, 0.008, 0.009], + [0.055, 0.014, 0.014], + [0.122, 0.018, 0.018], + [0.157, 0.069, 0.068], + [0.217, 0.124, 0.121], + [0.020, 0.008, 0.008], + [0.013, 0.008, 0.007], + [0.284, 0.213, 0.217], + [0.312, 0.241, 0.239], + [0.164, 0.087, 0.090], + [0.168, 0.092, 0.092], + [0.318, 0.256, 0.251], + [0.436, 0.323, 0.325], + [0.385, 0.327, 0.327], + [0.419, 0.402, 0.398], + [1.061, 0.918, 0.935], + [0.523, 0.474, 0.598], + [1.706, 1.621, 1.657], + [0.124, 0.037, 0.024], + [1.192, 0.263, 0.258], + [1.362, 0.303, 0.302], + [2.473, 0.730, 0.735], + [2.857, 0.451, 0.382], + [0.374, 0.106, 0.103], + [0.204, 0.079, 0.078], + [0.381, 0.114, 0.109], + [1.185, 0.327, 0.318], + [1.011, 0.408, 0.415], + [0.895, 0.925, 0.880], + [0.406, 0.277, 0.274], + [0.846, 0.386, 0.389], + [3.174, 2.500, 2.533], + [1.758, 1.311, 1.315], + [1.766, 1.332, 1.355], + [0.469, 0.449, 0.462], + [0.201, 0.169, 0.170], + [0.069, 0.061, 0.061], + [0.064, 0.056, 0.057], + [0.362, 0.365, 0.369], + [0.035, 0.022, 0.024], + [0.030, 0.019, 0.019], + [0.009, 0.005, 0.005] + ] + } +] diff --git a/website/benchmark/hardware/results/002_yandex_cloud_cascade_lake_64_vcpu.json b/website/benchmark/hardware/results/002_yandex_cloud_cascade_lake_64_vcpu.json new file mode 100644 index 00000000000..cf51a1a3b6e --- /dev/null +++ b/website/benchmark/hardware/results/002_yandex_cloud_cascade_lake_64_vcpu.json @@ -0,0 +1,103 @@ +[ + { + "system": "Yandex Cloud Cascade Lake, 64 vCPU (32 threads), 128 GB RAM, 400 GB SSD", + "time": "2020-01-11 00:00:00", + "result": + [ + [0.037, 0.002, 0.002], + [0.113, 0.008, 0.014], + [0.275, 0.016, 0.016], + [0.448, 0.022, 0.022], + [0.472, 0.083, 0.082], + [0.808, 0.137, 0.136], + [0.063, 0.016, 0.010], + [0.055, 0.008, 0.008], + [0.753, 0.280, 0.327], + [0.850, 0.308, 0.305], + [0.540, 0.110, 0.105], + [0.533, 0.113, 0.112], + [0.930, 0.326, 0.314], + [1.463, 0.392, 0.374], + [0.994, 0.389, 0.386], + [0.763, 0.471, 0.499], + [1.890, 1.271, 1.280], + [1.335, 0.642, 1.776], + [3.886, 2.448, 2.370], + [0.452, 0.052, 0.029], + [5.834, 0.380, 0.374], + [6.645, 0.509, 0.385], + [12.401, 1.204, 1.146], + [12.727, 0.502, 0.513], + [1.628, 0.154, 0.128], + [0.739, 0.121, 0.144], + [1.632, 0.173, 0.127], + [5.856, 0.495, 0.568], + [4.960, 0.685, 0.784], + [1.619, 0.974, 1.638], + [1.420, 0.377, 0.361], + [3.379, 0.596, 0.580], + [5.797, 4.241, 4.280], + [6.864, 2.376, 2.224], + [6.834, 2.112, 2.118], + [0.996, 0.890, 0.947], + [0.286, 0.226, 0.218], + [0.110, 0.080, 0.077], + [0.146, 0.075, 0.068], + [0.531, 0.500, 0.438], + [0.076, 0.025, 0.027], + [0.054, 0.021, 0.022], + [0.033, 0.004, 0.004] + ] + }, + + { + "system": "Yandex Cloud Cascade Lake, 64 vCPU (32 threads), 128 GB RAM, 4 TB SSD", + "time": "2020-01-13 00:00:00", + "result": + [ + [0.054, 0.002, 0.002], + [0.140, 0.009, 0.015], + [0.139, 0.017, 0.020], + [0.430, 0.022, 0.022], + [0.453, 0.083, 0.082], + [0.839, 0.160, 0.159], + [0.058, 0.010, 0.010], + [0.048, 0.009, 0.008], + [0.706, 0.307, 0.288], + [0.821, 0.328, 0.301], + [0.509, 0.108, 0.106], + [0.534, 0.117, 0.116], + [0.905, 0.318, 0.313], + [1.573, 0.429, 0.413], + [0.960, 0.410, 0.403], + [0.769, 0.619, 0.521], + [1.914, 1.335, 1.272], + [1.279, 0.657, 1.215], + [3.839, 2.264, 2.481], + [0.425, 0.064, 0.027], + [5.605, 0.344, 0.367], + [6.389, 0.382, 0.403], + [11.794, 0.894, 0.878], + [11.730, 0.536, 0.436], + [1.540, 0.120, 0.109], + [0.715, 0.091, 0.106], + [1.553, 0.132, 0.132], + [5.580, 0.375, 0.350], + [4.720, 0.511, 0.480], + [1.025, 0.953, 1.008], + [1.475, 0.359, 0.357], + [3.457, 0.504, 0.495], + [4.688, 3.581, 3.673], + [6.325, 1.913, 1.865], + [6.338, 1.933, 2.030], + [0.961, 0.785, 0.847], + [0.267, 0.221, 0.215], + [0.095, 0.071, 0.078], + [0.148, 0.065, 0.071], + [0.516, 0.471, 0.432], + [0.076, 0.028, 0.025], + [0.053, 0.018, 0.021], + [0.034, 0.004, 0.004] + ] + } +] diff --git a/website/benchmark/hardware/results/003_yandex_cloud_cascade_lake_4_vcpu.json b/website/benchmark/hardware/results/003_yandex_cloud_cascade_lake_4_vcpu.json new file mode 100644 index 00000000000..e10d567d86b --- /dev/null +++ b/website/benchmark/hardware/results/003_yandex_cloud_cascade_lake_4_vcpu.json @@ -0,0 +1,52 @@ +[ + { + "system": "Yandex Cloud Cascade Lake, 4 vCPU (2 threads), 16 GB RAM, 30 GB SSD", + "time": "2020-01-13 00:00:00", + "result": + [ + [0.621, 0.002, 0.002], + [0.288, 0.035, 0.030], + [1.023, 0.126, 0.132], + [5.152, 0.219, 0.194], + [0.458, 0.427, 0.447], + [6.848, 1.223, 1.232], + [0.271, 0.077, 0.058], + [0.130, 0.044, 0.032], + [3.722, 2.145, 2.159], + [2.571, 2.459, 2.490], + [0.764, 0.679, 0.721], + [0.892, 0.816, 0.816], + [5.743, 3.467, 3.294], + [5.177, 4.540, 4.596], + [5.294, 4.565, 4.510], + [5.109, 3.902, 3.845], + [14.256, 12.943, 12.882], + [8.741, 8.056, 9.738], + [30.649, 26.987, 26.702], + [2.063, 0.183, 0.239], + [54.740, 3.602, 3.559], + [54.077, 6.038, 4.264], + [107.285, 11.156, 9.986], + [114.734, 4.735, 4.673], + [15.581, 1.257, 1.249], + [3.779, 1.002, 0.992], + [4.864, 1.305, 1.305], + [55.450, 3.348, 3.230], + [46.372, 5.424, 5.263], + [6.437, 6.404, 6.179], + [11.933, 3.524, 3.546], + [20.803, 5.352, 5.216], + [43.065, 41.106, 41.870], + [58.396, 16.545, 16.610], + [51.752, 16.329, 16.221], + [6.722, 6.256, 6.391], + [0.533, 0.241, 0.237], + [0.113, 0.085, 0.077], + [0.093, 0.083, 0.074], + [0.624, 0.497, 0.492], + [0.286, 0.036, 0.028], + [0.088, 0.022, 0.021], + [0.099, 0.005, 0.005] + ] + } +] diff --git a/website/benchmark/hardware/results/004_yandex_cloud_broadwell_4_vcpu.json b/website/benchmark/hardware/results/004_yandex_cloud_broadwell_4_vcpu.json new file mode 100644 index 00000000000..c3b4f230aab --- /dev/null +++ b/website/benchmark/hardware/results/004_yandex_cloud_broadwell_4_vcpu.json @@ -0,0 +1,52 @@ +[ + { + "system": "Yandex Cloud Broadwell, 4 vCPU (2 threads), 16 GB RAM, 30 GB SSD", + "time": "2020-01-14 00:00:00", + "result": + [ + [0.507, 0.002, 0.002], + [0.267, 0.035, 0.034], + [0.970, 0.120, 0.121], + [4.600, 0.200, 0.194], + [0.554, 0.469, 0.462], + [7.314, 1.276, 1.251], + [0.164, 0.062, 0.063], + [0.159, 0.035, 0.036], + [5.551, 1.935, 1.946], + [2.291, 2.170, 2.188], + [0.718, 0.653, 0.686], + [0.841, 0.796, 0.776], + [7.636, 2.906, 2.849], + [6.644, 4.234, 3.796], + [3.847, 3.080, 3.029], + [4.308, 3.285, 3.214], + [9.768, 8.793, 8.694], + [6.103, 5.225, 6.072], + [20.421, 17.609, 17.372], + [2.141, 0.182, 0.189], + [55.415, 3.527, 3.553], + [55.961, 4.545, 4.011], + [106.069, 9.063, 8.975], + [116.871, 4.638, 4.542], + [16.100, 1.818, 1.186], + [2.543, 0.950, 0.933], + [5.086, 1.199, 1.192], + [55.720, 3.259, 3.240], + [46.784, 5.170, 5.190], + [6.505, 6.229, 6.191], + [11.382, 2.817, 2.863], + [22.205, 4.495, 4.348], + [34.430, 27.314, 27.662], + [58.643, 14.066, 14.196], + [50.675, 14.220, 13.868], + [5.674, 5.107, 5.219], + [0.577, 0.293, 0.272], + [0.151, 0.098, 0.094], + [0.107, 0.094, 0.089], + [0.692, 0.582, 0.610], + [0.302, 0.040, 0.036], + [0.101, 0.027, 0.026], + [0.094, 0.006, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/005_amd_epyc_7551.json b/website/benchmark/hardware/results/005_amd_epyc_7551.json new file mode 100644 index 00000000000..1af296d47e8 --- /dev/null +++ b/website/benchmark/hardware/results/005_amd_epyc_7551.json @@ -0,0 +1,52 @@ +[ + { + "system": "Dell PowerEdge R6415 DX180 AMD EPYC™ 7551P 32-Core Naples (Zen), 128 GB RAM, 2x SSD 960 GB RAID-1", + "time": "2020-01-13 00:00:00", + "result": + [ + [0.007, 0.002, 0.001], + [0.030, 0.016, 0.014], + [0.042, 0.026, 0.026], + [0.078, 0.043, 0.042], + [0.143, 0.120, 0.117], + [0.239, 0.198, 0.198], + [0.022, 0.014, 0.014], + [0.016, 0.013, 0.015], + [0.388, 0.380, 0.384], + [0.476, 0.429, 0.411], + [0.201, 0.192, 0.191], + [0.204, 0.207, 0.192], + [0.676, 0.654, 0.637], + [0.890, 0.932, 0.940], + [0.730, 0.789, 0.738], + [0.658, 0.641, 0.678], + [1.556, 1.430, 1.529], + [0.819, 1.096, 0.906], + [3.569, 3.626, 3.508], + [0.083, 0.047, 0.077], + [0.812, 1.010, 0.601], + [1.097, 0.847, 0.864], + [2.654, 3.146, 3.169], + [1.595, 0.922, 0.877], + [0.259, 0.227, 0.236], + [0.206, 0.187, 0.181], + [0.245, 0.235, 0.232], + [0.974, 1.018, 1.012], + [1.280, 1.398, 1.243], + [2.171, 2.270, 2.284], + [0.594, 0.592, 0.602], + [0.976, 0.946, 0.966], + [4.543, 4.471, 4.364], + [3.844, 4.052, 3.858], + [3.932, 3.961, 3.982], + [1.128, 1.117, 1.146], + [0.233, 0.216, 0.221], + [0.088, 0.082, 0.085], + [0.075, 0.070, 0.070], + [0.465, 0.445, 0.435], + [0.036, 0.026, 0.031], + [0.028, 0.024, 0.021], + [0.010, 0.006, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/006_xeon_sp_gold.json b/website/benchmark/hardware/results/006_xeon_sp_gold.json new file mode 100644 index 00000000000..a0fc33cb0b2 --- /dev/null +++ b/website/benchmark/hardware/results/006_xeon_sp_gold.json @@ -0,0 +1,52 @@ +[ + { + "system": "Dell PowerEdge R640 DX292 2x Xeon SP Gold 16-Core 2.10GHz, 196 GB RAM, 2x SSD 960 GB RAID-1", + "time": "2020-01-13 00:00:00", + "result": + [ + [0.005, 0.003, 0.003], + [0.035, 0.013, 0.016], + [0.043, 0.023, 0.023], + [0.076, 0.030, 0.027], + [0.109, 0.087, 0.098], + [0.184, 0.154, 0.151], + [0.030, 0.017, 0.016], + [0.018, 0.017, 0.016], + [0.346, 0.357, 0.375], + [0.467, 0.397, 0.410], + [0.165, 0.135, 0.137], + [0.166, 0.146, 0.143], + [0.452, 0.432, 0.415], + [0.543, 0.523, 0.527], + [0.508, 0.489, 0.472], + [0.638, 0.551, 0.549], + [1.280, 1.231, 1.272], + [0.680, 0.748, 0.611], + [2.380, 2.465, 2.351], + [0.073, 0.065, 0.040], + [0.724, 0.371, 0.376], + [0.805, 0.474, 0.450], + [1.547, 1.064, 1.117], + [1.798, 0.543, 0.507], + [0.217, 0.145, 0.142], + [0.139, 0.122, 0.133], + [0.221, 0.161, 0.159], + [0.730, 0.440, 0.449], + [0.875, 0.744, 0.721], + [1.307, 1.259, 1.318], + [0.457, 0.401, 0.404], + [0.716, 0.688, 0.617], + [4.147, 4.251, 3.844], + [2.082, 1.950, 2.187], + [2.109, 2.095, 1.930], + [0.875, 0.851, 0.848], + [0.233, 0.235, 0.221], + [0.103, 0.087, 0.086], + [0.087, 0.078, 0.078], + [0.452, 0.407, 0.403], + [0.047, 0.041, 0.054], + [0.036, 0.034, 0.035], + [0.013, 0.010, 0.010] + ] + } +] diff --git a/website/benchmark/hardware/results/007_xeon_e5_2650.json b/website/benchmark/hardware/results/007_xeon_e5_2650.json new file mode 100644 index 00000000000..5c2d79a34c2 --- /dev/null +++ b/website/benchmark/hardware/results/007_xeon_e5_2650.json @@ -0,0 +1,52 @@ +[ + { + "system": "E5-2650 v2 @ 2.60GHz, 2 sockets, 16 threads, 8xHDD RAID-5", + "time": "2020-01-12 00:00:00", + "result": + [ + [0.101, 0.002, 0.002], + [0.196, 0.019, 0.021], + [0.486, 0.035, 0.029], + [0.413, 0.045, 0.043], + [0.368, 0.134, 0.105], + [0.563, 0.282, 0.269], + [0.078, 0.030, 0.025], + [0.070, 0.019, 0.014], + [0.751, 0.522, 0.558], + [0.856, 0.549, 0.547], + [0.458, 0.155, 0.163], + [0.439, 0.169, 0.190], + [0.929, 0.699, 0.608], + [1.494, 0.863, 0.902], + [1.379, 0.778, 0.794], + [1.032, 0.832, 0.851], + [2.364, 1.974, 1.914], + [1.284, 1.140, 1.043], + [4.745, 4.279, 4.294], + [0.713, 0.085, 0.071], + [4.133, 0.775, 0.729], + [3.485, 0.924, 0.880], + [7.568, 1.808, 1.853], + [9.496, 1.115, 1.119], + [1.130, 0.209, 0.243], + [0.643, 0.225, 0.211], + [1.338, 0.293, 0.233], + [4.353, 0.803, 0.759], + [2.667, 1.158, 1.070], + [2.612, 1.753, 1.721], + [1.370, 0.641, 0.704], + [2.348, 0.977, 1.015], + [6.154, 5.822, 5.696], + [4.553, 3.076, 3.232], + [4.647, 2.960, 3.249], + [1.441, 1.424, 1.285], + [0.560, 0.303, 0.245], + [0.223, 0.082, 0.084], + [0.275, 0.078, 0.076], + [0.929, 0.487, 0.416], + [0.362, 0.033, 0.049], + [0.179, 0.035, 0.022], + [0.075, 0.013, 0.013] + ] + } +] diff --git a/website/benchmark/hardware/results/008_skylake_kvm.json b/website/benchmark/hardware/results/008_skylake_kvm.json new file mode 100644 index 00000000000..2ddff47a460 --- /dev/null +++ b/website/benchmark/hardware/results/008_skylake_kvm.json @@ -0,0 +1,52 @@ +[ + { + "system": "Time4vps.eu VPS (KVM) Linux Ubuntu 4 Core (Skylake) 16GB RAM 160GB Disk", + "time": "2020-01-13 00:00:00", + "result": + [ + [0.068, 0.002, 0.002], + [0.124, 0.021, 0.025], + [0.594, 0.089, 0.077], + [2.300, 0.133, 0.090], + [2.710, 0.205, 0.212], + [5.203, 0.603, 0.610], + [0.090, 0.029, 0.036], + [0.118, 0.021, 0.022], + [5.977, 1.295, 1.206], + [3.909, 1.415, 1.452], + [2.551, 0.336, 0.324], + [3.123, 0.446, 0.409], + [4.075, 1.743, 1.661], + [6.427, 2.499, 2.487], + [5.775, 2.156, 2.431], + [3.322, 2.288, 2.276], + [8.642, 6.463, 6.690], + [6.365, 3.852, 3.757], + [20.426, 13.849, 13.695], + [2.507, 0.105, 0.100], + [30.691, 1.747, 1.699], + [30.206, 2.010, 1.943], + [57.155, 4.699, 4.859], + [50.924, 2.173, 2.119], + [10.907, 0.660, 0.686], + [3.636, 0.505, 0.524], + [8.388, 0.683, 0.627], + [27.423, 1.650, 1.703], + [21.309, 2.824, 2.821], + [4.227, 4.053, 4.037], + [8.198, 1.797, 1.776], + [18.853, 2.927, 2.881], + [22.254, 21.156, 20.854], + [29.323, 8.728, 8.621], + [27.889, 8.759, 9.063], + [4.121, 3.837, 3.934], + [0.452, 0.292, 0.247], + [0.221, 0.093, 0.090], + [0.331, 0.069, 0.074], + [0.703, 0.469, 0.506], + [0.211, 0.026, 0.027], + [0.134, 0.021, 0.021], + [0.121, 0.007, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/009_core_i5_3210M_lenovo_b580.json b/website/benchmark/hardware/results/009_core_i5_3210M_lenovo_b580.json new file mode 100644 index 00000000000..dbbbf93b338 --- /dev/null +++ b/website/benchmark/hardware/results/009_core_i5_3210M_lenovo_b580.json @@ -0,0 +1,52 @@ +[ + { + "system": "Lenovo B580 Laptop (i5-3210M)", + "time": "2020-01-11 00:00:00", + "result": + [ + [0.035, 0.003, 0.005], + [0.093, 0.064, 0.060], + [0.265, 0.170, 0.167], + [0.880, 0.251, 0.266], + [0.954, 0.593, 0.561], + [2.140, 1.506, 1.525], + [0.148, 0.096, 0.105], + [0.064, 0.048, 0.044], + [2.727, 2.330, 2.280], + [3.386, 3.210, 2.951], + [1.218, 0.787, 0.749], + [1.293, 0.915, 0.904], + [3.713, 3.224, 3.190], + [4.943, 4.338, 4.310], + [4.503, 3.999, 3.918], + [4.001, 3.686, 4.144], + [10.714, 10.011, 10.035], + [7.456, 6.556, 6.675], + [20.201, 19.238, 19.135], + [0.888, 0.217, 0.209], + [9.685, 4.144, 4.023], + [11.201, 4.648, 4.636], + [21.037, 10.712, 10.571], + [18.186, 4.743, 4.743], + [2.844, 1.379, 1.358], + [1.623, 1.138, 1.130], + [2.861, 1.394, 1.417], + [9.691, 4.191, 4.129], + [10.285, 7.381, 7.379], + [6.879, 6.871, 6.829], + [4.131, 3.336, 3.240], + [7.157, 4.666, 4.616], + [29.371, 36.392, 29.946], + [17.929, 14.223, 14.127], + [17.058, 13.998, 14.055], + [5.667, 5.460, 5.408], + [0.325, 0.230, 0.217], + [0.115, 0.101, 0.094], + [0.148, 0.093, 0.084], + [0.585, 0.464, 0.459], + [0.078, 0.042, 0.035], + [0.057, 0.038, 0.032], + [0.024, 0.011, 0.010] + ] + } +] diff --git a/website/benchmark/hardware/results/010_xeon_e5_2640v4.json b/website/benchmark/hardware/results/010_xeon_e5_2640v4.json new file mode 100644 index 00000000000..d81c9a2a2a0 --- /dev/null +++ b/website/benchmark/hardware/results/010_xeon_e5_2640v4.json @@ -0,0 +1,52 @@ +[ + { + "system": "Dell PowerEdge R730xd, 2 socket 10 cores E5-2640 v4, HW RAID5 3TBx12 SATA", + "time": "2020-01-14 00:00:00", + "result": + [ + [0.225, 0.001, 0.002], + [0.534, 0.010, 0.010], + [0.229, 0.025, 0.026], + [0.530, 0.042, 0.040], + [0.265, 0.094, 0.090], + [0.685, 0.224, 0.219], + [0.172, 0.013, 0.013], + [0.181, 0.010, 0.011], + [0.908, 0.418, 0.424], + [0.725, 0.450, 0.462], + [0.517, 0.138, 0.139], + [0.445, 0.168, 0.161], + [1.065, 0.585, 0.584], + [1.325, 0.756, 0.747], + [1.184, 0.627, 0.637], + [0.905, 0.676, 0.699], + [2.101, 1.848, 1.775], + [1.275, 0.927, 0.988], + [5.285, 4.201, 4.088], + [0.465, 0.040, 0.071], + [7.380, 0.557, 0.538], + [7.636, 0.665, 0.718], + [13.905, 1.685, 1.645], + [18.739, 0.828, 0.790], + [1.950, 0.195, 0.185], + [0.549, 0.163, 0.156], + [1.384, 0.205, 0.190], + [7.199, 0.650, 0.611], + [6.514, 0.935, 0.885], + [2.154, 2.034, 2.031], + [1.538, 0.546, 0.525], + [3.711, 0.916, 0.936], + [5.993, 4.973, 5.183], + [8.215, 2.759, 2.741], + [8.162, 2.795, 2.772], + [1.347, 1.074, 1.051], + [0.478, 0.208, 0.204], + [0.147, 0.077, 0.074], + [0.197, 0.066, 0.066], + [0.694, 0.438, 0.453], + [0.217, 0.024, 0.033], + [0.137, 0.032, 0.020], + [0.058, 0.006, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/011_yandex_managed_clickhouse_s3_3xlarge.json b/website/benchmark/hardware/results/011_yandex_managed_clickhouse_s3_3xlarge.json new file mode 100644 index 00000000000..5b0c38ec60d --- /dev/null +++ b/website/benchmark/hardware/results/011_yandex_managed_clickhouse_s3_3xlarge.json @@ -0,0 +1,103 @@ +[ + { + "system": "Yandex Managed ClickHouse, s3.3xlarge, Cascade Lake 32 vCPU, 128 GB RAM, 1 TB local SSD", + "time": "2020-01-14 00:00:00", + "result": + [ + [0.039, 0.041, 0.046], + [0.079, 0.084, 0.081], + [0.069, 0.068, 0.060], + [0.067, 0.065, 0.073], + [0.114, 0.116, 0.115], + [0.176, 0.171, 0.166], + [0.055, 0.058, 0.065], + [0.060, 0.061, 0.057], + [0.290, 0.290, 0.289], + [0.323, 0.315, 0.309], + [0.128, 0.124, 0.137], + [0.144, 0.136, 0.136], + [0.344, 0.326, 0.325], + [0.402, 0.392, 0.431], + [0.430, 0.436, 0.414], + [0.509, 0.485, 0.485], + [1.233, 1.151, 1.167], + [0.652, 0.651, 0.631], + [2.078, 2.076, 2.046], + [0.070, 0.071, 0.070], + [0.358, 0.327, 0.355], + [0.428, 0.403, 0.405], + [0.974, 0.986, 0.976], + [0.532, 0.549, 0.500], + [0.164, 0.169, 0.158], + [0.128, 0.130, 0.138], + [0.166, 0.169, 0.159], + [0.428, 0.449, 0.471], + [0.586, 0.598, 0.568], + [1.115, 1.115, 1.147], + [0.342, 0.344, 0.342], + [0.481, 0.473, 0.470], + [4.436, 3.273, 3.320], + [1.661, 1.542, 1.545], + [1.573, 1.875, 1.576], + [0.600, 0.566, 0.586], + [0.261, 0.255, 0.251], + [0.134, 0.136, 0.120], + [0.135, 0.133, 0.132], + [0.525, 0.531, 0.521], + [0.073, 0.071, 0.068], + [0.060, 0.071, 0.071], + [0.051, 0.057, 0.050] + ] + }, + + { + "system": "Yandex Managed ClickHouse, s3.3xlarge, Cascade Lake 32 vCPU, 128 GB RAM, 12.5 TB local HDD", + "time": "2020-01-14 00:00:00", + "result": + [ + [0.049, 0.049, 0.045], + [0.297, 0.079, 0.068], + [0.350, 0.059, 0.065], + [1.099, 0.068, 0.064], + [0.120, 0.123, 0.117], + [1.847, 0.202, 0.191], + [0.124, 0.056, 0.060], + [0.062, 0.058, 0.055], + [0.681, 0.321, 0.312], + [0.346, 0.349, 0.368], + [0.200, 0.140, 0.145], + [0.227, 0.142, 0.152], + [0.447, 0.410, 0.414], + [0.499, 0.570, 0.487], + [0.658, 0.467, 0.465], + [0.587, 0.576, 0.580], + [1.376, 1.340, 1.290], + [0.746, 0.748, 0.717], + [3.608, 2.427, 2.455], + [0.073, 0.106, 0.076], + [9.138, 0.422, 0.386], + [0.615, 0.463, 0.449], + [12.166, 1.067, 1.094], + [13.856, 0.735, 0.548], + [0.194, 0.187, 0.179], + [0.145, 0.140, 0.151], + [0.185, 0.180, 0.182], + [0.493, 0.507, 0.498], + [9.379, 0.618, 0.617], + [1.182, 1.187, 1.268], + [1.088, 0.402, 0.407], + [3.943, 0.646, 0.538], + [4.547, 3.858, 4.407], + [13.240, 1.776, 1.808], + [1.760, 1.823, 1.918], + [0.719, 0.693, 0.729], + [0.346, 0.309, 0.279], + [0.151, 0.143, 0.134], + [0.191, 0.141, 0.149], + [0.804, 0.550, 0.686], + [0.147, 0.093, 0.086], + [0.099, 0.085, 0.084], + [0.056, 0.059, 0.057] + ] + } +] diff --git a/website/benchmark/hardware/results/012_dell_r530.json b/website/benchmark/hardware/results/012_dell_r530.json new file mode 100644 index 00000000000..93d7234df55 --- /dev/null +++ b/website/benchmark/hardware/results/012_dell_r530.json @@ -0,0 +1,52 @@ +[ + { + "system": "Dell R530, 128GB DDR4, 2x480 GB SATA SSD, Perc H730 RAID-1", + "time": "2020-01-14 00:00:00", + "result": + [ + [0.027, 0.002, 0.002], + [0.147, 0.017, 0.016], + [0.328, 0.034, 0.033], + [1.059, 0.050, 0.044], + [1.334, 0.123, 0.118], + [2.579, 0.239, 0.264], + [0.057, 0.020, 0.019], + [0.036, 0.019, 0.018], + [2.079, 0.648, 0.569], + [2.012, 0.631, 0.634], + [1.454, 0.158, 0.160], + [1.502, 0.178, 0.185], + [3.095, 0.722, 0.661], + [3.675, 0.816, 0.809], + [2.900, 0.903, 0.810], + [2.005, 0.861, 0.842], + [4.103, 1.983, 2.004], + [2.948, 1.200, 1.160], + [7.687, 4.411, 4.239], + [1.087, 0.054, 0.062], + [14.186, 0.651, 0.757], + [16.497, 0.739, 0.676], + [23.165, 1.703, 1.700], + [22.803, 0.898, 0.919], + [4.247, 0.317, 0.267], + [2.519, 0.214, 0.246], + [4.115, 0.316, 0.274], + [13.759, 0.805, 0.827], + [16.473, 1.215, 1.062], + [2.034, 1.870, 2.016], + [3.152, 0.677, 0.697], + [6.630, 1.216, 1.019], + [9.651, 6.131, 6.017], + [23.506, 3.416, 3.294], + [23.271, 3.547, 3.411], + [1.763, 1.344, 1.308], + [0.317, 0.215, 0.227], + [0.122, 0.090, 0.087], + [0.168, 0.074, 0.090], + [0.565, 0.419, 0.450], + [0.079, 0.037, 0.030], + [0.059, 0.032, 0.032], + [0.025, 0.015, 0.010] + ] + } +] diff --git a/website/benchmark/hardware/results/013_xeon_2176g.json b/website/benchmark/hardware/results/013_xeon_2176g.json new file mode 100644 index 00000000000..7df8050fc81 --- /dev/null +++ b/website/benchmark/hardware/results/013_xeon_2176g.json @@ -0,0 +1,52 @@ +[ + { + "system": "Xeon 2176G, 64GB RAM, 2xSSD 960GB (SAMSUNG MZQLB960HAJR-00007), ZFS RAID-1", + "time": "2020-01-14 00:00:00", + "result": + [ + [0.001, 0.001, 0.001], + [0.010, 0.011, 0.009], + [0.035, 0.031, 0.033], + [0.058, 0.056, 0.058], + [0.113, 0.126, 0.121], + [0.296, 0.300, 0.301], + [0.017, 0.016, 0.016], + [0.009, 0.009, 0.011], + [0.660, 0.659, 0.655], + [0.775, 0.746, 0.737], + [0.185, 0.181, 0.184], + [0.219, 0.237, 0.243], + [0.943, 0.933, 0.952], + [1.228, 1.185, 1.201], + [0.975, 0.963, 0.971], + [1.068, 1.086, 1.077], + [2.704, 2.713, 2.725], + [1.596, 1.564, 1.562], + [5.653, 5.571, 5.581], + [0.072, 0.065, 0.062], + [1.209, 0.958, 0.951], + [1.383, 1.222, 1.224], + [3.261, 2.771, 2.776], + [1.586, 1.210, 1.196], + [0.417, 0.392, 0.325], + [0.271, 0.268, 0.267], + [0.340, 0.338, 0.337], + [1.376, 1.160, 1.134], + [1.928, 1.643, 1.697], + [3.167, 3.135, 3.149], + [0.947, 0.859, 0.858], + [1.566, 1.446, 1.467], + [8.005, 8.065, 7.980], + [4.640, 4.322, 4.277], + [4.410, 4.330, 4.300], + [1.811, 1.749, 1.767], + [0.138, 0.142, 0.144], + [0.052, 0.047, 0.048], + [0.042, 0.043, 0.041], + [0.271, 0.249, 0.245], + [0.030, 0.016, 0.016], + [0.014, 0.013, 0.013], + [0.004, 0.004, 0.004] + ] + } +] diff --git a/website/benchmark/hardware/results/014_azure_ds3v2.json b/website/benchmark/hardware/results/014_azure_ds3v2.json new file mode 100644 index 00000000000..ed8d21be847 --- /dev/null +++ b/website/benchmark/hardware/results/014_azure_ds3v2.json @@ -0,0 +1,102 @@ +[ + { + "system": "Azure DS3v2 4vcpu 14GB RAM 1TB Standard SSD", + "time": "2020-01-15 00:00:00", + "result": + [ + [0.709, 0.004, 0.004], + [1.052, 0.028, 0.025], + [2.075, 0.077, 0.080], + [2.700, 0.104, 0.101], + [2.858, 0.267, 0.259], + [4.058, 0.737, 0.718], + [0.597, 0.038, 0.049], + [0.598, 0.025, 0.024], + [3.786, 1.324, 1.313], + [3.982, 1.579, 1.562], + [2.995, 0.395, 0.395], + [3.279, 0.467, 0.470], + [4.301, 1.674, 1.690], + [6.499, 2.126, 2.132], + [4.774, 1.886, 1.927], + [3.484, 1.872, 1.818], + [7.813, 4.801, 5.006], + [6.032, 3.162, 3.106], + [13.991, 10.573, 10.665], + [2.750, 0.118, 0.101], + [25.608, 1.978, 1.960], + [29.117, 2.297, 2.303], + [53.220, 5.367, 5.325], + [51.767, 2.669, 2.465], + [7.509, 0.890, 0.865], + [3.827, 0.666, 0.653], + [7.574, 0.918, 0.899], + [25.753, 1.904, 1.898], + [21.624, 3.269, 3.192], + [5.454, 4.966, 4.975], + [6.569, 1.870, 1.912], + [14.536, 2.844, 2.863], + [18.908, 16.591, 16.820], + [27.527, 7.790, 7.738], + [27.556, 7.694, 7.695], + [4.168, 3.568, 3.426], + [1.185, 0.307, 0.252], + [0.483, 0.096, 0.093], + [0.519, 0.086, 0.088], + [1.274, 0.525, 0.496], + [1.048, 0.033, 0.034], + [0.379, 0.027, 0.036], + [0.599, 0.010, 0.009] + ] + }, + { + "system": "Azure DS3v2 4vcpu 14GB RAM 1TB Premium SSD", + "time": "2020-01-15 00:00:00", + "result": + [ + [0.047, 0.004, 0.003], + [0.078, 0.023, 0.023], + [0.312, 0.077, 0.077], + [1.202, 0.105, 0.103], + [1.216, 0.260, 0.264], + [1.896, 0.751, 0.726], + [0.122, 0.041, 0.038], + [0.095, 0.028, 0.025], + [1.848, 1.304, 1.375], + [2.104, 1.534, 1.535], + [1.298, 0.394, 0.397], + [1.363, 0.469, 0.479], + [2.296, 1.728, 1.650], + [3.540, 2.320, 2.177], + [2.542, 1.863, 1.847], + [2.047, 1.861, 1.873], + [5.203, 4.830, 4.882], + [3.466, 3.131, 3.197], + [10.795, 10.396, 10.516], + [1.244, 0.111, 0.105], + [13.163, 2.019, 1.932], + [14.969, 2.346, 2.340], + [27.664, 5.259, 5.309], + [26.819, 2.589, 2.464], + [3.795, 0.902, 0.866], + [1.867, 0.665, 0.672], + [3.822, 0.919, 0.903], + [13.173, 1.916, 1.886], + [11.168, 3.253, 3.214], + [5.126, 5.290, 4.982], + [3.465, 1.866, 1.875], + [7.902, 3.009, 2.803], + [17.132, 17.154, 17.387], + [15.132, 7.755, 7.678], + [15.054, 7.779, 8.068], + [3.598, 3.590, 3.501], + [0.483, 0.279, 0.263], + [0.183, 0.094, 0.095], + [0.174, 0.084, 0.096], + [0.693, 0.480, 0.503], + [0.237, 0.038, 0.031], + [0.108, 0.029, 0.028], + [0.096, 0.010, 0.009] + ] + } +] diff --git a/website/benchmark/hardware/results/015_aws_i3_8xlarge.json b/website/benchmark/hardware/results/015_aws_i3_8xlarge.json new file mode 100644 index 00000000000..ddb87c457a5 --- /dev/null +++ b/website/benchmark/hardware/results/015_aws_i3_8xlarge.json @@ -0,0 +1,52 @@ +[ + { + "system": "AWS i3.8xlarge 32vCPU 244GiB 4x1900 NVMe SSD", + "time": "2020-01-15 00:00:00", + "result": + [ + [0.009, 0.002, 0.002], + [0.053, 0.040, 0.021], + [0.043, 0.028, 0.027], + [0.109, 0.036, 0.035], + [0.147, 0.108, 0.100], + [0.296, 0.239, 0.239], + [0.017, 0.013, 0.015], + [0.013, 0.010, 0.011], + [0.524, 0.460, 0.445], + [0.589, 0.519, 0.510], + [0.186, 0.142, 0.140], + [0.210, 0.167, 0.164], + [0.659, 0.584, 0.529], + [0.781, 0.679, 0.665], + [0.709, 0.630, 0.613], + [0.642, 0.590, 0.588], + [1.723, 1.564, 1.557], + [1.027, 0.925, 0.909], + [3.618, 3.432, 3.411], + [0.123, 0.037, 0.049], + [1.318, 0.587, 0.570], + [1.368, 0.655, 0.646], + [2.847, 1.518, 1.495], + [2.431, 0.812, 0.764], + [0.366, 0.213, 0.193], + [0.237, 0.167, 0.158], + [0.374, 0.204, 0.211], + [1.310, 0.590, 0.597], + [1.260, 0.877, 0.870], + [1.966, 1.952, 1.967], + [0.692, 0.571, 0.566], + [1.080, 0.823, 0.827], + [5.017, 4.816, 4.843], + [3.072, 2.661, 2.726], + [3.006, 2.711, 2.688], + [1.071, 0.999, 1.024], + [0.231, 0.221, 0.221], + [0.094, 0.090, 0.086], + [0.093, 0.085, 0.075], + [0.488, 0.432, 0.451], + [0.046, 0.029, 0.030], + [0.030, 0.023, 0.022], + [0.012, 0.007, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/016_aws_m5d_24xlarge.json b/website/benchmark/hardware/results/016_aws_m5d_24xlarge.json new file mode 100644 index 00000000000..f1b80c6269b --- /dev/null +++ b/website/benchmark/hardware/results/016_aws_m5d_24xlarge.json @@ -0,0 +1,52 @@ +[ + { + "system": "AWS m5d.24xlarge 96vCPU 384GiB 4x900 NVMe SSD", + "time": "2020-01-15 00:00:00", + "result": + [ + [0.012, 0.002, 0.002], + [0.061, 0.017, 0.008], + [0.043, 0.014, 0.014], + [0.160, 0.017, 0.016], + [0.193, 0.074, 0.075], + [0.300, 0.120, 0.118], + [0.023, 0.009, 0.009], + [0.015, 0.009, 0.009], + [0.321, 0.206, 0.203], + [0.351, 0.238, 0.244], + [0.205, 0.113, 0.112], + [0.211, 0.106, 0.091], + [0.394, 0.213, 0.211], + [0.519, 0.270, 0.259], + [0.439, 0.292, 0.286], + [0.394, 0.301, 0.296], + [1.195, 0.829, 0.806], + [0.561, 0.743, 0.418], + [1.841, 1.660, 1.650], + [0.163, 0.041, 0.026], + [1.632, 0.251, 0.269], + [1.885, 0.265, 0.265], + [3.425, 0.644, 0.620], + [3.839, 0.431, 0.367], + [0.486, 0.092, 0.086], + [0.256, 0.081, 0.091], + [0.493, 0.107, 0.106], + [1.646, 0.275, 0.255], + [1.445, 0.332, 0.332], + [0.768, 0.702, 0.721], + [0.509, 0.280, 0.268], + [1.071, 0.382, 0.374], + [2.800, 2.452, 2.389], + [2.159, 1.134, 1.181], + [2.153, 1.145, 1.200], + [0.516, 0.457, 0.493], + [0.256, 0.182, 0.188], + [0.091, 0.073, 0.070], + [0.121, 0.063, 0.064], + [0.506, 0.399, 0.421], + [0.055, 0.030, 0.027], + [0.041, 0.019, 0.023], + [0.016, 0.006, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/017_aws_i3en_24xlarge.json b/website/benchmark/hardware/results/017_aws_i3en_24xlarge.json new file mode 100644 index 00000000000..7f33b99b127 --- /dev/null +++ b/website/benchmark/hardware/results/017_aws_i3en_24xlarge.json @@ -0,0 +1,52 @@ +[ + { + "system": "AWS i3en.24xlarge 96vCPU 768GiB 8x7500 NVMe SSD", + "time": "2020-01-15 00:00:00", + "result": + [ + [0.010, 0.002, 0.002], + [0.067, 0.009, 0.009], + [0.040, 0.014, 0.013], + [0.120, 0.017, 0.017], + [0.159, 0.076, 0.077], + [0.240, 0.116, 0.119], + [0.020, 0.010, 0.009], + [0.015, 0.010, 0.009], + [0.279, 0.195, 0.197], + [0.299, 0.230, 0.258], + [0.199, 0.088, 0.111], + [0.185, 0.094, 0.094], + [0.327, 0.212, 0.206], + [0.439, 0.271, 0.267], + [0.370, 0.281, 0.280], + [0.367, 0.306, 0.312], + [1.092, 0.931, 1.022], + [0.533, 0.599, 0.413], + [1.629, 1.921, 1.572], + [0.130, 0.031, 0.026], + [1.451, 0.264, 0.269], + [1.714, 0.273, 0.261], + [3.668, 0.636, 0.669], + [3.837, 0.472, 0.402], + [0.378, 0.107, 0.079], + [0.199, 0.070, 0.088], + [0.381, 0.104, 0.086], + [1.426, 0.284, 0.272], + [1.246, 0.363, 0.360], + [0.737, 0.708, 0.741], + [0.426, 0.246, 0.284], + [0.877, 0.420, 0.384], + [2.698, 2.390, 2.375], + [1.918, 1.223, 1.122], + [1.909, 1.234, 1.217], + [0.486, 0.482, 0.473], + [0.235, 0.187, 0.200], + [0.083, 0.069, 0.072], + [0.111, 0.063, 0.062], + [0.473, 0.433, 0.406], + [0.050, 0.028, 0.027], + [0.038, 0.022, 0.021], + [0.012, 0.006, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/018_huawei_taishan_2280_v2.json b/website/benchmark/hardware/results/018_huawei_taishan_2280_v2.json new file mode 100644 index 00000000000..329ebe782b9 --- /dev/null +++ b/website/benchmark/hardware/results/018_huawei_taishan_2280_v2.json @@ -0,0 +1,52 @@ +[ + { + "system": "Huawei TaiShan 2280 v2 (AArch64) 64 core (2-die), one physical HDD", + "time": "2020-01-15 00:00:00", + "result": + [ + [0.356, 0.002, 0.002], + [0.333, 0.018, 0.017], + [0.608, 0.021, 0.021], + [1.885, 0.032, 0.032], + [0.598, 0.099, 0.097], + [2.884, 0.165, 0.167], + [0.356, 0.016, 0.014], + [0.349, 0.015, 0.015], + [0.981, 0.283, 0.296], + [0.783, 0.326, 0.328], + [0.580, 0.135, 0.136], + [0.511, 0.142, 0.142], + [1.060, 0.434, 0.438], + [1.069, 0.569, 0.566], + [1.116, 0.479, 0.479], + [0.825, 0.478, 0.486], + [1.899, 1.574, 1.590], + [1.260, 0.874, 0.849], + [5.456, 2.869, 2.903], + [0.418, 0.037, 0.034], + [19.336, 0.478, 0.494], + [22.442, 0.595, 0.595], + [45.958, 8.735, 1.363], + [41.321, 0.675, 0.706], + [6.074, 0.167, 0.159], + [0.925, 0.133, 0.133], + [1.151, 0.153, 0.152], + [19.627, 0.607, 0.622], + [16.496, 0.792, 0.787], + [1.770, 2.045, 1.242], + [4.827, 0.471, 0.466], + [7.695, 0.701, 0.647], + [5.246, 4.741, 4.676], + [20.496, 2.676, 2.628], + [20.338, 2.559, 2.557], + [1.696, 0.701, 0.724], + [0.665, 0.294, 0.302], + [0.402, 0.140, 0.137], + [0.366, 0.082, 0.086], + [0.867, 0.575, 0.552], + [0.334, 0.025, 0.025], + [0.333, 0.023, 0.022], + [0.340, 0.007, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/019_aws_m5ad_24xlarge.json b/website/benchmark/hardware/results/019_aws_m5ad_24xlarge.json new file mode 100644 index 00000000000..2b691153822 --- /dev/null +++ b/website/benchmark/hardware/results/019_aws_m5ad_24xlarge.json @@ -0,0 +1,52 @@ +[ + { + "system": "AWS m5ad.24xlarge 96vCPU 384GiB 4x900 NVMe SSD, AMD EPYC 7000 series 2.5 GHz", + "time": "2020-01-17 00:00:00", + "result": + [ + [0.013, 0.002, 0.002], + [0.055, 0.020, 0.025], + [0.054, 0.027, 0.026], + [0.154, 0.035, 0.035], + [0.221, 0.117, 0.118], + [0.325, 0.171, 0.166], + [0.042, 0.021, 0.017], + [0.025, 0.017, 0.018], + [0.353, 0.253, 0.253], + [0.477, 0.610, 0.720], + [0.257, 0.154, 0.139], + [0.251, 0.130, 0.114], + [0.513, 0.293, 0.286], + [0.618, 0.360, 0.350], + [0.468, 0.336, 0.329], + [0.390, 0.333, 0.411], + [1.112, 0.936, 1.497], + [2.434, 1.350, 0.886], + [2.590, 2.069, 2.331], + [0.160, 0.048, 0.036], + [1.638, 0.334, 0.312], + [1.841, 0.423, 0.373], + [3.673, 1.122, 1.078], + [3.808, 0.912, 0.494], + [0.480, 0.112, 0.120], + [0.248, 0.107, 0.099], + [0.470, 0.118, 0.114], + [1.648, 0.544, 0.469], + [1.418, 0.583, 0.624], + [0.966, 1.231, 0.999], + [0.539, 0.311, 0.370], + [1.159, 0.712, 0.716], + [3.755, 2.772, 2.973], + [2.748, 2.033, 2.242], + [2.842, 2.150, 2.019], + [0.784, 0.616, 0.641], + [0.304, 0.273, 0.235], + [0.106, 0.086, 0.093], + [0.117, 0.073, 0.075], + [0.604, 0.453, 0.502], + [0.050, 0.036, 0.034], + [0.043, 0.023, 0.027], + [0.013, 0.008, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/020_core_i7_8550u_lenovo_x1.json b/website/benchmark/hardware/results/020_core_i7_8550u_lenovo_x1.json new file mode 100644 index 00000000000..57d1d8198ee --- /dev/null +++ b/website/benchmark/hardware/results/020_core_i7_8550u_lenovo_x1.json @@ -0,0 +1,52 @@ +[ + { + "system": "Lenovo Thinkpad X1 Carbon 6th Gen i7-8550U CPU @ 1.80GHz 4 threads, 16 GiB", + "time": "2020-01-18 00:00:00", + "result": + [ + [0.006, 0.002, 0.002], + [0.031, 0.019, 0.020], + [0.082, 0.078, 0.080], + [0.157, 0.093, 0.092], + [0.274, 0.214, 0.206], + [0.601, 0.513, 0.513], + [0.038, 0.045, 0.041], + [0.023, 0.018, 0.018], + [1.394, 1.378, 1.323], + [1.567, 1.496, 1.483], + [0.406, 0.328, 0.327], + [0.468, 0.414, 0.397], + [1.846, 1.753, 1.737], + [2.492, 2.423, 2.404], + [2.136, 2.064, 2.078], + [2.038, 1.971, 1.971], + [5.794, 5.679, 5.708], + [3.430, 3.498, 3.356], + [11.946, 11.738, 11.700], + [0.158, 0.105, 0.091], + [2.151, 1.551, 1.593], + [2.581, 1.990, 1.985], + [6.101, 5.390, 5.320], + [3.528, 2.341, 2.322], + [0.772, 0.699, 0.701], + [0.606, 0.583, 0.587], + [0.877, 0.723, 0.728], + [2.398, 1.916, 1.924], + [3.634, 3.272, 3.247], + [4.102, 4.082, 4.078], + [1.885, 1.784, 1.741], + [2.994, 2.691, 2.707], + [19.060, 18.852, 18.929], + [8.745, 8.476, 8.553], + [8.685, 8.406, 8.946], + [3.416, 3.426, 3.397], + [0.238, 0.234, 0.210], + [0.080, 0.071, 0.072], + [0.078, 0.066, 0.066], + [0.470, 0.407, 0.396], + [0.034, 0.030, 0.029], + [0.025, 0.021, 0.021], + [0.010, 0.007, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/021_xeon_e5645.json b/website/benchmark/hardware/results/021_xeon_e5645.json new file mode 100644 index 00000000000..2f3f96232de --- /dev/null +++ b/website/benchmark/hardware/results/021_xeon_e5645.json @@ -0,0 +1,52 @@ +[ + { + "system": "E5645 @ 2.40GHz, 2 sockets, 12 threads, 96 GiB, 14 x 2TB HDD RAID-10", + "time": "2020-01-18 00:00:00", + "result": + [ + [0.061, 0.003, 0.003], + [0.203, 0.026, 0.019], + [0.231, 0.056, 0.060], + [0.533, 0.080, 0.099], + [0.458, 0.202, 0.213], + [0.723, 0.468, 0.411], + [0.143, 0.034, 0.029], + [0.117, 0.025, 0.023], + [1.033, 0.810, 0.745], + [1.165, 0.916, 0.898], + [0.514, 0.249, 0.297], + [0.600, 0.343, 0.385], + [1.294, 1.156, 1.221], + [1.859, 1.459, 1.384], + [1.627, 1.349, 1.346], + [1.414, 1.269, 1.306], + [3.798, 3.774, 3.631], + [2.177, 2.054, 2.016], + [7.002, 6.187, 6.263], + [0.461, 0.081, 0.116], + [3.860, 1.296, 1.330], + [4.705, 1.587, 1.503], + [9.533, 3.887, 3.564], + [11.468, 1.932, 1.712], + [1.362, 0.451, 0.403], + [0.648, 0.374, 0.414], + [1.195, 0.437, 0.418], + [4.187, 1.686, 1.474], + [3.289, 2.146, 2.159], + [3.919, 4.242, 4.208], + [1.673, 1.084, 1.040], + [3.264, 1.496, 1.629], + [8.883, 8.965, 9.027], + [5.813, 5.225, 5.365], + [5.874, 5.376, 5.353], + [2.053, 1.910, 1.951], + [0.478, 0.324, 0.325], + [0.206, 0.132, 0.124], + [0.222, 0.105, 0.111], + [0.699, 0.599, 0.563], + [0.213, 0.041, 0.040], + [0.133, 0.032, 0.040], + [0.062, 0.010, 0.010] + ] + } +] diff --git a/website/benchmark/hardware/results/022_amd_epyc_7402p.json b/website/benchmark/hardware/results/022_amd_epyc_7402p.json new file mode 100644 index 00000000000..1f03baeb94e --- /dev/null +++ b/website/benchmark/hardware/results/022_amd_epyc_7402p.json @@ -0,0 +1,152 @@ +[ + { + "system": "AMD EPYC 7402P 2.8 GHz, 128 GB DDR4, SSD RAID1 2×1920 GB SSD (INTEL SSDSC2KB019T7)", + "time": "2020-01-26 00:00:00", + "result": + [ + [0.014, 0.002, 0.002], + [0.031, 0.014, 0.010], + [0.077, 0.015, 0.015], + [0.255, 0.020, 0.019], + [0.286, 0.075, 0.073], + [0.452, 0.136, 0.135], + [0.025, 0.012, 0.012], + [0.021, 0.011, 0.011], + [0.431, 0.188, 0.188], + [0.491, 0.213, 0.214], + [0.308, 0.099, 0.097], + [0.319, 0.102, 0.098], + [0.491, 0.247, 0.248], + [0.786, 0.323, 0.316], + [0.574, 0.291, 0.291], + [0.414, 0.266, 0.267], + [1.097, 0.847, 0.835], + [0.748, 0.507, 0.505], + [1.977, 1.467, 1.488], + [0.264, 0.018, 0.029], + [2.937, 0.281, 0.254], + [3.288, 0.301, 0.283], + [6.502, 0.698, 0.687], + [7.260, 0.358, 0.351], + [0.796, 0.096, 0.095], + [0.399, 0.084, 0.083], + [0.873, 0.099, 0.101], + [3.215, 0.318, 0.300], + [2.680, 0.394, 0.391], + [1.099, 1.058, 1.055], + [0.802, 0.250, 0.251], + [1.823, 0.340, 0.341], + [2.750, 2.168, 2.157], + [3.638, 1.301, 1.267], + [3.583, 1.289, 1.288], + [0.455, 0.392, 0.393], + [0.279, 0.170, 0.159], + [0.089, 0.068, 0.066], + [0.135, 0.063, 0.061], + [0.479, 0.329, 0.341], + [0.059, 0.021, 0.020], + [0.042, 0.018, 0.020], + [0.011, 0.006, 0.006] + ] + }, + { + "system": "AMD EPYC 7402P 2.8 GHz, 128 GB DDR4, HDD RAID1 2×8000 GB HDD (TOSHIBA MG06ACA800E)", + "time": "2020-01-26 00:00:00", + "result": + [ + [0.149, 0.002, 0.002], + [0.263, 0.012, 0.011], + [0.631, 0.017, 0.016], + [1.829, 0.023, 0.020], + [2.073, 0.078, 0.076], + [2.981, 0.176, 0.138], + [0.204, 0.022, 0.012], + [0.195, 0.011, 0.011], + [2.652, 0.195, 0.193], + [2.949, 0.226, 0.218], + [2.124, 0.101, 0.099], + [2.369, 0.106, 0.102], + [2.978, 0.254, 0.248], + [4.546, 0.328, 0.321], + [3.391, 0.298, 0.297], + [2.211, 0.269, 0.268], + [4.889, 0.850, 0.842], + [4.627, 0.514, 0.505], + [8.805, 1.506, 1.432], + [1.979, 0.034, 0.044], + [18.744, 0.315, 0.248], + [22.946, 0.301, 0.276], + [41.584, 0.703, 0.692], + [42.963, 0.392, 0.335], + [5.992, 0.130, 0.096], + [3.050, 0.096, 0.085], + [6.390, 0.115, 0.101], + [20.038, 0.319, 0.296], + [17.610, 0.408, 0.396], + [1.187, 1.056, 1.055], + [5.134, 0.254, 0.249], + [10.690, 0.348, 0.341], + [9.296, 2.190, 2.149], + [20.999, 1.258, 1.258], + [22.020, 1.256, 1.254], + [1.715, 0.400, 0.390], + [0.403, 0.169, 0.164], + [0.147, 0.069, 0.069], + [0.137, 0.063, 0.062], + [0.568, 0.344, 0.359], + [0.152, 0.027, 0.021], + [0.076, 0.018, 0.017], + [0.021, 0.006, 0.006] + ] + }, + { + "system": "AMD EPYC 7502P / 128G DDR4 / 2NVME SAMSUNG MZQLB960HAJR", + "time": "2020-03-05 00:00:00", + "result": + [ + [0.012, 0.019, 0.009], + [0.042, 0.026, 0.038], + [0.026, 0.032, 0.017], + [0.058, 0.025, 0.027], + [0.095, 0.080, 0.087], + [0.143, 0.125, 0.124], + [0.018, 0.010, 0.016], + [0.013, 0.012, 0.013], + [0.201, 0.182, 0.182], + [0.228, 0.204, 0.204], + [0.093, 0.078, 0.077], + [0.100, 0.080, 0.081], + [0.241, 0.222, 0.218], + [0.291, 0.265, 0.270], + [0.268, 0.254, 0.256], + [0.255, 0.241, 0.242], + [0.623, 0.593, 0.599], + [0.373, 0.343, 0.339], + [1.354, 1.318, 1.311], + [0.054, 0.020, 0.022], + [0.495, 0.247, 0.242], + [0.520, 0.258, 0.248], + [0.957, 0.646, 0.652], + [null, null, null], + [0.149, 0.105, 0.099], + [0.091, 0.070, 0.069], + [0.150, 0.096, 0.094], + [0.499, 0.315, 0.309], + [0.437, 0.354, 0.357], + [1.002, 0.996, 0.991], + [0.234, 0.205, 0.207], + [0.380, 0.305, 0.305], + [1.733, 1.651, 1.655], + [1.230, 1.134, 1.132], + [1.217, 1.130, 1.114], + [0.396, 0.385, 0.383], + [0.156, 0.148, 0.160], + [0.065, 0.062, 0.063], + [0.057, 0.052, 0.052], + [0.368, 0.342, 0.336], + [0.030, 0.025, 0.027], + [0.022, 0.017, 0.019], + [0.005, 0.004, 0.004] + ] + } +] diff --git a/website/benchmark/hardware/results/023_xeon_e5_1650v3.json b/website/benchmark/hardware/results/023_xeon_e5_1650v3.json new file mode 100644 index 00000000000..78c3e4061ca --- /dev/null +++ b/website/benchmark/hardware/results/023_xeon_e5_1650v3.json @@ -0,0 +1,52 @@ +[ + { + "system": "Intel Xeon E5-1650v3 3.5 GHz, 64 GB DDR4, RAID1 2×480 GB SSD (INTEL SSDSC2BB480G4)", + "time": "2020-01-26 00:00:00", + "result": + [ + [0.005, 0.001, 0.001], + [0.025, 0.015, 0.016], + [0.085, 0.039, 0.034], + [0.302, 0.052, 0.057], + [0.313, 0.137, 0.136], + [0.639, 0.426, 0.434], + [0.036, 0.027, 0.027], + [0.021, 0.014, 0.014], + [0.871, 0.707, 0.707], + [1.060, 0.812, 0.811], + [0.521, 0.285, 0.290], + [0.548, 0.331, 0.339], + [1.358, 0.987, 0.983], + [1.832, 1.317, 1.326], + [1.464, 1.165, 1.134], + [1.311, 1.115, 1.111], + [3.426, 3.060, 3.069], + [2.113, 1.832, 1.849], + [7.000, 6.235, 6.309], + [0.293, 0.067, 0.074], + [3.325, 1.062, 1.030], + [3.765, 1.296, 1.311], + [7.438, 2.870, 2.831], + [7.820, 1.318, 1.277], + [1.139, 0.412, 0.405], + [0.704, 0.372, 0.362], + [1.154, 0.428, 0.427], + [3.557, 1.103, 1.060], + [3.249, 1.680, 1.666], + [2.411, 2.364, 2.312], + [1.365, 0.993, 1.003], + [2.696, 1.535, 1.523], + [9.226, 8.948, 8.734], + [6.053, 4.337, 4.321], + [6.020, 4.341, 4.300], + [1.919, 1.790, 1.759], + [0.220, 0.160, 0.161], + [0.085, 0.067, 0.061], + [0.116, 0.061, 0.061], + [0.431, 0.335, 0.320], + [0.050, 0.037, 0.025], + [0.083, 0.016, 0.019], + [0.029, 0.006, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/024_selectel_cloud_16vcpu.json b/website/benchmark/hardware/results/024_selectel_cloud_16vcpu.json new file mode 100644 index 00000000000..d41778e47b2 --- /dev/null +++ b/website/benchmark/hardware/results/024_selectel_cloud_16vcpu.json @@ -0,0 +1,152 @@ +[ + { + "system": "Selectel Cloud, 16 vCPU, 32 GB RAM, 'fast disk'", + "time": "2020-01-26 00:00:00", + "result": + [ + [0.017, 0.002, 0.002], + [0.047, 0.011, 0.012], + [0.070, 0.024, 0.024], + [0.350, 0.033, 0.031], + [0.353, 0.087, 0.091], + [0.623, 0.233, 0.226], + [0.040, 0.018, 0.019], + [0.021, 0.011, 0.011], + [0.628, 0.420, 0.415], + [0.725, 0.466, 0.468], + [0.389, 0.152, 0.151], + [0.416, 0.181, 0.172], + [0.819, 0.481, 0.495], + [1.332, 0.714, 0.719], + [1.044, 0.853, 0.796], + [0.865, 0.827, 0.846], + [2.010, 1.816, 1.798], + [1.097, 0.940, 0.927], + [4.414, 4.188, 4.310], + [0.336, 0.034, 0.030], + [4.807, 0.541, 0.527], + [5.494, 0.633, 0.608], + [10.233, 1.448, 1.469], + [9.897, 0.721, 0.676], + [1.322, 0.212, 0.213], + [0.594, 0.177, 0.175], + [1.319, 0.218, 0.216], + [4.804, 0.615, 0.567], + [4.093, 0.855, 0.801], + [1.428, 1.378, 1.300], + [1.299, 0.649, 0.646], + [2.921, 0.835, 0.809], + [5.717, 5.883, 6.368], + [5.655, 2.715, 2.666], + [5.943, 3.008, 2.795], + [1.091, 1.011, 1.089], + [0.264, 0.212, 0.221], + [0.097, 0.085, 0.081], + [0.083, 0.081, 0.075], + [0.498, 0.531, 0.446], + [0.063, 0.031, 0.033], + [0.029, 0.022, 0.022], + [0.006, 0.006, 0.005] + ] + }, + { + "system": "Selectel Cloud, 16 vCPU, 32 GB RAM, 'basic disk'", + "time": "2020-01-26 00:00:00", + "result": + [ + [0.142, 0.002, 0.002], + [0.090, 0.014, 0.013], + [0.478, 0.023, 0.023], + [2.305, 0.032, 0.032], + [2.371, 0.090, 0.087], + [3.377, 0.228, 0.224], + [0.080, 0.020, 0.017], + [0.034, 0.009, 0.009], + [3.210, 0.425, 0.414], + [3.389, 0.471, 0.459], + [2.446, 0.157, 0.152], + [2.331, 0.187, 0.177], + [3.826, 0.503, 0.505], + [5.749, 0.725, 0.701], + [4.521, 0.752, 0.705], + [2.345, 0.572, 0.588], + [6.360, 1.697, 1.512], + [5.928, 0.949, 0.908], + [11.325, 3.598, 3.829], + [6.485, 0.126, 0.033], + [24.494, 0.550, 0.519], + [27.711, 0.615, 0.594], + [52.599, 1.455, 1.435], + [50.820, 1.001, 0.648], + [6.933, 0.206, 0.203], + [3.278, 0.177, 0.172], + [7.082, 0.216, 0.206], + [24.399, 0.564, 0.541], + [20.916, 0.812, 0.801], + [1.574, 1.323, 1.358], + [6.717, 0.558, 0.550], + [13.772, 0.830, 0.801], + [12.547, 5.084, 4.637], + [25.178, 2.601, 2.337], + [25.118, 2.460, 2.432], + [1.626, 0.901, 0.895], + [0.286, 0.206, 0.209], + [0.130, 0.081, 0.083], + [0.220, 0.075, 0.076], + [6.502, 0.495, 0.520], + [0.223, 0.024, 0.024], + [0.045, 0.028, 0.030], + [0.006, 0.005, 0.004] + ] + }, + { + "system": "Selectel Cloud, 16 vCPU, 32 GB RAM, 'universal disk'", + "time": "2020-01-26 00:00:00", + "result": + [ + [0.016, 0.002, 0.002], + [0.055, 0.023, 0.009], + [0.252, 0.020, 0.019], + [1.337, 0.038, 0.031], + [1.342, 0.088, 0.086], + [2.192, 0.241, 0.222], + [0.047, 0.017, 0.016], + [0.017, 0.009, 0.009], + [1.946, 0.410, 0.412], + [2.262, 0.454, 0.455], + [1.427, 0.158, 0.150], + [1.487, 0.181, 0.172], + [2.327, 0.481, 0.464], + [3.794, 0.657, 0.633], + [2.582, 0.596, 0.602], + [1.708, 0.673, 0.670], + [4.431, 1.545, 1.622], + [3.661, 0.890, 0.871], + [7.950, 3.954, 3.844], + [1.327, 0.055, 0.031], + [16.200, 0.538, 0.523], + [18.455, 0.613, 0.599], + [34.059, 1.428, 1.436], + [31.534, 0.713, 0.653], + [4.576, 0.205, 0.201], + [2.155, 0.177, 0.172], + [4.574, 0.208, 0.206], + [16.201, 0.554, 0.534], + [13.787, 0.840, 0.809], + [1.443, 1.408, 1.331], + [3.815, 0.561, 0.536], + [8.831, 0.778, 0.761], + [9.031, 4.476, 4.428], + [16.938, 2.347, 2.287], + [17.093, 2.722, 2.257], + [1.133, 0.897, 0.888], + [0.240, 0.207, 0.202], + [0.101, 0.083, 0.082], + [0.086, 0.075, 0.074], + [0.467, 0.450, 0.440], + [0.060, 0.028, 0.024], + [0.028, 0.020, 0.020], + [0.006, 0.005, 0.005] + ] + } +] diff --git a/website/benchmark/hardware/results/035_aws_a1_4xlarge.json b/website/benchmark/hardware/results/035_aws_a1_4xlarge.json new file mode 100644 index 00000000000..9c3e32abd0e --- /dev/null +++ b/website/benchmark/hardware/results/035_aws_a1_4xlarge.json @@ -0,0 +1,52 @@ +[ + { + "system": "AWS a1.4xlarge (Graviton) 16 vCPU, 2.3 GHz, 32 GiB RAM, EBS", + "time": "2020-02-13 00:00:00", + "result": + [ + [0.012, 0.003, 0.003], + [0.073, 0.031, 0.031], + [0.098, 0.053, 0.053], + [0.209, 0.139, 0.141], + [0.251, 0.200, 0.202], + [0.662, 0.439, 0.436], + [0.062, 0.041, 0.041], + [0.040, 0.033, 0.032], + [3.379, 0.720, 0.722], + [0.934, 0.847, 0.845], + [0.436, 0.379, 0.377], + [0.500, 0.417, 0.430], + [1.536, 1.381, 1.373], + [1.956, 1.832, 1.855], + [1.527, 1.458, 1.466], + [1.613, 1.576, 1.581], + [3.644, 3.490, 3.530], + [2.143, 1.982, 1.965], + [7.808, 7.617, 7.764], + [0.390, 0.179, 0.168], + [8.797, 2.308, 2.257], + [10.138, 2.533, 2.517], + [19.626, 5.738, 5.707], + [20.183, 2.195, 2.156], + [1.841, 0.577, 0.578], + [0.535, 0.479, 0.476], + [1.830, 0.578, 0.577], + [8.786, 2.521, 2.524], + [7.364, 2.941, 2.926], + [3.373, 3.186, 3.203], + [1.641, 1.213, 1.209], + [4.890, 1.964, 1.913], + [10.442, 10.410, 10.427], + [11.183, 7.431, 7.402], + [11.175, 7.460, 7.487], + [2.317, 2.232, 2.221], + [0.473, 0.406, 0.418], + [0.201, 0.187, 0.183], + [0.193, 0.144, 0.160], + [0.901, 0.811, 0.836], + [0.090, 0.046, 0.041], + [0.053, 0.032, 0.033], + [0.015, 0.012, 0.012] + ] + } +] diff --git a/website/benchmark/hardware/results/036_xeon_x5675.json b/website/benchmark/hardware/results/036_xeon_x5675.json new file mode 100644 index 00000000000..56d16a46d08 --- /dev/null +++ b/website/benchmark/hardware/results/036_xeon_x5675.json @@ -0,0 +1,102 @@ +[ + { + "system": "ProLiant DL380 G7, 12Gb RAM, 2x Xeon X5675 3.07GHz, 8x300GB SAS soft RAID5", + "time": "2020-02-18 00:00:00", + "result": + [ + [0.041, 0.005, 0.005], + [0.084, 0.020, 0.019], + [0.403, 0.046, 0.043], + [0.190, 0.081, 0.082], + [0.192, 0.127, 0.131], + [0.388, 0.324, 0.309], + [0.078, 0.028, 0.038], + [0.055, 0.019, 0.019], + [0.677, 0.614, 0.604], + [0.808, 0.706, 0.727], + [0.282, 0.190, 0.181], + [0.312, 0.223, 0.229], + [0.997, 0.895, 0.891], + [1.167, 1.155, 1.115], + [1.155, 1.088, 1.143], + [1.119, 1.090, 1.109], + [3.451, 3.222, 3.153], + [1.743, 1.770, 1.655], + [9.346, 6.206, 6.436], + [0.352, 0.108, 0.105], + [2.985, 0.993, 0.976], + [3.594, 1.211, 1.195], + [6.626, 2.829, 2.800], + [10.086, 1.331, 1.318], + [1.072, 0.348, 0.332], + [0.535, 0.298, 0.269], + [1.046, 0.362, 0.334], + [3.487, 1.221, 1.165], + [2.718, 1.742, 1.719], + [3.200, 3.158, 3.116], + [1.346, 0.901, 0.917], + [2.336, 1.285, 1.285], + [8.876, 64.491, 123.728], + [10.200, 5.127, 4.743], + [5.196, 4.783, 4.659], + [1.628, 1.544, 1.527], + [0.476, 0.296, 0.285], + [0.172, 0.127, 0.097], + [0.170, 0.078, 0.083], + [0.670, 0.529, 0.511], + [0.181, 0.065, 0.039], + [0.123, 0.029, 0.033], + [0.045, 0.011, 0.011] + ] + }, + { + "system": "ProLiant DL380 G7, 12Gb RAM, 1x Xeon X5675 3.07GHz, 8x300GB SAS Soft RAID5", + "time": "2020-02-18 00:00:00", + "result": + [ + [0.048, 0.005, 0.005], + [0.092, 0.026, 0.026], + [0.167, 0.067, 0.073], + [0.200, 0.117, 0.116], + [0.263, 0.185, 0.203], + [0.587, 0.586, 0.586], + [0.094, 0.043, 0.043], + [0.067, 0.025, 0.026], + [1.371, 1.299, 1.298], + [1.638, 1.546, 1.548], + [0.441, 0.341, 0.337], + [0.482, 0.405, 0.385], + [2.682, 2.680, 2.630], + [3.189, 3.207, 3.167], + [2.634, 2.525, 2.556], + [3.181, 3.200, 3.213], + [7.793, 7.714, 7.768], + [3.802, 3.819, 3.960], + [19.101, 16.177, 15.840], + [0.320, 0.153, 0.134], + [3.108, 2.188, 2.115], + [4.515, 3.139, 3.069], + [7.712, 6.856, 6.906], + [11.063, 2.630, 2.567], + [1.015, 0.739, 0.723], + [0.738, 0.644, 0.623], + [1.048, 0.717, 0.736], + [3.371, 2.905, 2.903], + [4.772, 4.539, 4.518], + [11.700, 11.656, 11.589], + [2.217, 2.083, 2.072], + [4.329, 4.153, 3.889], + [21.212, 21.887, 21.417], + [12.816, 12.501, 12.664], + [13.192, 12.624, 12.820], + [5.454, 5.447, 5.462], + [0.376, 0.280, 0.288], + [0.152, 0.097, 0.113], + [0.171, 0.093, 0.100], + [0.594, 0.484, 0.464], + [0.129, 0.043, 0.036], + [0.098, 0.027, 0.045], + [0.033, 0.025, 0.011] + ] + } +] diff --git a/website/benchmark/hardware/results/037_pinebook_pro.json b/website/benchmark/hardware/results/037_pinebook_pro.json new file mode 100644 index 00000000000..d82d6533d7a --- /dev/null +++ b/website/benchmark/hardware/results/037_pinebook_pro.json @@ -0,0 +1,52 @@ +[ + { + "system": "Pinebook Pro (AArch64, 4 GiB RAM)", + "time": "2020-03-08 00:00:00", + "result": + [ + [0.021, 0.009, 0.007], + [0.195, 0.135, 0.144], + [0.439, 0.264, 0.273], + [1.266, 0.672, 0.706], + [1.337, 0.795, 0.790], + [2.706, 1.989, 1.947], + [0.246, 0.198, 0.197], + [0.157, 0.142, 0.133], + [4.150, 3.769, 3.617], + [5.223, 4.405, 4.234], + [2.391, 1.815, 1.785], + [2.534, 2.158, 2.042], + [7.895, 6.890, 7.003], + [10.338, 9.311, 9.410], + [8.139, 7.441, 7.312], + [8.532, 8.035, 8.011], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null] + ] + } +] diff --git a/website/benchmark/hardware/results/038_amd_ryzen_9_3950x.json b/website/benchmark/hardware/results/038_amd_ryzen_9_3950x.json new file mode 100644 index 00000000000..3c2c042e538 --- /dev/null +++ b/website/benchmark/hardware/results/038_amd_ryzen_9_3950x.json @@ -0,0 +1,52 @@ +[ + { + "system": "AMD Ryzen 9 3950X 16-Core Processor, 64 GiB RAM, Intel Optane 900P 280 GB", + "time": "2020-03-14 00:00:00", + "result": + [ + [0.002, 0.001, 0.001], + [0.018, 0.013, 0.012], + [0.041, 0.027, 0.026], + [0.091, 0.040, 0.041], + [0.115, 0.075, 0.074], + [0.201, 0.157, 0.153], + [0.017, 0.015, 0.014], + [0.013, 0.013, 0.013], + [0.363, 0.321, 0.313], + [0.441, 0.390, 0.389], + [0.189, 0.164, 0.162], + [0.201, 0.172, 0.178], + [0.879, 0.809, 0.802], + [1.081, 1.030, 1.027], + [0.875, 0.832, 0.837], + [1.042, 1.013, 1.019], + [2.604, 2.544, 2.529], + [1.435, 1.396, 1.414], + [4.208, 4.148, 4.132], + [0.094, 0.052, 0.037], + [0.965, 0.451, 0.450], + [1.141, 0.762, 0.763], + [2.813, 2.378, 2.383], + [2.127, 0.749, 0.733], + [0.301, 0.228, 0.228], + [0.218, 0.180, 0.174], + [0.301, 0.228, 0.227], + [1.241, 0.965, 0.958], + [1.469, 1.271, 1.274], + [3.782, 3.780, 3.762], + [0.738, 0.656, 0.655], + [1.147, 1.015, 1.009], + [7.114, 6.989, 7.074], + [4.277, 4.085, 4.069], + [4.256, 4.032, 4.073], + [1.811, 1.787, 1.785], + [0.138, 0.128, 0.129], + [0.057, 0.051, 0.053], + [0.058, 0.051, 0.047], + [0.283, 0.284, 0.278], + [0.023, 0.024, 0.027], + [0.024, 0.015, 0.021], + [0.007, 0.013, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/039_azure_e32s.json b/website/benchmark/hardware/results/039_azure_e32s.json new file mode 100644 index 00000000000..bf1c8c6af8b --- /dev/null +++ b/website/benchmark/hardware/results/039_azure_e32s.json @@ -0,0 +1,52 @@ +[ + { + "system": "Azure E32s v3 32 256 GiB 512 GiB", + "time": "2020-03-23 00:00:00", + "result": + [ + [0.003, 0.002, 0.003], + [0.114, 0.014, 0.013], + [0.230, 0.031, 0.029], + [0.893, 0.043, 0.042], + [0.915, 0.123, 0.143], + [1.475, 0.263, 0.264], + [0.055, 0.016, 0.017], + [0.056, 0.013, 0.013], + [1.467, 0.523, 0.523], + [1.661, 0.614, 0.608], + [0.999, 0.169, 0.176], + [1.058, 0.188, 0.190], + [1.839, 0.658, 0.697], + [2.753, 0.892, 0.881], + [2.034, 0.895, 0.895], + [1.425, 0.860, 0.879], + [3.401, 2.070, 2.091], + [2.573, 1.183, 1.208], + [6.376, 4.374, 4.442], + [0.922, 0.044, 0.043], + [10.137, 0.653, 0.691], + [11.589, 0.711, 0.805], + [21.234, 1.841, 1.827], + [22.035, 0.973, 0.940], + [2.983, 0.238, 0.237], + [1.493, 0.201, 0.186], + [3.016, 0.262, 0.259], + [10.139, 0.696, 0.676], + [8.723, 1.017, 1.008], + [1.561, 1.439, 1.563], + [2.688, 0.713, 0.728], + [5.942, 1.075, 1.063], + [7.803, 5.871, 5.826], + [11.131, 2.860, 2.798], + [11.089, 2.898, 2.847], + [1.550, 1.276, 1.256], + [0.851, 0.303, 0.280], + [0.376, 0.100, 0.108], + [0.571, 0.084, 0.082], + [1.654, 0.560, 0.533], + [0.356, 0.029, 0.027], + [0.232, 0.022, 0.024], + [0.032, 0.009, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/040_core_i7_macbook_pro_2018.json b/website/benchmark/hardware/results/040_core_i7_macbook_pro_2018.json new file mode 100644 index 00000000000..e0ad42f423b --- /dev/null +++ b/website/benchmark/hardware/results/040_core_i7_macbook_pro_2018.json @@ -0,0 +1,52 @@ +[ + { + "system": "MacBook Pro 2018, 2.7 GHz Quad-Core Intel Core i7, 16 GiB RAM, 1TB SSD", + "time": "2020-04-04 00:00:00", + "result": + [ + [0.002, 0.002, 0.002], + [0.028, 0.031, 0.025], + [0.060, 0.058, 0.047], + [0.125, 0.101, 0.070], + [0.164, 0.185, 0.168], + [0.672, 0.568, 0.557], + [0.072, 0.038, 0.037], + [0.031, 0.021, 0.021], + [0.849, 0.836, 0.820], + [0.941, 0.938, 0.942], + [0.423, 0.444, 0.457], + [0.617, 0.556, 0.555], + [1.761, 1.694, 1.641], + [2.190, 2.277, 2.226], + [1.964, 1.895, 1.934], + [1.956, 1.978, 1.884], + [6.029, 5.977, 5.975], + [3.372, 3.436, 3.439], + [12.883, 12.778, 12.572], + [0.116, 0.080, 0.076], + [1.874, 1.372, 1.467], + [2.321, 2.356, 2.238], + [5.304, 4.955, 4.912], + [2.474, 1.993, 2.033], + [0.744, 0.708, 0.719], + [0.562, 0.568, 0.602], + [0.737, 0.742, 0.719], + [1.547, 1.580, 1.583], + [3.074, 2.665, 2.697], + [5.466, 5.560, 5.693], + [1.658, 1.562, 1.543], + [2.935, 2.802, 2.743], + [19.141, 19.674, 19.212], + [8.738, 8.334, 8.302], + [8.268, 8.276, 8.364], + [3.311, 3.288, 3.243], + [0.182, 0.169, 0.169], + [0.075, 0.066, 0.066], + [0.066, 0.057, 0.053], + [0.353, 0.324, 0.327], + [0.030, 0.018, 0.018], + [0.018, 0.015, 0.015], + [0.011, 0.007, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/041_amd_epyc_7702.json b/website/benchmark/hardware/results/041_amd_epyc_7702.json new file mode 100644 index 00000000000..ae332605155 --- /dev/null +++ b/website/benchmark/hardware/results/041_amd_epyc_7702.json @@ -0,0 +1,52 @@ +[ + { + "system": "AMD EPYC 7702, 256 cores, 512 GiB, NVMe SSD", + "time": "2020-04-09 00:00:00", + "result": + [ + [0.006, 0.002, 0.002], + [0.252, 0.072, 0.057], + [0.113, 0.066, 0.057], + [0.197, 0.055, 0.065], + [0.311, 0.199, 0.217], + [0.360, 0.200, 0.183], + [0.119, 0.050, 0.045], + [0.066, 0.061, 0.057], + [0.320, 0.150, 0.144], + [0.346, 0.170, 0.162], + [0.226, 0.117, 0.115], + [0.265, 0.112, 0.118], + [0.402, 0.249, 0.250], + [0.561, 0.327, 0.332], + [0.397, 0.267, 0.257], + [0.323, 0.221, 0.233], + [0.710, 0.527, 0.517], + [0.667, 0.437, 0.443], + [1.269, 0.936, 0.957], + [0.189, 0.043, 0.043], + [1.673, 0.206, 0.169], + [1.937, 0.214, 0.184], + [3.527, 0.755, 0.737], + [3.197, 0.551, 0.523], + [0.519, 0.076, 0.086], + [0.268, 0.060, 0.080], + [0.522, 0.075, 0.079], + [1.693, 0.345, 0.351], + [1.466, 0.330, 0.318], + [1.078, 0.974, 1.019], + [0.501, 0.196, 0.200], + [1.032, 0.266, 0.271], + [1.621, 1.156, 1.169], + [2.089, 0.998, 0.972], + [2.106, 0.974, 0.959], + [0.366, 0.305, 0.305], + [0.190, 0.187, 0.183], + [0.071, 0.066, 0.075], + [0.072, 0.068, 0.062], + [0.415, 0.353, 0.457], + [0.034, 0.032, 0.028], + [0.031, 0.027, 0.032], + [0.024, 0.007, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/042_core_i7_6770hq_intel_nuc.json b/website/benchmark/hardware/results/042_core_i7_6770hq_intel_nuc.json new file mode 100644 index 00000000000..94b0e25d7ba --- /dev/null +++ b/website/benchmark/hardware/results/042_core_i7_6770hq_intel_nuc.json @@ -0,0 +1,52 @@ +[ + { + "system": "Intel NUC, 4 cores (Intel i7-6770HQ), 32 GiB RAM, 1 TB NVMe SSD", + "time": "2020-04-15 00:00:00", + "result": + [ + [0.003, 0.002, 0.001], + [0.025, 0.016, 0.018], + [0.084, 0.058, 0.057], + [0.158, 0.092, 0.085], + [0.273, 0.211, 0.190], + [0.671, 0.555, 0.539], + [0.031, 0.033, 0.033], + [0.026, 0.019, 0.017], + [1.183, 1.110, 1.090], + [1.330, 1.246, 1.254], + [0.352, 0.297, 0.296], + [0.441, 0.375, 0.352], + [1.611, 1.491, 1.439], + [2.130, 2.022, 1.976], + [1.903, 1.795, 1.819], + [1.927, 1.851, 1.861], + [5.282, 5.155, 5.172], + [3.246, 3.313, 3.189], + [12.059, 11.378, 10.562], + [0.146, 0.092, 0.090], + [2.103, 1.496, 1.477], + [2.447, 1.777, 1.734], + [5.123, 3.999, 3.955], + [3.733, 1.808, 1.775], + [0.685, 0.530, 0.523], + [0.525, 0.446, 0.438], + [0.755, 0.545, 0.547], + [2.052, 1.416, 1.403], + [2.976, 2.441, 2.423], + [2.197, 2.189, 2.164], + [1.748, 1.596, 1.607], + [2.773, 2.481, 2.466], + [18.903, 19.166, 16.563], + [7.457, 7.116, 6.943], + [7.311, 6.957, 6.958], + [3.036, 3.005, 2.991], + [0.247, 0.186, 0.162], + [0.100, 0.063, 0.065], + [0.098, 0.061, 0.056], + [0.434, 0.344, 0.331], + [0.040, 0.025, 0.025], + [0.049, 0.026, 0.026], + [0.022, 0.008, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/043_amd_epyc_7502p.json b/website/benchmark/hardware/results/043_amd_epyc_7502p.json new file mode 100644 index 00000000000..95ea5d2d568 --- /dev/null +++ b/website/benchmark/hardware/results/043_amd_epyc_7502p.json @@ -0,0 +1,52 @@ +[ + { + "system": "AMD EPYC 7502P 32-Core Processor with HT (64 thread) / 512 Gb RAM / mdadm RAID1 SAMSUNG MZQLB3T8HALS-00007 + LVM", + "time": "2020-04-16 00:00:00", + "result": + [ + [0.007, 0.002, 0.002], + [0.022, 0.011, 0.011], + [0.028, 0.017, 0.017], + [0.050, 0.022, 0.022], + [0.098, 0.079, 0.078], + [0.149, 0.125, 0.126], + [0.020, 0.014, 0.014], + [0.015, 0.013, 0.013], + [0.169, 0.148, 0.148], + [0.237, 0.171, 0.168], + [0.103, 0.084, 0.082], + [0.099, 0.085, 0.084], + [0.262, 0.221, 0.221], + [0.312, 0.281, 0.282], + [0.274, 0.259, 0.255], + [0.255, 0.237, 0.237], + [0.616, 0.592, 0.589], + [0.398, 0.364, 0.359], + [1.358, 1.301, 1.292], + [0.056, 0.025, 0.022], + [0.485, 0.265, 0.263], + [0.510, 0.271, 0.255], + [0.938, 0.693, 0.674], + [1.262, 0.396, 0.367], + [0.144, 0.082, 0.081], + [0.088, 0.066, 0.065], + [0.141, 0.084, 0.082], + [0.488, 0.329, 0.325], + [0.441, 0.376, 0.374], + [1.054, 1.049, 1.068], + [0.222, 0.190, 0.189], + [0.386, 0.321, 0.319], + [1.703, 1.625, 1.660], + [1.272, 1.202, 1.207], + [1.276, 1.185, 1.204], + [0.398, 0.382, 0.382], + [0.171, 0.160, 0.160], + [0.071, 0.058, 0.059], + [0.059, 0.055, 0.053], + [0.364, 0.341, 0.354], + [0.028, 0.022, 0.027], + [0.027, 0.020, 0.018], + [0.010, 0.008, 0.005] + ] + } +] diff --git a/website/benchmark_hardware.html b/website/benchmark_hardware.html deleted file mode 100644 index ab75e7ca063..00000000000 --- a/website/benchmark_hardware.html +++ /dev/null @@ -1,2876 +0,0 @@ - - - - - Performance Comparison Of ClickHouse On Various Hardware - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - -

Performance Comparison Of ClickHouse On Various Hardware

-
- -
- -
- -

Relative query processing time (lower is better):

-
- -
-

Full results:

- -
- -
- -
-Submit your own results: https://clickhouse.yandex/docs/en/operations/performance_test/
-
-Results for Lenovo B580 Laptop are from Ragıp Ünal. 16GB RAM 1600 GHz, 240GB SSD, Intel(R) Core(TM) i5-3210M CPU @ 2.50GHz (2 Core / 4 HT)
-Results for Time4vps.eu are from Ragıp Ünal.
-Results for Dell PowerEdge R640, R641 (in Hetzner) are from Dmirty Titov.
-Results for Dell PowerEdge R730 are from Amos Bird.
-Results for Dell R530 are from Yuriy Zolkin.
-Results for Xeon 2176G are from Sergey Golod.
-Results for Azure DS3v2 are from Boris Granveaud.
-Results for AWS are from Wolf Kreuzerkrieg.
-Results for Huawei Taishan are from Peng Gao in sina.com.
-Results for Selectel and AMD EPYC 7402P are from Andrey Dudin.
-Results for ProLiant are from Denis Ustinov.
-Results for AMD EPYC 7502P are from Kostiantyn Velychkovskyi.
-Results for Pinebook Pro are from Aleksey R. @kITerE.
-Results for AMD Ryzen are from Alexey Milovidov. Firefox was running in background.
-Results for Azure E32s are from Piotr Maśko.
-Results for MacBook Pro are from Denis Glazachev. MacOS Catalina Version 10.15.4 (19E266). For "drop caches", the "Free Up RAM" in CleanMyMac is used.
-Xeon Gold 6230 server is using 4 x SAMSUNG datacenter class SSD in RAID-10.
-Results for Yandex Managed ClickHouse for "cold cache" are biased and should not be compared, because cache was not flushed for every next query.
-
- - - - - diff --git a/website/css/base.css b/website/css/base.css index 4926bd3af8a..e7fe6d4723d 100644 --- a/website/css/base.css +++ b/website/css/base.css @@ -11,6 +11,19 @@ a:hover, a:active { text-decoration: underline; } +.btn.disabled, .btn:disabled { + cursor: default; + opacity: 0.4; +} + +.fake-btn { + display: inline-block; + padding: 0.375rem 0.75rem; + text-align: center; + font-size: 1rem; + line-height: 1.5; +} + #logo-icon, #docs-logo-icon { width: 40px; } @@ -26,20 +39,16 @@ a:hover, a:active { line-height: 1.2; } -.bg-dark-alt { +.bg-dark-alt, .bg-dark-alt:focus { background: #36363F; } -.bg-dark-alt:focus { - background: #36363F; -} - -.bg-secondary-alt { +.bg-secondary-alt, .bg-secondary-alt:focus { background: #444451; } -.bg-secondary-alt:focus { - background: #444451; +.bg-orange, .bg-orange:focus { + background: #f14600; } .text-dark-alt { @@ -161,3 +170,27 @@ a.btn-outline-yellow { .dots-cc { background: #fff url('/images/dots.svg') repeat-y 50% -17.5%; } + +.benchmark-query-cell { + width: 20rem; + white-space: pre; + overflow-x: hidden; +} + +.benchmark-query-cell:hover { + width: auto; + background-color: #efefef; + position: absolute; + padding: 0.5rem; + margin: -0.5rem 0 0 -0.5rem; + overflow-x: auto; + white-space: normal; +} + +.benchmark-query-cell-wrapper { + width: 22rem; +} + +.w-15 { + width: 15% !important; +} diff --git a/website/css/docs.css b/website/css/docs.css index f4c4857bc7e..7e4e1040848 100644 --- a/website/css/docs.css +++ b/website/css/docs.css @@ -1,3 +1,14 @@ +details { + background: #444451; + padding: 1rem; + margin-bottom: 1rem; + margin-top: 1rem; +} + +summary { + font-weight: bold; +} + #sidebar { position: fixed; z-index: 50; @@ -109,6 +120,15 @@ border-color: #333; } +.algolia-autocomplete .algolia-docsearch-suggestion--content { + background-color: #333; +} + +.algolia-autocomplete .algolia-docsearch-suggestion--content:hover, +.algolia-autocomplete .ds-dropdown-menu .ds-suggestion.ds-cursor .algolia-docsearch-suggestion:not(.suggestion-layout-simple) .algolia-docsearch-suggestion--content { + background-color: #444451 !important; +} + .algolia-autocomplete .algolia-docsearch-suggestion--category-header, .algolia-autocomplete .algolia-docsearch-suggestion--subcategory-column, .algolia-autocomplete .algolia-docsearch-suggestion--title, @@ -184,4 +204,3 @@ color: #bbb; } } - diff --git a/website/images/clickhouse-3069x1531.png b/website/images/clickhouse-3069x1531.png new file mode 100644 index 00000000000..f143c1d1518 Binary files /dev/null and b/website/images/clickhouse-3069x1531.png differ diff --git a/website/images/flags/tr.svg b/website/images/flags/tr.svg new file mode 100644 index 00000000000..7f9f90a079d --- /dev/null +++ b/website/images/flags/tr.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/website/index.html b/website/index.html index b249fc31285..e2ac6e31441 100644 --- a/website/index.html +++ b/website/index.html @@ -1,3 +1,7 @@ +{% set prefetch_items = [ + ('/docs/en/', 'document') +] %} + {% extends "templates/base.html" %} {% block content %} diff --git a/website/js/base.js b/website/js/base.js index 2c43e435f48..3e91a01092f 100644 --- a/website/js/base.js +++ b/website/js/base.js @@ -47,16 +47,23 @@ }, 70); } + $('img').each(function() { + var src = $(this).attr('data-src'); + if (src) { + $(this).attr('src', src); + } + }); (function (d, w, c) { (w[c] = w[c] || []).push(function() { + var is_single_page = $('html').attr('data-single-page') === 'true'; try { w.yaCounter18343495 = new Ya.Metrika2({ - id:18343495, - clickmap:true, - trackLinks:true, - accurateTrackBounce:true, - webvisor:true + id: 18343495, + clickmap: !is_single_page, + trackLinks: !is_single_page, + accurateTrackBounce: !is_single_page, + webvisor: !is_single_page }); } catch(e) { } }); @@ -66,11 +73,14 @@ f = function () { n.parentNode.insertBefore(s, n); }; s.type = "text/javascript"; s.async = true; - s.src = "https://mc.yandex.ru/metrika/tag.js"; - - if (w.opera == "[object Opera]") { - d.addEventListener("DOMContentLoaded", f, false); - } else { f(); } + s.src = "/js/metrika.js"; + if (window.location.hostname.endsWith('clickhouse.tech')) { + if (w.opera == "[object Opera]") { + d.addEventListener("DOMContentLoaded", f, false); + } else { + f(); + } + } })(document, window, "yandex_metrika_callbacks2"); var beforePrint = function() { diff --git a/website/locale/en/LC_MESSAGES/messages.po b/website/locale/en/LC_MESSAGES/messages.po index c272d156de6..ef9726c101c 100644 --- a/website/locale/en/LC_MESSAGES/messages.po +++ b/website/locale/en/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language: en\n" @@ -80,11 +80,15 @@ msgstr "" "codebase and waits for fellow native speakers to make it more human-" "readable." +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "You can also use the original English version as a reference." + #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "Help ClickHouse documentation by editing this page" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "Built from" @@ -184,15 +188,15 @@ msgstr "Email discussions" msgid "Like ClickHouse?" msgstr "Like ClickHouse?" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "Help to spread the word about it via" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "and" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "Hosting ClickHouse Meetups" diff --git a/website/locale/es/LC_MESSAGES/messages.mo b/website/locale/es/LC_MESSAGES/messages.mo index e7a8e7bd13c..6f29d0fd3df 100644 Binary files a/website/locale/es/LC_MESSAGES/messages.mo and b/website/locale/es/LC_MESSAGES/messages.mo differ diff --git a/website/locale/es/LC_MESSAGES/messages.po b/website/locale/es/LC_MESSAGES/messages.po index 1a8706e60fd..96632f2b7f3 100644 --- a/website/locale/es/LC_MESSAGES/messages.po +++ b/website/locale/es/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: es\n" @@ -79,11 +79,15 @@ msgstr "" "la base de código principal de ClickHouse y espera a que otros hablantes " "nativos lo hagan más legible por humanos." +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "También puede usar la versión original en inglés como referencia." + #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "Ayuda a la documentación de ClickHouse editando esta página" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "Construido a partir de" @@ -187,15 +191,15 @@ msgstr "Discusiones por correo electrónico" msgid "Like ClickHouse?" msgstr "¿Te gusta ClickHouse?" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "Ayuda a correr la voz al respecto a través de" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "y" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "Grupos de Meetup de Hosting ClickHouse" diff --git a/website/locale/fa/LC_MESSAGES/messages.mo b/website/locale/fa/LC_MESSAGES/messages.mo index 6d2a415b7fc..d474a31e03a 100644 Binary files a/website/locale/fa/LC_MESSAGES/messages.mo and b/website/locale/fa/LC_MESSAGES/messages.mo differ diff --git a/website/locale/fa/LC_MESSAGES/messages.po b/website/locale/fa/LC_MESSAGES/messages.po index 5e4205412d1..03a7d9baf3f 100644 --- a/website/locale/fa/LC_MESSAGES/messages.po +++ b/website/locale/fa/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: fa\n" @@ -78,11 +78,15 @@ msgstr "" " کنار پایگاه داده اصلی خانه کلیک زندگی می کند و منتظر سخنرانان مادری " "همکار به انسان بیشتر قابل خواندن است." +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "شما همچنین می توانید نسخه اصلی انگلیسی به عنوان یک مرجع استفاده کنید." + #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "راهنما مستندات تاتر با ویرایش این صفحه" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "ساخته شده از" @@ -182,15 +186,15 @@ msgstr "بحث های ایمیل" msgid "Like ClickHouse?" msgstr "مانند خانه کلیک?" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "کمک به گسترش این کلمه از طریق" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "و" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "میزبانی اکسسوری تاتر" diff --git a/website/locale/fr/LC_MESSAGES/messages.mo b/website/locale/fr/LC_MESSAGES/messages.mo index 7d67a68c6e5..c5f2ca8793f 100644 Binary files a/website/locale/fr/LC_MESSAGES/messages.mo and b/website/locale/fr/LC_MESSAGES/messages.mo differ diff --git a/website/locale/fr/LC_MESSAGES/messages.po b/website/locale/fr/LC_MESSAGES/messages.po index 38cd1031b49..0797104a1a5 100644 --- a/website/locale/fr/LC_MESSAGES/messages.po +++ b/website/locale/fr/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: 2020-03-30 15:12+0300\n" "Last-Translator: FULL NAME \n" "Language: fr\n" @@ -79,11 +79,17 @@ msgstr "" "code ClickHouse principale et attend que d'autres locuteurs natifs le " "rendent plus lisible par l'homme." +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "" +"Vous pouvez également utiliser la version originale anglaise comme " +"référence." + #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "Aide clickhouse documentation en éditant cette page" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "Construit à partir de" @@ -185,15 +191,15 @@ msgstr "Discussions par courriel" msgid "Like ClickHouse?" msgstr "Comme ClickHouse?" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "Aider à passer le mot à ce sujet via" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "et" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "Accueil Clickhouse Meetups" diff --git a/website/locale/ja/LC_MESSAGES/messages.mo b/website/locale/ja/LC_MESSAGES/messages.mo index ec416b79c0a..96cc1b834bd 100644 Binary files a/website/locale/ja/LC_MESSAGES/messages.mo and b/website/locale/ja/LC_MESSAGES/messages.mo differ diff --git a/website/locale/ja/LC_MESSAGES/messages.po b/website/locale/ja/LC_MESSAGES/messages.po index e0c901ea399..9ef50c6d5b9 100644 --- a/website/locale/ja/LC_MESSAGES/messages.po +++ b/website/locale/ja/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: ja\n" @@ -71,11 +71,15 @@ msgstr "" "このドキュメントページの以下の内容は機械翻訳されています。 異な他のウェブサイトで行われませんができます。 " "この翻訳テキスト生活をGitHubリポジトリとClickHouseコードベース、待ち員ネイティブスピーカーで人間が読む." +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "また、参照として、元の英語版を使用することができます。" + #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "このページを編集してClickHouseの文書を編集する" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "から構築" @@ -175,15 +179,15 @@ msgstr "メールでの議論" msgid "Like ClickHouse?" msgstr "ClickHouseのような？" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "それについての言葉を広めるのに役立ちます" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "と" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "開催ClickHouse Meetups" diff --git a/website/locale/messages.pot b/website/locale/messages.pot index a09273f2907..8465be2bf7c 100644 --- a/website/locale/messages.pot +++ b/website/locale/messages.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -64,11 +64,15 @@ msgid "" "readable." msgstr "" +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "" + #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "" @@ -168,15 +172,15 @@ msgstr "" msgid "Like ClickHouse?" msgstr "" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "" diff --git a/website/locale/ru/LC_MESSAGES/messages.mo b/website/locale/ru/LC_MESSAGES/messages.mo index d6e0e5eaffe..8aecc12706d 100644 Binary files a/website/locale/ru/LC_MESSAGES/messages.mo and b/website/locale/ru/LC_MESSAGES/messages.mo differ diff --git a/website/locale/ru/LC_MESSAGES/messages.po b/website/locale/ru/LC_MESSAGES/messages.po index 32241543cbd..27a36517edd 100644 --- a/website/locale/ru/LC_MESSAGES/messages.po +++ b/website/locale/ru/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: ru\n" @@ -64,7 +64,7 @@ msgstr "ООО «Яндекс»" #: templates/docs/content.html:6 msgid "Help wanted!" -msgstr "Помощь нужна!" +msgstr "Нужна помощь!" #: templates/docs/content.html:7 msgid "" @@ -74,17 +74,23 @@ msgid "" "codebase and waits for fellow native speakers to make it more human-" "readable." msgstr "" -"Следующее содержание этой страницы документации было переведено на " -"машинный язык. Но в отличие от других сайтов, это делается не на лету. " +"Нижеследующее содержание этой страницы документации было создано через " +"машинный перевод. Но в отличие от других сайтов, это делается не на лету. " "Этот переведенный текст живет в репозитории GitHub вместе с основной " "кодовой базой ClickHouse и ждет, когда другие носители языка сделают его " -"более удобочитаемым для человека." +"более удобочитаемым." + +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "" +"Вы также можете использовать оригинальную английскую версию в качестве " +"образца." #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "Помогите документации ClickHouse, отредактировав эту страницу" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "Собрано из" @@ -184,15 +190,15 @@ msgstr "Обсуждения по электронной почте" msgid "Like ClickHouse?" msgstr "Нравится ClickHouse?" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "Помогите распространить информацию о нём через" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "и" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "Таких Встреч ClickHouse " diff --git a/website/locale/tr/LC_MESSAGES/messages.mo b/website/locale/tr/LC_MESSAGES/messages.mo new file mode 100644 index 00000000000..c8aa99e505d Binary files /dev/null and b/website/locale/tr/LC_MESSAGES/messages.mo differ diff --git a/website/locale/tr/LC_MESSAGES/messages.po b/website/locale/tr/LC_MESSAGES/messages.po new file mode 100644 index 00000000000..562fde71a59 --- /dev/null +++ b/website/locale/tr/LC_MESSAGES/messages.po @@ -0,0 +1,271 @@ +# Turkish translations for PROJECT. +# Copyright (C) 2020 ORGANIZATION +# This file is distributed under the same license as the PROJECT project. +# FIRST AUTHOR , 2020. +# +msgid "" +msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" +"PO-Revision-Date: 2020-04-15 13:17+0000\n" +"Last-Translator: FULL NAME \n" +"Language: tr\n" +"Language-Team: tr \n" +"Plural-Forms: nplurals=1; plural=0\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.8.0\n" + +#: templates/common_meta.html:5 templates/common_meta.html:9 +#: templates/common_meta.html:16 +msgid "ClickHouse DBMS" +msgstr "ClickHouse DBMS" + +#: templates/common_meta.html:11 +msgid "" +"ClickHouse is an open source column-oriented database management system " +"that allows generating analytical data reports in real time using SQL " +"queries." +msgstr "" +"ClickHouse SQL sorguları kullanarak gerçek zamanlı olarak analitik veri " +"raporları üreten sağlayan bir açık kaynak sütun odaklı veritabanı yönetim" +" sistemidir." + +#: templates/common_meta.html:19 +msgid "" +"ClickHouse is an open source distributed column-oriented database " +"management system that allows generating analytical data reports in real " +"time using SQL queries. Сreated by Yandex ClickHouse manages extremely " +"large volumes of data in a stable and sustainable manner." +msgstr "" +"ClickHouse is an open source distributed column-oriented database " +"management system that allows generating analytical data reports in real " +"time using SQL queries. Сreated by Yandex ClickHouse manages extremely " +"large volumes of data in a stable and sustainable manner." + +#: templates/footer.html:8 +msgid "ClickHouse source code is published under the Apache 2.0 License." +msgstr "ClickHouse kaynak kodu Apache 2.0 Lisansı altında yayınlandı." + +#: templates/footer.html:8 +msgid "" +"Software is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR " +"CONDITIONS OF ANY KIND, either express or implied." +msgstr "" +"Yazılım \"OLDUĞU GİBİ\" BAZINDA dağıtılır, GARANTİ VEYA HERHANGİ bir şart" +" OLMADAN, Açık veya zımni." + +#: templates/footer.html:11 +msgid "Yandex LLC" +msgstr "Yandex LLC" + +#: templates/docs/content.html:6 +msgid "Help wanted!" +msgstr "Yardım istedi!" + +#: templates/docs/content.html:7 +msgid "" +"The following content of this documentation page has been machine-" +"translated. But unlike other websites, it is not done on the fly. This " +"translated text lives on GitHub repository alongside main ClickHouse " +"codebase and waits for fellow native speakers to make it more human-" +"readable." +msgstr "" +"Bu Dokümantasyon sayfasının aşağıdaki içeriği makine tarafından " +"çevrilmiştir. Ancak diğer web sitelerinin aksine, anında yapılmaz. Bu " +"çevrilmiş metin, ana ClickHouse kod tabanının yanında GitHub deposunda " +"yaşar ve diğer anadili konuşanların daha insan tarafından okunabilir " +"olmasını bekler." + +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "Orijinal İngilizce sürümünü referans olarak da kullanabilirsiniz." + +#: templates/docs/content.html:10 +msgid "Help ClickHouse documentation by editing this page" +msgstr "Bu sayfayı düzenleyerek ClickHouse belgelerine yardım edin" + +#: templates/docs/content.html:27 +msgid "Built from" +msgstr "Dahili" + +#: templates/docs/sidebar.html:3 +msgid "Multi-page or single-page" +msgstr "Çok sayfalı veya tek sayfalı" + +#: templates/docs/sidebar.html:5 +msgid "Multi-page version" +msgstr "Çok sayfalı sürüm" + +#: templates/docs/sidebar.html:8 +msgid "Single-page version" +msgstr "Tek sayfalık sürüm" + +#: templates/docs/sidebar.html:13 +msgid "Version" +msgstr "Sürüm" + +#: templates/docs/sidebar.html:13 templates/docs/sidebar.html:19 +msgid "latest" +msgstr "son" + +#: templates/docs/sidebar.html:36 +msgid "PDF version" +msgstr "PDF versiyonu" + +#: templates/docs/toc.html:8 +msgid "Table of Contents" +msgstr "İçindekiler tablosu" + +#: templates/index/community.html:4 +msgid "ClickHouse community" +msgstr "ClickHouse topluluğu" + +#: templates/index/community.html:13 templates/index/community.html:14 +msgid "ClickHouse YouTube Channel" +msgstr "ClickHouse YouTube Kanalı" + +#: templates/index/community.html:25 templates/index/community.html:26 +msgid "ClickHouse Official Twitter Account" +msgstr "ClickHouse Resmi Twitter Hesabı" + +#: templates/index/community.html:36 templates/index/community.html:37 +msgid "ClickHouse at Telegram" +msgstr "Telegram'da ClickHouse" + +#: templates/index/community.html:41 +msgid "Chat with real users in " +msgstr "Gerçek kullanıcılar ile sohbet " + +#: templates/index/community.html:44 templates/index/community.html:116 +msgid "English" +msgstr "İngilizce" + +#: templates/index/community.html:45 +msgid "or in" +msgstr "veya içinde" + +#: templates/index/community.html:47 templates/index/community.html:117 +msgid "Russian" +msgstr "Rusça" + +#: templates/index/community.html:65 +msgid "Open GitHub issue to ask for help or to file a feature request" +msgstr "Yardım istemek veya bir özellik isteği göndermek için GitHub sorununu açın" + +#: templates/index/community.html:76 templates/index/community.html:77 +msgid "ClickHouse Slack Workspace" +msgstr "ClickHouse Slack Çalışma Alanı" + +#: templates/index/community.html:82 +msgid "Multipurpose public hangout" +msgstr "Çok amaçlı kamu hangout" + +#: templates/index/community.html:101 +msgid "Ask any questions" +msgstr "Herhangi bir soru sorun" + +#: templates/index/community.html:115 +msgid "ClickHouse Blog" +msgstr "ClickHouse Blog" + +#: templates/index/community.html:116 +msgid "in" +msgstr "içinde" + +#: templates/index/community.html:128 templates/index/community.html:129 +msgid "ClickHouse at Google Groups" +msgstr "Google Grupları'nda ClickHouse" + +#: templates/index/community.html:133 +msgid "Email discussions" +msgstr "E-posta tartış discussionsmaları" + +#: templates/index/community.html:142 +msgid "Like ClickHouse?" +msgstr "ClickHouse Gibi Mi?" + +#: templates/index/community.html:143 +msgid "Help to spread the word about it via" +msgstr "Aracılığıyla bu konuda kelime yaymak için yardım" + +#: templates/index/community.html:144 +msgid "and" +msgstr "ve" + +#: templates/index/community.html:153 +msgid "Hosting ClickHouse Meetups" +msgstr "ClickHouse Buluşmaları Barındırma" + +#: templates/index/community.html:157 +msgid "" +"ClickHouse meetups are essential for strengthening community worldwide, " +"but they couldn't be possible without the help of local organizers. " +"Please, feel this form if you want to become one or want to meet " +"ClickHouse core team for any other reason." +msgstr "" +"ClickHouse buluşmalar dünya çapında toplumu güçlendirmek için gereklidir," +" ancak yerel organizatörlerin yardımı olmadan mümkün olamazdı. Eğer biri " +"olmak istiyorsanız veya başka bir nedenle ClickHouse çekirdek ekibi " +"karşılamak istiyorsanız, bu formu hissedin." + +#: templates/index/community.html:159 +msgid "ClickHouse Meetup" +msgstr "ClickHouse Meetup" + +#: templates/index/community.html:165 +msgid "Name" +msgstr "Ad" + +#: templates/index/community.html:168 +msgid "Email" +msgstr "Posta" + +#: templates/index/community.html:171 +msgid "Company" +msgstr "Şirket" + +#: templates/index/community.html:174 +msgid "City" +msgstr "Şehir" + +#: templates/index/community.html:179 +msgid "We'd like to host a public ClickHouse Meetup" +msgstr "Halka açık bir ClickHouse buluşmasına ev sahipliği yapmak istiyoruz" + +#: templates/index/community.html:185 +msgid "We'd like to invite Yandex ClickHouse team to our office" +msgstr "Yandex ClickHouse ekibini ofisimize davet etmek istiyoruz" + +#: templates/index/community.html:191 +msgid "We'd like to invite Yandex ClickHouse team to another event we organize" +msgstr "" +"Yandex ClickHouse ekibini organize ettiğimiz başka bir etkinliğe davet " +"etmek istiyoruz" + +#: templates/index/community.html:197 +msgid "We're interested in commercial consulting, support or managed service" +msgstr "Ticari danışmanlık, destek veya yönetilen hizmetle ilgileniyoruz" + +#: templates/index/community.html:201 +msgid "Additional comments" +msgstr "Ek yorumlar" + +#: templates/index/community.html:203 +msgid "Send" +msgstr "Göndermek" + +#: templates/index/community.html:212 +msgid "" +"If you have any more thoughts or questions, feel free to contact Yandex " +"ClickHouse team directly at" +msgstr "" +"Daha fazla düşünceniz veya sorunuz varsa, Yandex ClickHouse ekibiyle " +"doğrudan iletişime geçmekten çekinmeyin" + +#: templates/index/community.html:213 +msgid "turn on JavaScript to see email address" +msgstr "e-posta adresini görmek için JavaScript'i açın" + diff --git a/website/locale/zh/LC_MESSAGES/messages.mo b/website/locale/zh/LC_MESSAGES/messages.mo index 3a2aa7f01c6..896fedc0658 100644 Binary files a/website/locale/zh/LC_MESSAGES/messages.mo and b/website/locale/zh/LC_MESSAGES/messages.mo differ diff --git a/website/locale/zh/LC_MESSAGES/messages.po b/website/locale/zh/LC_MESSAGES/messages.po index f86d756797b..21ec6c77696 100644 --- a/website/locale/zh/LC_MESSAGES/messages.po +++ b/website/locale/zh/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-03-30 15:12+0300\n" +"POT-Creation-Date: 2020-04-15 13:19+0000\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: zh\n" @@ -71,11 +71,15 @@ msgstr "" "本文档页面的以下内容已经过机器翻译。 但与其他网站不同，它不是在飞行中完成。 " "这个翻译的文本生活在github存储库旁边的主ClickHouse代码库，并等待同胞母语，使其更具人类可读性。" +#: templates/docs/content.html:7 +msgid "You can also use the original English version as a reference." +msgstr "您也可以使用原始的英文版本作为参考。" + #: templates/docs/content.html:10 msgid "Help ClickHouse documentation by editing this page" msgstr "通过编辑此页面帮助ClickHouse文档" -#: templates/docs/content.html:31 +#: templates/docs/content.html:27 msgid "Built from" msgstr "建于" @@ -175,15 +179,15 @@ msgstr "电子邮件讨论" msgid "Like ClickHouse?" msgstr "像克里克豪斯？" -#: templates/index/community.html:142 +#: templates/index/community.html:143 msgid "Help to spread the word about it via" msgstr "帮助通过传播这个词" -#: templates/index/community.html:143 +#: templates/index/community.html:144 msgid "and" msgstr "和" -#: templates/index/community.html:155 +#: templates/index/community.html:153 msgid "Hosting ClickHouse Meetups" msgstr "碌莽禄Hosting拢Hosting0755-88888888" diff --git a/website/main.html b/website/main.html index fe44b74f2db..eede52a11c6 100644 --- a/website/main.html +++ b/website/main.html @@ -4,6 +4,7 @@ {% set direction = config.theme.direction %} {% set single_page = config.extra.single_page %} {% set no_footer = True %} +{% set og_type = 'article' %} {% if page and page.meta and page.meta.title %} {% set title = page.meta.title %} diff --git a/website/nginx/default.conf b/website/nginx/default.conf deleted file mode 100644 index a5b54e75cb7..00000000000 --- a/website/nginx/default.conf +++ /dev/null @@ -1,10 +0,0 @@ -server { - listen 80 default; - listen [::]:80 default ipv6only=on; - server_name localhost clickhouse.yandex clickhouse.yandex.ru clickhouse.yandex.com; - return 301 https://clickhouse.tech$request_uri; -} -server { - server_name test.clickhouse.yandex; - return 301 https://test.clickhouse.tech$request_uri; -} diff --git a/website/nginx/nginx.conf b/website/nginx/nginx.conf deleted file mode 100644 index 9468004b061..00000000000 --- a/website/nginx/nginx.conf +++ /dev/null @@ -1,41 +0,0 @@ - -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - - -events { - worker_connections 4096; - multi_accept on; - use epoll; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - - keepalive_timeout 65; - - gzip on; - gzip_comp_level 5; - gzip_min_length 256; - - add_header X-Content-Type-Options nosniff always; - add_header X-Frame-Options DENY always; - add_header X-XSS-Protection "1; mode=block" always; - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} diff --git a/website/sitemap_static.xml b/website/sitemap_static.xml index 751ad4e8ce2..6d6b41e5827 100644 --- a/website/sitemap_static.xml +++ b/website/sitemap_static.xml @@ -5,11 +5,15 @@ daily - https://clickhouse.tech/benchmark.html + https://clickhouse.tech/benchmark/dbms/ weekly - https://clickhouse.tech/benchmark_hardware.html + https://clickhouse.tech/benchmark/hardware/ weekly + + https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html + daily + diff --git a/website/templates/common_css.html b/website/templates/common_css.html index 4aa9114e06d..ac10b233f25 100644 --- a/website/templates/common_css.html +++ b/website/templates/common_css.html @@ -1 +1,5 @@ + +{% for src in extra_css %} + +{% endfor %} diff --git a/website/templates/common_js.html b/website/templates/common_js.html index b2bed146503..72421f00562 100644 --- a/website/templates/common_js.html +++ b/website/templates/common_js.html @@ -1,4 +1,8 @@ - + + +{% for src in extra_js %} + +{% endfor %}
{% endif %}
+{% if single_page and page.content %} + +(function() { + {% for chunk in page.content|chunks %} + document.write({{ chunk|tojson|safe }}); + {% endfor %} +})(); + +{% endif %} diff --git a/website/templates/docs/nav.html b/website/templates/docs/nav.html index 42808819ef3..adc7231658f 100644 --- a/website/templates/docs/nav.html +++ b/website/templates/docs/nav.html @@ -6,7 +6,11 @@

- {{ page.title or config.site_name }} + {% if page.title != 'hidden' %} + {{ page.title or config.site_name }} + {% else %} + {{ page.meta.toc_folder_title or config.site_name }} + {% endif %}

diff --git a/website/templates/docs/sidebar.html b/website/templates/docs/sidebar.html index f3ce0b5be7e..a4ff0e780d8 100644 --- a/website/templates/docs/sidebar.html +++ b/website/templates/docs/sidebar.html @@ -21,7 +21,7 @@ {% for release in config.extra.stable_releases %} {{ release.0 }} + href="/docs/{{ release.0 }}/{{ language }}/">{{ release.0 }}{% if release.1.2 %} LTS{% endif %} {% endfor %} diff --git a/website/templates/footer.html b/website/templates/footer.html index 42f1e4263be..765ea63d528 100644 --- a/website/templates/footer.html +++ b/website/templates/footer.html @@ -8,7 +8,7 @@ {{ _('ClickHouse source code is published under the Apache 2.0 License.') }} {{ _('Software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.') }}
- © 2016–2020 {{ _('Yandex LLC') }}} + © 2016–2020 {{ _('Yandex LLC') }}
diff --git a/website/templates/index/community.html b/website/templates/index/community.html index e48edb311b6..e230cac8da9 100644 --- a/website/templates/index/community.html +++ b/website/templates/index/community.html @@ -9,7 +9,7 @@
- {{ _('ClickHouse YouTube Channel') }} @@ -21,7 +21,7 @@
- {{ _('ClickHouse Official Twitter Account') }} @@ -32,7 +32,7 @@
- {{ _('ClickHouse at Telegram') }} @@ -56,7 +56,7 @@
- ClickHouse GitHub @@ -69,10 +69,10 @@
-
- {{ _('ClickHouse Slack Workspace') }}
- ClickHouse at Stack Overflow @@ -107,7 +107,7 @@
- ClickHouse Blog @@ -124,7 +124,7 @@
- {{ _('ClickHouse at Google Groups') }} @@ -156,7 +156,7 @@

{{ _('ClickHouse meetups are essential for strengthening community worldwide, but they couldn\'t be possible without the help of local organizers. Please, feel this form if you want to become one or want to meet ClickHouse core team for any other reason.') }}

- {{ _('ClickHouse Meetup') }} + {{ _('ClickHouse Meetup') }}
diff --git a/website/templates/index/efficient.html b/website/templates/index/efficient.html index 240852696b4..97d3e0c20ca 100644 --- a/website/templates/index/efficient.html +++ b/website/templates/index/efficient.html @@ -9,7 +9,7 @@

Total cost of ownership could be further lowered by using commodity hardware with rotating disk drives instead of enterprise grade NVMe or SSD without significant sacrifices in latency for most kinds of queries.

- ClickHouse is hardware efficient + ClickHouse is hardware efficient
@@ -37,6 +37,6 @@
- ClickHouse не тормозит + ClickHouse не тормозит
diff --git a/website/templates/index/hero.html b/website/templates/index/hero.html index 86e2ed1b18c..e391ca0a8a1 100644 --- a/website/templates/index/hero.html +++ b/website/templates/index/hero.html @@ -1,7 +1,7 @@
-

ClickHouse is a fast open-source database management system

+

ClickHouse is a fast open-source OLAP database management system

It is column-oriented and allows to generate analytical reports using SQL queries in real-time.

diff --git a/website/templates/index/performance.html b/website/templates/index/performance.html index 9f31ad7a3ad..505059e42c6 100644 --- a/website/templates/index/performance.html +++ b/website/templates/index/performance.html @@ -3,20 +3,19 @@

ClickHouse works 100-1000x faster than traditional approaches

-

ClickHouse's performance exceeds comparable column-oriented DBMS currently available +

ClickHouse's performance exceeds comparable column-oriented DBMS currently available on the market. It processes hundreds of millions to more than a billion rows and tens of gigabytes of data per single server per second.

- Detailed comparison + Detailed comparison
-

Independent benchmarks

+

Independent benchmarks

-
- - + {## TODO: ##}
diff --git a/website/templates/index/scalable.html b/website/templates/index/scalable.html index deb63839131..672a02f202b 100644 --- a/website/templates/index/scalable.html +++ b/website/templates/index/scalable.html @@ -2,14 +2,14 @@
-

Linerarly scalable

+

Linearly scalable

ClickHouse scales well both vertically and horizontally. ClickHouse is easily adaptable to perform either on a cluster with hundreds or thousands of nodes or on a single server or even on a tiny virtual machine. Currently, there are installations with more multiple trillion rows or hundreds of terabytes of data per single node.

There are many ClickHouse clusters consisting of multiple hundred nodes, including few clusters of Yandex Metrica, while the largest known ClickHouse cluster is well over a thousand nodes.

- Lineraly scalable + Lineraly scalable
diff --git a/website/templates/index/success.html b/website/templates/index/success.html index 961dc859535..83b5c1427c9 100644 --- a/website/templates/index/success.html +++ b/website/templates/index/success.html @@ -13,7 +13,7 @@
- +

Yandex Metrica

diff --git a/website/templates/index/why.html b/website/templates/index/why.html index 53bde640c4f..a2917258923 100644 --- a/website/templates/index/why.html +++ b/website/templates/index/why.html @@ -7,7 +7,7 @@
- Blazing fast + Blazing fast

Blazing fast

@@ -15,7 +15,7 @@ processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.

- Fault tolerant + Fault tolerant

Fault-tolerant

@@ -24,14 +24,14 @@
- Easy to use + Easy to use

Easy to use

ClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some DBMS.

- Highly reliable + Highly reliable

Highly reliable

diff --git a/website/workers/events.js b/website/workers/events.js deleted file mode 100644 index 653139af9f9..00000000000 --- a/website/workers/events.js +++ /dev/null @@ -1,34 +0,0 @@ -addEventListener('fetch', event => { - event.respondWith(handleRequest(event.request)) -}) - -async function handleRequest(request) { - let raw = await fetch('https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/README.md'); - let text = await raw.text(); - let lines = text.split('\n'); - let skip = true; - let events = []; - for (let idx in lines) { - let line = lines[idx]; - if (skip) { - if (line.includes('Upcoming Events')) { - skip = false; - } - } else { - if (!line) { continue; }; - line = line.split(']('); - var tail = line[1].split(') '); - events.push({ - 'signup_link': tail[0], - 'event_name': line[0].replace('* [', ''), - 'event_date': tail[1].slice(0, -1).replace('on ', '') - }); - } - } - - let response = new Response(JSON.stringify({ - 'events': events - })); - response.headers.set('Content-Type', 'application/json'); - return response; -} diff --git a/website/workers/meet-form.js b/website/workers/meet-form.js deleted file mode 100644 index 6506d59522e..00000000000 --- a/website/workers/meet-form.js +++ /dev/null @@ -1,75 +0,0 @@ - -addEventListener('fetch', event => { - event.respondWith(handleRequest(event.request)) -}) - -async function handleRequest(request) { - if (request.method != 'POST') { - return new Response('Bad request', { - status: 400, - statusText: 'Bad request' - }); - } - let url = new URL('https://api.sendgrid.com/v3/mail/send'); - let newHdrs = new Headers(); - newHdrs.set('Authorization', 'Bearer ' + SENDGRID_TOKEN); - newHdrs.set('Content-Type', 'application/json'); - let args = await request.json(); - let subject = args['name'] + ' wants to meet'; - let content = ''; - let argsKeys = Object.keys(args); - if (['name', 'email', 'city', 'company'].filter(n=>!argsKeys.includes(n)).length) { - return new Response('Bad request', { - status: 400, - statusText: 'Bad request' - }); - } - for (let key in args) { - content += key.charAt(0).toUpperCase() + key.slice(1); - content += ':\r\n' + args[key] + '\r\n\r\n'; - } - let body = { - "personalizations": [ - { - "to": [ - { - "email": "clickhouse-feedback@yandex-team.ru", - "name": "ClickHouse Core Team" - } - ], - "subject": subject - } - ], "content": [ - { - "type": "text/plain", - "value": content - } - ], "from": { - "email": "no-reply@clickhouse.tech", - "name": "ClickHouse Website" - }, "reply_to": - { - "email": "no-reply@clickhouse.tech", - "name": "ClickHouse Website" - } - }; - const init = { - body: JSON.stringify(body), - headers: newHdrs, - method: 'POST' - } - - let response = await fetch(url, init); - let status = 200; - if (response.status != 202) { - status = 200; - } - - return new Response('{}', { - status: status, - statusText: response.statusText.replace('Accepted', 'OK'), - headers: new Headers({ - 'Content-Type': 'application/json' - }) - }) -} diff --git a/website/workers/play-api.js b/website/workers/play-api.js deleted file mode 100644 index 62792d37a4d..00000000000 --- a/website/workers/play-api.js +++ /dev/null @@ -1,24 +0,0 @@ -addEventListener('fetch', event => { - event.respondWith(handleRequest(event.request)) -}) - -async function handleRequest(request) { - let url = new URL(request.url); - url.hostname = 'play-api.clickhouse.tech'; - url.port = 8443; - url.pathname = url.pathname.replace('/api/', '/'); - let newHdrs = new Headers() - - const init = { - body: request.body, - headers: request.headers, - method: request.method - } - - let response = await fetch(url, init); - - return new Response(response.body, { - status: response.status, - statusText: response.statusText - }) -} diff --git a/website/workers/repo.js b/website/workers/repo.js deleted file mode 100644 index 470391cf225..00000000000 --- a/website/workers/repo.js +++ /dev/null @@ -1,10 +0,0 @@ -addEventListener('fetch', event => { - event.respondWith(handleRequest(event.request)) -}) - -async function handleRequest(request) { - let url = new URL(request.url); - url.hostname = 'repo.yandex.ru'; - url.pathname = '/clickhouse' + url.pathname; - return fetch(url) -} diff --git a/ya.make b/ya.make index 84f1a2163a4..7f7c1bbce2f 100644 --- a/ya.make +++ b/ya.make @@ -7,4 +7,6 @@ OWNER(g:clickhouse) RECURSE( base + programs + src )