diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0e914c656fc..330bab44533 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,16 +7,20 @@ Changelog category (leave one): - Performance Improvement - Backward Incompatible Change - Build/Testing/Packaging Improvement -- Documentation +- Documentation (changelog entry is not required) - Other -- Non-significant (changelog entry is not needed) +- Non-significant (changelog entry is not required) -Changelog entry (up to few sentences, required except for Non-significant/Documentation categories): +Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md): ... -Detailed description (optional): +Detailed description / Documentation draft: ... + +By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder. + +If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first. diff --git a/.gitmodules b/.gitmodules index 8333e5544bc..dab5b985760 100644 --- a/.gitmodules +++ b/.gitmodules @@ -134,6 +134,13 @@ [submodule "contrib/libc-headers"] path = contrib/libc-headers url = https://github.com/ClickHouse-Extras/libc-headers.git +[submodule "contrib/replxx"] + path = contrib/replxx + url = https://github.com/AmokHuginnsson/replxx.git [submodule "contrib/ryu"] path = contrib/ryu url = https://github.com/ClickHouse-Extras/ryu.git +[submodule "contrib/avro"] + path = contrib/avro + url = https://github.com/ClickHouse-Extras/avro.git + ignore = untracked diff --git a/AUTHORS b/AUTHORS index db769ac16ce..12838d7fa14 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,43 +1,2 @@ -The following authors have created the source code of "ClickHouse" -published and distributed by YANDEX LLC as the owner: - -Alexander Makarov -Alexander Prudaev -Alexey Arno -Alexey Milovidov -Alexey Tronov -Alexey Vasiliev -Alexey Zatelepin -Amy Krishnevsky -Andrey M -Andrey Mironov -Andrey Urusov -Anton Tikhonov -Dmitry Bilunov -Dmitry Galuza -Eugene Konkov -Evgeniy Gatov -Ilya Khomutov -Ilya Korolev -Ivan Blinkov -Maxim Nikulin -Michael Kolupaev -Michael Razuvaev -Nikolai Kochetov -Nikolay Vasiliev -Nikolay Volosatov -Pavel Artemkin -Pavel Kartaviy -Roman Nozdrin -Roman Peshkurov -Sergey Fedorov -Sergey Lazarev -Sergey Magidovich -Sergey Serebryanik -Sergey Veletskiy -Vasily Okunev -Vitaliy Lyudvichenko -Vladimir Chebotarev -Vsevolod Orlov -Vyacheslav Alipov -Yuriy Galitskiy +To see the list of authors who created the source code of ClickHouse, published and distributed by YANDEX LLC as the owner, +run "SELECT * FROM system.contributors;" query on any ClickHouse server. diff --git a/CHANGELOG.md b/CHANGELOG.md index f456a56f1be..cdfaed101b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,300 @@ +## ClickHouse release v20.1 + +### Backward Incompatible Change +* Make the setting `merge_tree_uniform_read_distribution` obsolete. The server still recognizes this setting but it has no effect. [#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Changed return type of the function `greatCircleDistance` to `Float32` because now the result of calculation is `Float32`. [#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now it's expected that query parameters are represented in "escaped" format. For example, to pass string `ab` you have to write `a\tb` or `a\b` and respectively, `a%5Ctb` or `a%5C%09b` in URL. This is needed to add the possibility to pass NULL as `\N`. This fixes [#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enable `use_minimalistic_part_header_in_zookeeper` setting for `ReplicatedMergeTree` by default. This will significantly reduce amount of data stored in ZooKeeper. This setting is supported since version 19.1 and we already use it in production in multiple services without any issues for more than half a year. Disable this setting if you have a chance to downgrade to versions older than 19.1. [#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Data skipping indices are production ready and enabled by default. The settings `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` and `allow_experimental_multiple_joins_emulation` are now obsolete and do nothing. [#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add new `ANY JOIN` logic for `StorageJoin` consistent with `JOIN` operation. To upgrade without changes in behaviour you need add `SETTINGS any_join_distinct_right_table_keys = 1` to Engine Join tables metadata or recreate these tables after upgrade. [#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) + +### New Feature +* Added information about part paths to `system.merges`. [#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Add ability to execute `SYSTEM RELOAD DICTIONARY` query in `ON CLUSTER` mode. [#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Add ability to execute `CREATE DICTIONARY` queries in `ON CLUSTER` mode. [#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) +* Now user's profile in `users.xml` can inherit multiple profiles. [#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +* Added `system.stack_trace` table that allows to look at stack traces of all server threads. This is useful for developers to introspect server state. This fixes [#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add `DateTime64` datatype with configurable sub-second precision. [#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) +* Add table function `clusterAllReplicas` which allows to query all the nodes in the cluster. [#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) +* Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature. [#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) +* Speed up parsing of data files in `CSV`, `TSV` and `JSONEachRow` format by doing it in parallel. [#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Add function `bankerRound` which performs banker's rounding. [#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) +* Support more languages in embedded dictionary for region names: 'ru', 'en', 'ua', 'uk', 'by', 'kz', 'tr', 'de', 'uz', 'lv', 'lt', 'et', 'pt', 'he', 'vi'. [#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improvements in consistency of `ANY JOIN` logic. Now `t1 ANY LEFT JOIN t2` equals `t2 ANY RIGHT JOIN t1`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Add setting `any_join_distinct_right_table_keys` which enables old behaviour for `ANY INNER JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Add new `SEMI` and `ANTI JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by asynchronous inserts into `Distributed` table. [#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Add optional reset column argument for `runningAccumulate` which allows to reset aggregation results for each new key value. [#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) +* Add ability to alter materialized views with `ALTER MODIFY QUERY `. [#7533](https://github.com/ClickHouse/ClickHouse/pull/7533) ([nvartolomei](https://github.com/nvartolomei)) +* Add ability to use ClickHouse as Prometheus endpoint. [#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) +* Add section `` in `config.xml` which restricts allowed hosts for remote table engines and table functions `URL`, `S3`, `HDFS`. [#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) +* Added function `greatCircleAngle` which calculates the distance on a sphere in degrees. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Changed Earth radius to be consistent with H3 library. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added `JSONCompactEachRow` and `JSONCompactEachRowWithNamesAndTypes` formats for input and output. [#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) +* Added feature for file-related table engines and table functions (`File`, `S3`, `URL`, `HDFS`) which allows to read and write `gzip` files based on additional engine parameter or file extension. [#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) +* Added the `randomASCII(length)` function, generating a string with a random set of [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) printable characters. [#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([BayoNet](https://github.com/BayoNet)) +* Added function `JSONExtractArrayRaw` which returns an array on unparsed json array elements from `JSON` string. [#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) +* Add `arrayZip` function which allows to combine multiple arrays of equal lengths into one array of tuples. [#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([Winter Zhang](https://github.com/zhang2014)) +* Add ability to move data between disks according to configured `TTL`-expressions for `*MergeTree` table engines family. [#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Added new aggregate function `avgWeighted` which allows to calculate weighted average. [#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) +* Now parallel parsing is enabled by default for `TSV`, `TSKV`, `CSV` and `JSONEachRow` formats. [#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Add several geo functions from `H3` library: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` and `h3kRing`. [#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) +* Added support for brotli (`br`) compression in file-related storages and table functions. This fixes [#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add `groupBit*` functions for the `SimpleAggregationFunction` type. [#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) + +### Bug Fix +* Fix rename of tables with `Distributed` engine. Fixes issue [#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) +* Now dictionaries support `EXPRESSION` for attributes in arbitrary string in non-ClickHouse SQL dialect. [#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) +* Fix broken `INSERT SELECT FROM mysql(...)` query. This fixes [#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) and [#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) +* Fix error "Mismatch column sizes" when inserting default `Tuple` from `JSONEachRow`. This fixes [#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) +* Now an exception will be thrown in case of using `WITH TIES` alongside `LIMIT BY`. Also add ability to use `TOP` with `LIMIT BY`. This fixes [#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix unintendent dependency from fresh glibc version in `clickhouse-odbc-bridge` binary. [#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Bird](https://github.com/amosbird)) +* Fix bug in check function of `*MergeTree` engines family. Now it doesn't fail in case when we have equal amount of rows in last granule and last mark (non-final). [#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) +* Fix insert into `Enum*` columns after `ALTER` query, when underlying numeric type is equal to table specified type. This fixes [#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) +* Allowed non-constant negative "size" argument for function `substring`. It was not allowed by mistake. This fixes [#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix parsing bug when wrong number of arguments passed to `(O|J)DBC` table engine. [#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) +* Using command name of the running clickhouse process when sending logs to syslog. In previous versions, empty string was used instead of command name. [#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) +* Fix check of allowed hosts for `localhost`. This PR fixes the solution provided in [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix rare crash in `argMin` and `argMax` functions for long string arguments, when result is used in `runningAccumulate` function. This fixes [#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) +* Fix memory overcommit for tables with `Buffer` engine. [#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) +* Fixed potential bug in functions that can take `NULL` as one of the arguments and return non-NULL. [#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better metrics calculations in thread pool for background processes for `MergeTree` table engines. [#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix function `IN` inside `WHERE` statement when row-level table filter is present. Fixes [#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) +* Now an exception is thrown if the integral value is not parsed completely for settings values. [#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) +* Fix exception when aggregate function is used in query to distributed table with more than two local shards. [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) +* Now bloom filter can handle zero length arrays and doesn't perform redundant calculations. [#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) +* Fixed checking if a client host is allowed by matching the client host to `host_regexp` specified in `users.xml`. [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) +* Relax ambiguous column check that leads to false positives in multiple `JOIN ON` section. [#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed possible server crash (`std::terminate`) when the server cannot send or write data in `JSON` or `XML` format with values of `String` data type (that require `UTF-8` validation) or when compressing result data with Brotli algorithm or in some other rare cases. This fixes [#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix race condition in `StorageDistributedDirectoryMonitor` found by CI. This fixes [#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now background merges in `*MergeTree` table engines family preserve storage policy volume order more accurately. [#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Now table engine `Kafka` works properly with `Native` format. This fixes [#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +* Fixed formats with headers (like `CSVWithNames`) which were throwing exception about EOF for table engine `Kafka`. [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +* Fixed a bug with making set from subquery in right part of `IN` section. This fixes [#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) and [#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix possible crash while reading from storage `File`. [#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed reading of the files in `Parquet` format containing columns of type `list`. [#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) +* Fix error `Not found column` for distributed queries with `PREWHERE` condition dependent on sampling key if `max_parallel_replicas > 1`. [#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix error `Not found column` if query used `PREWHERE` dependent on table's alias and the result set was empty because of primary key condition. [#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed return type for functions `rand` and `randConstant` in case of `Nullable` argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Disabled predicate push-down for `WITH FILL` expression. This fixes [#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Winter Zhang](https://github.com/zhang2014)) +* Fixed incorrect `count()` result for `SummingMergeTree` when `FINAL` section is used. [#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix possible incorrect result for constant functions from remote servers. It happened for queries with functions like `version()`, `uptime()`, etc. which returns different constant values for different servers. This fixes [#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix complicated bug in push-down predicate optimization which leads to wrong results. This fixes a lot of issues on push-down predicate optimization. [#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Winter Zhang](https://github.com/zhang2014)) +* Fix crash in `CREATE TABLE .. AS dictionary` query. [#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) +* Several improvements ClickHouse grammar in `.g4` file. [#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) +* Fix bug that leads to crashes in `JOIN`s with tables with engine `Join`. This fixes [#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix redundant dictionaries reload on `CREATE DATABASE`. [#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) +* Limit maximum number of streams for read from `StorageFile` and `StorageHDFS`. Fixes https://github.com/ClickHouse/ClickHouse/issues/7650. [#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) +* Fix bug in `ALTER ... MODIFY ... CODEC` query, when user specify both default expression and codec. Fixes [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) +* Fix error in background merge of columns with `SimpleAggregateFunction(LowCardinality)` type. [#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed type check in function `toDateTime64`. [#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) +* Now server do not crash on `LEFT` or `FULL JOIN` with and Join engine and unsupported `join_use_nulls` settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) +* Now `DROP DICTIONARY IF EXISTS db.dict` query doesn't throw exception if `db` doesn't exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix possible crashes in table functions (`file`, `mysql`, `remote`) caused by usage of reference to removed `IStorage` object. Fix incorrect parsing of columns specified at insertion into table function. [#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) +* Ensure network be up before starting `clickhouse-server`. This fixes [#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) +* Fix timeouts handling for secure connections, so queries doesn't hang indefenitely. This fixes [#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `clickhouse-copier`'s redundant contention between concurrent workers. [#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +* Now mutations doesn't skip attached parts, even if their mutation version were larger than current mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) +* Ignore redundant copies of `*MergeTree` data parts after move to another disk and server restart. [#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix crash in `FULL JOIN` with `LowCardinality` in `JOIN` key. [#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) +* Forbidden to use column name more than once in insert query like `INSERT INTO tbl (x, y, x)`. This fixes [#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) +* Added fallback for detection the number of physical CPU cores for unknown CPUs (using the number of logical CPU cores). This fixes [#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `There's no column` error for materialized and alias columns. [#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed sever crash when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier. Just like `EXISTS t`. This fixes [#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). This bug was introduced in version 19.17. [#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix rare bug with error `"Sizes of columns doesn't match"` that might appear when using `SimpleAggregateFunction` column. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +* Fix bug where user with empty `allow_databases` got access to all databases (and same for `allow_dictionaries`). [#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) +* Fix client crash when server already disconnected from client. [#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) +* Fix `ORDER BY` behaviour in case of sorting by primary key prefix and non primary key suffix. [#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) +* Check if qualified column present in the table. This fixes [#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed behavior with `ALTER MOVE` ran immediately after merge finish moves superpart of specified. Fixes [#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix possible server crash while using `UNION` with different number of columns. Fixes [#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix size of result substring for function `substr` with negative size. [#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now server does not execute part mutation in `MergeTree` if there are not enough free threads in background pool. [#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) +* Fix a minor typo on formatting `UNION ALL` AST. [#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) +* Fixed incorrect bloom filter results for negative numbers. This fixes [#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([Winter Zhang](https://github.com/zhang2014)) +* Fixed potential buffer overflow in decompress. Malicious user can pass fabricated compressed data that will cause read after buffer. This issue was found by Eldar Zaitov from Yandex information security team. [#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix incorrect result because of integers overflow in `arrayIntersect`. [#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now `OPTIMIZE TABLE` query will not wait for offline replicas to perform the operation. [#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) +* Fixed `ALTER TTL` parser for `Replicated*MergeTree` tables. [#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix communication between server and client, so server read temporary tables info after query failure. [#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) +* Fix `bitmapAnd` function error when intersecting an aggregated bitmap and a scalar bitmap. [#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) +* Refine the definition of `ZXid` according to the ZooKeeper Programmer's Guide which fixes bug in `clickhouse-cluster-copier`. [#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +* `odbc` table function now respects `external_table_functions_use_nulls` setting. [#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) +* Fixed bug that lead to a rare data race. [#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) +* Now `SYSTEM RELOAD DICTIONARY` reloads a dictionary completely, ignoring `update_field`. This fixes [#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) +* Add ability to check if dictionary exists in create query. [#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alesapin](https://github.com/alesapin)) +* Fix `Float*` parsing in `Values` format. This fixes [#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) +* Fix crash when we cannot reserve space in some background operations of `*MergeTree` table engines family. [#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix crash of merge operation when table contains `SimpleAggregateFunction(LowCardinality)` column. This fixes [#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) +* Restore support of all ICU locales and add the ability to apply collations for constant expressions. Also add language name to `system.collations` table. [#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) +* Fix bug when external dictionaries with zero minimal lifetime (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) don't update in background. [#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alesapin](https://github.com/alesapin)) +* Fix crash when external dictionary with ClickHouse source has subquery in query. [#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix incorrect parsing of file extension in table with engine `URL`. This fixes [#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) +* Fix `CHECK TABLE` query for `*MergeTree` tables without key. Fixes [#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) +* Fixed conversion of `Float64` to MySQL type. [#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) +* Now if table was not completely dropped because of server crash, server will try to restore and load it. [#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) +* Fixed crash in table function `file` while inserting into file that doesn't exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +* Fix rare deadlock which can happen when `trace_log` is in enabled. [#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) +* Add ability to work with different types besides `Date` in `RangeHashed` external dictionary created from DDL query. Fixes [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) +* Fixes crash when `now64()` is called with result of another function. [#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) +* Fixed bug with detecting client IP for connections through mysql wire protocol. [#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) +* Fix empty array handling in `arraySplit` function. This fixes [#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) +* Fixed the issue when `pid-file` of another running `clickhouse-server` may be deleted. [#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) +* Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) +* Fixed error in function `arrayReduce` that may lead to "double free" and error in aggregate function combinator `Resample` that may lead to memory leak. Added aggregate function `aggThrow`. This function can be used for testing purposes. [#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### Improvement +* Improved logging when working with `S3` table engine. [#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +* Printed help message when no arguments are passed when calling `clickhouse-local`. This fixes [#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) +* Add setting `mutations_sync` which allows to wait `ALTER UPDATE/DELETE` queries synchronously. [#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) +* Allow to set up relative `user_files_path` in `config.xml` (in the way similar to `format_schema_path`). [#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) +* Add exception for illegal types for conversion functions with `-OrZero` postfix. [#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) +* Simplify format of the header of data sending to a shard in a distributed query. [#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) +* `Live View` table engine refactoring. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) +* Add additional checks for external dictionaries created from DDL-queries. [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) +* Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now table the first argument of `joinGet` function can be table indentifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) +* Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) +* Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) +* `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) +* Stack traces now display physical addresses (offsets in object file) instead of virtual memory addresses (where the object file was loaded). That allows the use of `addr2line` when binary is position independent and ASLR is active. This fixes [#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Support new syntax for row-level security filters: `…
`. Fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) +* Now `cityHash` function can work with `Decimal` and `UUID` types. Fixes [#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) +* Removed fixed index granularity (it was 1024) from system logs because it's obsolete after implementation of adaptive granularity. [#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enabled MySQL compatibility server when ClickHouse is compiled without SSL. [#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([Yuriy Baranov](https://github.com/yurriy)) +* Now server checksums distributed batches, which gives more verbose errors in case of corrupted data in batch. [#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) +* Support `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` and `ATTACH TABLE` for `MySQL` database engine. [#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([Winter Zhang](https://github.com/zhang2014)) +* Add authentication in S3 table function and table engine. [#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Added check for extra parts of `MergeTree` at different disks, in order to not allow to miss data parts at undefined disks. [#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Enable SSL support for Mac client and server. [#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([Ivan](https://github.com/abyss7)) +* Now ClickHouse can work as MySQL federated server (see https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) +* `clickhouse-client` now only enable `bracketed-paste` when multiquery is on and multiline is off. This fixes (#7757)[https://github.com/ClickHouse/ClickHouse/issues/7757]. [#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Bird](https://github.com/amosbird)) +* Support `Array(Decimal)` in `if` function. [#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) +* Support Decimals in `arrayDifference`, `arrayCumSum` and `arrayCumSumNegative` functions. [#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) +* Added `lifetime` column to `system.dictionaries` table. [#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) +* Improved check for existing parts on different disks for `*MergeTree` table engines. Addresses [#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Integration with `AWS SDK` for `S3` interactions which allows to use all S3 features out of the box. [#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) +* Added support for subqueries in `Live View` tables. [#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) +* Check for using `Date` or `DateTime` column from `TTL` expressions was removed. [#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Information about disk was added to `system.detached_parts` table. [#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Now settings `max_(table|partition)_size_to_drop` can be changed without a restart. [#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +* Slightly better usability of error messages. Ask user not to remove the lines below `Stack trace:`. [#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better reading messages from `Kafka` engine in various formats after [#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([Ivan](https://github.com/abyss7)) +* Better compatibility with MySQL clients which don't support `sha2_password` auth plugin. [#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) +* Support more column types in MySQL compatibility server. [#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) +* Implement `ORDER BY` optimization for `Merge`, `Buffer` and `Materilized View` storages with underlying `MergeTree` tables. [#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) +* Now we always use POSIX implementation of `getrandom` to have better compatibility with old kernels (< 3.17). [#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Bird](https://github.com/amosbird)) +* Better check for valid destination in a move TTL rule. [#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Better checks for broken insert batches for `Distributed` table engine. [#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) +* Add column with array of parts name which mutations must process in future to `system.mutations` table. [#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) +* Parallel merge sort optimization for processors. [#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* The settings `mark_cache_min_lifetime` is now obsolete and does nothing. In previous versions, mark cache can grow in memory larger than `mark_cache_size` to accomodate data within `mark_cache_min_lifetime` seconds. That was leading to confusion and higher memory usage than expected, that is especially bad on memory constrained systems. If you will see performance degradation after installing this release, you should increase the `mark_cache_size`. [#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Preparation to use `tid` everywhere. This is needed for [#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### Performance Improvement +* Performance optimizations in processors pipeline. [#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Non-blocking updates of expired keys in cache dictionaries (with permission to read old ones). [#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Compile ClickHouse without `-fno-omit-frame-pointer` globally to spare one more register. [#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Bird](https://github.com/amosbird)) +* Speedup `greatCircleDistance` function and add performance tests for it. [#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) +* Improved performance of function `roundDown`. [#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improved performance of `max`, `min`, `argMin`, `argMax` for `DateTime64` data type. [#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) +* Improved performance of sorting without a limit or with big limit and external sorting. [#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improved performance of formatting floating point numbers up to 6 times. [#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improved performance of `modulo` function. [#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Bird](https://github.com/amosbird)) +* Optimized `ORDER BY` and merging with single column key. [#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better implementation for `arrayReduce`, `-Array` and `-State` combinators. [#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Bird](https://github.com/amosbird)) +* Now `PREWHERE` should be optimized to be at least as efficient as `WHERE`. [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) +* Improve the way `round` and `roundBankers` handling negative numbers. [#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) +* Improved decoding performance of `DoubleDelta` and `Gorilla` codecs by roughly 30-40%. This fixes [#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) +* Improved performance of `base64` related functions. [#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added a function `geoDistance`. It is similar to `greatCircleDistance` but uses approximation to WGS-84 ellipsoid model. The performance of both functions are near the same. [#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Faster `min` and `max` aggregation functions for `Decimal` data type. [#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) +* Vectorize processing `arrayReduce`. [#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) +* `if` chains are now optimized as `multiIf`. [#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) +* Fix performance regression of `Kafka` table engine introduced in 19.15. This fixes [#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) +* Removed "pie" code generation that `gcc` from Debian packages occasionally brings by default. [#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Parallel parsing data formats [#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Enable optimized parser of `Values` with expressions by default (`input_format_values_deduce_templates_of_expressions=1`). [#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) + +### Build/Testing/Packaging Improvement +* Build fixes for `ARM` and in minimal mode. [#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) +* Add coverage file flush for `clickhouse-server` when std::atexit is not called. Also slightly improved logging in stateless tests with coverage. [#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) +* Update LLVM library in contrib. Avoid using LLVM from OS packages. [#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make bundled `curl` build fully quiet. [#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) +* Fix some `MemorySanitizer` warnings. [#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Use `add_warning` and `no_warning` macros in `CMakeLists.txt`. [#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([Ivan](https://github.com/abyss7)) +* Add support of Minio S3 Compatible object (https://min.io/) for better integration tests. [#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) +* Imported `libc` headers to contrib. It allows to make builds more consistent across various systems (only for `x86_64-linux-gnu`). [#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove `-fPIC` from some libraries. [#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Clean `CMakeLists.txt` for curl. See https://github.com/ClickHouse/ClickHouse/pull/8011#issuecomment-569478910 [#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Silent warnings in `CapNProto` library. [#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add performance tests for short string optimized hash tables. [#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Bird](https://github.com/amosbird)) +* Now ClickHouse will build on `AArch64` even if `MADV_FREE` is not available. This fixes [#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Bird](https://github.com/amosbird)) +* Update `zlib-ng` to fix memory sanitizer problems. [#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually doesn't work at all. This fixes [#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed build on some systems after enabling `libc++`. This supersedes [#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make `Field` methods more type-safe to find more errors. [#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Added missing files to the `libc-headers` submodule. [#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix wrong `JSON` quoting in performance test output. [#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now stack trace is displayed for `std::exception` and `Poco::Exception`. In previous versions it was available only for `DB::Exception`. This improves diagnostics. [#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Porting `clock_gettime` and `clock_nanosleep` for fresh glibc versions. [#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Bird](https://github.com/amosbird)) +* Enable `part_log` in example config for developers. [#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix async nature of reload in `01036_no_superfluous_dict_reload_on_create_database*`. [#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) +* Fixed codec performance tests. [#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) +* Add install scripts for `.tgz` build and documentation for them. [#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) +* Removed old `ZSTD` test (it was created in year 2016 to reproduce the bug that pre 1.0 version of ZSTD has had). This fixes [#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed build on Mac OS Catalina. [#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) +* Increased number of rows in codec performance tests to make results noticeable. [#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) +* In debug builds, treat `LOGICAL_ERROR` exceptions as assertion failures, so that they are easier to notice. [#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Make formats-related performance test more deterministic. [#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update `lz4` to fix a MemorySanitizer failure. [#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Suppress a known MemorySanitizer false positive in exception handling. [#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Update `gcc` and `g++` to version 9 in `build/docker/build.sh` [#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) +* Add performance test case to test that `PREWHERE` is worse than `WHERE`. [#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Bird](https://github.com/amosbird)) +* Progress towards fixing one flacky test. [#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid MemorySanitizer report for data from `libunwind`. [#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Updated `libc++` to the latest version. [#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Build ICU library from sources. This fixes [#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Switched from `libressl` to `openssl`. ClickHouse should support TLS 1.3 and SNI after this change. This fixes [#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed UBSan report when using `chacha20_poly1305` from SSL (happens on connect to https://yandex.ru/). [#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix mode of default password file for `.deb` linux distros. [#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) +* Improved expression for getting `clickhouse-server` PID in `clickhouse-test`. [#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) +* Updated contrib/googletest to v1.10.0. [#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) +* Fixed ThreadSaninitizer report in `base64` library. Also updated this library to the latest version, but it doesn't matter. This fixes [#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `00600_replace_running_query` for processors. [#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Remove support for `tcmalloc` to make `CMakeLists.txt` simpler. [#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Release gcc builds now use `libc++` instead of `libstdc++`. Recently `libc++` was used only with clang. This will improve consistency of build configurations and portability. [#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enable ICU library for build with MemorySanitizer. [#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Suppress warnings from `CapNProto` library. [#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Removed special cases of code for `tcmalloc`, because it's no longer supported. [#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* In CI coverage task, kill the server gracefully to allow it to save the coverage report. This fixes incomplete coverage reports we've been seeing lately. [#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alesapin](https://github.com/alesapin)) +* Performance tests for all codecs against `Float64` and `UInt64` values. [#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) +* `termcap` is very much deprecated and lead to various problems (f.g. missing "up" cap and echoing `^J` instead of multi line) . Favor `terminfo` or bundled `ncurses`. [#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Bird](https://github.com/amosbird)) +* Fix `test_storage_s3` integration test. [#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Support `StorageFile(, null) ` to insert block into given format file without actually write to disk. This is required for performance tests. [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) +* Added argument `--print-time` to functional tests which prints execution time per test. [#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Added asserts to `KeyCondition` while evaluating RPN. This will fix warning from gcc-9. [#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Dump cmake options in CI builds. [#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Don't generate debug info for some fat libraries. [#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make `log_to_console.xml` always log to stderr, regardless of is it interactive or not. [#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Removed some unused features from `clickhouse-performance-test` tool. [#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Now we will also search for `lld-X` with corresponding `clang-X` version. [#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alesapin](https://github.com/alesapin)) +* Parquet build improvement. [#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) +* More GCC warnings [#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) +* Package for Arch Linux now allows to run ClickHouse server, and not only client. [#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix test with processors. Tiny performance fixes. [#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Update contrib/protobuf. [#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) +* In preparation of switching to c++20 as a new year celebration. "May the C++ force be with ClickHouse." [#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Bird](https://github.com/amosbird)) + +### Experimental Feature +* Added experimental setting `min_bytes_to_use_mmap_io`. It allows to read big files without copying data from kernel to userspace. The setting is disabled by default. Recommended threshold is about 64 MB, because mmap/munmap is slow. [#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Reworked quotas as a part of access control system. Added new table `system.quotas`, new functions `currentQuota`, `currentQuotaKey`, new SQL syntax `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) +* Allow skipping unknown settings with warnings instead of throwing exceptions. [#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) +* Reworked row policies as a part of access control system. Added new table `system.row_policies`, new function `currentRowPolicies()`, new SQL syntax `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) + +### Security Fix +* Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) + ## ClickHouse release v19.17 ### ClickHouse release v19.17.6.36, 2019-12-27 diff --git a/CMakeLists.txt b/CMakeLists.txt index cd32288ec9c..c194ea5bdc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,7 +328,6 @@ include (cmake/find/xxhash.cmake) include (cmake/find/sparsehash.cmake) include (cmake/find/rt.cmake) include (cmake/find/execinfo.cmake) -include (cmake/find/readline_edit.cmake) include (cmake/find/re2.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/rdkafka.cmake) @@ -353,6 +352,7 @@ include (cmake/find/simdjson.cmake) include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) include (cmake/find/orc.cmake) +include (cmake/find/avro.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5110e7c4999..e16957b759d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,24 +1,30 @@ # Contributing to ClickHouse -## Technical info -Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation: -https://clickhouse.yandex/docs/en/development/architecture/ +ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better. -## Legal info +Thank you. -In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here: +## Technical Info + +We have a [developer's guide](https://clickhouse.yandex/docs/en/development/developer_instruction/) for writing code for ClickHouse. Besides this guide, you can find [Overview of ClickHouse Architecture](https://clickhouse.yandex/docs/en/development/architecture/) and instructions on how to build ClickHouse in different environments. + +If you want to contribute to documentation, read the [Contributing to ClickHouse Documentation](docs/README.md) guide. + +## Legal Info + +In order for us (YANDEX LLC) to accept patches and other contributions from you, you may adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here: 1) https://yandex.ru/legal/cla/?lang=en (in English) and 2) https://yandex.ru/legal/cla/?lang=ru (in Russian). By adopting the CLA, you state the following: * You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA, -* You has read the terms and conditions of the CLA and agree with them in full, +* You have read the terms and conditions of the CLA and agree with them in full, * You are legally able to provide and license your contributions as stated, * We may use your contributions for our open source projects and for any other our project too, -* We rely on your assurances concerning the rights of third parties in relation to your contributes. +* We rely on your assurances concerning the rights of third parties in relation to your contributions. -If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA. +If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA. If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it: @@ -31,4 +37,7 @@ Replace the bracketed text as follows: It is enough to provide us such notification once. -If you don't agree with the CLA, you still can open a pull request to provide your contributions. +As an alternative, you can provide DCO instead of CLA. You can find the text of DCO here: https://developercertificate.org/ +It is enough to read and copy it verbatim to your pull request. + +If you don't agree with the CLA and don't want to provide DCO, you still can open a pull request to provide your contributions. diff --git a/LICENSE b/LICENSE index ef36c40c4b0..f79538892b8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2016-2019 Yandex LLC +Copyright 2016-2020 Yandex LLC Apache License Version 2.0, January 2004 @@ -188,7 +188,7 @@ Copyright 2016-2019 Yandex LLC same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2016-2019 Yandex LLC + Copyright 2016-2020 Yandex LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmake/find/avro.cmake b/cmake/find/avro.cmake new file mode 100644 index 00000000000..cdb3fc84d3d --- /dev/null +++ b/cmake/find/avro.cmake @@ -0,0 +1,28 @@ +option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES}) + +if (ENABLE_AVRO) + +option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED}) + +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") + if(USE_INTERNAL_AVRO_LIBRARY) + message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init --recursive") + endif() + set(MISSING_INTERNAL_AVRO_LIBRARY 1) + set(USE_INTERNAL_AVRO_LIBRARY 0) +endif() + +if (NOT USE_INTERNAL_AVRO_LIBRARY) +elseif(NOT MISSING_INTERNAL_AVRO_LIBRARY) + include(cmake/find/snappy.cmake) + set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include") + set(AVROCPP_LIBRARY avrocpp) +endif () + +if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR) + set(USE_AVRO 1) +endif() + +endif() + +message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_INCLUDE_DIR} : ${AVROCPP_LIBRARY}") diff --git a/cmake/find/base64.cmake b/cmake/find/base64.cmake index 022a78478ea..f72397597d7 100644 --- a/cmake/find/base64.cmake +++ b/cmake/find/base64.cmake @@ -11,7 +11,6 @@ if (ENABLE_BASE64) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64") message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") else() - set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include) set (BASE64_LIBRARY base64) set (USE_BASE64 1) endif() diff --git a/cmake/find/boost.cmake b/cmake/find/boost.cmake index 6776d0cea06..ec10a34d839 100644 --- a/cmake/find/boost.cmake +++ b/cmake/find/boost.cmake @@ -31,6 +31,7 @@ if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY) set (Boost_SYSTEM_LIBRARY boost_system_internal) set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal) set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal ${Boost_SYSTEM_LIBRARY}) + set (Boost_IOSTREAMS_LIBRARY boost_iostreams_internal) set (Boost_REGEX_LIBRARY boost_regex_internal) set (Boost_INCLUDE_DIRS) @@ -48,4 +49,4 @@ if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY) list (APPEND Boost_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/boost") endif () -message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_REGEX_LIBRARY}") +message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_IOSTREAMS_LIBRARY},${Boost_REGEX_LIBRARY}") diff --git a/cmake/find/poco.cmake b/cmake/find/poco.cmake index b44d2932276..0c676d374f1 100644 --- a/cmake/find/poco.cmake +++ b/cmake/find/poco.cmake @@ -14,6 +14,7 @@ if (NOT ENABLE_LIBRARIES) set (ENABLE_POCO_REDIS ${ENABLE_LIBRARIES} CACHE BOOL "") set (ENABLE_POCO_ODBC ${ENABLE_LIBRARIES} CACHE BOOL "") set (ENABLE_POCO_SQL ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_JSON ${ENABLE_LIBRARIES} CACHE BOOL "") endif () set (POCO_COMPONENTS Net XML SQL Data) @@ -34,6 +35,9 @@ if (NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC) list (APPEND POCO_COMPONENTS DataODBC) list (APPEND POCO_COMPONENTS SQLODBC) endif () +if (NOT DEFINED ENABLE_POCO_JSON OR ENABLE_POCO_JSON) + list (APPEND POCO_COMPONENTS JSON) +endif () if (NOT USE_INTERNAL_POCO_LIBRARY) find_package (Poco COMPONENTS ${POCO_COMPONENTS}) @@ -112,6 +116,11 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) endif () endif () + if (NOT DEFINED ENABLE_POCO_JSON OR ENABLE_POCO_JSON) + set (Poco_JSON_LIBRARY PocoJSON) + set (Poco_JSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/JSON/include/") + endif () + if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL)) set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES}) set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES}) @@ -145,8 +154,11 @@ endif () if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND) set (USE_POCO_SQLODBC 1) endif () +if (Poco_JSON_LIBRARY) + set (USE_POCO_JSON 1) +endif () -message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}") +message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY},${Poco_JSON_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}, JSON=${USE_POCO_JSON}") # How to make sutable poco: # use branch: diff --git a/cmake/find/readline_edit.cmake b/cmake/find/readline_edit.cmake deleted file mode 100644 index 96518a66887..00000000000 --- a/cmake/find/readline_edit.cmake +++ /dev/null @@ -1,60 +0,0 @@ -include (CMakePushCheckState) -cmake_push_check_state () - -option (ENABLE_READLINE "Enable readline" ${ENABLE_LIBRARIES}) -if (ENABLE_READLINE) - -set (READLINE_PATHS "/usr/local/opt/readline/lib") -# First try find custom lib for macos users (default lib without history support) -find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH) -if (NOT READLINE_LIB) - find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS}) -endif () - -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.2) - -find_library (EDIT_LIB NAMES edit) - -set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include") -if (READLINE_LIB AND TERMCAP_LIBRARY) - find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH) - if (NOT READLINE_INCLUDE_DIR) - find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS}) - endif () - if (READLINE_INCLUDE_DIR AND READLINE_LIB) - set (USE_READLINE 1) - set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIBRARY}) - message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}") - endif () -elseif (EDIT_LIB AND TERMCAP_LIBRARY) - find_library (CURSES_LIB NAMES curses) - find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS}) - if (CURSES_LIB AND READLINE_INCLUDE_DIR) - set (USE_LIBEDIT 1) - set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIBRARY}) - message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}") - endif () -endif () - -endif () - -if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR) - include (CheckCXXSourceRuns) - - set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LINE_EDITING_LIBS}) - set (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${READLINE_INCLUDE_DIR}) - check_cxx_source_runs (" - #include - #include - #include - int main() { - add_history(NULL); - append_history(1,NULL); - return 0; - } - " HAVE_READLINE_HISTORY) -else () - message (STATUS "Not using any library for line editing.") -endif () - -cmake_pop_check_state () diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index cb099ade7f5..13947425f7b 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -48,12 +48,12 @@ if (SANITIZE) set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "") set (USE_INTERNAL_CAPNP_LIBRARY 0 CACHE BOOL "") set (USE_SIMDJSON 0 CACHE BOOL "") - set (ENABLE_READLINE 0 CACHE BOOL "") set (ENABLE_ORC 0 CACHE BOOL "") set (ENABLE_PARQUET 0 CACHE BOOL "") set (USE_CAPNP 0 CACHE BOOL "") set (USE_INTERNAL_ORC_LIBRARY 0 CACHE BOOL "") set (USE_ORC 0 CACHE BOOL "") + set (USE_AVRO 0 CACHE BOOL "") set (ENABLE_SSL 0 CACHE BOOL "") elseif (SANITIZE STREQUAL "thread") diff --git a/cmake/target.cmake b/cmake/target.cmake index 9233af62dcf..1f40e28e76b 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -15,7 +15,6 @@ if (CMAKE_CROSSCOMPILING) set (USE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "") - set (ENABLE_READLINE OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "") elseif (OS_LINUX) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 53ad9a0c138..7c9db5bb06f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -146,6 +146,20 @@ if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY) add_subdirectory (icu-cmake) endif () +if(USE_INTERNAL_SNAPPY_LIBRARY) + set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "") + if (NOT MAKE_STATIC_LIBRARIES) + set(BUILD_SHARED_LIBS 1) # TODO: set at root dir + endif() + + add_subdirectory(snappy) + + set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") + if(SANITIZE STREQUAL "undefined") + target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined) + endif() +endif() + if (USE_INTERNAL_PARQUET_LIBRARY) if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) # We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time @@ -189,20 +203,6 @@ if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) endif() else() - if(USE_INTERNAL_SNAPPY_LIBRARY) - set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "") - if (NOT MAKE_STATIC_LIBRARIES) - set(BUILD_SHARED_LIBS 1) # TODO: set at root dir - endif() - - add_subdirectory(snappy) - - set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") - if(SANITIZE STREQUAL "undefined") - target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined) - endif() - endif() - add_subdirectory(arrow-cmake) # The library is large - avoid bloat. @@ -212,6 +212,10 @@ else() endif() endif() +if (USE_INTERNAL_AVRO_LIBRARY) + add_subdirectory(avro-cmake) +endif() + if (USE_INTERNAL_POCO_LIBRARY) set (POCO_VERBOSE_MESSAGES 0 CACHE INTERNAL "") set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) @@ -331,3 +335,5 @@ endif() if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() + +add_subdirectory(replxx-cmake) diff --git a/contrib/avro b/contrib/avro new file mode 160000 index 00000000000..5b2752041c8 --- /dev/null +++ b/contrib/avro @@ -0,0 +1 @@ +Subproject commit 5b2752041c8d2f75eb5c1dbec8b4c25fc0e24d12 diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt new file mode 100644 index 00000000000..f544b3c50cd --- /dev/null +++ b/contrib/avro-cmake/CMakeLists.txt @@ -0,0 +1,70 @@ +set(AVROCPP_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/avro/lang/c++) +set(AVROCPP_INCLUDE_DIR ${AVROCPP_ROOT_DIR}/api) +set(AVROCPP_SOURCE_DIR ${AVROCPP_ROOT_DIR}/impl) + +set (CMAKE_CXX_STANDARD 17) + +if (EXISTS ${AVROCPP_ROOT_DIR}/../../share/VERSION.txt) + file(READ "${AVROCPP_ROOT_DIR}/../../share/VERSION.txt" + AVRO_VERSION) +endif() + +string(REPLACE "\n" "" AVRO_VERSION ${AVRO_VERSION}) +set (AVRO_VERSION_MAJOR ${AVRO_VERSION}) +set (AVRO_VERSION_MINOR "0") + +set (AVROCPP_SOURCE_FILES + ${AVROCPP_SOURCE_DIR}/Compiler.cc + ${AVROCPP_SOURCE_DIR}/Node.cc + ${AVROCPP_SOURCE_DIR}/LogicalType.cc + ${AVROCPP_SOURCE_DIR}/NodeImpl.cc + ${AVROCPP_SOURCE_DIR}/ResolverSchema.cc + ${AVROCPP_SOURCE_DIR}/Schema.cc + ${AVROCPP_SOURCE_DIR}/Types.cc + ${AVROCPP_SOURCE_DIR}/ValidSchema.cc + ${AVROCPP_SOURCE_DIR}/Zigzag.cc + ${AVROCPP_SOURCE_DIR}/BinaryEncoder.cc + ${AVROCPP_SOURCE_DIR}/BinaryDecoder.cc + ${AVROCPP_SOURCE_DIR}/Stream.cc + ${AVROCPP_SOURCE_DIR}/FileStream.cc + ${AVROCPP_SOURCE_DIR}/Generic.cc + ${AVROCPP_SOURCE_DIR}/GenericDatum.cc + ${AVROCPP_SOURCE_DIR}/DataFile.cc + ${AVROCPP_SOURCE_DIR}/parsing/Symbol.cc + ${AVROCPP_SOURCE_DIR}/parsing/ValidatingCodec.cc + ${AVROCPP_SOURCE_DIR}/parsing/JsonCodec.cc + ${AVROCPP_SOURCE_DIR}/parsing/ResolvingDecoder.cc + ${AVROCPP_SOURCE_DIR}/json/JsonIO.cc + ${AVROCPP_SOURCE_DIR}/json/JsonDom.cc + ${AVROCPP_SOURCE_DIR}/Resolver.cc + ${AVROCPP_SOURCE_DIR}/Validator.cc + ) + +add_library (avrocpp ${AVROCPP_SOURCE_FILES}) +set_target_properties (avrocpp PROPERTIES VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) + +target_include_directories(avrocpp SYSTEM PUBLIC ${AVROCPP_INCLUDE_DIR}) + +target_include_directories(avrocpp SYSTEM PUBLIC ${Boost_INCLUDE_DIRS}) +target_link_libraries (avrocpp ${Boost_IOSTREAMS_LIBRARY}) + +if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) + target_compile_definitions (avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) + target_include_directories (avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) + target_link_libraries (avrocpp ${SNAPPY_LIBRARY}) +endif () + +if (COMPILER_GCC) + set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) +elseif (COMPILER_CLANG) + set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) +endif () + +target_compile_options(avrocpp PRIVATE ${SUPPRESS_WARNINGS}) + +# create a symlink to include headers with +ADD_CUSTOM_TARGET(avro_symlink_headers ALL + COMMAND ${CMAKE_COMMAND} -E make_directory ${AVROCPP_ROOT_DIR}/include + COMMAND ${CMAKE_COMMAND} -E create_symlink ${AVROCPP_ROOT_DIR}/api ${AVROCPP_ROOT_DIR}/include/avro +) +add_dependencies(avrocpp avro_symlink_headers) \ No newline at end of file diff --git a/contrib/aws-s3-cmake/CMakeLists.txt b/contrib/aws-s3-cmake/CMakeLists.txt index e86ac0cb5a6..ef9107e0d4f 100644 --- a/contrib/aws-s3-cmake/CMakeLists.txt +++ b/contrib/aws-s3-cmake/CMakeLists.txt @@ -74,7 +74,6 @@ file(GLOB S3_UNIFIED_SRC ) set(S3_INCLUDES - "${CMAKE_CURRENT_SOURCE_DIR}/include/" "${AWS_COMMON_LIBRARY_DIR}/include/" "${AWS_EVENT_STREAM_LIBRARY_DIR}/include/" "${AWS_S3_LIBRARY_DIR}/include/" @@ -96,7 +95,7 @@ target_compile_definitions(aws_s3 PUBLIC -DENABLE_CURL_CLIENT) target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231") -target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES} "${CMAKE_BINARY_DIR}/install") +target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES}) if (OPENSSL_FOUND) target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION) diff --git a/contrib/boost b/contrib/boost index 830e51edb59..86be2aef20b 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 830e51edb59c4f37a8638138581e1e56c29ac44f +Subproject commit 86be2aef20bee2356b744e5569eed6eaded85dbe diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index d9a8a70ef17..54dcd750320 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -37,3 +37,8 @@ target_link_libraries(boost_filesystem_internal PRIVATE boost_system_internal) if (USE_INTERNAL_PARQUET_LIBRARY) add_boost_lib(regex) endif() + +if (USE_INTERNAL_AVRO_LIBRARY) + add_boost_lib(iostreams) + target_link_libraries(boost_iostreams_internal PUBLIC ${ZLIB_LIBRARIES}) +endif() diff --git a/contrib/murmurhash/include/murmurhash3.h b/contrib/murmurhash/include/murmurhash3.h index 256da1ad9da..eb16425576a 100644 --- a/contrib/murmurhash/include/murmurhash3.h +++ b/contrib/murmurhash/include/murmurhash3.h @@ -23,6 +23,10 @@ typedef unsigned __int64 uint64_t; #endif // !defined(_MSC_VER) +#ifdef __cplusplus +extern "C" { +#endif + //----------------------------------------------------------------------------- void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); @@ -32,3 +36,7 @@ void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); //----------------------------------------------------------------------------- + +#ifdef __cplusplus +} +#endif diff --git a/contrib/replxx b/contrib/replxx new file mode 160000 index 00000000000..37582f0bb8c --- /dev/null +++ b/contrib/replxx @@ -0,0 +1 @@ +Subproject commit 37582f0bb8c52513c6c6b76797c02d852d701dad diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt new file mode 100644 index 00000000000..1240eb56b39 --- /dev/null +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -0,0 +1,57 @@ +option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES}) + +if (ENABLE_REPLXX) + option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED}) + + if (USE_INTERNAL_REPLXX) + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") + + set(SRCS + ${LIBRARY_DIR}/src/conversion.cxx + ${LIBRARY_DIR}/src/ConvertUTF.cpp + ${LIBRARY_DIR}/src/escape.cxx + ${LIBRARY_DIR}/src/history.cxx + ${LIBRARY_DIR}/src/io.cxx + ${LIBRARY_DIR}/src/prompt.cxx + ${LIBRARY_DIR}/src/replxx_impl.cxx + ${LIBRARY_DIR}/src/replxx.cxx + ${LIBRARY_DIR}/src/util.cxx + ${LIBRARY_DIR}/src/wcwidth.cpp + ) + + add_library (replxx ${SRCS}) + target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include) + else () + find_library(LIBRARY_REPLXX NAMES replxx replxx-static) + find_path(INCLUDE_REPLXX replxx.hxx) + + add_library(replxx UNKNOWN IMPORTED) + set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX}) + target_include_directories(replxx PUBLIC ${INCLUDE_REPLXX}) + + set(CMAKE_REQUIRED_LIBRARIES replxx) + check_cxx_source_compiles( + " + #include + int main() { + replxx::Replxx rx; + } + " + EXTERNAL_REPLXX_WORKS + ) + + if (NOT EXTERNAL_REPLXX_WORKS) + message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}") + endif () + endif () + + target_compile_options(replxx PUBLIC -Wno-documentation) + target_compile_definitions(replxx PUBLIC USE_REPLXX=1) + + message (STATUS "Using replxx") +else () + add_library(replxx INTERFACE) + target_compile_definitions(replxx INTERFACE USE_REPLXX=0) + + message (STATUS "Not using replxx (Beware! Runtime fallback to readline is possible!)") +endif () diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index d7072371487..eeda7aa6a1f 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -142,10 +142,10 @@ elseif (COMPILER_GCC) add_cxx_compile_options(-Wmaybe-uninitialized) # Warn when the indentation of the code does not reflect the block structure add_cxx_compile_options(-Wmisleading-indentation) - # Warn if a global function is defined without a previous declaration + # Warn if a global function is defined without a previous declaration - disabled because of build times # add_cxx_compile_options(-Wmissing-declarations) # Warn if a user-supplied include directory does not exist - # add_cxx_compile_options(-Wmissing-include-dirs) + add_cxx_compile_options(-Wmissing-include-dirs) # Obvious add_cxx_compile_options(-Wnon-virtual-dtor) # Obvious @@ -177,7 +177,7 @@ elseif (COMPILER_GCC) # Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof add_cxx_compile_options(-Wsizeof-pointer-memaccess) # Warn about overriding virtual functions that are not marked with the override keyword - # add_cxx_compile_options(-Wsuggest-override) + add_cxx_compile_options(-Wsuggest-override) # Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type add_cxx_compile_options(-Wswitch-bool) # Warn if a self-comparison always evaluates to true or false @@ -504,6 +504,10 @@ if (USE_POCO_NETSSL) dbms_target_link_libraries (PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) endif() +if (USE_POCO_JSON) + dbms_target_link_libraries (PRIVATE ${Poco_JSON_LIBRARY}) +endif() + dbms_target_link_libraries (PRIVATE ${Poco_Foundation_LIBRARY}) if (USE_ICU) @@ -522,6 +526,11 @@ if (USE_PARQUET) endif () endif () +if (USE_AVRO) + dbms_target_link_libraries(PRIVATE ${AVROCPP_LIBRARY}) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${AVROCPP_INCLUDE_DIR}) +endif () + if (OPENSSL_CRYPTO_LIBRARY) dbms_target_link_libraries (PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) target_link_libraries (clickhouse_common_io PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) @@ -563,7 +572,7 @@ if (USE_JEMALLOC) endif() endif () -dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) +dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) diff --git a/dbms/benchmark/clickhouse/benchmark-chyt.sh b/dbms/benchmark/clickhouse/benchmark-chyt.sh new file mode 100755 index 00000000000..efc790a029a --- /dev/null +++ b/dbms/benchmark/clickhouse/benchmark-chyt.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +QUERIES_FILE="queries.sql" +TABLE=$1 +TRIES=3 + +cat "$QUERIES_FILE" | sed "s|{table}|\"${TABLE}\"|g" | while read query; do + + echo -n "[" + for i in $(seq 1 $TRIES); do + while true; do + RES=$(command time -f %e -o /dev/stdout curl -sS --location-trusted -H "Authorization: OAuth $YT_TOKEN" "$YT_PROXY.yt.yandex.net/query?default_format=Null&database=*$YT_CLIQUE_ID" --data-binary @- <<< "$query" 2>/dev/null) && break; + done + + [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" + [[ "$i" != $TRIES ]] && echo -n ", " + done + echo "]," +done diff --git a/dbms/benchmark/clickhouse/benchmark-yql.sh b/dbms/benchmark/clickhouse/benchmark-yql.sh new file mode 100755 index 00000000000..7d30d39e7d3 --- /dev/null +++ b/dbms/benchmark/clickhouse/benchmark-yql.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +QUERIES_FILE="queries.sql" +TABLE=$1 +TRIES=3 + +cat "$QUERIES_FILE" | sed "s|{table}|\"${TABLE}\"|g" | while read query; do + + echo -n "[" + for i in $(seq 1 $TRIES); do + while true; do + RES=$(command time -f %e -o time ./yql --clickhouse --syntax-version 1 -f empty <<< "USE chyt.hume; PRAGMA max_memory_usage = 100000000000; PRAGMA max_memory_usage_for_all_queries = 100000000000; $query" >/dev/null 2>&1 && cat time) && break; + done + + [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" + [[ "$i" != $TRIES ]] && echo -n ", " + done + echo "]," +done diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 1c98ecd1333..3dd89cd1710 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -101,7 +101,7 @@ public: } - void initialize(Poco::Util::Application & self [[maybe_unused]]) + void initialize(Poco::Util::Application & self [[maybe_unused]]) override { std::string home_path; const char * home_path_cstr = getenv("HOME"); @@ -111,7 +111,7 @@ public: configReadClient(config(), home_path); } - int main(const std::vector &) + int main(const std::vector &) override { if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results Poco::File(json_path).remove(); @@ -418,7 +418,7 @@ private: std::cerr << percent << "%\t\t"; for (const auto & info : infos) { - std::cerr << info->sampler.quantileInterpolated(percent / 100.0) << " sec." << "\t"; + std::cerr << info->sampler.quantileNearest(percent / 100.0) << " sec." << "\t"; } std::cerr << "\n"; }; @@ -453,7 +453,7 @@ private: auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true) { - json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileInterpolated(percent / 100.0) << (with_comma ? ",\n" : "\n"); + json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileNearest(percent / 100.0) << (with_comma ? ",\n" : "\n"); }; json_out << "{\n"; @@ -492,7 +492,7 @@ private: public: - ~Benchmark() + ~Benchmark() override { shutdown = true; } diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index dc5cf787adf..11ade559a8d 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -1,14 +1,10 @@ set(CLICKHOUSE_CLIENT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Suggest.cpp ) -set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) -set(CLICKHOUSE_CLIENT_INCLUDE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) - -if (READLINE_INCLUDE_DIR) - set(CLICKHOUSE_CLIENT_INCLUDE ${CLICKHOUSE_CLIENT_INCLUDE} SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) -endif () +set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY}) include(CheckSymbolExists) check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index b37acab77ea..c41a26a6c63 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1,7 +1,13 @@ #include "TestHint.h" #include "ConnectionParameters.h" +#include "Suggest.h" + +#if USE_REPLXX +# include +#else +# include +#endif -#include #include #include #include @@ -18,8 +24,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -69,10 +75,6 @@ #include #include -#if USE_READLINE -#include "Suggest.h" -#endif - #ifndef __clang__ #pragma GCC optimize("-fno-var-tracking-assignments") #endif @@ -89,39 +91,6 @@ #define DISABLE_LINE_WRAPPING "\033[?7l" #define ENABLE_LINE_WRAPPING "\033[?7h" -#if USE_READLINE && RL_VERSION_MAJOR >= 7 - -#define BRACK_PASTE_PREF "\033[200~" -#define BRACK_PASTE_SUFF "\033[201~" - -#define BRACK_PASTE_LAST '~' -#define BRACK_PASTE_SLEN 6 - -/// This handler bypasses some unused macro/event checkings. -static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */) -{ - std::string buf; - buf.reserve(128); - - RL_SETSTATE(RL_STATE_MOREINPUT); - SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT)); - int c; - while ((c = rl_read_key()) >= 0) - { - if (c == '\r') - c = '\n'; - buf.push_back(c); - if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF) - { - buf.resize(buf.size() - BRACK_PASTE_SLEN); - break; - } - } - return static_cast(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1; -} - -#endif - namespace DB { @@ -136,7 +105,6 @@ namespace ErrorCodes extern const int UNEXPECTED_PACKET_FROM_SERVER; extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED; extern const int CANNOT_SET_SIGNAL_HANDLER; - extern const int CANNOT_READLINE; extern const int SYSTEM_ERROR; extern const int INVALID_USAGE_OF_INPUT; } @@ -157,7 +125,7 @@ private: "учшеж", "йгшеж", "дщпщгеж", "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй" }; - bool is_interactive = true; /// Use either readline interface or batch mode. + bool is_interactive = true; /// Use either interactive line editing interface or batch mode. bool need_render_progress = true; /// Render query execution progress. bool echo_queries = false; /// Print queries before execution in batch mode. bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode. @@ -242,7 +210,7 @@ private: ConnectionParameters connection_parameters; - void initialize(Poco::Util::Application & self) + void initialize(Poco::Util::Application & self) override { Poco::Util::Application::initialize(self); @@ -270,7 +238,7 @@ private: } - int main(const std::vector & /*args*/) + int main(const std::vector & /*args*/) override { try { @@ -514,26 +482,10 @@ private: if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); -#if USE_READLINE - SCOPE_EXIT({ Suggest::instance().finalize(); }); - if (server_revision >= Suggest::MIN_SERVER_REVISION - && !config().getBool("disable_suggestion", false)) - { + if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false)) /// Load suggestion data from the server. Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); - /// Added '.' to the default list. Because it is used to separate database and table. - rl_basic_word_break_characters = " \t\n\r\"\\'`@$><=;|&{(."; - - /// Not append whitespace after single suggestion. Because whitespace after function name is meaningless. - rl_completion_append_character = '\0'; - - rl_completion_entry_function = Suggest::generator; - } - else - /// Turn tab completion off. - rl_bind_key('\t', rl_insert); -#endif /// Load command history if present. if (config().has("history_file")) history_file = config().getString("history_file"); @@ -546,70 +498,56 @@ private: history_file = home_path + "/.clickhouse-client-history"; } - if (!history_file.empty()) - { - if (Poco::File(history_file).exists()) - { -#if USE_READLINE - int res = read_history(history_file.c_str()); - if (res) - std::cerr << "Cannot read history from file " + history_file + ": "+ errnoToString(ErrorCodes::CANNOT_READ_HISTORY); + if (!history_file.empty() && !Poco::File(history_file).exists()) + Poco::File(history_file).createFile(); + +#if USE_REPLXX + ReplxxLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0); +#else + LineReader lr(history_file, '\\', config().has("multiline") ? ';' : 0); #endif + + do + { + auto input = lr.readLine(prompt(), ":-] "); + if (input.empty()) + break; + + has_vertical_output_suffix = false; + if (input.ends_with("\\G")) + { + input.resize(input.size() - 2); + has_vertical_output_suffix = true; + } + + try + { + if (!process(input)) + break; + } + catch (const Exception & e) + { + actual_client_error = e.code(); + if (!actual_client_error || actual_client_error != expected_client_error) + { + std::cerr << std::endl + << "Exception on client:" << std::endl + << "Code: " << e.code() << ". " << e.displayText() << std::endl; + + if (config().getBool("stacktrace", false)) + std::cerr << "Stack trace:" << std::endl << e.getStackTraceString() << std::endl; + + std::cerr << std::endl; + + } + + /// Client-side exception during query execution can result in the loss of + /// sync in the connection protocol. + /// So we reconnect and allow to enter the next query. + connect(); } - else /// Create history file. - Poco::File(history_file).createFile(); } - -#if USE_READLINE - /// Install Ctrl+C signal handler that will be used in interactive mode. - - if (rl_initialize()) - throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE); - -#if RL_VERSION_MAJOR >= 7 - /// Enable bracketed-paste-mode only when multiquery is enabled and multiline is - /// disabled, so that we are able to paste and execute multiline queries in a whole - /// instead of erroring out, while be less intrusive. - if (config().has("multiquery") && !config().has("multiline")) - { - /// When bracketed paste mode is set, pasted text is bracketed with control sequences so - /// that the program can differentiate pasted text from typed-in text. This helps - /// clickhouse-client so that without -m flag, one can still paste multiline queries, and - /// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for - /// more details. - rl_variable_bind("enable-bracketed-paste", "on"); - - /// Use our bracketed paste handler to get better user experience. See comments above. - rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin); - } -#endif - - auto clear_prompt_or_exit = [](int) - { - /// This is signal safe. - ssize_t res = write(STDOUT_FILENO, "\n", 1); - - /// Allow to quit client while query is in progress by pressing Ctrl+C twice. - /// (First press to Ctrl+C will try to cancel query by InterruptListener). - if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE)) - { - rl_replace_line("", 0); - if (rl_forced_update_display()) - _exit(0); - } - else - { - /// A little dirty, but we struggle to find better way to correctly - /// force readline to exit after returning from the signal handler. - _exit(0); - } - }; - - if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR) - throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER); -#endif - - loop(); + while (true); if (isNewYearMode()) std::cout << "Happy new year." << std::endl; @@ -621,17 +559,6 @@ private: } else { - /// This is intended for testing purposes. - if (config().getBool("always_load_suggestion_data", false)) - { -#if USE_READLINE - SCOPE_EXIT({ Suggest::instance().finalize(); }); - Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); -#else - throw Exception("Command line suggestions cannot work without readline", ErrorCodes::BAD_ARGUMENTS); -#endif - } - query_id = config().getString("query_id", ""); nonInteractive(); @@ -706,111 +633,11 @@ private: } - /// Check if multi-line query is inserted from the paste buffer. - /// Allows delaying the start of query execution until the entirety of query is inserted. - static bool hasDataInSTDIN() - { - timeval timeout = { 0, 0 }; - fd_set fds; - FD_ZERO(&fds); - FD_SET(STDIN_FILENO, &fds); - return select(1, &fds, nullptr, nullptr, &timeout) == 1; - } - inline const String prompt() const { return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default")); } - void loop() - { - String input; - String prev_input; - - while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] ")) - { - String line = line_; - free(line_); - - size_t ws = line.size(); - while (ws > 0 && isWhitespaceASCII(line[ws - 1])) - --ws; - - if (ws == 0 || line.empty()) - continue; - - bool ends_with_semicolon = line[ws - 1] == ';'; - bool ends_with_backslash = line[ws - 1] == '\\'; - - has_vertical_output_suffix = (ws >= 2) && (line[ws - 2] == '\\') && (line[ws - 1] == 'G'); - - if (ends_with_backslash) - line = line.substr(0, ws - 1); - - input += line; - - if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN()))) - { - // TODO: should we do sensitive data masking on client too? History file can be source of secret leaks. - if (input != prev_input) - { - /// Replace line breaks with spaces to prevent the following problem. - /// Every line of multi-line query is saved to history file as a separate line. - /// If the user restarts the client then after pressing the "up" button - /// every line of the query will be displayed separately. - std::string logged_query = input; - if (config().has("multiline")) - std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); - add_history(logged_query.c_str()); - -#if USE_READLINE && HAVE_READLINE_HISTORY - if (!history_file.empty() && append_history(1, history_file.c_str())) - std::cerr << "Cannot append history to file " + history_file + ": " + errnoToString(ErrorCodes::CANNOT_APPEND_HISTORY); -#endif - - prev_input = input; - } - - if (has_vertical_output_suffix) - input = input.substr(0, input.length() - 2); - - try - { - if (!process(input)) - break; - } - catch (const Exception & e) - { - actual_client_error = e.code(); - if (!actual_client_error || actual_client_error != expected_client_error) - { - std::cerr << std::endl - << "Exception on client:" << std::endl - << "Code: " << e.code() << ". " << e.displayText() << std::endl; - - if (config().getBool("stacktrace", false)) - std::cerr << "Stack trace:" << std::endl - << e.getStackTraceString() << std::endl; - - std::cerr << std::endl; - - } - - /// Client-side exception during query execution can result in the loss of - /// sync in the connection protocol. - /// So we reconnect and allow to enter the next query. - connect(); - } - - input = ""; - } - else - { - input += '\n'; - } - } - } - void nonInteractive() { @@ -2001,13 +1828,6 @@ public: server_logs_file = options["server_logs_file"].as(); if (options.count("disable_suggestion")) config().setBool("disable_suggestion", true); - if (options.count("always_load_suggestion_data")) - { - if (options.count("disable_suggestion")) - throw Exception("Command line parameters disable_suggestion (-A) and always_load_suggestion_data cannot be specified simultaneously", - ErrorCodes::BAD_ARGUMENTS); - config().setBool("always_load_suggestion_data", true); - } if (options.count("suggestion_limit")) config().setInt("suggestion_limit", options["suggestion_limit"].as()); diff --git a/dbms/programs/client/Suggest.cpp b/dbms/programs/client/Suggest.cpp new file mode 100644 index 00000000000..65487e07a93 --- /dev/null +++ b/dbms/programs/client/Suggest.cpp @@ -0,0 +1,144 @@ +#include "Suggest.h" + +#include +#include + +namespace DB +{ + +void Suggest::load(const ConnectionParameters & connection_parameters, size_t suggestion_limit) +{ + loading_thread = std::thread([connection_parameters, suggestion_limit, this] + { + try + { + Connection connection( + connection_parameters.host, + connection_parameters.port, + connection_parameters.default_database, + connection_parameters.user, + connection_parameters.password, + "client", + connection_parameters.compression, + connection_parameters.security); + + loadImpl(connection, connection_parameters.timeouts, suggestion_limit); + } + catch (...) + { + std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; + } + + /// Note that keyword suggestions are available even if we cannot load data from server. + + std::sort(words.begin(), words.end()); + ready = true; + }); +} + +Suggest::Suggest() +{ + /// Keywords may be not up to date with ClickHouse parser. + words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", + "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", + "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", + "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", + "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", + "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", + "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", + "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", + "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", + "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"}; +} + +void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit) +{ + std::stringstream query; + query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + "SELECT name FROM system.functions" + " UNION ALL " + "SELECT name FROM system.table_engines" + " UNION ALL " + "SELECT name FROM system.formats" + " UNION ALL " + "SELECT name FROM system.table_functions" + " UNION ALL " + "SELECT name FROM system.data_type_families" + " UNION ALL " + "SELECT name FROM system.settings" + " UNION ALL " + "SELECT cluster FROM system.clusters" + " UNION ALL " + "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate"; + + /// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero. + if (suggestion_limit > 0) + { + String limit_str = toString(suggestion_limit); + query << + " UNION ALL " + "SELECT name FROM system.databases LIMIT " << limit_str + << " UNION ALL " + "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str + << " UNION ALL " + "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str; + } + + query << ") WHERE notEmpty(res)"; + + fetch(connection, timeouts, query.str()); +} + +void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query) +{ + connection.sendQuery(timeouts, query); + + while (true) + { + Packet packet = connection.receivePacket(); + switch (packet.type) + { + case Protocol::Server::Data: + fillWordsFromBlock(packet.block); + continue; + + case Protocol::Server::Progress: + continue; + case Protocol::Server::ProfileInfo: + continue; + case Protocol::Server::Totals: + continue; + case Protocol::Server::Extremes: + continue; + case Protocol::Server::Log: + continue; + + case Protocol::Server::Exception: + packet.exception->rethrow(); + return; + + case Protocol::Server::EndOfStream: + return; + + default: + throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + } + } +} + +void Suggest::fillWordsFromBlock(const Block & block) +{ + if (!block) + return; + + if (block.columns() != 1) + throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR); + + const ColumnString & column = typeid_cast(*block.getByPosition(0).column); + + size_t rows = block.rows(); + for (size_t i = 0; i < rows; ++i) + words.emplace_back(column.getDataAt(i).toString()); +} + +} diff --git a/dbms/programs/client/Suggest.h b/dbms/programs/client/Suggest.h index 78cc8d94db0..bd4f239ddc7 100644 --- a/dbms/programs/client/Suggest.h +++ b/dbms/programs/client/Suggest.h @@ -2,18 +2,9 @@ #include "ConnectionParameters.h" -#include -#include -#include -#include -#include - -#include - -#include -#include #include #include +#include namespace DB @@ -24,141 +15,8 @@ namespace ErrorCodes extern const int UNKNOWN_PACKET_FROM_SERVER; } -class Suggest : private boost::noncopyable +class Suggest : public LineReader::Suggest, boost::noncopyable { -private: - /// The vector will be filled with completion words from the server and sorted. - using Words = std::vector; - - /// Keywords may be not up to date with ClickHouse parser. - Words words - { - "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", "MATERIALIZED", "ALIAS", "ENGINE", - "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", - "COPY", "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", "OUTFILE", "FORMAT", "TABLES", - "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", - "INSERT", "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", "LEFT", "RIGHT", "FULL", "OUTER", - "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN", - "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE" - }; - - /// Words are fetched asynchronously. - std::thread loading_thread; - std::atomic ready{false}; - - /// Points to current word to suggest. - Words::const_iterator pos; - /// Points after the last possible match. - Words::const_iterator end; - - /// Set iterators to the matched range of words if any. - void findRange(const char * prefix, size_t prefix_length) - { - std::string prefix_str(prefix); - std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str, - [prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; }); - } - - /// Iterates through matched range. - char * nextMatch() - { - if (pos >= end) - return nullptr; - - /// readline will free memory by itself. - char * word = strdup(pos->c_str()); - ++pos; - return word; - } - - void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit) - { - std::stringstream query; - query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" - "SELECT name FROM system.functions" - " UNION ALL " - "SELECT name FROM system.table_engines" - " UNION ALL " - "SELECT name FROM system.formats" - " UNION ALL " - "SELECT name FROM system.table_functions" - " UNION ALL " - "SELECT name FROM system.data_type_families" - " UNION ALL " - "SELECT name FROM system.settings" - " UNION ALL " - "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate"; - - /// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero. - if (suggestion_limit > 0) - { - String limit_str = toString(suggestion_limit); - query << - " UNION ALL " - "SELECT name FROM system.databases LIMIT " << limit_str - << " UNION ALL " - "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str - << " UNION ALL " - "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str; - } - - query << ") WHERE notEmpty(res)"; - - fetch(connection, timeouts, query.str()); - } - - void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query) - { - connection.sendQuery(timeouts, query); - - while (true) - { - Packet packet = connection.receivePacket(); - switch (packet.type) - { - case Protocol::Server::Data: - fillWordsFromBlock(packet.block); - continue; - - case Protocol::Server::Progress: - continue; - case Protocol::Server::ProfileInfo: - continue; - case Protocol::Server::Totals: - continue; - case Protocol::Server::Extremes: - continue; - case Protocol::Server::Log: - continue; - - case Protocol::Server::Exception: - packet.exception->rethrow(); - return; - - case Protocol::Server::EndOfStream: - return; - - default: - throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); - } - } - } - - void fillWordsFromBlock(const Block & block) - { - if (!block) - return; - - if (block.columns() != 1) - throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR); - - const ColumnString & column = typeid_cast(*block.getByPosition(0).column); - - size_t rows = block.rows(); - for (size_t i = 0; i < rows; ++i) - words.emplace_back(column.getDataAt(i).toString()); - } - public: static Suggest & instance() { @@ -166,64 +24,25 @@ public: return instance; } - /// More old server versions cannot execute the query above. + void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit); + + /// Older server versions cannot execute the query above. static constexpr int MIN_SERVER_REVISION = 54406; - void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit) - { - loading_thread = std::thread([connection_parameters, suggestion_limit, this] - { - try - { - Connection connection( - connection_parameters.host, - connection_parameters.port, - connection_parameters.default_database, - connection_parameters.user, - connection_parameters.password, - "client", - connection_parameters.compression, - connection_parameters.security); - - loadImpl(connection, connection_parameters.timeouts, suggestion_limit); - } - catch (...) - { - std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; - } - - /// Note that keyword suggestions are available even if we cannot load data from server. - - std::sort(words.begin(), words.end()); - ready = true; - }); - } - - void finalize() +private: + Suggest(); + ~Suggest() { if (loading_thread.joinable()) loading_thread.join(); } - /// A function for readline. - static char * generator(const char * text, int state) - { - Suggest & suggest = Suggest::instance(); - if (!suggest.ready) - return nullptr; - if (state == 0) - suggest.findRange(text, strlen(text)); + void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit); + void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query); + void fillWordsFromBlock(const Block & block); - /// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work. - rl_completion_suppress_append = 1; - - return suggest.nextMatch(); - } - - ~Suggest() - { - finalize(); - } + /// Words are fetched asynchronously. + std::thread loading_thread; }; } diff --git a/dbms/programs/client/readpassphrase/CMakeLists.txt b/dbms/programs/client/readpassphrase/CMakeLists.txt index 94ed9f54bdb..a10b54c377d 100644 --- a/dbms/programs/client/readpassphrase/CMakeLists.txt +++ b/dbms/programs/client/readpassphrase/CMakeLists.txt @@ -10,4 +10,4 @@ set_target_properties(readpassphrase PROPERTIES LINKER_LANGUAGE C ) # . to allow #include -target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/../include) +target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include) diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index cac561117b4..5cfceaeb592 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -111,7 +111,7 @@ void LocalServer::tryInitPath() /// In case of empty path set paths to helpful directories std::string cd = Poco::Path::current(); - context->setTemporaryPath(cd + "tmp"); + context->setTemporaryStorage(cd + "tmp"); context->setFlagsPath(cd + "flags"); context->setUserFilesPath(""); // user's files are everywhere } diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt index d03ff257562..350663eb9ef 100644 --- a/dbms/programs/odbc-bridge/CMakeLists.txt +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -15,20 +15,24 @@ set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemo if (USE_POCO_SQLODBC) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY}) - set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR}) + # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories` + # set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR}) endif () if (Poco_SQL_FOUND) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY}) - set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) + # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories` + # set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) endif () if (USE_POCO_DATAODBC) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY}) - set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR}) + # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories` + # set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR}) endif() if (Poco_Data_FOUND) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY}) - set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) + # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories` + # set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) endif () clickhouse_program_add_library(odbc-bridge) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index b2b3298693e..d28b4aee104 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -351,7 +352,8 @@ void HTTPHandler::processQuery( if (buffer_until_eof) { - std::string tmp_path_template = context.getTemporaryPath() + "http_buffers/"; + const std::string tmp_path(context.getTemporaryVolume()->getNextDisk()->getPath()); + const std::string tmp_path_template(tmp_path + "http_buffers/"); auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &) { @@ -590,7 +592,11 @@ void HTTPHandler::processQuery( customizeContext(context); executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, - [&response] (const String & content_type) { response.setContentType(content_type); }, + [&response] (const String & content_type, const String & format) + { + response.setContentType(content_type); + response.add("X-ClickHouse-Format", format); + }, [&response] (const String & current_query_id) { response.add("X-ClickHouse-Query-Id", current_query_id); }); if (used_output.hasDelayed()) @@ -610,6 +616,8 @@ void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_ { try { + response.set("X-ClickHouse-Exception-Code", toString(exception_code)); + /// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body /// to avoid reading part of the current request body in the next request. if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST diff --git a/dbms/programs/server/InterserverIOHTTPHandler.cpp b/dbms/programs/server/InterserverIOHTTPHandler.cpp index 5302302bb5b..407d3c41a9b 100644 --- a/dbms/programs/server/InterserverIOHTTPHandler.cpp +++ b/dbms/programs/server/InterserverIOHTTPHandler.cpp @@ -61,6 +61,10 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque ReadBufferFromIStream body(request.stream()); auto endpoint = server.context().getInterserverIOHandler().getEndpoint(endpoint_name); + /// Locked for read while query processing + std::shared_lock lock(endpoint->rwlock); + if (endpoint->blocker.isCancelled()) + throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); if (compress) { diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index a147ccafba0..9dd107f9d5f 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -79,21 +79,19 @@ void MySQLHandler::run() if (!connection_context.mysql.max_packet_size) connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH; -/* LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags - << "\nmax_packet_size: " + LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags + << ", max_packet_size: " << handshake_response.max_packet_size - << "\ncharacter_set: " - << handshake_response.character_set - << "\nuser: " + << ", character_set: " + << static_cast(handshake_response.character_set) + << ", user: " << handshake_response.username - << "\nauth_response length: " + << ", auth_response length: " << handshake_response.auth_response.length() - << "\nauth_response: " - << handshake_response.auth_response - << "\ndatabase: " + << ", database: " << handshake_response.database - << "\nauth_plugin_name: " - << handshake_response.auth_plugin_name);*/ + << ", auth_plugin_name: " + << handshake_response.auth_plugin_name); client_capability_flags = handshake_response.capability_flags; if (!(client_capability_flags & CLIENT_PROTOCOL_41)) @@ -284,7 +282,8 @@ void MySQLHandler::comQuery(ReadBuffer & payload) else { bool with_output = false; - std::function set_content_type = [&with_output](const String &) -> void { + std::function set_content_type_and_format = [&with_output](const String &, const String &) -> void + { with_output = true; }; @@ -307,7 +306,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload) ReadBufferFromString replacement(replacement_query); Context query_context = connection_context; - executeQuery(should_replace ? replacement : payload, *out, true, query_context, set_content_type, nullptr); + executeQuery(should_replace ? replacement : payload, *out, true, query_context, set_content_type_and_format, {}); if (!with_output) packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); diff --git a/dbms/programs/server/MySQLHandlerFactory.cpp b/dbms/programs/server/MySQLHandlerFactory.cpp index 987efbfa347..576ef4d5170 100644 --- a/dbms/programs/server/MySQLHandlerFactory.cpp +++ b/dbms/programs/server/MySQLHandlerFactory.cpp @@ -34,7 +34,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) } catch (...) { - LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false)); + LOG_TRACE(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false)); ssl_enabled = false; } #endif @@ -47,7 +47,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) } catch (...) { - LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false)); + LOG_TRACE(log, "Failed to read RSA key pair from server certificate. Error: " << getCurrentExceptionMessage(false)); generateRSAKeys(); } #endif @@ -104,7 +104,7 @@ void MySQLHandlerFactory::readRSAKeys() void MySQLHandlerFactory::generateRSAKeys() { - LOG_INFO(log, "Generating new RSA key."); + LOG_TRACE(log, "Generating new RSA key pair."); public_key.reset(RSA_new()); if (!public_key) throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 35168a4b606..0cd357bbd94 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -77,6 +77,31 @@ namespace CurrentMetrics extern const Metric VersionInteger; } +namespace +{ + +void setupTmpPath(Logger * log, const std::string & path) +{ + LOG_DEBUG(log, "Setting up " << path << " to store temporary data in it"); + + Poco::File(path).createDirectories(); + + /// Clearing old temporary files. + Poco::DirectoryIterator dir_end; + for (Poco::DirectoryIterator it(path); it != dir_end; ++it) + { + if (it->isFile() && startsWith(it.name(), "tmp")) + { + LOG_DEBUG(log, "Removing old temporary file " << it->path()); + it->remove(); + } + else + LOG_DEBUG(log, "Skipped file in temporary path " << it->path()); + } +} + +} + namespace DB { @@ -331,22 +356,14 @@ int Server::main(const std::vector & /*args*/) DateLUT::instance(); LOG_TRACE(log, "Initialized DateLUT with time zone '" << DateLUT::instance().getTimeZone() << "'."); - /// Directory with temporary data for processing of heavy queries. + + /// Storage with temporary data for processing of heavy queries. { std::string tmp_path = config().getString("tmp_path", path + "tmp/"); - global_context->setTemporaryPath(tmp_path); - Poco::File(tmp_path).createDirectories(); - - /// Clearing old temporary files. - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(tmp_path); it != dir_end; ++it) - { - if (it->isFile() && startsWith(it.name(), "tmp")) - { - LOG_DEBUG(log, "Removing old temporary file " << it->path()); - it->remove(); - } - } + std::string tmp_policy = config().getString("tmp_policy", ""); + const VolumePtr & volume = global_context->setTemporaryStorage(tmp_path, tmp_policy); + for (const DiskPtr & disk : volume->disks) + setupTmpPath(log, disk->getPath()); } /** Directory with 'flags': files indicating temporary settings for the server set by system administrator. @@ -436,8 +453,10 @@ int Server::main(const std::vector & /*args*/) main_config_zk_changed_event, [&](ConfigurationPtr config) { - setTextLog(global_context->getTextLog()); - buildLoggers(*config, logger()); + // FIXME logging-related things need synchronization -- see the 'Logger * log' saved + // in a lot of places. For now, disable updating log configuration without server restart. + //setTextLog(global_context->getTextLog()); + //buildLoggers(*config, logger()); global_context->setClustersConfig(config); global_context->setMacros(std::make_unique(*config, "macros")); @@ -862,6 +881,13 @@ int Server::main(const std::vector & /*args*/) for (auto & server : servers) server->start(); + { + String level_str = config().getString("text_log.level", ""); + int level = level_str.empty() ? INT_MAX : Poco::Logger::parseLevel(level_str); + setTextLog(global_context->getTextLog(), level); + } + buildLoggers(config(), logger()); + main_config_reloader->start(); users_config_reloader->start(); if (dns_cache_updater) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 01c5ae59cea..1975349fcf1 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -591,11 +591,9 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads) } }); - /// Wait in case of exception. Delete pipeline to release memory. + /// Wait in case of exception happened outside of pool. SCOPE_EXIT( - /// Clear queue in case if somebody is waiting lazy_format to push. lazy_format->finish(); - lazy_format->clearQueue(); try { @@ -604,72 +602,58 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads) catch (...) { /// If exception was thrown during pipeline execution, skip it while processing other exception. + tryLogCurrentException(log); } - - pipeline = QueryPipeline() ); - while (true) + while (!lazy_format->isFinished() && !exception) { - Block block; - - while (true) + if (isQueryCancelled()) { - if (isQueryCancelled()) - { - /// A packet was received requesting to stop execution of the request. - executor->cancel(); - - break; - } - else - { - if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) - { - /// Some time passed and there is a progress. - after_send_progress.restart(); - sendProgress(); - } - - sendLogs(); - - if ((block = lazy_format->getBlock(query_context->getSettingsRef().interactive_delay / 1000))) - break; - - if (lazy_format->isFinished()) - break; - - if (exception) - { - pool.wait(); - break; - } - } - } - - /** If data has run out, we will send the profiling data and total values to - * the last zero block to be able to use - * this information in the suffix output of stream. - * If the request was interrupted, then `sendTotals` and other methods could not be called, - * because we have not read all the data yet, - * and there could be ongoing calculations in other threads at the same time. - */ - if (!block && !isQueryCancelled()) - { - pool.wait(); - pipeline.finalize(); - - sendTotals(lazy_format->getTotals()); - sendExtremes(lazy_format->getExtremes()); - sendProfileInfo(lazy_format->getProfileInfo()); - sendProgress(); - sendLogs(); - } - - sendData(block); - if (!block) + /// A packet was received requesting to stop execution of the request. + executor->cancel(); break; + } + + if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) + { + /// Some time passed and there is a progress. + after_send_progress.restart(); + sendProgress(); + } + + sendLogs(); + + if (auto block = lazy_format->getBlock(query_context->getSettingsRef().interactive_delay / 1000)) + { + if (!state.io.null_format) + sendData(block); + } } + + /// Finish lazy_format before waiting. Otherwise some thread may write into it, and waiting will lock. + lazy_format->finish(); + pool.wait(); + + /** If data has run out, we will send the profiling data and total values to + * the last zero block to be able to use + * this information in the suffix output of stream. + * If the request was interrupted, then `sendTotals` and other methods could not be called, + * because we have not read all the data yet, + * and there could be ongoing calculations in other threads at the same time. + */ + if (!isQueryCancelled()) + { + pipeline.finalize(); + + sendTotals(lazy_format->getTotals()); + sendExtremes(lazy_format->getExtremes()); + sendProfileInfo(lazy_format->getProfileInfo()); + sendProgress(); + sendLogs(); + } + + sendData({}); } state.io.onFinish(); @@ -993,7 +977,7 @@ bool TCPHandler::receiveData(bool scalar) if (!(storage = query_context->tryGetExternalTable(name))) { NamesAndTypesList columns = block.getNamesAndTypesList(); - storage = StorageMemory::create("_external", name, ColumnsDescription{columns}, ConstraintsDescription{}); + storage = StorageMemory::create(StorageID("_external", name), ColumnsDescription{columns}, ConstraintsDescription{}); storage->startup(); query_context->addExternalTable(name, storage); } diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index 4ab9097b9bb..9ec40329e04 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -111,7 +111,7 @@ public: server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } - void run(); + void run() override; /// This method is called right before the query execution. virtual void customizeContext(DB::Context & /*context*/) {} diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index c1479eaa528..1ed9c75e5ec 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -3,27 +3,27 @@ NOTE: User and query level settings are set up in "users.xml" file. --> - - - + + + - - + + - + trace /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log @@ -34,6 +34,7 @@ 8123 9000 + 9004 /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ @@ -343,6 +355,11 @@ toStartOfHour(event_time) --> toYYYYMM(event_date) + + + 7500 @@ -377,10 +394,12 @@ diff --git a/dbms/programs/server/users.xml b/dbms/programs/server/users.xml index 87e6c406b0a..d631fbb0f8a 100644 --- a/dbms/programs/server/users.xml +++ b/dbms/programs/server/users.xml @@ -49,7 +49,7 @@ In first line will be password and in second - corresponding SHA256. How to generate double SHA1: - Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | openssl dgst -sha1 -binary | openssl dgst -sha1 + Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' In first line will be password and in second - corresponding double SHA1. --> diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index effff5eeb55..7c9fb2ede7b 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -487,10 +487,17 @@ void LogisticRegression::compute( size_t row_num) { Float64 derivative = bias; + + std::vector values(weights.size()); + for (size_t i = 0; i < weights.size(); ++i) { - auto value = (*columns[i]).getFloat64(row_num); - derivative += weights[i] * value; + values[i] = (*columns[i]).getFloat64(row_num); + } + + for (size_t i = 0; i < weights.size(); ++i) + { + derivative += weights[i] * values[i]; } derivative *= target; derivative = exp(derivative); @@ -498,8 +505,7 @@ void LogisticRegression::compute( batch_gradient[weights.size()] += target / (derivative + 1); for (size_t i = 0; i < weights.size(); ++i) { - auto value = (*columns[i]).getFloat64(row_num); - batch_gradient[i] += target * value / (derivative + 1) - 2 * l2_reg_coef * weights[i]; + batch_gradient[i] += target * values[i] / (derivative + 1) - 2 * l2_reg_coef * weights[i]; } } @@ -558,18 +564,25 @@ void LinearRegression::compute( size_t row_num) { Float64 derivative = (target - bias); + + std::vector values(weights.size()); + + for (size_t i = 0; i < weights.size(); ++i) { - auto value = (*columns[i]).getFloat64(row_num); - derivative -= weights[i] * value; + values[i] = (*columns[i]).getFloat64(row_num); + } + + for (size_t i = 0; i < weights.size(); ++i) + { + derivative -= weights[i] * values[i]; } derivative *= 2; batch_gradient[weights.size()] += derivative; for (size_t i = 0; i < weights.size(); ++i) { - auto value = (*columns[i]).getFloat64(row_num); - batch_gradient[i] += derivative * value - 2 * l2_reg_coef * weights[i]; + batch_gradient[i] += derivative * values[i] - 2 * l2_reg_coef * weights[i]; } } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h b/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h index 61fd28f2a70..47240db8b0d 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h @@ -309,7 +309,7 @@ protected: /// Uses a DFA based approach in order to better handle patterns without /// time assertions. /// - /// NOTE: This implementation relies on the assumption that the pattern are *small*. + /// NOTE: This implementation relies on the assumption that the pattern is *small*. /// /// This algorithm performs in O(mn) (with m the number of DFA states and N the number /// of events) with a memory consumption and memory allocations in O(m). It means that diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index a359d5b2f1b..1e275f4ec13 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -217,7 +217,7 @@ UInt64 ColumnVector::get64(size_t n) const } template -Float64 ColumnVector::getFloat64(size_t n) const +inline Float64 ColumnVector::getFloat64(size_t n) const { return static_cast(data[n]); } diff --git a/dbms/src/Common/Allocator.h b/dbms/src/Common/Allocator.h index 5d39d327243..12116f6d8d3 100644 --- a/dbms/src/Common/Allocator.h +++ b/dbms/src/Common/Allocator.h @@ -50,16 +50,21 @@ * * P.S. This is also required, because tcmalloc can not allocate a chunk of * memory greater than 16 GB. + * + * P.P.S. Note that MMAP_THRESHOLD symbol is intentionally made weak. It allows + * to override it during linkage when using ClickHouse as a library in + * third-party applications which may already use own allocator doing mmaps + * in the implementation of alloc/realloc. */ #ifdef NDEBUG - static constexpr size_t MMAP_THRESHOLD = 64 * (1ULL << 20); + __attribute__((__weak__)) extern const size_t MMAP_THRESHOLD = 64 * (1ULL << 20); #else /** * In debug build, use small mmap threshold to reproduce more memory * stomping bugs. Along with ASLR it will hopefully detect more issues than * ASan. The program may fail due to the limit on number of memory mappings. */ - static constexpr size_t MMAP_THRESHOLD = 4096; + __attribute__((__weak__)) extern const size_t MMAP_THRESHOLD = 4096; #endif static constexpr size_t MMAP_MIN_ALIGNMENT = 4096; diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index fd72f8c7ea7..fe5bca9f55a 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -387,7 +387,6 @@ namespace ErrorCodes extern const int PTHREAD_ERROR = 411; extern const int NETLINK_ERROR = 412; extern const int CANNOT_SET_SIGNAL_HANDLER = 413; - extern const int CANNOT_READLINE = 414; extern const int ALL_REPLICAS_LOST = 415; extern const int REPLICA_STATUS_CHANGED = 416; extern const int EXPECTED_ALL_OR_ANY = 417; @@ -479,6 +478,7 @@ namespace ErrorCodes extern const int FILE_ALREADY_EXISTS = 504; extern const int CANNOT_DELETE_DIRECTORY = 505; extern const int UNEXPECTED_ERROR_CODE = 506; + extern const int UNABLE_TO_SKIP_UNUSED_SHARDS = 507; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Common/Exception.cpp b/dbms/src/Common/Exception.cpp index 25da9674e4d..318da1a27f2 100644 --- a/dbms/src/Common/Exception.cpp +++ b/dbms/src/Common/Exception.cpp @@ -195,7 +195,7 @@ std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded << ", e.displayText() = " << e.displayText() << (with_stacktrace ? getExceptionStackTraceString(e) : "") << (with_extra_info ? getExtraExceptionInfo(e) : "") - << " (version " << VERSION_STRING << VERSION_OFFICIAL; + << " (version " << VERSION_STRING << VERSION_OFFICIAL << ")"; } catch (...) {} } diff --git a/dbms/src/Common/FileChecker.cpp b/dbms/src/Common/FileChecker.cpp index 7c31ee9a71b..e6fcfef2ad6 100644 --- a/dbms/src/Common/FileChecker.cpp +++ b/dbms/src/Common/FileChecker.cpp @@ -112,7 +112,7 @@ void FileChecker::save() const out->next(); } - disk->moveFile(tmp_files_info_path, files_info_path); + disk->replaceFile(tmp_files_info_path, files_info_path); } void FileChecker::load(Map & local_map, const String & path) const diff --git a/dbms/src/Common/QueryProfiler.cpp b/dbms/src/Common/QueryProfiler.cpp index e142be2e4d9..96d96b151a7 100644 --- a/dbms/src/Common/QueryProfiler.cpp +++ b/dbms/src/Common/QueryProfiler.cpp @@ -141,7 +141,15 @@ QueryProfilerBase::QueryProfilerBase(const Int32 thread_id, const sev._sigev_un._tid = thread_id; #endif if (timer_create(clock_type, &sev, &timer_id)) + { + /// In Google Cloud Run, the function "timer_create" is implemented incorrectly as of 2020-01-25. + /// https://mybranch.dev/posts/clickhouse-on-cloud-run/ + if (errno == 0) + throw Exception("Failed to create thread timer. The function 'timer_create' returned non-zero but didn't set errno. This is bug in your OS.", + ErrorCodes::CANNOT_CREATE_TIMER); + throwFromErrno("Failed to create thread timer", ErrorCodes::CANNOT_CREATE_TIMER); + } /// Randomize offset as uniform random value from 0 to period - 1. /// It will allow to sample short queries even if timer period is large. diff --git a/dbms/src/Common/RemoteHostFilter.cpp b/dbms/src/Common/RemoteHostFilter.cpp index 16aaac35dbe..4c4aa3bca81 100644 --- a/dbms/src/Common/RemoteHostFilter.cpp +++ b/dbms/src/Common/RemoteHostFilter.cpp @@ -1,12 +1,13 @@ #include -#include #include -#include #include +#include +#include #include #include #include + namespace DB { namespace ErrorCodes diff --git a/dbms/src/Common/RemoteHostFilter.h b/dbms/src/Common/RemoteHostFilter.h index 86743891051..48d9b2bda7c 100644 --- a/dbms/src/Common/RemoteHostFilter.h +++ b/dbms/src/Common/RemoteHostFilter.h @@ -1,17 +1,19 @@ #pragma once +#include #include #include -#include -#include +namespace Poco { class URI; } +namespace Poco { namespace Util { class AbstractConfiguration; } } + namespace DB { class RemoteHostFilter { /** - * This class checks if url is allowed. + * This class checks if URL is allowed. * If primary_hosts and regexp_hosts are empty all urls are allowed. */ public: @@ -25,6 +27,7 @@ private: std::unordered_set primary_hosts; /// Allowed primary () URL from config.xml std::vector regexp_hosts; /// Allowed regexp () URL from config.xml - bool checkForDirectEntry(const std::string & str) const; /// Checks if the primary_hosts and regexp_hosts contain str. If primary_hosts and regexp_hosts are empty return true. + /// Checks if the primary_hosts and regexp_hosts contain str. If primary_hosts and regexp_hosts are empty return true. + bool checkForDirectEntry(const std::string & str) const; }; } diff --git a/dbms/src/Common/checkStackSize.cpp b/dbms/src/Common/checkStackSize.cpp index 16074e74281..10e93a8356c 100644 --- a/dbms/src/Common/checkStackSize.cpp +++ b/dbms/src/Common/checkStackSize.cpp @@ -23,7 +23,14 @@ namespace DB static thread_local void * stack_address = nullptr; static thread_local size_t max_stack_size = 0; -void checkStackSize() +/** It works fine when interpreters are instantiated by ClickHouse code in properly prepared threads, + * but there are cases when ClickHouse runs as a library inside another application. + * If application is using user-space lightweight threads with manually allocated stacks, + * current implementation is not reasonable, as it has no way to properly check the remaining + * stack size without knowing the details of how stacks are allocated. + * We mark this function as weak symbol to be able to replace it in another ClickHouse-based products. + */ +__attribute__((__weak__)) void checkStackSize() { using namespace DB; diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h index ae8b07718dd..668c9e6a401 100644 --- a/dbms/src/Core/Block.h +++ b/dbms/src/Core/Block.h @@ -147,6 +147,14 @@ using BlocksList = std::list; using BlocksPtr = std::shared_ptr; using BlocksPtrs = std::shared_ptr>; +/// Extends block with extra data in derived classes +struct ExtraBlock +{ + Block block; +}; + +using ExtraBlockPtr = std::shared_ptr; + /// Compare number of columns, data types, column types, column names, and values of constant columns. bool blocksHaveEqualStructure(const Block & lhs, const Block & rhs); diff --git a/dbms/src/Core/ExternalTable.cpp b/dbms/src/Core/ExternalTable.cpp index 8befb4e054c..e87bc3b513b 100644 --- a/dbms/src/Core/ExternalTable.cpp +++ b/dbms/src/Core/ExternalTable.cpp @@ -160,7 +160,7 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, /// Create table NamesAndTypesList columns = sample_block.getNamesAndTypesList(); - StoragePtr storage = StorageMemory::create("_external", data.second, ColumnsDescription{columns}, ConstraintsDescription{}); + StoragePtr storage = StorageMemory::create(StorageID("_external", data.second), ColumnsDescription{columns}, ConstraintsDescription{}); storage->startup(); context.addExternalTable(data.second, storage); BlockOutputStreamPtr output = storage->write(ASTPtr(), context); diff --git a/dbms/src/Core/ExternalTable.h b/dbms/src/Core/ExternalTable.h index 6927f19b524..a9ebbb5f5df 100644 --- a/dbms/src/Core/ExternalTable.h +++ b/dbms/src/Core/ExternalTable.h @@ -99,7 +99,7 @@ class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable public: ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {} - void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream); + void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream) override; private: Context & context; diff --git a/dbms/src/Core/MySQLProtocol.h b/dbms/src/Core/MySQLProtocol.h index 9b2a2cba249..2afeff341ab 100644 --- a/dbms/src/Core/MySQLProtocol.h +++ b/dbms/src/Core/MySQLProtocol.h @@ -1030,6 +1030,7 @@ public: LOG_TRACE(log, "Authentication method match."); } + bool sent_public_key = false; if (auth_response == "\1") { LOG_TRACE(log, "Client requests public key."); @@ -1050,6 +1051,7 @@ public: AuthMoreData data(pem); packet_sender->sendPacket(data, true); + sent_public_key = true; AuthSwitchResponse response; packet_sender->receivePacket(response); @@ -1069,13 +1071,15 @@ public: */ if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1)) { - LOG_TRACE(log, "Received nonempty password"); + LOG_TRACE(log, "Received nonempty password."); auto ciphertext = reinterpret_cast(auth_response->data()); unsigned char plaintext[RSA_size(&private_key)]; int plaintext_size = RSA_private_decrypt(auth_response->size(), ciphertext, plaintext, &private_key, RSA_PKCS1_OAEP_PADDING); if (plaintext_size == -1) { + if (!sent_public_key) + LOG_WARNING(log, "Client could have encrypted password with different public key since it didn't request it from server."); throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); } diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 724b31ca642..92b9618af77 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -52,6 +52,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ M(SettingUInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ M(SettingUInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ + M(SettingUInt64, max_joined_block_size_rows, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algorithm supports it). 0 means unlimited.", 0) \ + M(SettingUInt64, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_alter_threads, 0, "The maximum number of threads to execute the ALTER requests. By default, it is determined automatically.", 0) \ M(SettingUInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ @@ -64,14 +66,14 @@ struct Settings : public SettingsCollection M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "", 0) \ M(SettingSeconds, tcp_keep_alive_timeout, 0, "The time in seconds the connection needs to remain idle before TCP starts sending keepalive probes", 0) \ M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.", 0) \ - M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when connection pool is full.", 0) \ + M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when the connection pool is full.", 0) \ M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \ M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \ M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \ M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \ M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.", 0) \ M(SettingUInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \ - M(SettingUInt64, s3_min_upload_part_size, 512*1024*1024, "The mininum size of part to upload during multipart upload to S3.", 0) \ + M(SettingUInt64, s3_min_upload_part_size, 512*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \ M(SettingBool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ M(SettingBool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \ M(SettingBool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \ @@ -110,6 +112,7 @@ struct Settings : public SettingsCollection \ M(SettingBool, distributed_group_by_no_merge, false, "Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards.", 0) \ M(SettingBool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ + M(SettingUInt64, force_optimize_skip_unused_shards, 0, "Throw an exception if unused shards cannot be skipped (1 - throw only if the table has the sharding key, 2 - always throw.", 0) \ \ M(SettingBool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \ M(SettingUInt64, min_chunk_bytes_for_parallel_parsing, (1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \ @@ -183,9 +186,10 @@ struct Settings : public SettingsCollection M(SettingBool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ M(SettingBool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \ \ - M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ - M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ + M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ + M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \ + M(SettingString, input_format_avro_schema_registry_url, "", "For AvroConfluent format: Confluent Schema Registry URL.", 0) \ \ M(SettingBool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \ \ @@ -197,6 +201,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ M(SettingBool, output_format_pretty_color, true, "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ M(SettingUInt64, output_format_parquet_row_group_size, 1000000, "Row group size in rows.", 0) \ + M(SettingString, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy'.", 0) \ + M(SettingUInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ \ M(SettingBool, use_client_time_zone, false, "Use client timezone for interpreting DateTime string values, instead of adopting server timezone.", 0) \ \ @@ -212,7 +218,7 @@ struct Settings : public SettingsCollection M(SettingBool, join_use_nulls, 0, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ \ M(SettingJoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ - M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys maping to be consistent with LEFT one.", IMPORTANT) \ + M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ \ M(SettingUInt64, preferred_block_size_bytes, 1000000, "", 0) \ \ @@ -249,8 +255,8 @@ struct Settings : public SettingsCollection M(SettingBool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ M(SettingBool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \ - M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Highly experimental. Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ - M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Highly experimental. Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ + M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ + M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ \ \ /** Limits during query execution are part of the settings. \ @@ -310,9 +316,8 @@ struct Settings : public SettingsCollection M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \ M(SettingBool, partial_merge_join, false, "Use partial merge join instead of hash join for LEFT and INNER JOINs.", 0) \ M(SettingBool, partial_merge_join_optimizations, false, "Enable optimizations in partial merge join", 0) \ - M(SettingUInt64, default_max_bytes_in_join, 100000000, "Maximum size of right-side table if limit's required but max_bytes_in_join is not set.", 0) \ + M(SettingUInt64, default_max_bytes_in_join, 100000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \ M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 10000, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ - M(SettingUInt64, partial_merge_join_rows_in_left_blocks, 10000, "Group left-hand joining data in bigger blocks. Setting it to a bigger value increase JOIN performance and memory usage.", 0) \ \ M(SettingUInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ M(SettingUInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ @@ -360,7 +365,7 @@ struct Settings : public SettingsCollection M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ \ - M(SettingBool, experimental_use_processors, false, "Use processors pipeline.", 0) \ + M(SettingBool, experimental_use_processors, true, "Use processors pipeline.", 0) \ \ M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ @@ -371,7 +376,7 @@ struct Settings : public SettingsCollection M(SettingBool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ \ M(SettingSeconds, distributed_replica_error_half_life, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD, "Time period reduces replica error counter by 2 times.", 0) \ - M(SettingUInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up increadible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \ + M(SettingUInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up an incredible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \ \ M(SettingBool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \ M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.", 0) \ @@ -394,6 +399,7 @@ struct Settings : public SettingsCollection M(SettingBool, allow_experimental_data_skipping_indices, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingBool, merge_tree_uniform_read_distribution, true, "Obsolete setting, does nothing. Will be removed after 2020-05-20", 0) \ M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ + M(SettingUInt64, max_parser_depth, 1000, "Maximum parser depth.", 0) \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Core/SettingsCollection.cpp b/dbms/src/Core/SettingsCollection.cpp index 60c18a19fa7..9e44e327fcc 100644 --- a/dbms/src/Core/SettingsCollection.cpp +++ b/dbms/src/Core/SettingsCollection.cpp @@ -62,7 +62,7 @@ void SettingNumber::set(const Field & x) template void SettingNumber::set(const String & x) { - set(completeParse(x)); + set(parseWithSizeSuffix(x)); } template <> diff --git a/dbms/src/Core/Types.h b/dbms/src/Core/Types.h index ea80ab7d427..4f350ba00d5 100644 --- a/dbms/src/Core/Types.h +++ b/dbms/src/Core/Types.h @@ -31,7 +31,6 @@ enum class TypeIndex Float64, Date, DateTime, - DateTime32 = DateTime, DateTime64, String, FixedString, @@ -158,8 +157,6 @@ using Decimal32 = Decimal; using Decimal64 = Decimal; using Decimal128 = Decimal; -// TODO (nemkov): consider making a strong typedef -//using DateTime32 = time_t; using DateTime64 = Decimal64; template <> struct TypeName { static const char * get() { return "Decimal32"; } }; diff --git a/dbms/src/Core/config_core.h.in b/dbms/src/Core/config_core.h.in index fdbd69decd3..2365340cf33 100644 --- a/dbms/src/Core/config_core.h.in +++ b/dbms/src/Core/config_core.h.in @@ -10,5 +10,6 @@ #cmakedefine01 USE_POCO_DATAODBC #cmakedefine01 USE_POCO_MONGODB #cmakedefine01 USE_POCO_REDIS +#cmakedefine01 USE_POCO_JSON #cmakedefine01 USE_INTERNAL_LLVM_LIBRARY #cmakedefine01 USE_SSL diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp index 00189513352..3a77b1f42be 100644 --- a/dbms/src/Core/iostream_debug_helpers.cpp +++ b/dbms/src/Core/iostream_debug_helpers.cpp @@ -47,7 +47,8 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what) std::ostream & operator<<(std::ostream & stream, const IStorage & what) { - stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {" + auto table_id = what.getStorageID(); + stream << "IStorage(name = " << what.getName() << ", tableName = " << table_id.table_name << ") {" << what.getColumns().getAllPhysical().toString() << "}"; return stream; } diff --git a/dbms/src/DataStreams/BlockIO.h b/dbms/src/DataStreams/BlockIO.h index 50af2922306..c043e297d1f 100644 --- a/dbms/src/DataStreams/BlockIO.h +++ b/dbms/src/DataStreams/BlockIO.h @@ -66,6 +66,8 @@ struct BlockIO finish_callback = rhs.finish_callback; exception_callback = rhs.exception_callback; + null_format = rhs.null_format; + return *this; } }; diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index 9673395a21a..5a4fe4ecf1d 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -44,4 +44,29 @@ Block ExpressionBlockInputStream::readImpl() return res; } +Block InflatingExpressionBlockInputStream::readImpl() +{ + if (!initialized) + { + if (expression->resultIsAlwaysEmpty()) + return {}; + + initialized = true; + } + + Block res; + if (likely(!not_processed)) + { + res = children.back()->read(); + if (res) + expression->execute(res, not_processed, action_number); + } + else + { + res = std::move(not_processed->block); + expression->execute(res, not_processed, action_number); + } + return res; +} + } diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index af0e2db589b..62141a060af 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -15,10 +15,9 @@ class ExpressionActions; */ class ExpressionBlockInputStream : public IBlockInputStream { -private: +public: using ExpressionActionsPtr = std::shared_ptr; -public: ExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_); String getName() const override; @@ -26,12 +25,29 @@ public: Block getHeader() const override; protected: + bool initialized = false; + ExpressionActionsPtr expression; + Block readImpl() override; private: - ExpressionActionsPtr expression; Block cached_header; - bool initialized = false; +}; + +/// ExpressionBlockInputStream that could generate many out blocks for single input block. +class InflatingExpressionBlockInputStream : public ExpressionBlockInputStream +{ +public: + InflatingExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_) + : ExpressionBlockInputStream(input, expression_) + {} + +protected: + Block readImpl() override; + +private: + ExtraBlockPtr not_processed; + size_t action_number = 0; }; } diff --git a/dbms/src/DataStreams/IBlockStream_fwd.h b/dbms/src/DataStreams/IBlockStream_fwd.h index 06319a55262..d74a9528ed9 100644 --- a/dbms/src/DataStreams/IBlockStream_fwd.h +++ b/dbms/src/DataStreams/IBlockStream_fwd.h @@ -12,5 +12,6 @@ class IBlockOutputStream; using BlockInputStreamPtr = std::shared_ptr; using BlockInputStreams = std::vector; using BlockOutputStreamPtr = std::shared_ptr; +using BlockOutputStreams = std::vector; } diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 52f85f1349c..21422d0fe54 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace ProfileEvents @@ -21,10 +22,10 @@ namespace DB MergeSortingBlockInputStream::MergeSortingBlockInputStream( const BlockInputStreamPtr & input, SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_, size_t min_free_disk_space_) + size_t max_bytes_before_external_sort_, VolumePtr tmp_volume_, size_t min_free_disk_space_) : description(description_), max_merged_block_size(max_merged_block_size_), limit(limit_), max_bytes_before_remerge(max_bytes_before_remerge_), - max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_), + max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_volume(tmp_volume_), min_free_disk_space(min_free_disk_space_) { children.push_back(input); @@ -78,10 +79,14 @@ Block MergeSortingBlockInputStream::readImpl() */ if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) { - if (!enoughSpaceInDirectory(tmp_path, sum_bytes_in_blocks + min_free_disk_space)) - throw Exception("Not enough space for external sort in " + tmp_path, ErrorCodes::NOT_ENOUGH_SPACE); + size_t size = sum_bytes_in_blocks + min_free_disk_space; + auto reservation = tmp_volume->reserve(size); + if (!reservation) + throw Exception("Not enough space for external sort in temporary storage", ErrorCodes::NOT_ENOUGH_SPACE); + const std::string tmp_path(reservation->getDisk()->getPath()); temporary_files.emplace_back(createTemporaryFile(tmp_path)); + const std::string & path = temporary_files.back()->path(); MergeSortingBlocksBlockInputStream block_in(blocks, description, max_merged_block_size, limit); diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.h b/dbms/src/DataStreams/MergeSortingBlockInputStream.h index ce82f6bb120..5b157310765 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.h +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.h @@ -18,6 +18,9 @@ namespace DB struct TemporaryFileStream; +class Volume; +using VolumePtr = std::shared_ptr; + namespace ErrorCodes { extern const int NOT_ENOUGH_SPACE; @@ -77,7 +80,7 @@ public: MergeSortingBlockInputStream(const BlockInputStreamPtr & input, SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_, + size_t max_bytes_before_external_sort_, VolumePtr tmp_volume_, size_t min_free_disk_space_); String getName() const override { return "MergeSorting"; } @@ -97,7 +100,7 @@ private: size_t max_bytes_before_remerge; size_t max_bytes_before_external_sort; - const std::string tmp_path; + VolumePtr tmp_volume; size_t min_free_disk_space; Logger * log = &Logger::get("MergeSortingBlockInputStream"); diff --git a/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h b/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h index 8ef64cc5e05..8fe05c387a3 100644 --- a/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h +++ b/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h @@ -21,9 +21,19 @@ class NullAndDoCopyBlockInputStream : public IBlockInputStream { public: NullAndDoCopyBlockInputStream(const BlockInputStreamPtr & input_, BlockOutputStreamPtr output_) - : input(input_), output(output_) { - children.push_back(input_); + input_streams.push_back(input_); + output_streams.push_back(output_); + + for (auto & input_stream : input_streams) + children.push_back(input_stream); + } + + NullAndDoCopyBlockInputStream(const BlockInputStreams & input_, BlockOutputStreams & output_) + : input_streams(input_), output_streams(output_) + { + for (auto & input_stream : input_) + children.push_back(input_stream); } /// Suppress readPrefix and readSuffix, because they are called by copyData. @@ -39,13 +49,20 @@ public: protected: Block readImpl() override { - copyData(*input, *output); + /// We do not use cancel flag here. + /// If query was cancelled, it will be processed by child streams. + /// Part of the data will be processed. + + if (input_streams.size() == 1 && output_streams.size() == 1) + copyData(*input_streams.at(0), *output_streams.at(0)); + else + copyData(input_streams, output_streams); return Block(); } private: - BlockInputStreamPtr input; - BlockOutputStreamPtr output; + BlockInputStreams input_streams; + BlockOutputStreams output_streams; }; } diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 3d5fb426218..90e2d95ad15 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -18,7 +18,7 @@ namespace DB { PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( - const String & database, const String & table, const StoragePtr & storage_, + const StoragePtr & storage_, const Context & context_, const ASTPtr & query_ptr_, bool no_destination) : storage(storage_), context(context_), query_ptr(query_ptr_) { @@ -32,47 +32,59 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks bool disable_deduplication_for_children = !no_destination && storage->supportsDeduplication(); - if (!table.empty()) + auto table_id = storage->getStorageID(); + Dependencies dependencies = context.getDependencies(table_id); + + /// We need special context for materialized views insertions + if (!dependencies.empty()) { - Dependencies dependencies = context.getDependencies(database, table); + views_context = std::make_unique(context); + // Do not deduplicate insertions into MV if the main insertion is Ok + if (disable_deduplication_for_children) + views_context->getSettingsRef().insert_deduplicate = false; + } - /// We need special context for materialized views insertions - if (!dependencies.empty()) + for (const auto & database_table : dependencies) + { + auto dependent_table = context.getTable(database_table); + + ASTPtr query; + BlockOutputStreamPtr out; + + if (auto * materialized_view = dynamic_cast(dependent_table.get())) { - views_context = std::make_unique(context); - // Do not deduplicate insertions into MV if the main insertion is Ok - if (disable_deduplication_for_children) - views_context->getSettingsRef().insert_deduplicate = false; + StoragePtr inner_table = materialized_view->getTargetTable(); + auto inner_table_id = inner_table->getStorageID(); + query = materialized_view->getInnerQuery(); + + std::unique_ptr insert = std::make_unique(); + insert->database = inner_table_id.database_name; + insert->table = inner_table_id.table_name; + + /// Get list of columns we get from select query. + auto header = InterpreterSelectQuery(query, *views_context, SelectQueryOptions().analyze()) + .getSampleBlock(); + + /// Insert only columns returned by select. + auto list = std::make_shared(); + for (auto & column : header) + /// But skip columns which storage doesn't have. + if (inner_table->hasColumn(column.name)) + list->children.emplace_back(std::make_shared(column.name)); + + insert->columns = std::move(list); + + ASTPtr insert_query_ptr(insert.release()); + InterpreterInsertQuery interpreter(insert_query_ptr, *views_context); + BlockIO io = interpreter.execute(); + out = io.out; } + else if (dynamic_cast(dependent_table.get())) + out = std::make_shared(dependent_table, *views_context, ASTPtr(), true); + else + out = std::make_shared(dependent_table, *views_context, ASTPtr()); - for (const auto & database_table : dependencies) - { - auto dependent_table = context.getTable(database_table.first, database_table.second); - - ASTPtr query; - BlockOutputStreamPtr out; - - if (auto * materialized_view = dynamic_cast(dependent_table.get())) - { - StoragePtr inner_table = materialized_view->getTargetTable(); - query = materialized_view->getInnerQuery(); - std::unique_ptr insert = std::make_unique(); - insert->database = inner_table->getDatabaseName(); - insert->table = inner_table->getTableName(); - ASTPtr insert_query_ptr(insert.release()); - InterpreterInsertQuery interpreter(insert_query_ptr, *views_context); - BlockIO io = interpreter.execute(); - out = io.out; - } - else if (dynamic_cast(dependent_table.get())) - out = std::make_shared( - database_table.first, database_table.second, dependent_table, *views_context, ASTPtr(), true); - else - out = std::make_shared( - database_table.first, database_table.second, dependent_table, *views_context, ASTPtr()); - - views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)}); - } + views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out)}); } /* Do not push to destination table if the flag is set */ @@ -161,7 +173,7 @@ void PushingToViewsBlockOutputStream::writePrefix() } catch (Exception & ex) { - ex.addMessage("while write prefix to view " + view.database + "." + view.table); + ex.addMessage("while write prefix to view " + view.table_id.getNameForLogs()); throw; } } @@ -180,7 +192,7 @@ void PushingToViewsBlockOutputStream::writeSuffix() } catch (Exception & ex) { - ex.addMessage("while write prefix to view " + view.database + "." + view.table); + ex.addMessage("while write prefix to view " + view.table_id.getNameForLogs()); throw; } } @@ -223,7 +235,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n /// InterpreterSelectQuery will do processing of alias columns. Context local_context = *views_context; local_context.addViewSource( - StorageValues::create(storage->getDatabaseName(), storage->getTableName(), storage->getColumns(), + StorageValues::create(storage->getStorageID(), storage->getColumns(), block)); select.emplace(view.query, local_context, SelectQueryOptions()); in = std::make_shared(select->execute().in); @@ -250,7 +262,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n } catch (Exception & ex) { - ex.addMessage("while pushing to view " + backQuoteIfNeed(view.database) + "." + backQuoteIfNeed(view.table)); + ex.addMessage("while pushing to view " + view.table_id.getNameForLogs()); throw; } } diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h index c9b0538e470..162c2e1b447 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -17,8 +17,7 @@ class ReplicatedMergeTreeBlockOutputStream; class PushingToViewsBlockOutputStream : public IBlockOutputStream { public: - PushingToViewsBlockOutputStream( - const String & database, const String & table, const StoragePtr & storage_, + PushingToViewsBlockOutputStream(const StoragePtr & storage_, const Context & context_, const ASTPtr & query_ptr_, bool no_destination = false); Block getHeader() const override; @@ -39,8 +38,7 @@ private: struct ViewInfo { ASTPtr query; - String database; - String table; + StorageID table_id; BlockOutputStreamPtr out; }; diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index 339f81321e4..c08abba3bdf 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -70,7 +70,7 @@ bool TTLBlockInputStream::isTTLExpired(time_t ttl) Block TTLBlockInputStream::readImpl() { /// Skip all data if table ttl is expired for part - if (storage.hasTableTTL() && isTTLExpired(old_ttl_infos.table_ttl.max)) + if (storage.hasRowsTTL() && isTTLExpired(old_ttl_infos.table_ttl.max)) { rows_removed = data_part->rows_count; return {}; @@ -80,7 +80,7 @@ Block TTLBlockInputStream::readImpl() if (!block) return block; - if (storage.hasTableTTL() && (force || isTTLExpired(old_ttl_infos.table_ttl.min))) + if (storage.hasRowsTTL() && (force || isTTLExpired(old_ttl_infos.table_ttl.min))) removeRowsWithExpiredTableTTL(block); removeValuesWithExpiredColumnTTL(block); @@ -106,10 +106,10 @@ void TTLBlockInputStream::readSuffixImpl() void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block) { - storage.ttl_table_entry.expression->execute(block); + storage.rows_ttl_entry.expression->execute(block); const IColumn * ttl_column = - block.getByName(storage.ttl_table_entry.result_column).column.get(); + block.getByName(storage.rows_ttl_entry.result_column).column.get(); const auto & column_names = header.getNames(); MutableColumns result_columns; diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 9d17596fc8d..fd4bfab28d8 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -1,6 +1,10 @@ +#include #include #include #include +#include +#include +#include namespace DB @@ -51,6 +55,79 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall inline void doNothing(const Block &) {} +namespace +{ + + +struct ParallelInsertsHandler +{ + using CencellationHook = std::function; + + explicit ParallelInsertsHandler(BlockOutputStreams & output_streams, CencellationHook cancellation_hook_, size_t num_threads) + : outputs(output_streams.size()), cancellation_hook(std::move(cancellation_hook_)) + { + exceptions.resize(num_threads); + + for (auto & output : output_streams) + outputs.push(output.get()); + } + + void onBlock(Block & block, size_t /*thread_num*/) + { + IBlockOutputStream * out = nullptr; + + outputs.pop(out); + out->write(block); + outputs.push(out); + } + + void onFinishThread(size_t /*thread_num*/) {} + void onFinish() {} + + void onException(std::exception_ptr & exception, size_t thread_num) + { + exceptions[thread_num] = exception; + cancellation_hook(); + } + + void rethrowFirstException() + { + for (auto & exception : exceptions) + if (exception) + std::rethrow_exception(exception); + } + + ConcurrentBoundedQueue outputs; + std::vector exceptions; + CencellationHook cancellation_hook; +}; + +} + +static void copyDataImpl(BlockInputStreams & inputs, BlockOutputStreams & outputs) +{ + for (auto & output : outputs) + output->writePrefix(); + + using Processor = ParallelInputsProcessor; + Processor * processor_ptr = nullptr; + + ParallelInsertsHandler handler(outputs, [&processor_ptr]() { processor_ptr->cancel(false); }, inputs.size()); + ParallelInputsProcessor processor(inputs, nullptr, inputs.size(), handler); + processor_ptr = &processor; + + processor.process(); + processor.wait(); + handler.rethrowFirstException(); + + /// readPrefix is called in ParallelInputsProcessor. + for (auto & input : inputs) + input->readSuffix(); + + for (auto & output : outputs) + output->writeSuffix(); +} + void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled) { auto is_cancelled_pred = [is_cancelled] () @@ -61,6 +138,10 @@ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic & is_cancelled) { diff --git a/dbms/src/DataStreams/copyData.h b/dbms/src/DataStreams/copyData.h index f2bce8f411b..ae72dbd2421 100644 --- a/dbms/src/DataStreams/copyData.h +++ b/dbms/src/DataStreams/copyData.h @@ -16,6 +16,8 @@ class Block; */ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled = nullptr); +void copyData(BlockInputStreams & inputs, BlockOutputStreams & outputs); + void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function & is_cancelled); void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function & is_cancelled, diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index 21ab50a7c88..7ef0d931279 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -18,6 +19,9 @@ public: bool equals(const IDataType & rhs) const override { return typeid(rhs) == typeid(*this); } bool isParametric() const override { return true; } + // Used for expressions analysis. + MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } + // Used only for debugging, making it DUMPABLE Field getDefault() const override { return Tuple(); } }; diff --git a/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp b/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp index f70405dd765..f334ab222a4 100644 --- a/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp +++ b/dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp @@ -60,7 +60,7 @@ std::ostream & operator<<(std::ostream & ostr, const TypesTestCase & test_case) class TypeTest : public ::testing::TestWithParam { public: - void SetUp() + void SetUp() override { const auto & p = GetParam(); from_types = typesFromString(p.from_types); diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 9409fdc584a..9323a5914f1 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -52,7 +52,7 @@ Tables DatabaseDictionary::listTables(const Context & context, const FilterByNam auto dict_name = dict_ptr->getName(); const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - tables[dict_name] = StorageDictionary::create(getDatabaseName(), dict_name, ColumnsDescription{columns}, context, true, dict_name); + tables[dict_name] = StorageDictionary::create(StorageID(getDatabaseName(), dict_name), ColumnsDescription{columns}, context, true, dict_name); } } return tables; @@ -74,7 +74,7 @@ StoragePtr DatabaseDictionary::tryGetTable( { const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - return StorageDictionary::create(getDatabaseName(), table_name, ColumnsDescription{columns}, context, true, table_name); + return StorageDictionary::create(StorageID(getDatabaseName(), table_name), ColumnsDescription{columns}, context, true, table_name); } return {}; @@ -109,11 +109,12 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context, buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")"; } + auto settings = context.getSettingsRef(); ParserCreateQuery parser; const char * pos = query.data(); std::string error_message; auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, - /* hilite = */ false, "", /* allow_multi_statements = */ false, 0); + /* hilite = */ false, "", /* allow_multi_statements = */ false, 0, settings.max_parser_depth); if (!ast && throw_on_error) throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); @@ -121,15 +122,16 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context, return ast; } -ASTPtr DatabaseDictionary::getCreateDatabaseQuery() const +ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & context) const { String query; { WriteBufferFromString buffer(query); buffer << "CREATE DATABASE " << backQuoteIfNeed(database_name) << " ENGINE = Dictionary"; } + auto settings = context.getSettingsRef(); ParserCreateQuery parser; - return parseQuery(parser, query.data(), query.data() + query.size(), "", 0); + return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth); } void DatabaseDictionary::shutdown() diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 3155e12b862..cd5dde3177c 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -41,7 +41,7 @@ public: bool empty(const Context & context) const override; - ASTPtr getCreateDatabaseQuery() const override; + ASTPtr getCreateDatabaseQuery(const Context & context) const override; void shutdown() override; diff --git a/dbms/src/Databases/DatabaseLazy.cpp b/dbms/src/Databases/DatabaseLazy.cpp index fc71b3a63a7..48904f9eac8 100644 --- a/dbms/src/Databases/DatabaseLazy.cpp +++ b/dbms/src/Databases/DatabaseLazy.cpp @@ -122,7 +122,7 @@ StoragePtr DatabaseLazy::tryGetTable( std::lock_guard lock(mutex); auto it = tables_cache.find(table_name); if (it == tables_cache.end()) - throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + return {}; if (it->second.table) { @@ -230,7 +230,7 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table StoragePtr table; Context context_copy(context); /// some tables can change context, but not LogTables - auto ast = parseQueryFromMetadata(table_metadata_path, /*throw_on_error*/ true, /*remove_empty*/false); + auto ast = parseQueryFromMetadata(context, table_metadata_path, /*throw_on_error*/ true, /*remove_empty*/false); if (ast) { auto & ast_create = ast->as(); diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 996d5ca7c84..28a6a8a7af0 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -9,6 +9,7 @@ namespace DB DatabaseMemory::DatabaseMemory(const String & name_) : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")") + , data_path("data/" + escapeForFileName(database_name) + "/") {} void DatabaseMemory::createTable( @@ -27,7 +28,7 @@ void DatabaseMemory::removeTable( detachTable(table_name); } -ASTPtr DatabaseMemory::getCreateDatabaseQuery() const +ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context & /*context*/) const { auto create_query = std::make_shared(); create_query->database = database_name; diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 5609e6053ce..01789c11901 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include namespace Poco { class Logger; } @@ -31,7 +33,17 @@ public: const Context & context, const String & table_name) override; - ASTPtr getCreateDatabaseQuery() const override; + ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const override; + + /// DatabaseMemory allows to create tables, which store data on disk. + /// It's needed to create such tables in default database of clickhouse-local. + /// TODO May be it's better to use DiskMemory for such tables. + /// To save data on disk it's possible to explicitly CREATE DATABASE db ENGINE=Ordinary in clickhouse-local. + String getTableDataPath(const String & table_name) const override { return data_path + escapeForFileName(table_name) + "/"; } + String getTableDataPath(const ASTCreateQuery & query) const override { return getTableDataPath(query.table); } + +private: + String data_path; }; } diff --git a/dbms/src/Databases/DatabaseMySQL.cpp b/dbms/src/Databases/DatabaseMySQL.cpp index 9ae42f08d8a..db4ef926822 100644 --- a/dbms/src/Databases/DatabaseMySQL.cpp +++ b/dbms/src/Databases/DatabaseMySQL.cpp @@ -132,9 +132,9 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr { /// init create query. - - create_table_query->table = storage->getTableName(); - create_table_query->database = storage->getDatabaseName(); + auto table_id = storage->getStorageID(); + create_table_query->table = table_id.table_name; + create_table_query->database = table_id.database_name; for (const auto & column_type_and_name : storage->getColumns().getOrdinary()) { @@ -144,7 +144,7 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr columns_expression_list->children.emplace_back(column_declaration); } - auto mysql_table_name = std::make_shared(storage->getTableName()); + auto mysql_table_name = std::make_shared(table_id.table_name); auto storage_engine_arguments = table_storage_define->as()->engine->arguments; storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, mysql_table_name); } @@ -181,7 +181,7 @@ time_t DatabaseMySQL::getObjectMetadataModificationTime(const String & table_nam return time_t(local_tables_cache[table_name].first); } -ASTPtr DatabaseMySQL::getCreateDatabaseQuery() const +ASTPtr DatabaseMySQL::getCreateDatabaseQuery(const Context & /*context*/) const { const auto & create_query = std::make_shared(); create_query->database = database_name; @@ -239,7 +239,7 @@ void DatabaseMySQL::fetchLatestTablesStructureIntoCache(const std::map createTableFromAST( ast_create_query.table, StorageFactory::instance().get( ast_create_query, - table_data_path_relative, ast_create_query.table, database_name, context, context.getGlobalContext(), - columns, constraints, - true, has_force_restore_data_flag) + table_data_path_relative, + context, + context.getGlobalContext(), + columns, + constraints, + has_force_restore_data_flag) }; } @@ -211,7 +214,7 @@ void DatabaseOnDisk::renameTable( if (!table) throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - ASTPtr ast = parseQueryFromMetadata(getObjectMetadataPath(table_name)); + ASTPtr ast = parseQueryFromMetadata(context, getObjectMetadataPath(table_name)); if (!ast) throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST); auto & create = ast->as(); @@ -244,7 +247,7 @@ ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const St ASTPtr ast; auto table_metadata_path = getObjectMetadataPath(table_name); - ast = getCreateQueryFromMetadata(table_metadata_path, throw_on_error); + ast = getCreateQueryFromMetadata(context, table_metadata_path, throw_on_error); if (!ast && throw_on_error) { /// Handle system.* tables for which there are no table.sql files. @@ -260,20 +263,21 @@ ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const St return ast; } -ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const +ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const Context & context) const { ASTPtr ast; + auto settings = context.getSettingsRef(); auto metadata_dir_path = getMetadataPath(); auto database_metadata_path = metadata_dir_path.substr(0, metadata_dir_path.size() - 1) + ".sql"; - ast = getCreateQueryFromMetadata(database_metadata_path, true); + ast = getCreateQueryFromMetadata(context, database_metadata_path, true); if (!ast) { /// Handle databases (such as default) for which there are no database.sql files. /// If database.sql doesn't exist, then engine is Ordinary String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary"; ParserCreateQuery parser; - ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0); + ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth); } return ast; @@ -353,7 +357,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati } } -ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const +ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const Context & context, const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const { String query; @@ -380,11 +384,12 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, return nullptr; } + auto settings = context.getSettingsRef(); ParserCreateQuery parser; const char * pos = query.data(); std::string error_message; auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false, - "in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0); + "in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0, settings.max_parser_depth); if (!ast && throw_on_error) throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); @@ -394,9 +399,9 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, return ast; } -ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const String & database_metadata_path, bool throw_on_error) const +ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const Context & context, const String & database_metadata_path, bool throw_on_error) const { - ASTPtr ast = parseQueryFromMetadata(database_metadata_path, throw_on_error); + ASTPtr ast = parseQueryFromMetadata(context, database_metadata_path, throw_on_error); if (ast) { diff --git a/dbms/src/Databases/DatabaseOnDisk.h b/dbms/src/Databases/DatabaseOnDisk.h index fa77e80a3a4..079d0aefd10 100644 --- a/dbms/src/Databases/DatabaseOnDisk.h +++ b/dbms/src/Databases/DatabaseOnDisk.h @@ -52,7 +52,7 @@ public: const String & to_table_name, TableStructureWriteLockHolder & lock) override; - ASTPtr getCreateDatabaseQuery() const override; + ASTPtr getCreateDatabaseQuery(const Context & context) const override; void drop(const Context & context) override; @@ -74,8 +74,8 @@ protected: const String & table_name, bool throw_on_error) const override; - ASTPtr parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const; - ASTPtr getCreateQueryFromMetadata(const String & metadata_path, bool throw_on_error) const; + ASTPtr parseQueryFromMetadata(const Context & context, const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const; + ASTPtr getCreateQueryFromMetadata(const Context & context, const String & metadata_path, bool throw_on_error) const; const String metadata_path; diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index f9f6983604c..2b4102495d7 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -122,12 +122,12 @@ void DatabaseOrdinary::loadStoredObjects( FileNames file_names; size_t total_dictionaries = 0; - iterateMetadataFiles(context, [&file_names, &total_dictionaries, this](const String & file_name) + iterateMetadataFiles(context, [&context, &file_names, &total_dictionaries, this](const String & file_name) { String full_path = getMetadataPath() + file_name; try { - auto ast = parseQueryFromMetadata(full_path, /*throw_on_error*/ true, /*remove_empty*/false); + auto ast = parseQueryFromMetadata(context, full_path, /*throw_on_error*/ true, /*remove_empty*/false); if (ast) { auto * create_query = ast->as(); diff --git a/dbms/src/Databases/DatabaseWithDictionaries.cpp b/dbms/src/Databases/DatabaseWithDictionaries.cpp index 716ed32b676..3559067da0f 100644 --- a/dbms/src/Databases/DatabaseWithDictionaries.cpp +++ b/dbms/src/Databases/DatabaseWithDictionaries.cpp @@ -222,7 +222,7 @@ StoragePtr DatabaseWithDictionaries::getDictionaryStorage(const Context & contex { const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - return StorageDictionary::create(database_name, table_name, ColumnsDescription{columns}, context, true, dict_name); + return StorageDictionary::create(StorageID(database_name, table_name), ColumnsDescription{columns}, context, true, dict_name); } return nullptr; } @@ -235,7 +235,7 @@ ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl( ASTPtr ast; auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name); - ast = getCreateQueryFromMetadata(dictionary_metadata_path, throw_on_error); + ast = getCreateQueryFromMetadata(context, dictionary_metadata_path, throw_on_error); if (!ast && throw_on_error) { /// Handle system.* tables for which there are no table.sql files. diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index e9211560c51..1841d6e1e7f 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -56,13 +56,13 @@ public: DatabaseTablesSnapshotIterator(Tables && tables_) : tables(tables_), it(tables.begin()) {} - void next() { ++it; } + void next() override { ++it; } - bool isValid() const { return it != tables.end(); } + bool isValid() const override { return it != tables.end(); } - const String & name() const { return it->first; } + const String & name() const override { return it->first; } - const StoragePtr & table() const { return it->second; } + const StoragePtr & table() const override { return it->second; } }; /// Copies list of dictionaries and iterates through such snapshot. @@ -262,7 +262,7 @@ public: } /// Get the CREATE DATABASE query for current database. - virtual ASTPtr getCreateDatabaseQuery() const = 0; + virtual ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const = 0; /// Get name of database. String getDatabaseName() const { return database_name; } diff --git a/dbms/src/Dictionaries/DictionaryFactory.h b/dbms/src/Dictionaries/DictionaryFactory.h index b0d0c246ef5..3d3f793e2b2 100644 --- a/dbms/src/Dictionaries/DictionaryFactory.h +++ b/dbms/src/Dictionaries/DictionaryFactory.h @@ -23,6 +23,8 @@ namespace DB class Context; +/** Create dictionary according to its layout. + */ class DictionaryFactory : private boost::noncopyable { public: diff --git a/dbms/src/Disks/DiskLocal.cpp b/dbms/src/Disks/DiskLocal.cpp index 2998f9834d7..aed2f71f344 100644 --- a/dbms/src/Disks/DiskLocal.cpp +++ b/dbms/src/Disks/DiskLocal.cpp @@ -15,7 +15,57 @@ namespace ErrorCodes extern const int PATH_ACCESS_DENIED; } -std::mutex DiskLocal::mutex; +std::mutex DiskLocal::reservation_mutex; + + +using DiskLocalPtr = std::shared_ptr; + +class DiskLocalReservation : public IReservation +{ +public: + DiskLocalReservation(const DiskLocalPtr & disk_, UInt64 size_) + : disk(disk_), size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size_) + { + } + + UInt64 getSize() const override { return size; } + + DiskPtr getDisk() const override { return disk; } + + void update(UInt64 new_size) override; + + ~DiskLocalReservation() override; + +private: + DiskLocalPtr disk; + UInt64 size; + CurrentMetrics::Increment metric_increment; +}; + + +class DiskLocalDirectoryIterator : public IDiskDirectoryIterator +{ +public: + explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_) : + dir_path(dir_path_), iter(disk_path_ + dir_path_) {} + + void next() override { ++iter; } + + bool isValid() const override { return iter != Poco::DirectoryIterator(); } + + String path() const override + { + if (iter->isDirectory()) + return dir_path + iter.name() + '/'; + else + return dir_path + iter.name(); + } + +private: + String dir_path; + Poco::DirectoryIterator iter; +}; + ReservationPtr DiskLocal::reserve(UInt64 bytes) { @@ -26,7 +76,7 @@ ReservationPtr DiskLocal::reserve(UInt64 bytes) bool DiskLocal::tryReserve(UInt64 bytes) { - std::lock_guard lock(mutex); + std::lock_guard lock(DiskLocal::reservation_mutex); if (bytes == 0) { LOG_DEBUG(&Logger::get("DiskLocal"), "Reserving 0 bytes on disk " << backQuote(name)); @@ -71,7 +121,7 @@ UInt64 DiskLocal::getAvailableSpace() const UInt64 DiskLocal::getUnreservedSpace() const { - std::lock_guard lock(mutex); + std::lock_guard lock(DiskLocal::reservation_mutex); auto available_space = getAvailableSpace(); available_space -= std::min(available_space, reserved_bytes); return available_space; @@ -161,20 +211,31 @@ std::unique_ptr DiskLocal::writeFile(const String & path, size_t bu return std::make_unique(disk_path + path, buf_size, flags); } +void DiskLocal::remove(const String & path) +{ + Poco::File(disk_path + path).remove(false); +} + +void DiskLocal::removeRecursive(const String & path) +{ + Poco::File(disk_path + path).remove(true); +} + void DiskLocalReservation::update(UInt64 new_size) { - std::lock_guard lock(DiskLocal::mutex); + std::lock_guard lock(DiskLocal::reservation_mutex); disk->reserved_bytes -= size; size = new_size; disk->reserved_bytes += size; } + DiskLocalReservation::~DiskLocalReservation() { try { - std::lock_guard lock(DiskLocal::mutex); + std::lock_guard lock(DiskLocal::reservation_mutex); if (disk->reserved_bytes < size) { disk->reserved_bytes = 0; diff --git a/dbms/src/Disks/DiskLocal.h b/dbms/src/Disks/DiskLocal.h index c5f5a1cb2d9..85838d44298 100644 --- a/dbms/src/Disks/DiskLocal.h +++ b/dbms/src/Disks/DiskLocal.h @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -71,6 +70,10 @@ public: std::unique_ptr writeFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) override; + void remove(const String & path) override; + + void removeRecursive(const String & path) override; + private: bool tryReserve(UInt64 bytes); @@ -79,61 +82,10 @@ private: const String disk_path; const UInt64 keep_free_space_bytes; - /// Used for reservation counters modification - static std::mutex mutex; UInt64 reserved_bytes = 0; UInt64 reservation_count = 0; + + static std::mutex reservation_mutex; }; -using DiskLocalPtr = std::shared_ptr; - - -class DiskLocalDirectoryIterator : public IDiskDirectoryIterator -{ -public: - explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_) : - dir_path(dir_path_), iter(disk_path_ + dir_path_) {} - - void next() override { ++iter; } - - bool isValid() const override { return iter != Poco::DirectoryIterator(); } - - String path() const override - { - if (iter->isDirectory()) - return dir_path + iter.name() + '/'; - else - return dir_path + iter.name(); - } - -private: - String dir_path; - Poco::DirectoryIterator iter; -}; - -class DiskLocalReservation : public IReservation -{ -public: - DiskLocalReservation(const DiskLocalPtr & disk_, UInt64 size_) - : disk(disk_), size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size_) - { - } - - UInt64 getSize() const override { return size; } - - DiskPtr getDisk() const override { return disk; } - - void update(UInt64 new_size) override; - - ~DiskLocalReservation() override; - -private: - DiskLocalPtr disk; - UInt64 size; - CurrentMetrics::Increment metric_increment; -}; - -class DiskFactory; -void registerDiskLocal(DiskFactory & factory); - } diff --git a/dbms/src/Disks/DiskMemory.cpp b/dbms/src/Disks/DiskMemory.cpp index d491cf3bea0..46a73719b5f 100644 --- a/dbms/src/Disks/DiskMemory.cpp +++ b/dbms/src/Disks/DiskMemory.cpp @@ -10,14 +10,33 @@ namespace DB { namespace ErrorCodes { - extern const int UNKNOWN_ELEMENT_IN_CONFIG; - extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int FILE_DOESNT_EXIST; extern const int FILE_ALREADY_EXISTS; extern const int DIRECTORY_DOESNT_EXIST; extern const int CANNOT_DELETE_DIRECTORY; } + +class DiskMemoryDirectoryIterator : public IDiskDirectoryIterator +{ +public: + explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) + : dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin()) + { + } + + void next() override { ++iter; } + + bool isValid() const override { return iter != dir_file_paths.end(); } + + String path() const override { return *iter; } + +private: + std::vector dir_file_paths; + std::vector::iterator iter; +}; + + ReservationPtr DiskMemory::reserve(UInt64 /*bytes*/) { throw Exception("Method reserve is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED); @@ -73,7 +92,7 @@ size_t DiskMemory::getFileSize(const String & path) const auto iter = files.find(path); if (iter == files.end()) - throw Exception("File " + path + " does not exist", ErrorCodes::FILE_DOESNT_EXIST); + throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST); return iter->second.data.size(); } @@ -88,7 +107,7 @@ void DiskMemory::createDirectory(const String & path) String parent_path = parentPath(path); if (!parent_path.empty() && files.find(parent_path) == files.end()) throw Exception( - "Failed to create directory " + path + ". Parent directory " + parent_path + " does not exist", + "Failed to create directory '" + path + "'. Parent directory " + parent_path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); files.emplace(path, FileData{FileType::Directory}); @@ -118,7 +137,7 @@ void DiskMemory::clearDirectory(const String & path) std::lock_guard lock(mutex); if (files.find(path) == files.end()) - throw Exception("Directory " + path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); + throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); for (auto iter = files.begin(); iter != files.end();) { @@ -130,7 +149,7 @@ void DiskMemory::clearDirectory(const String & path) if (iter->second.type == FileType::Directory) throw Exception( - "Failed to clear directory " + path + ". " + iter->first + " is a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY); + "Failed to clear directory '" + path + "'. " + iter->first + " is a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY); files.erase(iter++); } @@ -146,7 +165,7 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path) std::lock_guard lock(mutex); if (!path.empty() && files.find(path) == files.end()) - throw Exception("Directory " + path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); + throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); std::vector dir_file_paths; for (const auto & file : files) @@ -205,7 +224,7 @@ std::unique_ptr DiskMemory::readFile(const String & path, size_t /*b auto iter = files.find(path); if (iter == files.end()) - throw Exception("File " + path + " does not exist", ErrorCodes::FILE_DOESNT_EXIST); + throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST); return std::make_unique(iter->second.data); } @@ -220,7 +239,7 @@ std::unique_ptr DiskMemory::writeFile(const String & path, size_t / String parent_path = parentPath(path); if (!parent_path.empty() && files.find(parent_path) == files.end()) throw Exception( - "Failed to create file " + path + ". Directory " + parent_path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); + "Failed to create file '" + path + "'. Directory " + parent_path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); iter = files.emplace(path, FileData{FileType::File}).first; } @@ -231,6 +250,46 @@ std::unique_ptr DiskMemory::writeFile(const String & path, size_t / return std::make_unique(iter->second.data); } +void DiskMemory::remove(const String & path) +{ + std::lock_guard lock(mutex); + + auto file_it = files.find(path); + if (file_it == files.end()) + throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); + + if (file_it->second.type == FileType::Directory) + { + files.erase(file_it); + if (std::any_of(files.begin(), files.end(), [path](const auto & file) { return parentPath(file.first) == path; })) + throw Exception("Directory '" + path + "' is not empty", ErrorCodes::CANNOT_DELETE_DIRECTORY); + } + else + { + files.erase(file_it); + } +} + +void DiskMemory::removeRecursive(const String & path) +{ + std::lock_guard lock(mutex); + + auto file_it = files.find(path); + if (file_it == files.end()) + throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); + + for (auto iter = files.begin(); iter != files.end();) + { + if (iter->first.size() >= path.size() && std::string_view(iter->first.data(), path.size()) == path) + iter = files.erase(iter); + else + ++iter; + } +} + + +using DiskMemoryPtr = std::shared_ptr; + void registerDiskMemory(DiskFactory & factory) { diff --git a/dbms/src/Disks/DiskMemory.h b/dbms/src/Disks/DiskMemory.h index 0fc7a5846d0..4ef5b230b1a 100644 --- a/dbms/src/Disks/DiskMemory.h +++ b/dbms/src/Disks/DiskMemory.h @@ -1,19 +1,24 @@ #pragma once #include -#include -#include #include +#include #include namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} +class ReadBuffer; +class WriteBuffer; + + +/** Implementation of Disk intended only for testing purposes. + * All filesystem objects are stored in memory and lost on server restart. + * + * NOTE Work in progress. Currently the interface is not viable enough to support MergeTree or even StripeLog tables. + * Please delete this interface if it will not be finished after 2020-06-18. + */ class DiskMemory : public IDisk { public: @@ -62,6 +67,10 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) override; + void remove(const String & path) override; + + void removeRecursive(const String & path) override; + private: void createDirectoriesImpl(const String & path); void replaceFileImpl(const String & from_path, const String & to_path); @@ -88,30 +97,4 @@ private: mutable std::mutex mutex; }; -using DiskMemoryPtr = std::shared_ptr; - - -class DiskMemoryDirectoryIterator : public IDiskDirectoryIterator -{ -public: - explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) - : dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin()) - { - } - - void next() override { ++iter; } - - bool isValid() const override { return iter != dir_file_paths.end(); } - - String path() const override { return *iter; } - -private: - std::vector dir_file_paths; - std::vector::iterator iter; -}; - - -class DiskFactory; -void registerDiskMemory(DiskFactory & factory); - } diff --git a/dbms/src/Disks/DiskS3.cpp b/dbms/src/Disks/DiskS3.cpp new file mode 100644 index 00000000000..9f20cee15fb --- /dev/null +++ b/dbms/src/Disks/DiskS3.cpp @@ -0,0 +1,439 @@ +#include "DiskS3.h" + +#if USE_AWS_S3 +# include "DiskFactory.h" + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include +# include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int FILE_ALREADY_EXISTS; + extern const int FILE_DOESNT_EXIST; + extern const int PATH_ACCESS_DENIED; +} + +namespace +{ + template + void throwIfError(Aws::Utils::Outcome && response) + { + if (!response.IsSuccess()) + { + const auto & err = response.GetError(); + throw Exception(err.GetMessage(), static_cast(err.GetErrorType())); + } + } + + String readKeyFromFile(const String & path) + { + String key; + ReadBufferFromFile buf(path, 1024); /* reasonable buffer size for small file */ + readStringUntilEOF(key, buf); + return key; + } + + void writeKeyToFile(const String & key, const String & path) + { + WriteBufferFromFile buf(path, 1024); + writeString(key, buf); + buf.next(); + } + + /// Stores data in S3 and the object key in file in local filesystem. + class WriteIndirectBufferFromS3 : public WriteBufferFromS3 + { + public: + WriteIndirectBufferFromS3( + std::shared_ptr & client_ptr_, + const String & bucket_, + const String & metadata_path_, + const String & s3_path_, + size_t buf_size_) + : WriteBufferFromS3(client_ptr_, bucket_, s3_path_, DEFAULT_BLOCK_SIZE, buf_size_) + , metadata_path(metadata_path_) + , s3_path(s3_path_) + { + } + + void finalize() override + { + WriteBufferFromS3::finalize(); + writeKeyToFile(s3_path, metadata_path); + finalized = true; + } + + ~WriteIndirectBufferFromS3() override + { + if (finalized) + return; + + try + { + finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + private: + bool finalized = false; + const String metadata_path; + const String s3_path; + }; +} + + +class DiskS3DirectoryIterator : public IDiskDirectoryIterator +{ +public: + DiskS3DirectoryIterator(const String & full_path, const String & folder_path_) : iter(full_path), folder_path(folder_path_) {} + + void next() override { ++iter; } + + bool isValid() const override { return iter != Poco::DirectoryIterator(); } + + String path() const override + { + if (iter->isDirectory()) + return folder_path + iter.name() + '/'; + else + return folder_path + iter.name(); + } + +private: + Poco::DirectoryIterator iter; + String folder_path; +}; + + +using DiskS3Ptr = std::shared_ptr; + +class DiskS3Reservation : public IReservation +{ +public: + DiskS3Reservation(const DiskS3Ptr & disk_, UInt64 size_) + : disk(disk_), size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size_) + { + } + + UInt64 getSize() const override { return size; } + + DiskPtr getDisk() const override { return disk; } + + void update(UInt64 new_size) override + { + std::lock_guard lock(disk->reservation_mutex); + disk->reserved_bytes -= size; + size = new_size; + disk->reserved_bytes += size; + } + + ~DiskS3Reservation() override; + +private: + DiskS3Ptr disk; + UInt64 size; + CurrentMetrics::Increment metric_increment; +}; + + +DiskS3::DiskS3(String name_, std::shared_ptr client_, String bucket_, String s3_root_path_, String metadata_path_) + : name(std::move(name_)) + , client(std::move(client_)) + , bucket(std::move(bucket_)) + , s3_root_path(std::move(s3_root_path_)) + , metadata_path(std::move(metadata_path_)) +{ +} + +ReservationPtr DiskS3::reserve(UInt64 bytes) +{ + if (!tryReserve(bytes)) + return {}; + return std::make_unique(std::static_pointer_cast(shared_from_this()), bytes); +} + +bool DiskS3::exists(const String & path) const +{ + return Poco::File(metadata_path + path).exists(); +} + +bool DiskS3::isFile(const String & path) const +{ + return Poco::File(metadata_path + path).isFile(); +} + +bool DiskS3::isDirectory(const String & path) const +{ + return Poco::File(metadata_path + path).isDirectory(); +} + +size_t DiskS3::getFileSize(const String & path) const +{ + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(getS3Path(path)); + auto outcome = client->GetObject(request); + if (!outcome.IsSuccess()) + { + auto & err = outcome.GetError(); + throw Exception(err.GetMessage(), static_cast(err.GetErrorType())); + } + else + { + return outcome.GetResult().GetContentLength(); + } +} + +void DiskS3::createDirectory(const String & path) +{ + Poco::File(metadata_path + path).createDirectory(); +} + +void DiskS3::createDirectories(const String & path) +{ + Poco::File(metadata_path + path).createDirectories(); +} + +DiskDirectoryIteratorPtr DiskS3::iterateDirectory(const String & path) +{ + return std::make_unique(metadata_path + path, path); +} + +void DiskS3::clearDirectory(const String & path) +{ + for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) + if (isFile(it->path())) + remove(it->path()); +} + +void DiskS3::moveFile(const String & from_path, const String & to_path) +{ + if (exists(to_path)) + throw Exception("File already exists " + to_path, ErrorCodes::FILE_ALREADY_EXISTS); + Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path); +} + +void DiskS3::replaceFile(const String & from_path, const String & to_path) +{ + Poco::File from_file(metadata_path + from_path); + Poco::File to_file(metadata_path + to_path); + if (to_file.exists()) + { + Poco::File tmp_file(metadata_path + to_path + ".old"); + to_file.renameTo(tmp_file.path()); + from_file.renameTo(metadata_path + to_path); + remove(to_path + ".old"); + } + else + from_file.renameTo(to_file.path()); +} + +void DiskS3::copyFile(const String & from_path, const String & to_path) +{ + if (exists(to_path)) + remove(to_path); + + String s3_from_path = readKeyFromFile(metadata_path + from_path); + String s3_to_path = s3_root_path + getRandomName(); + + Aws::S3::Model::CopyObjectRequest req; + req.SetBucket(bucket); + req.SetCopySource(s3_from_path); + req.SetKey(s3_to_path); + throwIfError(client->CopyObject(req)); + writeKeyToFile(s3_to_path, metadata_path + to_path); +} + +std::unique_ptr DiskS3::readFile(const String & path, size_t buf_size) const +{ + return std::make_unique(client, bucket, getS3Path(path), buf_size); +} + +std::unique_ptr DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode) +{ + if (!exists(path) || mode == WriteMode::Rewrite) + { + String new_s3_path = s3_root_path + getRandomName(); + return std::make_unique(client, bucket, metadata_path + path, new_s3_path, buf_size); + } + else + { + auto old_s3_path = getS3Path(path); + ReadBufferFromS3 read_buffer(client, bucket, old_s3_path, buf_size); + auto writeBuffer = std::make_unique(client, bucket, metadata_path + path, old_s3_path, buf_size); + std::vector buffer(buf_size); + while (!read_buffer.eof()) + writeBuffer->write(buffer.data(), read_buffer.read(buffer.data(), buf_size)); + return writeBuffer; + } +} + +void DiskS3::remove(const String & path) +{ + Poco::File file(metadata_path + path); + if (file.isFile()) + { + Aws::S3::Model::DeleteObjectRequest request; + request.SetBucket(bucket); + request.SetKey(getS3Path(path)); + throwIfError(client->DeleteObject(request)); + } + file.remove(); +} + +void DiskS3::removeRecursive(const String & path) +{ + checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. + + Poco::File file(metadata_path + path); + if (file.isFile()) + { + Aws::S3::Model::DeleteObjectRequest request; + request.SetBucket(bucket); + request.SetKey(getS3Path(path)); + throwIfError(client->DeleteObject(request)); + } + else + { + for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) + removeRecursive(it->path()); + } + file.remove(); +} + +String DiskS3::getS3Path(const String & path) const +{ + if (!exists(path)) + throw Exception("File not found: " + path, ErrorCodes::FILE_DOESNT_EXIST); + + return readKeyFromFile(metadata_path + path); +} + +String DiskS3::getRandomName() const +{ + std::uniform_int_distribution distribution('a', 'z'); + String res(32, ' '); /// The number of bits of entropy should be not less than 128. + for (auto & c : res) + c = distribution(thread_local_rng); + return res; +} + +bool DiskS3::tryReserve(UInt64 bytes) +{ + std::lock_guard lock(reservation_mutex); + if (bytes == 0) + { + LOG_DEBUG(&Logger::get("DiskS3"), "Reserving 0 bytes on s3 disk " << backQuote(name)); + ++reservation_count; + return true; + } + + auto available_space = getAvailableSpace(); + UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes); + if (unreserved_space >= bytes) + { + LOG_DEBUG( + &Logger::get("DiskS3"), + "Reserving " << formatReadableSizeWithBinarySuffix(bytes) << " on disk " << backQuote(name) << ", having unreserved " + << formatReadableSizeWithBinarySuffix(unreserved_space) << "."); + ++reservation_count; + reserved_bytes += bytes; + return true; + } + return false; +} + + +DiskS3Reservation::~DiskS3Reservation() +{ + try + { + std::lock_guard lock(disk->reservation_mutex); + if (disk->reserved_bytes < size) + { + disk->reserved_bytes = 0; + LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '" + disk->getName() + "'."); + } + else + { + disk->reserved_bytes -= size; + } + + if (disk->reservation_count == 0) + LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '" + disk->getName() + "'."); + else + --disk->reservation_count; + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + +void registerDiskS3(DiskFactory & factory) +{ + auto creator = [](const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + const Context & context) -> DiskPtr { + Poco::File disk{context.getPath() + "disks/" + name}; + disk.createDirectories(); + + S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint"))); + auto client = S3::ClientFactory::instance().create( + uri.endpoint, + config.getString(config_prefix + ".access_key_id", ""), + config.getString(config_prefix + ".secret_access_key", "")); + + if (uri.key.back() != '/') + throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::LOGICAL_ERROR); + + String metadata_path = context.getPath() + "disks/" + name + "/"; + + auto s3disk = std::make_shared(name, client, uri.bucket, uri.key, metadata_path); + + /// This code is used only to check access to the corresponding disk. + + { + auto file = s3disk->writeFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + file->write("test", 4); + } + { + auto file = s3disk->readFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE); + String buf(4, '0'); + file->readStrict(buf.data(), 4); + if (buf != "test") + throw Exception("No read accecss to S3 bucket in disk " + name, ErrorCodes::PATH_ACCESS_DENIED); + } + { + s3disk->remove("test_acl"); + } + + return s3disk; + }; + factory.registerDiskType("s3", creator); +} + +} + +#endif diff --git a/dbms/src/Disks/DiskS3.h b/dbms/src/Disks/DiskS3.h new file mode 100644 index 00000000000..c7341fa2183 --- /dev/null +++ b/dbms/src/Disks/DiskS3.h @@ -0,0 +1,93 @@ +#pragma once + +#include + +#if USE_AWS_S3 +# include "DiskFactory.h" + +# include +# include + + +namespace DB +{ +/** + * Storage for persisting data in S3 and metadata on the local disk. + * Files are represented by file in local filesystem (clickhouse_root/disks/disk_name/path/to/file) + * that contains S3 object key with actual data. + */ +class DiskS3 : public IDisk +{ +public: + friend class DiskS3Reservation; + + DiskS3(String name_, std::shared_ptr client_, String bucket_, String s3_root_path_, String metadata_path_); + + const String & getName() const override { return name; } + + const String & getPath() const override { return s3_root_path; } + + ReservationPtr reserve(UInt64 bytes) override; + + UInt64 getTotalSpace() const override { return std::numeric_limits::max(); } + + UInt64 getAvailableSpace() const override { return std::numeric_limits::max(); } + + UInt64 getUnreservedSpace() const override { return std::numeric_limits::max(); } + + UInt64 getKeepingFreeSpace() const override { return 0; } + + bool exists(const String & path) const override; + + bool isFile(const String & path) const override; + + bool isDirectory(const String & path) const override; + + size_t getFileSize(const String & path) const override; + + void createDirectory(const String & path) override; + + void createDirectories(const String & path) override; + + void clearDirectory(const String & path) override; + + void moveDirectory(const String & from_path, const String & to_path) override { moveFile(from_path, to_path); } + + DiskDirectoryIteratorPtr iterateDirectory(const String & path) override; + + void moveFile(const String & from_path, const String & to_path) override; + + void replaceFile(const String & from_path, const String & to_path) override; + + void copyFile(const String & from_path, const String & to_path) override; + + std::unique_ptr readFile(const String & path, size_t buf_size) const override; + + std::unique_ptr writeFile(const String & path, size_t buf_size, WriteMode mode) override; + + void remove(const String & path) override; + + void removeRecursive(const String & path) override; + +private: + String getS3Path(const String & path) const; + + String getRandomName() const; + + bool tryReserve(UInt64 bytes); + +private: + const String name; + std::shared_ptr client; + const String bucket; + const String s3_root_path; + const String metadata_path; + + UInt64 reserved_bytes = 0; + UInt64 reservation_count = 0; + std::mutex reservation_mutex; +}; + +} + +#endif diff --git a/dbms/src/Disks/DiskSpaceMonitor.cpp b/dbms/src/Disks/DiskSpaceMonitor.cpp index 59b8c21119a..6cc6d7e04db 100644 --- a/dbms/src/Disks/DiskSpaceMonitor.cpp +++ b/dbms/src/Disks/DiskSpaceMonitor.cpp @@ -111,6 +111,12 @@ Volume::Volume( << " < " << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")"); } +DiskPtr Volume::getNextDisk() +{ + size_t start_from = last_used.fetch_add(1u, std::memory_order_relaxed); + size_t index = start_from % disks.size(); + return disks[index]; +} ReservationPtr Volume::reserve(UInt64 expected_size) { diff --git a/dbms/src/Disks/DiskSpaceMonitor.h b/dbms/src/Disks/DiskSpaceMonitor.h index 3d2216b545b..cb00944e149 100644 --- a/dbms/src/Disks/DiskSpaceMonitor.h +++ b/dbms/src/Disks/DiskSpaceMonitor.h @@ -67,6 +67,13 @@ public: const String & config_prefix, const DiskSelector & disk_selector); + /// Next disk (round-robin) + /// + /// - Used with policy for temporary data + /// - Ignores all limitations + /// - Shares last access with reserve() + DiskPtr getNextDisk(); + /// Uses Round-robin to choose disk for reservation. /// Returns valid reservation or nullptr if there is no space left on any disk. ReservationPtr reserve(UInt64 bytes) override; diff --git a/dbms/src/Disks/IDisk.cpp b/dbms/src/Disks/IDisk.cpp index 3270fe8483b..48b080e1704 100644 --- a/dbms/src/Disks/IDisk.cpp +++ b/dbms/src/Disks/IDisk.cpp @@ -2,6 +2,7 @@ namespace DB { + bool IDisk::isDirectoryEmpty(const String & path) { return !iterateDirectory(path)->isValid(); diff --git a/dbms/src/Disks/IDisk.h b/dbms/src/Disks/IDisk.h index c1cda47b6eb..6ac2745225a 100644 --- a/dbms/src/Disks/IDisk.h +++ b/dbms/src/Disks/IDisk.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -97,7 +98,7 @@ public: /// Create directory and all parent directories if necessary. virtual void createDirectories(const String & path) = 0; - /// Remove all files from the directory. + /// Remove all files from the directory. Directories are not removed. virtual void clearDirectory(const String & path) = 0; /// Move directory from `from_path` to `to_path`. @@ -125,6 +126,12 @@ public: /// Open the file for write and return WriteBuffer object. virtual std::unique_ptr writeFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) = 0; + + /// Remove file or directory. Throws exception if file doesn't exists or if directory is not empty. + virtual void remove(const String & path) = 0; + + /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. + virtual void removeRecursive(const String & path) = 0; }; using DiskPtr = std::shared_ptr; @@ -151,7 +158,7 @@ public: /** * Information about reserved size on particular disk. */ -class IReservation +class IReservation : boost::noncopyable { public: /// Get reservation size. diff --git a/dbms/src/Disks/registerDisks.cpp b/dbms/src/Disks/registerDisks.cpp index f95bc557d9a..f28918cdbd0 100644 --- a/dbms/src/Disks/registerDisks.cpp +++ b/dbms/src/Disks/registerDisks.cpp @@ -1,10 +1,16 @@ -#include "DiskFactory.h" #include "registerDisks.h" +#include "DiskFactory.h" + +#include + namespace DB { void registerDiskLocal(DiskFactory & factory); void registerDiskMemory(DiskFactory & factory); +#if USE_AWS_S3 +void registerDiskS3(DiskFactory & factory); +#endif void registerDisks() { @@ -12,6 +18,9 @@ void registerDisks() registerDiskLocal(factory); registerDiskMemory(factory); +#if USE_AWS_S3 + registerDiskS3(factory); +#endif } } diff --git a/dbms/src/Disks/registerDisks.h b/dbms/src/Disks/registerDisks.h index 0ffd0ad8972..8c68cc52bde 100644 --- a/dbms/src/Disks/registerDisks.h +++ b/dbms/src/Disks/registerDisks.h @@ -3,5 +3,4 @@ namespace DB { void registerDisks(); - } diff --git a/dbms/src/Disks/tests/gtest_disk.cpp b/dbms/src/Disks/tests/gtest_disk.cpp index 8f0ba520d77..e694242ad5d 100644 --- a/dbms/src/Disks/tests/gtest_disk.cpp +++ b/dbms/src/Disks/tests/gtest_disk.cpp @@ -5,6 +5,10 @@ #include #include +#if !__clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif template DB::DiskPtr createDisk(); diff --git a/dbms/src/Formats/FormatFactory.cpp b/dbms/src/Formats/FormatFactory.cpp index 240e591123f..f812b56aa5d 100644 --- a/dbms/src/Formats/FormatFactory.cpp +++ b/dbms/src/Formats/FormatFactory.cpp @@ -68,6 +68,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; + format_settings.avro.schema_registry_url = settings.input_format_avro_schema_registry_url; return format_settings; } @@ -99,6 +100,8 @@ static FormatSettings getOutputFormatSetting(const Settings & settings, const Co format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; + format_settings.avro.output_codec = settings.output_format_avro_codec; + format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; return format_settings; } @@ -325,6 +328,8 @@ FormatFactory::FormatFactory() registerInputFormatProcessorORC(*this); registerInputFormatProcessorParquet(*this); registerOutputFormatProcessorParquet(*this); + registerInputFormatProcessorAvro(*this); + registerOutputFormatProcessorAvro(*this); registerInputFormatProcessorTemplate(*this); registerOutputFormatProcessorTemplate(*this); diff --git a/dbms/src/Formats/FormatFactory.h b/dbms/src/Formats/FormatFactory.h index cbf64afeaec..345ceaee690 100644 --- a/dbms/src/Formats/FormatFactory.h +++ b/dbms/src/Formats/FormatFactory.h @@ -166,6 +166,8 @@ void registerInputFormatProcessorORC(FormatFactory & factory); void registerOutputFormatProcessorParquet(FormatFactory & factory); void registerInputFormatProcessorProtobuf(FormatFactory & factory); void registerOutputFormatProcessorProtobuf(FormatFactory & factory); +void registerInputFormatProcessorAvro(FormatFactory & factory); +void registerOutputFormatProcessorAvro(FormatFactory & factory); void registerInputFormatProcessorTemplate(FormatFactory & factory); void registerOutputFormatProcessorTemplate(FormatFactory &factory); diff --git a/dbms/src/Formats/FormatSettings.h b/dbms/src/Formats/FormatSettings.h index 6219edf6e6d..cc6f7f4dbb3 100644 --- a/dbms/src/Formats/FormatSettings.h +++ b/dbms/src/Formats/FormatSettings.h @@ -110,6 +110,16 @@ struct FormatSettings }; Custom custom; + + struct Avro + { + String schema_registry_url; + String output_codec; + UInt64 output_sync_interval = 16 * 1024; + }; + + Avro avro; + }; } diff --git a/dbms/src/Formats/ParsedTemplateFormatString.cpp b/dbms/src/Formats/ParsedTemplateFormatString.cpp index 981d43089a2..af6fc39c8fd 100644 --- a/dbms/src/Formats/ParsedTemplateFormatString.cpp +++ b/dbms/src/Formats/ParsedTemplateFormatString.cpp @@ -16,11 +16,11 @@ namespace ErrorCodes ParsedTemplateFormatString::ParsedTemplateFormatString(const FormatSchemaInfo & schema, const ColumnIdxGetter & idx_by_name) { + ReadBufferFromFile schema_file(schema.absoluteSchemaPath(), 4096); + String format_string; + readStringUntilEOF(format_string, schema_file); try { - ReadBufferFromFile schema_file(schema.absoluteSchemaPath(), 4096); - String format_string; - readStringUntilEOF(format_string, schema_file); parse(format_string, idx_by_name); } catch (DB::Exception & e) @@ -193,7 +193,7 @@ const char * ParsedTemplateFormatString::readMayBeQuotedColumnNameInto(const cha String ParsedTemplateFormatString::dump() const { WriteBufferFromOwnString res; - res << "Delimiter " << 0 << ": "; + res << "\nDelimiter " << 0 << ": "; verbosePrintString(delimiters.front().c_str(), delimiters.front().c_str() + delimiters.front().size(), res); size_t num_columns = std::max(formats.size(), format_idx_to_column_idx.size()); diff --git a/dbms/src/Formats/config_formats.h.in b/dbms/src/Formats/config_formats.h.in index 1ddd0e18aa9..308ded92b5d 100644 --- a/dbms/src/Formats/config_formats.h.in +++ b/dbms/src/Formats/config_formats.h.in @@ -2,6 +2,7 @@ // .h autogenerated by cmake! +#cmakedefine01 USE_AVRO #cmakedefine01 USE_CAPNP #cmakedefine01 USE_SNAPPY #cmakedefine01 USE_PARQUET diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index 9eed1061349..f2a27974cbb 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -32,7 +32,6 @@ if (OPENSSL_CRYPTO_LIBRARY) target_link_libraries(clickhouse_functions PUBLIC ${OPENSSL_CRYPTO_LIBRARY}) endif() -target_include_directories(clickhouse_functions PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) target_include_directories(clickhouse_functions SYSTEM PRIVATE ${DIVIDE_INCLUDE_DIR} ${METROHASH_INCLUDE_DIR} ${SPARSEHASH_INCLUDE_DIR}) if (CONSISTENT_HASHING_INCLUDE_DIR) diff --git a/dbms/src/Functions/FunctionsLogical.cpp b/dbms/src/Functions/FunctionsLogical.cpp index 75c602df088..b5c64ce9602 100644 --- a/dbms/src/Functions/FunctionsLogical.cpp +++ b/dbms/src/Functions/FunctionsLogical.cpp @@ -282,29 +282,32 @@ template < typename Op, template typename OperationApplierImpl, size_t N = 10> struct OperationApplier { - template - static void apply(Columns & in, ResultColumn & result) + template + static void apply(Columns & in, ResultData & result_data, bool use_result_data_as_input = false) { - while (in.size() > 1) - { - doBatchedApply(in, result->getData()); - in.push_back(result.get()); - } + if (!use_result_data_as_input) + doBatchedApply(in, result_data); + while (in.size() > 0) + doBatchedApply(in, result_data); } - template + template static void NO_INLINE doBatchedApply(Columns & in, ResultData & result_data) { if (N > in.size()) { - OperationApplier::doBatchedApply(in, result_data); + OperationApplier + ::template doBatchedApply(in, result_data); return; } const OperationApplierImpl operationApplierImpl(in); size_t i = 0; for (auto & res : result_data) - res = operationApplierImpl.apply(i++); + if constexpr (CarryResult) + res = Op::apply(res, operationApplierImpl.apply(i++)); + else + res = operationApplierImpl.apply(i++); in.erase(in.end() - N, in.end()); } @@ -312,9 +315,9 @@ struct OperationApplier template < typename Op, template typename OperationApplierImpl> -struct OperationApplier +struct OperationApplier { - template + template static void NO_INLINE doBatchedApply(Columns &, Result &) { throw Exception( @@ -332,7 +335,7 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn const bool has_consts = extractConstColumnsTernary(arguments, const_3v_value); /// If the constant value uniquely determines the result, return it. - if (has_consts && (arguments.empty() || (Op::isSaturable() && Op::isSaturatedValue(const_3v_value)))) + if (has_consts && (arguments.empty() || Op::isSaturatedValue(const_3v_value))) { result_info.column = ColumnConst::create( convertFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()), @@ -341,16 +344,10 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn return; } - const auto result_column = ColumnUInt8::create(input_rows_count); - MutableColumnPtr const_column_holder; - if (has_consts) - { - const_column_holder = - convertFromTernaryData(UInt8Container(input_rows_count, const_3v_value), const_3v_value == Ternary::Null); - arguments.push_back(const_column_holder.get()); - } + const auto result_column = has_consts ? + ColumnUInt8::create(input_rows_count, const_3v_value) : ColumnUInt8::create(input_rows_count); - OperationApplier::apply(arguments, result_column); + OperationApplier::apply(arguments, result_column->getData(), has_consts); result_info.column = convertFromTernaryData(result_column->getData(), result_info.type->isNullable()); } @@ -425,19 +422,8 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re if (has_consts && Op::apply(const_val, 0) == 0 && Op::apply(const_val, 1) == 1) has_consts = false; - UInt8ColumnPtrs uint8_args; - - auto col_res = ColumnUInt8::create(); - UInt8Container & vec_res = col_res->getData(); - if (has_consts) - { - vec_res.assign(input_rows_count, const_val); - uint8_args.push_back(col_res.get()); - } - else - { - vec_res.resize(input_rows_count); - } + auto col_res = has_consts ? + ColumnUInt8::create(input_rows_count, const_val) : ColumnUInt8::create(input_rows_count); /// FastPath detection goes in here if (arguments.size() == (has_consts ? 1 : 2)) @@ -452,7 +438,8 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re } /// Convert all columns to UInt8 - Columns converted_columns; + UInt8ColumnPtrs uint8_args; + Columns converted_columns_holder; for (const IColumn * column : arguments) { if (auto uint8_column = checkAndGetColumn(column)) @@ -462,15 +449,11 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re auto converted_column = ColumnUInt8::create(input_rows_count); convertColumnToUInt8(column, converted_column->getData()); uint8_args.push_back(converted_column.get()); - converted_columns.emplace_back(std::move(converted_column)); + converted_columns_holder.emplace_back(std::move(converted_column)); } } - OperationApplier::apply(uint8_args, col_res); - - /// This is possible if there is exactly one non-constant among the arguments, and it is of type UInt8. - if (uint8_args[0] != col_res.get()) - vec_res.assign(uint8_args[0]->getData()); + OperationApplier::apply(uint8_args, col_res->getData(), has_consts); result_info.column = std::move(col_res); } diff --git a/dbms/src/Functions/FunctionsLogical.h b/dbms/src/Functions/FunctionsLogical.h index e671c7184d8..814c9319f52 100644 --- a/dbms/src/Functions/FunctionsLogical.h +++ b/dbms/src/Functions/FunctionsLogical.h @@ -83,12 +83,7 @@ struct XorImpl static inline constexpr bool isSaturable() { return false; } static inline constexpr bool isSaturatedValue(bool) { return false; } - /** Considering that CH uses UInt8 for representation of boolean values this function - * returns 255 as "true" but the current implementation of logical functions suggests that - * any nonzero value is "true" as well. Also the current code provides no guarantee - * for "true" to be represented with the value of 1. - */ - static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return (a != b) ? Ternary::True : Ternary::False; } + static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return !!a != !!b; } static inline constexpr bool specialImplementationForNulls() { return false; } #if USE_EMBEDDED_COMPILER diff --git a/dbms/src/Functions/GatherUtils/Sources.h b/dbms/src/Functions/GatherUtils/Sources.h index 7c881bba0c5..39cd4ac49aa 100644 --- a/dbms/src/Functions/GatherUtils/Sources.h +++ b/dbms/src/Functions/GatherUtils/Sources.h @@ -121,6 +121,11 @@ struct NumericArraySource : public ArraySourceImpl> } }; +#if !__clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif + template struct ConstSource : public Base { @@ -199,6 +204,10 @@ struct ConstSource : public Base } }; +#if !__clang__ +#pragma GCC diagnostic pop +#endif + struct StringSource { using Slice = NumericArraySlice; diff --git a/dbms/src/Functions/URL/CMakeLists.txt b/dbms/src/Functions/URL/CMakeLists.txt index 92bdbd8fcab..fabfccae230 100644 --- a/dbms/src/Functions/URL/CMakeLists.txt +++ b/dbms/src/Functions/URL/CMakeLists.txt @@ -2,7 +2,6 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_functions_url .) add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers}) target_link_libraries(clickhouse_functions_url PRIVATE dbms) -target_include_directories(clickhouse_functions_url PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) # ${CMAKE_CURRENT_BINARY_DIR}/include if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. diff --git a/dbms/src/Functions/array/arrayZip.cpp b/dbms/src/Functions/array/arrayZip.cpp index 20fca29bae8..b191a055468 100644 --- a/dbms/src/Functions/array/arrayZip.cpp +++ b/dbms/src/Functions/array/arrayZip.cpp @@ -14,6 +14,7 @@ namespace ErrorCodes { extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_COLUMN; } /// arrayZip(['a', 'b', 'c'], ['d', 'e', 'f']) = [('a', 'd'), ('b', 'e'), ('c', 'f')] @@ -44,9 +45,8 @@ public: const DataTypeArray * array_type = checkAndGetDataType(arguments[index].type.get()); if (!array_type) - throw Exception( - "Argument " + toString(index + 1) + " of function must be array. Found " + arguments[0].type->getName() + " instead.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Argument " + toString(index + 1) + " of function " + getName() + + " must be array. Found " + arguments[0].type->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); arguments_types.emplace_back(array_type->getNestedType()); } @@ -56,26 +56,37 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override { - auto first_argument = block.getByPosition(arguments[0]); - const auto & first_array_column = checkAndGetColumn(first_argument.column.get()); + size_t num_arguments = arguments.size(); - Columns res_tuple_columns(arguments.size()); - res_tuple_columns[0] = first_array_column->getDataPtr(); + ColumnPtr first_array_column; + Columns tuple_columns(num_arguments); - for (size_t index = 1; index < arguments.size(); ++index) + for (size_t i = 0; i < num_arguments; ++i) { - const auto & argument_type_and_column = block.getByPosition(arguments[index]); - const auto & argument_array_column = checkAndGetColumn(argument_type_and_column.column.get()); + /// Constant columns cannot be inside tuple. It's only possible to have constant tuple as a whole. + ColumnPtr holder = block.getByPosition(arguments[i]).column->convertToFullColumnIfConst(); - if (!first_array_column->hasEqualOffsets(*argument_array_column)) - throw Exception("The argument 1 and argument " + toString(index + 1) + " of function have different array sizes", - ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + const ColumnArray * column_array = checkAndGetColumn(holder.get()); - res_tuple_columns[index] = argument_array_column->getDataPtr(); + if (!column_array) + throw Exception("Argument " + toString(i + 1) + " of function " + getName() + " must be array." + " Found column " + holder->getName() + " instead.", ErrorCodes::ILLEGAL_COLUMN); + + if (i == 0) + { + first_array_column = holder; + } + else if (!column_array->hasEqualOffsets(static_cast(*first_array_column))) + { + throw Exception("The argument 1 and argument " + toString(i + 1) + " of function " + getName() + " have different array sizes", + ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + } + + tuple_columns[i] = column_array->getDataPtr(); } block.getByPosition(result).column = ColumnArray::create( - ColumnTuple::create(res_tuple_columns), first_array_column->getOffsetsPtr()); + ColumnTuple::create(tuple_columns), static_cast(*first_array_column).getOffsetsPtr()); } }; diff --git a/dbms/src/Functions/bitCount.cpp b/dbms/src/Functions/bitCount.cpp new file mode 100644 index 00000000000..78bca5c4f1b --- /dev/null +++ b/dbms/src/Functions/bitCount.cpp @@ -0,0 +1,54 @@ +#include +#include +#include + + +namespace DB +{ + +template +struct BitCountImpl +{ + using ResultType = UInt8; + + static inline ResultType apply(A a) + { + /// We count bits in the value representation in memory. For example, we support floats. + /// We need to avoid sign-extension when converting signed numbers to larger type. So, uint8_t(-1) has 8 bits. + + if constexpr (std::is_same_v || std::is_same_v) + return __builtin_popcountll(a); + if constexpr (std::is_same_v || std::is_same_v || std::is_unsigned_v) + return __builtin_popcount(a); + if constexpr (std::is_same_v) + return __builtin_popcount(static_cast(a)); + if constexpr (std::is_same_v) + return __builtin_popcount(static_cast(a)); + else + return __builtin_popcountll(ext::bit_cast(a)); + } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; +#endif +}; + +struct NameBitCount { static constexpr auto name = "bitCount"; }; +using FunctionBitCount = FunctionUnaryArithmetic; + +/// The function has no ranges of monotonicity. +template <> struct FunctionUnaryArithmeticMonotonicity +{ + static bool has() { return false; } + static IFunction::Monotonicity get(const Field &, const Field &) + { + return {}; + } +}; + +void registerFunctionBitCount(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/ifNotFinite.cpp b/dbms/src/Functions/ifNotFinite.cpp new file mode 100644 index 00000000000..605eeddf515 --- /dev/null +++ b/dbms/src/Functions/ifNotFinite.cpp @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/// ifNotFinite(x, y) is equivalent to isFinite(x) ? x : y. +class FunctionIfNotFinite : public IFunction +{ +public: + static constexpr auto name = "ifNotFinite"; + + FunctionIfNotFinite(const Context & context_) : context(context_) {} + + static FunctionPtr create(const Context & context) + { + return std::make_shared(context); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 2; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + auto is_finite_type = FunctionFactory::instance().get("isFinite", context)->build({arguments[0]})->getReturnType(); + auto if_type = FunctionFactory::instance().get("if", context)->build({{nullptr, is_finite_type, ""}, arguments[0], arguments[1]})->getReturnType(); + return if_type; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + Block temp_block = block; + + auto is_finite = FunctionFactory::instance().get("isFinite", context)->build( + {temp_block.getByPosition(arguments[0])}); + + size_t is_finite_pos = temp_block.columns(); + temp_block.insert({nullptr, is_finite->getReturnType(), ""}); + + auto func_if = FunctionFactory::instance().get("if", context)->build( + {temp_block.getByPosition(is_finite_pos), temp_block.getByPosition(arguments[0]), temp_block.getByPosition(arguments[1])}); + + is_finite->execute(temp_block, {arguments[0]}, is_finite_pos, input_rows_count); + + func_if->execute(temp_block, {is_finite_pos, arguments[0], arguments[1]}, result, input_rows_count); + + block.getByPosition(result).column = std::move(temp_block.getByPosition(result).column); + } + +private: + const Context & context; +}; + + +void registerFunctionIfNotFinite(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/dbms/src/Functions/registerFunctionsArithmetic.cpp b/dbms/src/Functions/registerFunctionsArithmetic.cpp index eb68fc32fa1..88350b4fac7 100644 --- a/dbms/src/Functions/registerFunctionsArithmetic.cpp +++ b/dbms/src/Functions/registerFunctionsArithmetic.cpp @@ -20,6 +20,7 @@ void registerFunctionBitShiftLeft(FunctionFactory & factory); void registerFunctionBitShiftRight(FunctionFactory & factory); void registerFunctionBitRotateLeft(FunctionFactory & factory); void registerFunctionBitRotateRight(FunctionFactory & factory); +void registerFunctionBitCount(FunctionFactory & factory); void registerFunctionLeast(FunctionFactory & factory); void registerFunctionGreatest(FunctionFactory & factory); void registerFunctionBitTest(FunctionFactory & factory); @@ -58,6 +59,7 @@ void registerFunctionsArithmetic(FunctionFactory & factory) registerFunctionBitShiftRight(factory); registerFunctionBitRotateLeft(factory); registerFunctionBitRotateRight(factory); + registerFunctionBitCount(factory); registerFunctionLeast(factory); registerFunctionGreatest(factory); registerFunctionBitTest(factory); diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index ef93d4554d5..23d95f8d5b1 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -36,6 +36,7 @@ void registerFunctionHasColumnInTable(FunctionFactory &); void registerFunctionIsFinite(FunctionFactory &); void registerFunctionIsInfinite(FunctionFactory &); void registerFunctionIsNaN(FunctionFactory &); +void registerFunctionIfNotFinite(FunctionFactory &); void registerFunctionThrowIf(FunctionFactory &); void registerFunctionVersion(FunctionFactory &); void registerFunctionUptime(FunctionFactory &); @@ -93,6 +94,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionIsFinite(factory); registerFunctionIsInfinite(factory); registerFunctionIsNaN(factory); + registerFunctionIfNotFinite(factory); registerFunctionThrowIf(factory); registerFunctionVersion(factory); registerFunctionUptime(factory); diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index fc8e444330c..4bf33ac93ba 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -746,6 +746,23 @@ inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf) inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); } +template +inline std::enable_if_t && (sizeof(T) <= 8), void> +readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian architecture. +{ + readPODBinary(x, buf); + + if constexpr (sizeof(x) == 1) + return; + else if constexpr (sizeof(x) == 2) + x = __builtin_bswap16(x); + else if constexpr (sizeof(x) == 4) + x = __builtin_bswap32(x); + else if constexpr (sizeof(x) == 8) + x = __builtin_bswap64(x); +} + + /// Generic methods to read value in text tab-separated format. template inline std::enable_if_t, void> @@ -955,28 +972,78 @@ inline T parse(const char * data, size_t size) return res; } -/// Read something from text format, but expect complete parse of given text -/// For example: 723145 -- ok, 213MB -- not ok template -inline T completeParse(const char * data, size_t size) +inline std::enable_if_t, void> +readTextWithSizeSuffix(T & x, ReadBuffer & buf) { readText(x, buf); } + +template +inline std::enable_if_t, void> +readTextWithSizeSuffix(T & x, ReadBuffer & buf) +{ + readIntText(x, buf); + if (buf.eof()) + return; + + /// Updates x depending on the suffix + auto finish = [&buf, &x] (UInt64 base, int power_of_two) mutable + { + ++buf.position(); + if (buf.eof()) + { + x *= base; /// For decimal suffixes, such as k, M, G etc. + } + else if (*buf.position() == 'i') + { + x = (x << power_of_two); /// For binary suffixes, such as ki, Mi, Gi, etc. + ++buf.position(); + } + return; + }; + + switch (*buf.position()) + { + case 'k': [[fallthrough]]; + case 'K': + finish(1000, 10); + break; + case 'M': + finish(1000000, 20); + break; + case 'G': + finish(1000000000, 30); + break; + case 'T': + finish(1000000000000ULL, 40); + break; + default: + return; + } + return; +} + +/// Read something from text format and trying to parse the suffix. +/// If the suffix is not valid gives an error +/// For example: 723145 -- ok, 213MB -- not ok, but 213Mi -- ok +template +inline T parseWithSizeSuffix(const char * data, size_t size) { T res; ReadBufferFromMemory buf(data, size); - readText(res, buf); + readTextWithSizeSuffix(res, buf); assertEOF(buf); return res; } template -inline T completeParse(const String & s) +inline T parseWithSizeSuffix(const String & s) { - return completeParse(s.data(), s.size()); + return parseWithSizeSuffix(s.data(), s.size()); } template -inline T completeParse(const char * data) +inline T parseWithSizeSuffix(const char * data) { - return completeParse(data, strlen(data)); + return parseWithSizeSuffix(data, strlen(data)); } template diff --git a/dbms/src/IO/S3Common.cpp b/dbms/src/IO/S3Common.cpp index b981c34c2d2..f41d5c3b253 100644 --- a/dbms/src/IO/S3Common.cpp +++ b/dbms/src/IO/S3Common.cpp @@ -93,11 +93,10 @@ namespace S3 if (!endpoint.empty()) cfg.endpointOverride = endpoint; - auto cred_provider = std::make_shared(access_key_id, - secret_access_key); + Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); return std::make_shared( - std::move(cred_provider), // Credentials provider. + credentials, // Aws credentials. std::move(cfg), // Client configuration. Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, // Sign policy. endpoint.empty() // Use virtual addressing only if endpoint is not specified. @@ -105,7 +104,7 @@ namespace S3 } - URI::URI(Poco::URI & uri_) + URI::URI(const Poco::URI & uri_) { static const std::regex BUCKET_KEY_PATTERN("([^/]+)/(.*)"); diff --git a/dbms/src/IO/S3Common.h b/dbms/src/IO/S3Common.h index d05a2d6e977..c9db8d9ea1d 100644 --- a/dbms/src/IO/S3Common.h +++ b/dbms/src/IO/S3Common.h @@ -49,7 +49,7 @@ struct URI String bucket; String key; - explicit URI (Poco::URI & uri_); + explicit URI(const Poco::URI & uri_); }; } diff --git a/dbms/src/IO/WriteBufferFromHTTP.h b/dbms/src/IO/WriteBufferFromHTTP.h index c68b8f88d3e..a01233d8717 100644 --- a/dbms/src/IO/WriteBufferFromHTTP.h +++ b/dbms/src/IO/WriteBufferFromHTTP.h @@ -29,7 +29,7 @@ public: size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE); /// Receives response from the server after sending all data. - void finalize(); + void finalize() override; }; } diff --git a/dbms/src/IO/WriteBufferFromS3.cpp b/dbms/src/IO/WriteBufferFromS3.cpp index a5932c20ada..ee18ad719d4 100644 --- a/dbms/src/IO/WriteBufferFromS3.cpp +++ b/dbms/src/IO/WriteBufferFromS3.cpp @@ -69,6 +69,7 @@ void WriteBufferFromS3::nextImpl() void WriteBufferFromS3::finalize() { + next(); temporary_buffer->finalize(); if (!buffer_string.empty()) { diff --git a/dbms/src/IO/readDecimalText.h b/dbms/src/IO/readDecimalText.h index 4dad5fdb557..5b2a3f76481 100644 --- a/dbms/src/IO/readDecimalText.h +++ b/dbms/src/IO/readDecimalText.h @@ -101,7 +101,13 @@ inline bool readDigits(ReadBuffer & buf, T & x, unsigned int & digits, int & exp { ++buf.position(); Int32 addition_exp = 0; - readIntText(addition_exp, buf); + if (!tryReadIntText(addition_exp, buf)) + { + if constexpr (_throw_on_error) + throw Exception("Cannot parse exponent while reading decimal", ErrorCodes::CANNOT_PARSE_NUMBER); + else + return false; + } exponent += addition_exp; stop = true; continue; diff --git a/dbms/src/IO/tests/ryu_test.cpp b/dbms/src/IO/tests/ryu_test.cpp index 0866f6afd3d..d8c385f2d0b 100644 --- a/dbms/src/IO/tests/ryu_test.cpp +++ b/dbms/src/IO/tests/ryu_test.cpp @@ -1,5 +1,6 @@ #include #include +#include #include diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 8b2bff5414e..8118f2d1c54 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace ProfileEvents @@ -681,22 +682,25 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData && current_memory_usage > static_cast(params.max_bytes_before_external_group_by) && worth_convert_to_two_level) { - if (!enoughSpaceInDirectory(params.tmp_path, current_memory_usage + params.min_free_disk_space)) - throw Exception("Not enough space for external aggregation in " + params.tmp_path, ErrorCodes::NOT_ENOUGH_SPACE); + size_t size = current_memory_usage + params.min_free_disk_space; + auto reservation = params.tmp_volume->reserve(size); + if (!reservation) + throw Exception("Not enough space for external aggregation in temporary storage", ErrorCodes::NOT_ENOUGH_SPACE); - writeToTemporaryFile(result); + const std::string tmp_path(reservation->getDisk()->getPath()); + writeToTemporaryFile(result, tmp_path); } return true; } -void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) +void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, const String & tmp_path) { Stopwatch watch; size_t rows = data_variants.size(); - auto file = createTemporaryFile(params.tmp_path); + auto file = createTemporaryFile(tmp_path); const std::string & path = file->path(); WriteBufferFromFile file_buf(path); CompressedWriteBuffer compressed_buf(file_buf); @@ -753,6 +757,10 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) << (uncompressed_bytes / elapsed_seconds / 1048576.0) << " MiB/sec. uncompressed, " << (compressed_bytes / elapsed_seconds / 1048576.0) << " MiB/sec. compressed)"); } +void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) +{ + return writeToTemporaryFile(data_variants, params.tmp_volume->getNextDisk()->getPath()); +} template @@ -781,7 +789,8 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( ManyAggregatedDataVariants & variants, Arena * arena, bool final, - size_t bucket) const + size_t bucket, + std::atomic * is_cancelled) const { auto & merged_data = *variants[0]; auto method = merged_data.type; @@ -792,6 +801,8 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( else if (method == AggregatedDataVariants::Type::NAME) \ { \ mergeBucketImpl(variants, bucket, arena); \ + if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) \ + return {}; \ block = convertOneBucketToBlock(merged_data, *merged_data.NAME, final, bucket); \ } @@ -1482,12 +1493,15 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl( template void NO_INLINE Aggregator::mergeBucketImpl( - ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena) const + ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic * is_cancelled) const { /// We merge all aggregation results to the first. AggregatedDataVariantsPtr & res = data[0]; for (size_t result_num = 1, size = data.size(); result_num < size; ++result_num) { + if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + return; + AggregatedDataVariants & current = *data[result_num]; mergeDataImpl( diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 889b04fc700..cdb1b96f4e8 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -46,6 +46,8 @@ namespace ErrorCodes class IBlockOutputStream; +class Volume; +using VolumePtr = std::shared_ptr; /** Different data structures that can be used for aggregation * For efficiency, the aggregation data itself is put into the pool. @@ -860,7 +862,7 @@ public: /// Return empty result when aggregating without keys on empty set. bool empty_result_for_aggregation_by_empty_set; - const std::string tmp_path; + VolumePtr tmp_volume; /// Settings is used to determine cache size. No threads are created. size_t max_threads; @@ -873,7 +875,7 @@ public: size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, size_t max_bytes_before_external_group_by_, bool empty_result_for_aggregation_by_empty_set_, - const std::string & tmp_path_, size_t max_threads_, + VolumePtr tmp_volume_, size_t max_threads_, size_t min_free_disk_space_) : src_header(src_header_), keys(keys_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()), @@ -881,7 +883,7 @@ public: group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_), max_bytes_before_external_group_by(max_bytes_before_external_group_by_), empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_), - tmp_path(tmp_path_), max_threads(max_threads_), + tmp_volume(tmp_volume_), max_threads(max_threads_), min_free_disk_space(min_free_disk_space_) { } @@ -889,7 +891,7 @@ public: /// Only parameters that matter during merge. Params(const Block & intermediate_header_, const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) - : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, "", max_threads_, 0) + : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0) { intermediate_header = intermediate_header_; } @@ -955,6 +957,7 @@ public: void setCancellationHook(const CancellationHook cancellation_hook); /// For external aggregation. + void writeToTemporaryFile(AggregatedDataVariants & data_variants, const String & tmp_path); void writeToTemporaryFile(AggregatedDataVariants & data_variants); bool hasTemporaryFiles() const { return !temporary_files.empty(); } @@ -1170,7 +1173,8 @@ protected: ManyAggregatedDataVariants & variants, Arena * arena, bool final, - size_t bucket) const; + size_t bucket, + std::atomic * is_cancelled = nullptr) const; Block prepareBlockAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const; Block prepareBlockAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const; @@ -1206,7 +1210,7 @@ protected: template void mergeBucketImpl( - ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena) const; + ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic * is_cancelled = nullptr) const; template void convertBlockToTwoLevelImpl( diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index 5e4bf1fe53b..c6c89b565f1 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -19,14 +19,15 @@ namespace ErrorCodes extern const int PARAMETER_OUT_OF_BOUND; } -AnalyzedJoin::AnalyzedJoin(const Settings & settings, const String & tmp_path_) +AnalyzedJoin::AnalyzedJoin(const Settings & settings, VolumePtr tmp_volume_) : size_limits(SizeLimits{settings.max_rows_in_join, settings.max_bytes_in_join, settings.join_overflow_mode}) , default_max_bytes(settings.default_max_bytes_in_join) , join_use_nulls(settings.join_use_nulls) + , max_joined_block_rows(settings.max_joined_block_size_rows) , partial_merge_join(settings.partial_merge_join) , partial_merge_join_optimizations(settings.partial_merge_join_optimizations) , partial_merge_join_rows_in_right_blocks(settings.partial_merge_join_rows_in_right_blocks) - , tmp_path(tmp_path_) + , tmp_volume(tmp_volume_) {} void AnalyzedJoin::addUsingKey(const ASTPtr & ast) diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index 677662d949c..4a83fe77fc9 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -21,6 +21,9 @@ class Block; struct Settings; +class Volume; +using VolumePtr = std::shared_ptr; + class AnalyzedJoin { /** Query of the form `SELECT expr(x) AS k FROM t1 ANY LEFT JOIN (SELECT expr(x) AS k FROM t2) USING k` @@ -40,6 +43,7 @@ class AnalyzedJoin const SizeLimits size_limits; const size_t default_max_bytes; const bool join_use_nulls; + const size_t max_joined_block_rows = 0; const bool partial_merge_join = false; const bool partial_merge_join_optimizations = false; const size_t partial_merge_join_rows_in_right_blocks = 0; @@ -61,10 +65,10 @@ class AnalyzedJoin /// Original name -> name. Only ranamed columns. std::unordered_map renames; - String tmp_path; + VolumePtr tmp_volume; public: - AnalyzedJoin(const Settings &, const String & tmp_path); + AnalyzedJoin(const Settings &, VolumePtr tmp_volume); /// for StorageJoin AnalyzedJoin(SizeLimits limits, bool use_nulls, ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness, @@ -81,11 +85,12 @@ public: ASTTableJoin::Kind kind() const { return table_join.kind; } ASTTableJoin::Strictness strictness() const { return table_join.strictness; } const SizeLimits & sizeLimits() const { return size_limits; } - const String & getTemporaryPath() const { return tmp_path; } + VolumePtr getTemporaryVolume() { return tmp_volume; } bool forceNullableRight() const { return join_use_nulls && isLeftOrFull(table_join.kind); } bool forceNullableLeft() const { return join_use_nulls && isRightOrFull(table_join.kind); } size_t defaultMaxBytes() const { return default_max_bytes; } + size_t maxJoinedBlockRows() const { return max_joined_block_rows; } size_t maxRowsInRightBlock() const { return partial_merge_join_rows_in_right_blocks; } bool enablePartialMergeJoinOptimizations() const { return partial_merge_join_optimizations; } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 724cdd10f69..04d01a24cc1 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +96,7 @@ namespace ErrorCodes extern const int SCALAR_ALREADY_EXISTS; extern const int UNKNOWN_SCALAR; extern const int NOT_ENOUGH_PRIVILEGES; + extern const int UNKNOWN_POLICY; } @@ -123,12 +125,14 @@ struct ContextShared String interserver_scheme; /// http or https String path; /// Path to the data directory, with a slash at the end. - String tmp_path; /// The path to the temporary files that occur when processing the request. String flags_path; /// Path to the directory with some control flags for server maintenance. String user_files_path; /// Path to the directory with user provided files, usable by 'file' table function. String dictionaries_lib_path; /// Path to the directory with user provided binaries and libraries for external dictionaries. ConfigurationPtr config; /// Global configuration settings. + String tmp_path; /// Path to the temporary files that occur when processing the request. + mutable VolumePtr tmp_volume; /// Volume for the the temporary files that occur when processing the request. + Databases databases; /// List of databases and tables in them. mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::optional external_dictionaries_loader; @@ -151,9 +155,9 @@ struct ContextShared std::unique_ptr ddl_worker; /// Process ddl commands from zk. /// Rules for selecting the compression settings, depending on the size of the part. mutable std::unique_ptr compression_codec_selector; - /// Storage disk chooser + /// Storage disk chooser for MergeTree engines mutable std::unique_ptr merge_tree_disk_selector; - /// Storage policy chooser + /// Storage policy chooser for MergeTree engines mutable std::unique_ptr merge_tree_storage_policy_selector; std::optional merge_tree_settings; /// Settings of MergeTree* engines. @@ -527,12 +531,6 @@ String Context::getPath() const return shared->path; } -String Context::getTemporaryPath() const -{ - auto lock = getLock(); - return shared->tmp_path; -} - String Context::getFlagsPath() const { auto lock = getLock(); @@ -551,13 +549,19 @@ String Context::getDictionariesLibPath() const return shared->dictionaries_lib_path; } +VolumePtr Context::getTemporaryVolume() const +{ + auto lock = getLock(); + return shared->tmp_volume; +} + void Context::setPath(const String & path) { auto lock = getLock(); shared->path = path; - if (shared->tmp_path.empty()) + if (shared->tmp_path.empty() && !shared->tmp_volume) shared->tmp_path = shared->path + "tmp/"; if (shared->flags_path.empty()) @@ -570,10 +574,31 @@ void Context::setPath(const String & path) shared->dictionaries_lib_path = shared->path + "dictionaries_lib/"; } -void Context::setTemporaryPath(const String & path) +VolumePtr Context::setTemporaryStorage(const String & path, const String & policy_name) { auto lock = getLock(); - shared->tmp_path = path; + + if (policy_name.empty()) + { + shared->tmp_path = path; + if (!shared->tmp_path.ends_with('/')) + shared->tmp_path += '/'; + + auto disk = std::make_shared("_tmp_default", shared->tmp_path, 0); + shared->tmp_volume = std::make_shared("_tmp_default", std::vector{disk}, 0); + } + else + { + StoragePolicyPtr tmp_policy = getStoragePolicySelector()[policy_name]; + if (tmp_policy->getVolumes().size() != 1) + throw Exception("Policy " + policy_name + " is used temporary files, such policy should have exactly one volume", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + shared->tmp_volume = tmp_policy->getVolume(0); + } + + if (!shared->tmp_volume->disks.size()) + throw Exception("No disks volume for temporary files", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + + return shared->tmp_volume; } void Context::setFlagsPath(const String & path) @@ -739,49 +764,50 @@ void Context::checkDatabaseAccessRightsImpl(const std::string & database_name) c throw Exception("Access denied to database " + database_name + " for user " + client_info.current_user , ErrorCodes::DATABASE_ACCESS_DENIED); } -void Context::addDependencyUnsafe(const DatabaseAndTableName & from, const DatabaseAndTableName & where) + +void Context::addDependencyUnsafe(const StorageID & from, const StorageID & where) { - checkDatabaseAccessRightsImpl(from.first); - checkDatabaseAccessRightsImpl(where.first); + checkDatabaseAccessRightsImpl(from.database_name); + checkDatabaseAccessRightsImpl(where.database_name); shared->view_dependencies[from].insert(where); // Notify table of dependencies change - auto table = tryGetTable(from.first, from.second); + auto table = tryGetTable(from); if (table != nullptr) table->updateDependencies(); } -void Context::addDependency(const DatabaseAndTableName & from, const DatabaseAndTableName & where) +void Context::addDependency(const StorageID & from, const StorageID & where) { auto lock = getLock(); addDependencyUnsafe(from, where); } -void Context::removeDependencyUnsafe(const DatabaseAndTableName & from, const DatabaseAndTableName & where) +void Context::removeDependencyUnsafe(const StorageID & from, const StorageID & where) { - checkDatabaseAccessRightsImpl(from.first); - checkDatabaseAccessRightsImpl(where.first); + checkDatabaseAccessRightsImpl(from.database_name); + checkDatabaseAccessRightsImpl(where.database_name); shared->view_dependencies[from].erase(where); // Notify table of dependencies change - auto table = tryGetTable(from.first, from.second); + auto table = tryGetTable(from); if (table != nullptr) table->updateDependencies(); } -void Context::removeDependency(const DatabaseAndTableName & from, const DatabaseAndTableName & where) +void Context::removeDependency(const StorageID & from, const StorageID & where) { auto lock = getLock(); removeDependencyUnsafe(from, where); } -Dependencies Context::getDependencies(const String & database_name, const String & table_name) const +Dependencies Context::getDependencies(const StorageID & from) const { auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); + String db = resolveDatabase(from.database_name, current_database); - if (database_name.empty() && tryGetExternalTable(table_name)) + if (from.database_name.empty() && tryGetExternalTable(from.table_name)) { /// Table is temporary. Access granted. } @@ -790,7 +816,7 @@ Dependencies Context::getDependencies(const String & database_name, const String checkDatabaseAccessRightsImpl(db); } - ViewDependencies::const_iterator iter = shared->view_dependencies.find(DatabaseAndTableName(db, table_name)); + ViewDependencies::const_iterator iter = shared->view_dependencies.find(StorageID(db, from.table_name, from.uuid)); if (iter == shared->view_dependencies.end()) return {}; @@ -919,24 +945,32 @@ StoragePtr Context::tryGetExternalTable(const String & table_name) const return jt->second.first; } - StoragePtr Context::getTable(const String & database_name, const String & table_name) const +{ + return getTable(StorageID(database_name, table_name)); +} + +StoragePtr Context::getTable(const StorageID & table_id) const { std::optional exc; - auto res = getTableImpl(database_name, table_name, &exc); + auto res = getTableImpl(table_id, &exc); if (!res) throw *exc; return res; } - StoragePtr Context::tryGetTable(const String & database_name, const String & table_name) const { - return getTableImpl(database_name, table_name, {}); + return getTableImpl(StorageID(database_name, table_name), {}); +} + +StoragePtr Context::tryGetTable(const StorageID & table_id) const +{ + return getTableImpl(table_id, {}); } -StoragePtr Context::getTableImpl(const String & database_name, const String & table_name, std::optional * exception) const +StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional * exception) const { String db; DatabasePtr database; @@ -944,14 +978,14 @@ StoragePtr Context::getTableImpl(const String & database_name, const String & ta { auto lock = getLock(); - if (database_name.empty()) + if (table_id.database_name.empty()) { - StoragePtr res = tryGetExternalTable(table_name); + StoragePtr res = tryGetExternalTable(table_id.table_name); if (res) return res; } - db = resolveDatabase(database_name, current_database); + db = resolveDatabase(table_id.database_name, current_database); checkDatabaseAccessRightsImpl(db); Databases::const_iterator it = shared->databases.find(db); @@ -965,11 +999,11 @@ StoragePtr Context::getTableImpl(const String & database_name, const String & ta database = it->second; } - auto table = database->tryGetTable(*this, table_name); + auto table = database->tryGetTable(*this, table_id.table_name); if (!table) { if (exception) - exception->emplace("Table " + backQuoteIfNeed(db) + "." + backQuoteIfNeed(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); return {}; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index c8ea63db114..dcce1a4772f 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -80,6 +80,7 @@ class ICompressionCodec; class AccessControlManager; class SettingsConstraints; class RemoteHostFilter; +struct StorageID; class IDisk; using DiskPtr = std::shared_ptr; class DiskSelector; @@ -90,18 +91,18 @@ class StoragePolicySelector; class IOutputFormat; using OutputFormatPtr = std::shared_ptr; +class Volume; +using VolumePtr = std::shared_ptr; + #if USE_EMBEDDED_COMPILER class CompiledExpressionCache; #endif -/// (database name, table name) -using DatabaseAndTableName = std::pair; - /// Table -> set of table-views that make SELECT from it. -using ViewDependencies = std::map>; -using Dependencies = std::vector; +using ViewDependencies = std::map>; +using Dependencies = std::vector; using TableAndCreateAST = std::pair; using TableAndCreateASTs = std::map; @@ -197,17 +198,19 @@ public: ~Context(); String getPath() const; - String getTemporaryPath() const; String getFlagsPath() const; String getUserFilesPath() const; String getDictionariesLibPath() const; + VolumePtr getTemporaryVolume() const; + void setPath(const String & path); - void setTemporaryPath(const String & path); void setFlagsPath(const String & path); void setUserFilesPath(const String & path); void setDictionariesLibPath(const String & path); + VolumePtr setTemporaryStorage(const String & path, const String & policy_name = ""); + using ConfigurationPtr = Poco::AutoPtr; /// Global application configuration settings. @@ -256,13 +259,13 @@ public: ClientInfo & getClientInfo() { return client_info; } const ClientInfo & getClientInfo() const { return client_info; } - void addDependency(const DatabaseAndTableName & from, const DatabaseAndTableName & where); - void removeDependency(const DatabaseAndTableName & from, const DatabaseAndTableName & where); - Dependencies getDependencies(const String & database_name, const String & table_name) const; + void addDependency(const StorageID & from, const StorageID & where); + void removeDependency(const StorageID & from, const StorageID & where); + Dependencies getDependencies(const StorageID & from) const; /// Functions where we can lock the context manually - void addDependencyUnsafe(const DatabaseAndTableName & from, const DatabaseAndTableName & where); - void removeDependencyUnsafe(const DatabaseAndTableName & from, const DatabaseAndTableName & where); + void addDependencyUnsafe(const StorageID & from, const StorageID & where); + void removeDependencyUnsafe(const StorageID & from, const StorageID & where); /// Checking the existence of the table/database. Database can be empty - in this case the current database is used. bool isTableExist(const String & database_name, const String & table_name) const; @@ -288,7 +291,9 @@ public: Tables getExternalTables() const; StoragePtr tryGetExternalTable(const String & table_name) const; StoragePtr getTable(const String & database_name, const String & table_name) const; + StoragePtr getTable(const StorageID & table_id) const; StoragePtr tryGetTable(const String & database_name, const String & table_name) const; + StoragePtr tryGetTable(const StorageID & table_id) const; void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {}); void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; @@ -594,7 +599,7 @@ private: EmbeddedDictionaries & getEmbeddedDictionariesImpl(bool throw_on_error) const; - StoragePtr getTableImpl(const String & database_name, const String & table_name, std::optional * exception) const; + StoragePtr getTableImpl(const StorageID & table_id, std::optional * exception) const; SessionKey getSessionKey(const String & session_id) const; diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index 32004e4e564..c1c2929d0b5 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -622,8 +622,13 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) if (auto query_with_table = dynamic_cast(rewritten_ast.get()); query_with_table) { - String database = query_with_table->database.empty() ? context.getCurrentDatabase() : query_with_table->database; - StoragePtr storage = context.tryGetTable(database, query_with_table->table); + StoragePtr storage; + if (!query_with_table->table.empty()) + { + /// It's not CREATE DATABASE + String database = query_with_table->database.empty() ? context.getCurrentDatabase() : query_with_table->database; + storage = context.tryGetTable(database, query_with_table->table); + } /// For some reason we check consistency of cluster definition only /// in case of ALTER query, but not in case of CREATE/DROP etc. diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 793dc7a8a3d..f10d4501d2d 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -346,7 +346,7 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, } -void ExpressionAction::execute(Block & block, bool dry_run) const +void ExpressionAction::execute(Block & block, bool dry_run, ExtraBlockPtr & not_processed) const { size_t input_rows_count = block.rows(); @@ -477,7 +477,7 @@ void ExpressionAction::execute(Block & block, bool dry_run) const case JOIN: { - join->joinBlock(block); + join->joinBlock(block, not_processed); break; } @@ -762,6 +762,21 @@ void ExpressionActions::execute(Block & block, bool dry_run) const } } +/// @warning It's a tricky method that allows to continue ONLY ONE action in reason of one-to-many ALL JOIN logic. +void ExpressionActions::execute(Block & block, ExtraBlockPtr & not_processed, size_t & start_action) const +{ + size_t i = start_action; + start_action = 0; + for (; i < actions.size(); ++i) + { + actions[i].execute(block, false, not_processed); + checkLimits(block); + + if (not_processed) + start_action = i; + } +} + bool ExpressionActions::hasTotalsInJoin() const { for (const auto & action : actions) diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index 90c01191a74..fbb3071e236 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -138,8 +138,16 @@ private: friend class ExpressionActions; void prepare(Block & sample_block, const Settings & settings, NameSet & names_not_for_constant_folding); - void execute(Block & block, bool dry_run) const; void executeOnTotals(Block & block) const; + + /// Executes action on block (modify it). Block could be splitted in case of JOIN. Then not_processed block is created. + void execute(Block & block, bool dry_run, ExtraBlockPtr & not_processed) const; + + void execute(Block & block, bool dry_run) const + { + ExtraBlockPtr extra; + execute(block, dry_run, extra); + } }; @@ -221,6 +229,9 @@ public: /// Execute the expression on the block. The block must contain all the columns returned by getRequiredColumns. void execute(Block & block, bool dry_run = false) const; + /// Execute the expression on the block with continuation. + void execute(Block & block, ExtraBlockPtr & not_processed, size_t & start_action) const; + /// Check if joined subquery has totals. bool hasTotalsInJoin() const; diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index e9cfe602437..a96f64c110f 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -540,6 +540,7 @@ public: Strings getAllTriedToLoadNames() const { + std::lock_guard lock{mutex}; Strings names; for (auto & [name, info] : infos) if (info.triedToLoad()) diff --git a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h index 6c380162af4..d49f4c8f4e5 100644 --- a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h @@ -95,7 +95,7 @@ public: Block sample = interpreter->getSampleBlock(); NamesAndTypesList columns = sample.getNamesAndTypesList(); - StoragePtr external_storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}, ConstraintsDescription{}); + StoragePtr external_storage = StorageMemory::create(StorageID("_external", external_table_name), ColumnsDescription{columns}, ConstraintsDescription{}); external_storage->startup(); /** We replace the subquery with the name of the temporary table. diff --git a/dbms/src/Interpreters/IJoin.h b/dbms/src/Interpreters/IJoin.h index c62f39e9edf..6d290d143da 100644 --- a/dbms/src/Interpreters/IJoin.h +++ b/dbms/src/Interpreters/IJoin.h @@ -11,6 +11,7 @@ namespace DB { class Block; +struct ExtraBlock; class IJoin { @@ -23,7 +24,7 @@ public: /// Join the block with data from left hand of JOIN to the right hand data (that was previously built by calls to addJoinedBlock). /// Could be called from different threads in parallel. - virtual void joinBlock(Block & block) = 0; + virtual void joinBlock(Block & block, std::shared_ptr & not_processed) = 0; virtual bool hasTotals() const = 0; virtual void setTotals(const Block & block) = 0; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index db48c3bed92..daffdfd71f8 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -638,13 +638,10 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, { res = StorageFactory::instance().get(create, database ? database->getTableDataPath(create) : "", - table_name, - create.database, context, context.getGlobalContext(), properties.columns, properties.constraints, - create.attach, false); } diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 6b1f3088636..c9e70581259 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -79,13 +79,14 @@ BlockIO InterpreterDropQuery::executeToTable( if (database_and_table.first && database_and_table.second) { + auto table_id = database_and_table.second->getStorageID(); if (kind == ASTDropQuery::Kind::Detach) { database_and_table.second->shutdown(); /// If table was already dropped by anyone, an exception will be thrown auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); /// Drop table from memory, don't touch data and metadata - database_and_table.first->detachTable(database_and_table.second->getTableName()); + database_and_table.first->detachTable(table_id.table_name); } else if (kind == ASTDropQuery::Kind::Truncate) { @@ -107,7 +108,7 @@ BlockIO InterpreterDropQuery::executeToTable( const std::string metadata_file_without_extension = database_and_table.first->getMetadataPath() - + escapeForFileName(database_and_table.second->getTableName()); + + escapeForFileName(table_id.table_name); const auto prev_metadata_name = metadata_file_without_extension + ".sql"; const auto drop_metadata_name = metadata_file_without_extension + ".sql.tmp_drop"; @@ -131,7 +132,7 @@ BlockIO InterpreterDropQuery::executeToTable( String table_data_path_relative = database_and_table.first->getTableDataPath(table_name); /// Delete table metadata and table itself from memory - database_and_table.first->removeTable(context, database_and_table.second->getTableName()); + database_and_table.first->removeTable(context, table_id.table_name); database_and_table.second->is_dropped = true; /// If it is not virtual database like Dictionary then drop remaining data dir diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index e01e6cd1c44..787b5d3717b 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -96,63 +96,95 @@ Block InterpreterInsertQuery::getSampleBlock(const ASTInsertQuery & query, const BlockIO InterpreterInsertQuery::execute() { + const Settings & settings = context.getSettingsRef(); + const auto & query = query_ptr->as(); checkAccess(query); + BlockIO res; StoragePtr table = getTable(query); auto table_lock = table->lockStructureForShare(true, context.getInitialQueryId()); - /// We create a pipeline of several streams, into which we will write data. - BlockOutputStreamPtr out; - - /// NOTE: we explicitly ignore bound materialized views when inserting into Kafka Storage. - /// Otherwise we'll get duplicates when MV reads same rows again from Kafka. - if (table->noPushingToViews() && !no_destination) - out = table->write(query_ptr, context); - else - out = std::make_shared(query.database, query.table, table, context, query_ptr, no_destination); - - /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. - /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). - if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote()) && !no_squash) - { - out = std::make_shared( - out, out->getHeader(), context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); - } - auto query_sample_block = getSampleBlock(query, table); - - /// Actually we don't know structure of input blocks from query/table, - /// because some clients break insertion protocol (columns != header) - out = std::make_shared( - out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); - - if (const auto & constraints = table->getConstraints(); !constraints.empty()) - out = std::make_shared(query.table, - out, query_sample_block, table->getConstraints(), context); - - auto out_wrapper = std::make_shared(out); - out_wrapper->setProcessListElement(context.getProcessListElement()); - out = std::move(out_wrapper); - - BlockIO res; - - /// What type of query: INSERT or INSERT SELECT? + BlockInputStreams in_streams; + size_t out_streams_size = 1; if (query.select) { /// Passing 1 as subquery_depth will disable limiting size of intermediate result. InterpreterSelectWithUnionQuery interpreter_select{query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; - /// BlockIO may hold StoragePtrs to temporary tables - res = interpreter_select.execute(); - res.out = nullptr; + if (table->supportsParallelInsert() && settings.max_insert_threads > 0) + { + in_streams = interpreter_select.executeWithMultipleStreams(res.pipeline); + out_streams_size = std::min(size_t(settings.max_insert_threads), in_streams.size()); + } + else + { + res = interpreter_select.execute(); + in_streams.emplace_back(res.in); + res.in = nullptr; + res.out = nullptr; + } + } - res.in = std::make_shared(context, res.in, out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); - res.in = std::make_shared(res.in, out); + BlockOutputStreams out_streams; + auto query_sample_block = getSampleBlock(query, table); + + for (size_t i = 0; i < out_streams_size; i++) + { + /// We create a pipeline of several streams, into which we will write data. + BlockOutputStreamPtr out; + + /// NOTE: we explicitly ignore bound materialized views when inserting into Kafka Storage. + /// Otherwise we'll get duplicates when MV reads same rows again from Kafka. + if (table->noPushingToViews() && !no_destination) + out = table->write(query_ptr, context); + else + out = std::make_shared(table, context, query_ptr, no_destination); + + /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. + /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). + if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote()) && !no_squash) + { + out = std::make_shared( + out, out->getHeader(), context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); + } + + /// Actually we don't know structure of input blocks from query/table, + /// because some clients break insertion protocol (columns != header) + out = std::make_shared( + out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); + + if (const auto & constraints = table->getConstraints(); !constraints.empty()) + out = std::make_shared(query.table, + out, query_sample_block, table->getConstraints(), context); + + auto out_wrapper = std::make_shared(out); + out_wrapper->setProcessListElement(context.getProcessListElement()); + out = std::move(out_wrapper); + out_streams.emplace_back(std::move(out)); + } + + /// What type of query: INSERT or INSERT SELECT? + if (query.select) + { + for (auto & in_stream : in_streams) + { + in_stream = std::make_shared( + context, in_stream, out_streams.at(0)->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); + } + + Block in_header = in_streams.at(0)->getHeader(); + if (in_streams.size() > 1) + { + for (size_t i = 1; i < in_streams.size(); ++i) + assertBlocksHaveEqualStructure(in_streams[i]->getHeader(), in_header, "INSERT SELECT"); + } + + res.in = std::make_shared(in_streams, out_streams); if (!allow_materialized) { - Block in_header = res.in->getHeader(); for (const auto & column : table->getColumns()) if (column.default_desc.kind == ColumnDefaultKind::Materialized && in_header.has(column.name)) throw Exception("Cannot insert column " + column.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); @@ -160,12 +192,12 @@ BlockIO InterpreterInsertQuery::execute() } else if (query.data && !query.has_tail) /// can execute without additional data { + // res.out = std::move(out_streams.at(0)); res.in = std::make_shared(query_ptr, nullptr, query_sample_block, context, nullptr); - res.in = std::make_shared(res.in, out); + res.in = std::make_shared(res.in, out_streams.at(0)); } else - res.out = std::move(out); - + res.out = std::move(out_streams.at(0)); res.pipeline.addStorageHolder(table); return res; diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index b147c5f4887..0e47a15bd80 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -75,6 +74,7 @@ #include #include #include +#include #include #include #include @@ -116,14 +116,11 @@ namespace ErrorCodes extern const int INVALID_WITH_FILL_EXPRESSION; } -namespace -{ - /// Assumes `storage` is set and the table filter (row-level security) is not empty. -String generateFilterActions(ExpressionActionsPtr & actions, const Context & context, const StoragePtr & storage, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) +String InterpreterSelectQuery::generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns) const { - const auto & db_name = storage->getDatabaseName(); - const auto & table_name = storage->getTableName(); + const auto & db_name = table_id.getDatabaseName(); + const auto & table_name = table_id.getTableName(); /// TODO: implement some AST builders for this kind of stuff ASTPtr query_ast = std::make_shared(); @@ -153,17 +150,15 @@ String generateFilterActions(ExpressionActionsPtr & actions, const Context & con table_expr->children.push_back(table_expr->database_and_table_name); /// Using separate expression analyzer to prevent any possible alias injection - auto syntax_result = SyntaxAnalyzer(context).analyze(query_ast, storage->getColumns().getAllPhysical()); - SelectQueryExpressionAnalyzer analyzer(query_ast, syntax_result, context); - ExpressionActionsChain new_chain(context); + auto syntax_result = SyntaxAnalyzer(*context).analyze(query_ast, storage->getColumns().getAllPhysical()); + SelectQueryExpressionAnalyzer analyzer(query_ast, syntax_result, *context); + ExpressionActionsChain new_chain(*context); analyzer.appendSelect(new_chain, false); actions = new_chain.getLastActions(); return expr_list->children.at(0)->getColumnName(); } -} - InterpreterSelectQuery::InterpreterSelectQuery( const ASTPtr & query_ptr_, const Context & context_, @@ -292,7 +287,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (auto view_source = context->getViewSource()) { auto & storage_values = static_cast(*view_source); - if (storage_values.getDatabaseName() == database_name && storage_values.getTableName() == table_name) + auto tmp_table_id = storage_values.getStorageID(); + if (tmp_table_id.database_name == database_name && tmp_table_id.table_name == table_name) { /// Read from view source. storage = context->getViewSource(); @@ -308,7 +304,10 @@ InterpreterSelectQuery::InterpreterSelectQuery( } if (storage) + { table_lock = storage->lockStructureForShare(false, context->getInitialQueryId()); + table_id = storage->getStorageID(); + } auto analyze = [&] () { @@ -370,11 +369,11 @@ InterpreterSelectQuery::InterpreterSelectQuery( source_header = storage->getSampleBlockForColumns(required_columns); /// Fix source_header for filter actions. - auto row_policy_filter = context->getRowPolicy()->getCondition(storage->getDatabaseName(), storage->getTableName(), RowPolicy::SELECT_FILTER); + auto row_policy_filter = context->getRowPolicy()->getCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER); if (row_policy_filter) { filter_info = std::make_shared(); - filter_info->column_name = generateFilterActions(filter_info->actions, *context, storage, row_policy_filter, required_columns); + filter_info->column_name = generateFilterActions(filter_info->actions, row_policy_filter, required_columns); source_header = storage->getSampleBlockForColumns(filter_info->actions->getRequiredColumns()); } } @@ -424,9 +423,9 @@ InterpreterSelectQuery::InterpreterSelectQuery( sanitizeBlock(result_header); /// Remove limits for some tables in the `system` database. - if (storage && (storage->getDatabaseName() == "system")) + if (storage && (table_id.getDatabaseName() == "system")) { - String table_name = storage->getTableName(); + String table_name = table_id.getTableName(); if ((table_name == "quotas") || (table_name == "quota_usage") || (table_name == "one")) { options.ignore_quota = true; @@ -510,7 +509,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl() /// PREWHERE optimization. /// Turn off, if the table filter (row-level security) is applied. - if (!context->getRowPolicy()->getCondition(storage->getDatabaseName(), storage->getTableName(), RowPolicy::SELECT_FILTER)) + if (!context->getRowPolicy()->getCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER)) { auto optimize_prewhere = [&](auto & merge_tree) { @@ -1105,7 +1104,12 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType type) { bool on_totals = type == QueryPipeline::StreamType::Totals; - return std::make_shared(header, expressions.before_join, on_totals, default_totals); + std::shared_ptr ret; + if (settings.partial_merge_join) + ret = std::make_shared(header, expressions.before_join, on_totals, default_totals); + else + ret = std::make_shared(header, expressions.before_join, on_totals, default_totals); + return ret; }); } else @@ -1113,14 +1117,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS header_before_join = pipeline.firstStream()->getHeader(); /// Applies to all sources except stream_with_non_joined_data. for (auto & stream : pipeline.streams) - stream = std::make_shared(stream, expressions.before_join); - - if (isMergeJoin(expressions.before_join->getTableJoinAlgo()) && settings.partial_merge_join_optimizations) - { - if (size_t rows_in_block = settings.partial_merge_join_rows_in_left_blocks) - for (auto & stream : pipeline.streams) - stream = std::make_shared(stream, rows_in_block, 0, true); - } + stream = std::make_shared(stream, expressions.before_join); } if (JoinPtr join = expressions.before_join->getTableJoinAlgo()) @@ -1366,12 +1363,12 @@ void InterpreterSelectQuery::executeFetchColumns( if (storage) { /// Append columns from the table filter to required - auto row_policy_filter = context->getRowPolicy()->getCondition(storage->getDatabaseName(), storage->getTableName(), RowPolicy::SELECT_FILTER); + auto row_policy_filter = context->getRowPolicy()->getCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER); if (row_policy_filter) { auto initial_required_columns = required_columns; ExpressionActionsPtr actions; - generateFilterActions(actions, *context, storage, row_policy_filter, initial_required_columns); + generateFilterActions(actions, row_policy_filter, initial_required_columns); auto required_columns_from_filter = actions->getRequiredColumns(); for (const auto & column : required_columns_from_filter) @@ -1874,7 +1871,7 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); + context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); /// If there are several sources, then we perform parallel aggregation if (pipeline.streams.size() > 1) @@ -1940,7 +1937,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); + context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); auto transform_params = std::make_shared(params, final); @@ -1950,7 +1947,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const if (pipeline.getNumStreams() > 1) { /// Add resize transform to uniformly distribute data between aggregating streams. - pipeline.resize(pipeline.getNumStreams(), true); + if (!(storage && storage->hasEvenlyDistributedRead())) + pipeline.resize(pipeline.getNumStreams(), true, true); auto many_data = std::make_shared(pipeline.getNumStreams()); auto merge_threads = settings.aggregation_memory_efficient_merge_threads @@ -2165,7 +2163,7 @@ void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificato false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); + context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); if (modificator == Modificator::ROLLUP) pipeline.firstStream() = std::make_shared(pipeline.firstStream(), params); @@ -2194,7 +2192,7 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modif false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); + context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); auto transform_params = std::make_shared(params, true); @@ -2278,7 +2276,7 @@ void InterpreterSelectQuery::executeOrder(Pipeline & pipeline, InputSortingInfoP sorting_stream, output_order_descr, settings.max_block_size, limit, settings.max_bytes_before_remerge_sort, settings.max_bytes_before_external_sort / pipeline.streams.size(), - context->getTemporaryPath(), settings.min_free_disk_space_for_temporary_data); + context->getTemporaryVolume(), settings.min_free_disk_space_for_temporary_data); stream = merging_stream; }); @@ -2360,7 +2358,8 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting return std::make_shared( header, output_order_descr, settings.max_block_size, limit, settings.max_bytes_before_remerge_sort / pipeline.getNumStreams(), - settings.max_bytes_before_external_sort, context->getTemporaryPath(), settings.min_free_disk_space_for_temporary_data); + settings.max_bytes_before_external_sort, context->getTemporaryVolume(), + settings.min_free_disk_space_for_temporary_data); }); /// If there are several streams, we merge them into one diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.h b/dbms/src/Interpreters/InterpreterSelectQuery.h index ca39d488102..da6e19bf2ab 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectQuery.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -244,6 +245,8 @@ private: void executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, std::unordered_map & subqueries_for_sets); void executeMergeSorted(QueryPipeline & pipeline, const SortDescription & sort_description, UInt64 limit); + String generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const; + /// Add ConvertingBlockInputStream to specified header. void unifyStreams(Pipeline & pipeline, Block header); @@ -293,6 +296,7 @@ private: /// Table from where to read data, if not subquery. StoragePtr storage; + StorageID table_id = StorageID::createEmpty(); /// Will be initialized if storage is not nullptr TableStructureReadLockHolder table_lock; /// Used when we read from prepared input, not table or subquery. diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp index 6f76016725f..4ffe3917a59 100644 --- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -55,7 +55,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() { if (show_query->temporary) throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR); - create_query = context.getDatabase(show_query->database)->getCreateDatabaseQuery(); + create_query = context.getDatabase(show_query->database)->getCreateDatabaseQuery(context); } else if ((show_query = query_ptr->as())) { diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index d5c04750302..150f828c80c 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -299,13 +299,10 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam StoragePtr table = StorageFactory::instance().get(create, database->getTableDataPath(create), - table_name, - database_name, system_context, system_context.getGlobalContext(), columns, constraints, - create.attach, false); database->createTable(system_context, table_name, table, create_ast); diff --git a/dbms/src/Interpreters/InterserverIOHandler.h b/dbms/src/Interpreters/InterserverIOHandler.h index 7cef5df9866..4651c8cb978 100644 --- a/dbms/src/Interpreters/InterserverIOHandler.h +++ b/dbms/src/Interpreters/InterserverIOHandler.h @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace Poco { namespace Net { class HTTPServerResponse; } } @@ -24,42 +25,6 @@ namespace ErrorCodes extern const int NO_SUCH_INTERSERVER_IO_ENDPOINT; } -/** Location of the service. - */ -struct InterserverIOEndpointLocation -{ -public: - InterserverIOEndpointLocation(const std::string & name_, const std::string & host_, UInt16 port_) - : name(name_), host(host_), port(port_) - { - } - - /// Creates a location based on its serialized representation. - InterserverIOEndpointLocation(const std::string & serialized_location) - { - ReadBufferFromString buf(serialized_location); - readBinary(name, buf); - readBinary(host, buf); - readBinary(port, buf); - assertEOF(buf); - } - - /// Serializes the location. - std::string toString() const - { - WriteBufferFromOwnString buf; - writeBinary(name, buf); - writeBinary(host, buf); - writeBinary(port, buf); - return buf.str(); - } - -public: - std::string name; - std::string host; - UInt16 port; -}; - /** Query processor from other servers. */ class InterserverIOEndpoint @@ -71,6 +36,7 @@ public: /// You need to stop the data transfer if blocker is activated. ActionBlocker blocker; + std::shared_mutex rwlock; }; using InterserverIOEndpointPtr = std::shared_ptr; @@ -90,11 +56,10 @@ public: throw Exception("Duplicate interserver IO endpoint: " + name, ErrorCodes::DUPLICATE_INTERSERVER_IO_ENDPOINT); } - void removeEndpoint(const String & name) + bool removeEndpointIfExists(const String & name) { std::lock_guard lock(mutex); - if (!endpoint_map.erase(name)) - throw Exception("No interserver IO endpoint named " + name, ErrorCodes::NO_SUCH_INTERSERVER_IO_ENDPOINT); + return endpoint_map.erase(name); } InterserverIOEndpointPtr getEndpoint(const String & name) @@ -115,41 +80,4 @@ private: std::mutex mutex; }; -/// In the constructor calls `addEndpoint`, in the destructor - `removeEndpoint`. -class InterserverIOEndpointHolder -{ -public: - InterserverIOEndpointHolder(const String & name_, InterserverIOEndpointPtr endpoint_, InterserverIOHandler & handler_) - : name(name_), endpoint(std::move(endpoint_)), handler(handler_) - { - handler.addEndpoint(name, endpoint); - } - - InterserverIOEndpointPtr getEndpoint() - { - return endpoint; - } - - ~InterserverIOEndpointHolder() - try - { - handler.removeEndpoint(name); - /// After destroying the object, `endpoint` can still live, since its ownership is acquired during the processing of the request, - /// see InterserverIOHTTPHandler.cpp - } - catch (...) - { - tryLogCurrentException("~InterserverIOEndpointHolder"); - } - - ActionBlocker & getBlocker() { return endpoint->blocker; } - -private: - String name; - InterserverIOEndpointPtr endpoint; - InterserverIOHandler & handler; -}; - -using InterserverIOEndpointHolderPtr = std::shared_ptr; - } diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 2cce5d7f51d..7e6f66f1f7f 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -1091,7 +1091,7 @@ void Join::joinGet(Block & block, const String & column_name) const } -void Join::joinBlock(Block & block) +void Join::joinBlock(Block & block, ExtraBlockPtr &) { std::shared_lock lock(data->rwlock); diff --git a/dbms/src/Interpreters/Join.h b/dbms/src/Interpreters/Join.h index c8c9376b926..337c18a5980 100644 --- a/dbms/src/Interpreters/Join.h +++ b/dbms/src/Interpreters/Join.h @@ -158,7 +158,7 @@ public: /** Join data from the map (that was previously built by calls to addJoinedBlock) to the block with data from "left" table. * Could be called from different threads in parallel. */ - void joinBlock(Block & block) override; + void joinBlock(Block & block, ExtraBlockPtr & not_processed) override; /// Infer the return type for joinGet function DataTypePtr joinGetReturnType(const String & column_name) const; diff --git a/dbms/src/Interpreters/MergeJoin.cpp b/dbms/src/Interpreters/MergeJoin.cpp index f301de17bc5..b28caa5d0f7 100644 --- a/dbms/src/Interpreters/MergeJoin.cpp +++ b/dbms/src/Interpreters/MergeJoin.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace DB { @@ -294,36 +295,56 @@ void joinEqualsAnyLeft(const Block & right_block, const Block & right_columns_to copyRightRange(right_block, right_columns_to_add, right_columns, range.right_start, range.left_length); } -void joinEquals(const Block & left_block, const Block & right_block, const Block & right_columns_to_add, - MutableColumns & left_columns, MutableColumns & right_columns, const Range & range, bool is_all) +template +bool joinEquals(const Block & left_block, const Block & right_block, const Block & right_columns_to_add, + MutableColumns & left_columns, MutableColumns & right_columns, Range & range, size_t max_rows [[maybe_unused]]) { - size_t left_rows_to_add = range.left_length; - size_t right_rows_to_add = is_all ? range.right_length : 1; + bool one_more = true; - size_t row_position = range.right_start; - for (size_t right_row = 0; right_row < right_rows_to_add; ++right_row, ++row_position) + if constexpr (is_all) { - copyLeftRange(left_block, left_columns, range.left_start, left_rows_to_add); - copyRightRange(right_block, right_columns_to_add, right_columns, row_position, left_rows_to_add); + size_t range_rows = range.left_length * range.right_length; + if (range_rows > max_rows) + { + /// We need progress. So we join at least one right row. + range.right_length = max_rows / range.left_length; + if (!range.right_length) + range.right_length = 1; + one_more = false; + } + + size_t left_rows_to_add = range.left_length; + size_t row_position = range.right_start; + for (size_t right_row = 0; right_row < range.right_length; ++right_row, ++row_position) + { + copyLeftRange(left_block, left_columns, range.left_start, left_rows_to_add); + copyRightRange(right_block, right_columns_to_add, right_columns, row_position, left_rows_to_add); + } } + else + { + size_t left_rows_to_add = range.left_length; + copyLeftRange(left_block, left_columns, range.left_start, left_rows_to_add); + copyRightRange(right_block, right_columns_to_add, right_columns, range.right_start, left_rows_to_add); + } + + return one_more; } -void appendNulls(MutableColumns & right_columns, size_t rows_to_add) -{ - for (auto & column : right_columns) - column->insertManyDefaults(rows_to_add); -} - +template void joinInequalsLeft(const Block & left_block, MutableColumns & left_columns, MutableColumns & right_columns, - size_t start, size_t end, bool copy_left) + size_t start, size_t end) { if (end <= start) return; size_t rows_to_add = end - start; - if (copy_left) + if constexpr (copy_left) copyLeftRange(left_block, left_columns, start, rows_to_add); - appendNulls(right_columns, rows_to_add); + + /// append nulls + for (auto & column : right_columns) + column->insertManyDefaults(rows_to_add); } Blocks blocksListToBlocks(const BlocksList & in_blocks) @@ -386,6 +407,8 @@ void MiniLSM::insert(const BlocksList & blocks) if (blocks.empty()) return; + const std::string path(volume->getNextDisk()->getPath()); + SortedFiles sorted_blocks; if (blocks.size() > 1) { @@ -414,6 +437,7 @@ void MiniLSM::merge(std::function callback) BlockInputStreams inputs = makeSortedInputStreams(sorted_files, sample_block); MergingSortedBlockInputStream sorted_stream(inputs, sort_description, rows_in_block); + const std::string path(volume->getNextDisk()->getPath()); SortedFiles out; flushStreamToFiles(path, sample_block, sorted_stream, out, callback); @@ -427,10 +451,11 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & ri , size_limits(table_join->sizeLimits()) , right_sample_block(right_sample_block_) , nullable_right_side(table_join->forceNullableRight()) - , is_all(table_join->strictness() == ASTTableJoin::Strictness::All) + , is_all_join(table_join->strictness() == ASTTableJoin::Strictness::All) , is_inner(isInner(table_join->kind())) , is_left(isLeft(table_join->kind())) , skip_not_intersected(table_join->enablePartialMergeJoinOptimizations()) + , max_joined_block_rows(table_join->maxJoinedBlockRows()) , max_rows_in_right_block(table_join->maxRowsInRightBlock()) { if (!isLeft(table_join->kind()) && !isInner(table_join->kind())) @@ -463,7 +488,7 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & ri makeSortAndMerge(table_join->keyNamesLeft(), left_sort_description, left_merge_description); makeSortAndMerge(table_join->keyNamesRight(), right_sort_description, right_merge_description); - lsm = std::make_unique(table_join->getTemporaryPath(), right_sample_block, right_sort_description, max_rows_in_right_block); + lsm = std::make_unique(table_join->getTemporaryVolume(), right_sample_block, right_sort_description, max_rows_in_right_block); } void MergeJoin::setTotals(const Block & totals_block) @@ -567,7 +592,7 @@ bool MergeJoin::addJoinedBlock(const Block & src_block) return saveRightBlock(std::move(block)); } -void MergeJoin::joinBlock(Block & block) +void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); @@ -575,13 +600,23 @@ void MergeJoin::joinBlock(Block & block) sortBlock(block, left_sort_description); if (is_in_memory) - joinSortedBlock(block); + { + if (is_all_join) + joinSortedBlock(block, not_processed); + else + joinSortedBlock(block, not_processed); + } else - joinSortedBlock(block); + { + if (is_all_join) + joinSortedBlock(block, not_processed); + else + joinSortedBlock(block, not_processed); + } } -template -void MergeJoin::joinSortedBlock(Block & block) +template +void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed) { std::shared_lock lock(rwlock); @@ -590,11 +625,22 @@ void MergeJoin::joinSortedBlock(Block & block) MutableColumns right_columns = makeMutableColumns(right_columns_to_add, rows_to_reserve); MergeJoinCursor left_cursor(block, left_merge_description); size_t left_key_tail = 0; + size_t skip_right = 0; size_t right_blocks_count = rightBlocksCount(); + size_t starting_right_block = 0; + if (not_processed) + { + auto & continuation = static_cast(*not_processed); + left_cursor.nextN(continuation.left_position); + skip_right = continuation.right_position; + starting_right_block = continuation.right_block; + not_processed.reset(); + } + if (is_left) { - for (size_t i = 0; i < right_blocks_count; ++i) + for (size_t i = starting_right_block; i < right_blocks_count; ++i) { if (left_cursor.atEnd()) break; @@ -610,11 +656,16 @@ void MergeJoin::joinSortedBlock(Block & block) std::shared_ptr right_block = loadRightBlock(i); - leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); + if (!leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail, skip_right)) + { + not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), + left_cursor.position(), skip_right, i); + return; + } } left_cursor.nextN(left_key_tail); - joinInequalsLeft(block, left_columns, right_columns, left_cursor.position(), left_cursor.end(), is_all); + joinInequalsLeft(block, left_columns, right_columns, left_cursor.position(), left_cursor.end()); //left_cursor.nextN(left_cursor.end() - left_cursor.position()); changeLeftColumns(block, std::move(left_columns)); @@ -622,7 +673,7 @@ void MergeJoin::joinSortedBlock(Block & block) } else if (is_inner) { - for (size_t i = 0; i < right_blocks_count; ++i) + for (size_t i = starting_right_block; i < right_blocks_count; ++i) { if (left_cursor.atEnd()) break; @@ -638,7 +689,12 @@ void MergeJoin::joinSortedBlock(Block & block) std::shared_ptr right_block = loadRightBlock(i); - innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); + if (!innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail, skip_right)) + { + not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), + left_cursor.position(), skip_right, i); + return; + } } left_cursor.nextN(left_key_tail); @@ -647,12 +703,30 @@ void MergeJoin::joinSortedBlock(Block & block) } } -void MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) +static size_t maxRangeRows(size_t current_rows, size_t max_rows) +{ + if (!max_rows) + return std::numeric_limits::max(); + if (current_rows >= max_rows) + return 0; + return max_rows - current_rows; +} + +template +bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, + size_t & skip_right [[maybe_unused]]) { MergeJoinCursor right_cursor(right_block, right_merge_description); left_cursor.setCompareNullability(right_cursor); + /// Set right cursor position in first continuation right block + if constexpr (is_all) + { + right_cursor.nextN(skip_right); + skip_right = 0; + } + while (!left_cursor.atEnd() && !right_cursor.atEnd()) { /// Not zero left_key_tail means there were equality for the last left key in previous leftJoin() call. @@ -662,56 +736,97 @@ void MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block Range range = left_cursor.getNextEqualRange(right_cursor); - joinInequalsLeft(left_block, left_columns, right_columns, left_unequal_position, range.left_start, is_all); + joinInequalsLeft(left_block, left_columns, right_columns, left_unequal_position, range.left_start); if (range.empty()) break; - if (is_all) - joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, is_all); + if constexpr (is_all) + { + size_t max_rows = maxRangeRows(left_columns[0]->size(), max_joined_block_rows); + + if (!joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, max_rows)) + { + right_cursor.nextN(range.right_length); + skip_right = right_cursor.position(); + return false; + } + } else joinEqualsAnyLeft(right_block, right_columns_to_add, right_columns, range); right_cursor.nextN(range.right_length); /// Do not run over last left keys for ALL JOIN (cause of possible duplicates in next right block) - if (is_all && right_cursor.atEnd()) + if constexpr (is_all) { - left_key_tail = range.left_length; - break; + if (right_cursor.atEnd()) + { + left_key_tail = range.left_length; + break; + } } left_cursor.nextN(range.left_length); } + + return true; } -void MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) +template +bool MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, + size_t & skip_right [[maybe_unused]]) { MergeJoinCursor right_cursor(right_block, right_merge_description); left_cursor.setCompareNullability(right_cursor); + /// Set right cursor position in first continuation right block + if constexpr (is_all) + { + right_cursor.nextN(skip_right); + skip_right = 0; + } + while (!left_cursor.atEnd() && !right_cursor.atEnd()) { Range range = left_cursor.getNextEqualRange(right_cursor); if (range.empty()) break; - joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, is_all); + if constexpr (is_all) + { + size_t max_rows = maxRangeRows(left_columns[0]->size(), max_joined_block_rows); + + if (!joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, max_rows)) + { + right_cursor.nextN(range.right_length); + skip_right = right_cursor.position(); + return false; + } + } + else + joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, 0); + right_cursor.nextN(range.right_length); /// Do not run over last left keys for ALL JOIN (cause of possible duplicates in next right block) - if (is_all && right_cursor.atEnd()) + if constexpr (is_all) { - left_key_tail = range.left_length; - break; + if (right_cursor.atEnd()) + { + left_key_tail = range.left_length; + break; + } } left_cursor.nextN(range.left_length); } + + return true; } void MergeJoin::changeLeftColumns(Block & block, MutableColumns && columns) { - if (is_left && !is_all) + if (is_left && !is_all_join) return; block.setColumns(std::move(columns)); } @@ -725,6 +840,27 @@ void MergeJoin::addRightColumns(Block & block, MutableColumns && right_columns) } } +/// Split block into processed (result) and not processed. Not processed block would be joined next time. +template +ExtraBlockPtr MergeJoin::extraBlock(Block & processed, MutableColumns && left_columns, MutableColumns && right_columns, + size_t left_position [[maybe_unused]], size_t right_position [[maybe_unused]], + size_t right_block_number [[maybe_unused]]) +{ + ExtraBlockPtr not_processed; + + if constexpr (is_all) + { + not_processed = std::make_shared( + NotProcessed{{processed.cloneEmpty()}, left_position, right_position, right_block_number}); + not_processed->block.swap(processed); + + changeLeftColumns(processed, std::move(left_columns)); + addRightColumns(processed, std::move(right_columns)); + } + + return not_processed; +} + template size_t MergeJoin::rightBlocksCount() { diff --git a/dbms/src/Interpreters/MergeJoin.h b/dbms/src/Interpreters/MergeJoin.h index 960ca31153d..a64ffd0a85a 100644 --- a/dbms/src/Interpreters/MergeJoin.h +++ b/dbms/src/Interpreters/MergeJoin.h @@ -17,20 +17,23 @@ class AnalyzedJoin; class MergeJoinCursor; struct MergeJoinEqualRange; +class Volume; +using VolumePtr = std::shared_ptr; + struct MiniLSM { using SortedFiles = std::vector>; - const String & path; + VolumePtr volume; const Block & sample_block; const SortDescription & sort_description; const size_t rows_in_block; const size_t max_size; std::vector sorted_files; - MiniLSM(const String & path_, const Block & sample_block_, const SortDescription & description, + MiniLSM(VolumePtr volume_, const Block & sample_block_, const SortDescription & description, size_t rows_in_block_, size_t max_size_ = 16) - : path(path_) + : volume(volume_) , sample_block(sample_block_) , sort_description(description) , rows_in_block(rows_in_block_) @@ -48,13 +51,20 @@ public: MergeJoin(std::shared_ptr table_join_, const Block & right_sample_block); bool addJoinedBlock(const Block & block) override; - void joinBlock(Block &) override; + void joinBlock(Block &, ExtraBlockPtr & not_processed) override; void joinTotals(Block &) const override; void setTotals(const Block &) override; bool hasTotals() const override { return totals; } size_t getTotalRowCount() const override { return right_blocks_row_count; } private: + struct NotProcessed : public ExtraBlock + { + size_t left_position; + size_t right_position; + size_t right_block; + }; + /// There're two size limits for right-hand table: max_rows_in_join, max_bytes_in_join. /// max_bytes is prefered. If it isn't set we approximate it as (max_rows * bytes/row). struct BlockByteWeight @@ -85,28 +95,35 @@ private: size_t right_blocks_bytes = 0; bool is_in_memory = true; const bool nullable_right_side; - const bool is_all; + const bool is_all_join; const bool is_inner; const bool is_left; const bool skip_not_intersected; + const size_t max_joined_block_rows; const size_t max_rows_in_right_block; void changeLeftColumns(Block & block, MutableColumns && columns); void addRightColumns(Block & block, MutableColumns && columns); + template + ExtraBlockPtr extraBlock(Block & processed, MutableColumns && left_columns, MutableColumns && right_columns, + size_t left_position, size_t right_position, size_t right_block_number); + void mergeRightBlocks(); template size_t rightBlocksCount(); - template - void joinSortedBlock(Block & block); + template + void joinSortedBlock(Block & block, ExtraBlockPtr & not_processed); template std::shared_ptr loadRightBlock(size_t pos); - void leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); - void innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); + template + bool leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, size_t & skip_right); + template + bool innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, size_t & skip_right); bool saveRightBlock(Block && block); void flushRightBlocks(); diff --git a/dbms/src/Interpreters/PartLog.cpp b/dbms/src/Interpreters/PartLog.cpp index d6a23f44904..52ad0e0f25c 100644 --- a/dbms/src/Interpreters/PartLog.cpp +++ b/dbms/src/Interpreters/PartLog.cpp @@ -110,7 +110,8 @@ bool PartLog::addNewParts(Context & current_context, const PartLog::MutableDataP try { - part_log = current_context.getPartLog(parts.front()->storage.getDatabaseName()); // assume parts belong to the same table + auto table_id = parts.front()->storage.getStorageID(); + part_log = current_context.getPartLog(table_id.database_name); // assume parts belong to the same table if (!part_log) return false; @@ -122,8 +123,8 @@ bool PartLog::addNewParts(Context & current_context, const PartLog::MutableDataP elem.event_time = time(nullptr); elem.duration_ms = elapsed_ns / 1000000; - elem.database_name = part->storage.getDatabaseName(); - elem.table_name = part->storage.getTableName(); + elem.database_name = table_id.database_name; + elem.table_name = table_id.table_name; elem.partition_id = part->info.partition_id; elem.part_name = part->name; elem.path_on_disk = part->getFullPath(); diff --git a/dbms/src/Interpreters/QueryLog.cpp b/dbms/src/Interpreters/QueryLog.cpp index d9b86ea91ea..b57b54ac710 100644 --- a/dbms/src/Interpreters/QueryLog.cpp +++ b/dbms/src/Interpreters/QueryLog.cpp @@ -49,6 +49,7 @@ Block QueryLogElement::createBlock() {std::make_shared(), "memory_usage"}, {std::make_shared(), "query"}, + {std::make_shared(), "exception_code"}, {std::make_shared(), "exception"}, {std::make_shared(), "stack_trace"}, @@ -107,6 +108,7 @@ void QueryLogElement::appendToBlock(Block & block) const columns[i++]->insert(memory_usage); columns[i++]->insertData(query.data(), query.size()); + columns[i++]->insert(exception_code); columns[i++]->insertData(exception.data(), exception.size()); columns[i++]->insertData(stack_trace.data(), stack_trace.size()); diff --git a/dbms/src/Interpreters/QueryLog.h b/dbms/src/Interpreters/QueryLog.h index 0bee61df394..f14691df64e 100644 --- a/dbms/src/Interpreters/QueryLog.h +++ b/dbms/src/Interpreters/QueryLog.h @@ -54,6 +54,7 @@ struct QueryLogElement String query; + Int32 exception_code{}; // because ErrorCodes are int String exception; String stack_trace; diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index a485bd7ad73..b929804d0ae 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -816,7 +816,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( SyntaxAnalyzerResult result; result.storage = storage; result.source_columns = source_columns_; - result.analyzed_join = std::make_shared(settings, context.getTemporaryPath()); /// TODO: move to select_query logic + result.analyzed_join = std::make_shared(settings, context.getTemporaryVolume()); /// TODO: move to select_query logic if (storage) collectSourceColumns(storage->getColumns(), result.source_columns, (select_query != nullptr)); diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index 6eb0ce69f4e..d347488ea50 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace { @@ -31,8 +36,19 @@ std::shared_ptr createSystemLog( String database = config.getString(config_prefix + ".database", default_database_name); String table = config.getString(config_prefix + ".table", default_table_name); - String partition_by = config.getString(config_prefix + ".partition_by", "toYYYYMM(event_date)"); - String engine = "ENGINE = MergeTree PARTITION BY (" + partition_by + ") ORDER BY (event_date, event_time)"; + + String engine; + if (config.has(config_prefix + ".engine")) + { + if (config.has(config_prefix + ".partition_by")) + throw Exception("If 'engine' is specified for system table, PARTITION BY parameters should be specified directly inside 'engine' and 'partition_by' setting doesn't make sense", ErrorCodes::BAD_ARGUMENTS); + engine = config.getString(config_prefix + ".engine"); + } + else + { + String partition_by = config.getString(config_prefix + ".partition_by", "toYYYYMM(event_date)"); + engine = "ENGINE = MergeTree PARTITION BY (" + partition_by + ") ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024"; + } size_t flush_interval_milliseconds = config.getUInt64(config_prefix + ".flush_interval_milliseconds", DEFAULT_SYSTEM_LOG_FLUSH_INTERVAL_MILLISECONDS); diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp index 0bf1ac36d3d..80a9d75cd80 100644 --- a/dbms/src/Interpreters/ThreadStatusExt.cpp +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -160,15 +160,23 @@ void ThreadStatus::initQueryProfiler() const auto & settings = query_context->getSettingsRef(); - if (settings.query_profiler_real_time_period_ns > 0) - query_profiler_real = std::make_unique( - /* thread_id */ os_thread_id, - /* period */ static_cast(settings.query_profiler_real_time_period_ns)); + try + { + if (settings.query_profiler_real_time_period_ns > 0) + query_profiler_real = std::make_unique( + /* thread_id */ os_thread_id, + /* period */ static_cast(settings.query_profiler_real_time_period_ns)); - if (settings.query_profiler_cpu_time_period_ns > 0) - query_profiler_cpu = std::make_unique( - /* thread_id */ os_thread_id, - /* period */ static_cast(settings.query_profiler_cpu_time_period_ns)); + if (settings.query_profiler_cpu_time_period_ns > 0) + query_profiler_cpu = std::make_unique( + /* thread_id */ os_thread_id, + /* period */ static_cast(settings.query_profiler_cpu_time_period_ns)); + } + catch (...) + { + /// QueryProfiler is optional. + tryLogCurrentException("ThreadStatus", "Cannot initialize QueryProfiler"); + } } void ThreadStatus::finalizeQueryProfiler() diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 69bbb6fcd41..b6741e17b8d 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -163,6 +163,7 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c elem.query_start_time = current_time; elem.query = query_for_logging; + elem.exception_code = getCurrentExceptionCode(); elem.exception = getCurrentExceptionMessage(false); elem.client_info = context.getClientInfo(); @@ -214,7 +215,7 @@ static std::tuple executeQueryImpl( try { /// TODO Parser should fail early when max_query_size limit is reached. - ast = parseQuery(parser, begin, end, "", max_query_size); + ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth); auto * insert_query = ast->as(); @@ -496,6 +497,7 @@ static std::tuple executeQueryImpl( elem.event_time = time(nullptr); elem.query_duration_ms = 1000 * (elem.event_time - elem.query_start_time); + elem.exception_code = getCurrentExceptionCode(); elem.exception = getCurrentExceptionMessage(false); QueryStatus * process_list_elem = context.getProcessListElement(); @@ -573,14 +575,17 @@ BlockIO executeQuery( BlockIO streams; std::tie(ast, streams) = executeQueryImpl(query.data(), query.data() + query.size(), context, internal, stage, !may_have_embedded_data, nullptr, allow_processors); - if (streams.in) + + if (const auto * ast_query_with_output = dynamic_cast(ast.get())) { - const auto * ast_query_with_output = dynamic_cast(ast.get()); - String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) - ? getIdentifierName(ast_query_with_output->format) : context.getDefaultFormat(); + String format_name = ast_query_with_output->format + ? getIdentifierName(ast_query_with_output->format) + : context.getDefaultFormat(); + if (format_name == "Null") streams.null_format = true; } + return streams; } @@ -590,7 +595,7 @@ void executeQuery( WriteBuffer & ostr, bool allow_into_outfile, Context & context, - std::function set_content_type, + std::function set_content_type_and_format, std::function set_query_id) { PODArray parse_buf; @@ -680,8 +685,8 @@ void executeQuery( out->onProgress(progress); }); - if (set_content_type) - set_content_type(out->getContentType()); + if (set_content_type_and_format) + set_content_type_and_format(out->getContentType(), format_name); if (set_query_id) set_query_id(context.getClientInfo().current_query_id); @@ -742,8 +747,8 @@ void executeQuery( out->onProgress(progress); }); - if (set_content_type) - set_content_type(out->getContentType()); + if (set_content_type_and_format) + set_content_type_and_format(out->getContentType(), format_name); if (set_query_id) set_query_id(context.getClientInfo().current_query_id); diff --git a/dbms/src/Interpreters/executeQuery.h b/dbms/src/Interpreters/executeQuery.h index 3cff461f6d6..59b555b9f94 100644 --- a/dbms/src/Interpreters/executeQuery.h +++ b/dbms/src/Interpreters/executeQuery.h @@ -19,7 +19,7 @@ void executeQuery( WriteBuffer & ostr, /// Where to write query output to. bool allow_into_outfile, /// If true and the query contains INTO OUTFILE section, redirect output to that file. Context & context, /// DB, tables, data types, storage engines, functions, aggregate functions... - std::function set_content_type, /// If non-empty callback is passed, it will be called with the Content-Type of the result. + std::function set_content_type_and_format, /// If non-empty callback is passed, it will be called with the Content-Type and the Format of the result. std::function set_query_id /// If non-empty callback is passed, it will be called with the query id. ); diff --git a/dbms/src/Interpreters/loadMetadata.cpp b/dbms/src/Interpreters/loadMetadata.cpp index de1ed00e74a..241385f97ec 100644 --- a/dbms/src/Interpreters/loadMetadata.cpp +++ b/dbms/src/Interpreters/loadMetadata.cpp @@ -139,7 +139,7 @@ void loadMetadataSystem(Context & context) Poco::File(global_path + "data/" SYSTEM_DATABASE).createDirectories(); Poco::File(global_path + "metadata/" SYSTEM_DATABASE).createDirectories(); - auto system_database = std::make_shared(SYSTEM_DATABASE, global_path + "metadata/" SYSTEM_DATABASE, context); + auto system_database = std::make_shared(SYSTEM_DATABASE, global_path + "metadata/" SYSTEM_DATABASE "/", context); context.addDatabase(SYSTEM_DATABASE, system_database); } diff --git a/dbms/src/Interpreters/tests/aggregate.cpp b/dbms/src/Interpreters/tests/aggregate.cpp index 4d4d964aa9a..df498d6039d 100644 --- a/dbms/src/Interpreters/tests/aggregate.cpp +++ b/dbms/src/Interpreters/tests/aggregate.cpp @@ -79,7 +79,7 @@ int main(int argc, char ** argv) Aggregator::Params params( stream->getHeader(), {0, 1}, aggregate_descriptions, - false, 0, OverflowMode::THROW, 0, 0, 0, false, "", 1, 0); + false, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, 1, 0); Aggregator aggregator(params); diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index 8369234416b..a46afb2c8c3 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -38,12 +38,9 @@ public: std::string getRemoteDatabaseName() const { return remote_database; } std::string getRemoteTableName() const { return remote_table; } - std::string getTableName() const override { return ""; } - std::string getDatabaseName() const override { return ""; } - protected: StorageDistributedFake(const std::string & remote_database_, const std::string & remote_table_, size_t shard_count_) - : remote_database(remote_database_), remote_table(remote_table_), shard_count(shard_count_) + : IStorage({"", ""}), remote_database(remote_database_), remote_table(remote_table_), shard_count(shard_count_) { } diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index f4575623c7e..fed9c92d630 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -182,10 +182,24 @@ void ASTAlterCommand::formatImpl( case PartDestinationType::VOLUME: settings.ostr << "VOLUME "; break; + case PartDestinationType::TABLE: + settings.ostr << "TABLE "; + if (!to_database.empty()) + { + settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(to_database) + << (settings.hilite ? hilite_none : "") << "."; + } + settings.ostr << (settings.hilite ? hilite_identifier : "") + << backQuoteIfNeed(to_table) + << (settings.hilite ? hilite_none : ""); + return; default: break; } - settings.ostr << quoteString(move_destination_name); + if (move_destination_type != PartDestinationType::TABLE) + { + settings.ostr << quoteString(move_destination_name); + } } else if (type == ASTAlterCommand::REPLACE_PARTITION) { diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 02e76d5555c..25688e3890f 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -146,6 +146,9 @@ public: String from_table; /// To distinguish REPLACE and ATTACH PARTITION partition FROM db.table bool replace = true; + /// MOVE PARTITION partition TO TABLE db.table + String to_database; + String to_table; String getID(char delim) const override { return "AlterCommand" + (delim + std::to_string(static_cast(type))); } diff --git a/dbms/src/Parsers/ASTQueryWithTableAndOutput.h b/dbms/src/Parsers/ASTQueryWithTableAndOutput.h index 594876ace7b..975d55e3e65 100644 --- a/dbms/src/Parsers/ASTQueryWithTableAndOutput.h +++ b/dbms/src/Parsers/ASTQueryWithTableAndOutput.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB @@ -15,6 +16,7 @@ class ASTQueryWithTableAndOutput : public ASTQueryWithOutput public: String database; String table; + UUID uuid; bool temporary{false}; protected: diff --git a/dbms/src/Parsers/ExpressionElementParsers.h b/dbms/src/Parsers/ExpressionElementParsers.h index fbcaeeb3d45..4fae3f0328f 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.h +++ b/dbms/src/Parsers/ExpressionElementParsers.h @@ -10,8 +10,8 @@ namespace DB class ParserArray : public IParserBase { protected: - const char * getName() const { return "array"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "array"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -22,8 +22,8 @@ protected: class ParserParenthesisExpression : public IParserBase { protected: - const char * getName() const { return "parenthesized expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "parenthesized expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -32,8 +32,8 @@ protected: class ParserSubquery : public IParserBase { protected: - const char * getName() const { return "SELECT subquery"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "SELECT subquery"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -42,8 +42,8 @@ protected: class ParserIdentifier : public IParserBase { protected: - const char * getName() const { return "identifier"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "identifier"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -52,16 +52,16 @@ protected: class ParserCompoundIdentifier : public IParserBase { protected: - const char * getName() const { return "compound identifier"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "compound identifier"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /// Just * class ParserAsterisk : public IParserBase { protected: - const char * getName() const { return "asterisk"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "asterisk"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** Something like t.* or db.table.* @@ -69,8 +69,8 @@ protected: class ParserQualifiedAsterisk : public IParserBase { protected: - const char * getName() const { return "qualified asterisk"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "qualified asterisk"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** COLUMNS('') @@ -78,8 +78,8 @@ protected: class ParserColumnsMatcher : public IParserBase { protected: - const char * getName() const { return "COLUMNS matcher"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "COLUMNS matcher"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** A function, for example, f(x, y + 1, g(z)). @@ -93,16 +93,16 @@ class ParserFunction : public IParserBase public: ParserFunction(bool allow_function_parameters_ = true) : allow_function_parameters(allow_function_parameters_) {} protected: - const char * getName() const { return "function"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "function"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool allow_function_parameters; }; class ParserCodecDeclarationList : public IParserBase { protected: - const char * getName() const { return "codec declaration list"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "codec declaration list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** Parse compression codec @@ -111,8 +111,8 @@ protected: class ParserCodec : public IParserBase { protected: - const char * getName() const { return "codec"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "codec"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; class ParserCastExpression : public IParserBase @@ -176,8 +176,8 @@ protected: class ParserNull : public IParserBase { protected: - const char * getName() const { return "NULL"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "NULL"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -186,8 +186,8 @@ protected: class ParserNumber : public IParserBase { protected: - const char * getName() const { return "number"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "number"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** Unsigned integer, used in right hand side of tuple access operator (x.1). @@ -195,8 +195,8 @@ protected: class ParserUnsignedInteger : public IParserBase { protected: - const char * getName() const { return "unsigned integer"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "unsigned integer"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -205,8 +205,8 @@ protected: class ParserStringLiteral : public IParserBase { protected: - const char * getName() const { return "string literal"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "string literal"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -219,8 +219,8 @@ protected: class ParserArrayOfLiterals : public IParserBase { protected: - const char * getName() const { return "array"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "array"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -229,8 +229,8 @@ protected: class ParserLiteral : public IParserBase { protected: - const char * getName() const { return "literal"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "literal"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -246,8 +246,8 @@ private: bool allow_alias_without_as_keyword; - const char * getName() const { return "alias"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "alias"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -257,8 +257,8 @@ private: class ParserSubstitution : public IParserBase { protected: - const char * getName() const { return "substitution"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "substitution"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -267,8 +267,8 @@ protected: class ParserExpressionElement : public IParserBase { protected: - const char * getName() const { return "element of expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "element of expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -283,8 +283,8 @@ protected: ParserPtr elem_parser; bool allow_alias_without_as_keyword; - const char * getName() const { return "element of expression with optional alias"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "element of expression with optional alias"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -296,8 +296,8 @@ protected: class ParserOrderByElement : public IParserBase { protected: - const char * getName() const { return "element of ORDER BY expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "element of ORDER BY expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** Parser for function with arguments like KEY VALUE (space separated) @@ -316,8 +316,8 @@ protected: class ParserIdentifierWithOptionalParameters : public IParserBase { protected: - const char * getName() const { return "identifier with optional parameters"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "identifier with optional parameters"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** Element of TTL expression - same as expression element, but in addition, @@ -326,8 +326,8 @@ protected: class ParserTTLElement : public IParserBase { protected: - const char * getName() const { return "element of TTL expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "element of TTL expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ExpressionListParsers.h b/dbms/src/Parsers/ExpressionListParsers.h index ed37807eb67..b0909fe8bf7 100644 --- a/dbms/src/Parsers/ExpressionListParsers.h +++ b/dbms/src/Parsers/ExpressionListParsers.h @@ -27,8 +27,8 @@ public: { } protected: - const char * getName() const { return "list of elements"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "list of elements"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; private: ParserPtr elem_parser; ParserPtr separator_parser; @@ -63,9 +63,9 @@ public: } protected: - const char * getName() const { return "list, delimited by binary operators"; } + const char * getName() const override { return "list, delimited by binary operators"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -86,9 +86,9 @@ public: } protected: - const char * getName() const { return "list, delimited by operator of variable arity"; } + const char * getName() const override { return "list, delimited by operator of variable arity"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -110,8 +110,8 @@ public: } protected: - const char * getName() const { return "expression with prefix unary operator"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "expression with prefix unary operator"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -121,9 +121,9 @@ private: static const char * operators[]; protected: - const char * getName() const { return "array element expression"; } + const char * getName() const override{ return "array element expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -133,9 +133,9 @@ private: static const char * operators[]; protected: - const char * getName() const { return "tuple element expression"; } + const char * getName() const override { return "tuple element expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -146,9 +146,9 @@ private: ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique()}; protected: - const char * getName() const { return "unary minus expression"; } + const char * getName() const override { return "unary minus expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -159,9 +159,9 @@ private: ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique()}; protected: - const char * getName() const { return "multiplicative expression"; } + const char * getName() const override{ return "multiplicative expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return operator_parser.parse(pos, node, expected); } @@ -174,8 +174,8 @@ class ParserIntervalOperatorExpression : public IParserBase protected: ParserMultiplicativeExpression next_parser; - const char * getName() const { return "INTERVAL operator expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "INTERVAL operator expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -186,9 +186,9 @@ private: ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique()}; protected: - const char * getName() const { return "additive expression"; } + const char * getName() const override{ return "additive expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return operator_parser.parse(pos, node, expected); } @@ -200,9 +200,9 @@ class ParserConcatExpression : public IParserBase ParserVariableArityOperatorList operator_parser {"||", "concat", std::make_unique()}; protected: - const char * getName() const { return "string concatenation expression"; } + const char * getName() const override { return "string concatenation expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return operator_parser.parse(pos, node, expected); } @@ -215,9 +215,9 @@ private: ParserConcatExpression elem_parser; protected: - const char * getName() const { return "BETWEEN expression"; } + const char * getName() const override { return "BETWEEN expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -228,9 +228,9 @@ private: ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique()}; protected: - const char * getName() const { return "comparison expression"; } + const char * getName() const override{ return "comparison expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return operator_parser.parse(pos, node, expected); } @@ -257,9 +257,9 @@ private: ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique()}; protected: - const char * getName() const { return "logical-NOT expression"; } + const char * getName() const override{ return "logical-NOT expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return operator_parser.parse(pos, node, expected); } @@ -272,9 +272,9 @@ private: ParserVariableArityOperatorList operator_parser {"AND", "and", std::make_unique()}; protected: - const char * getName() const { return "logical-AND expression"; } + const char * getName() const override { return "logical-AND expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return operator_parser.parse(pos, node, expected); } @@ -287,9 +287,9 @@ private: ParserVariableArityOperatorList operator_parser {"OR", "or", std::make_unique()}; protected: - const char * getName() const { return "logical-OR expression"; } + const char * getName() const override { return "logical-OR expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return operator_parser.parse(pos, node, expected); } @@ -305,9 +305,9 @@ private: ParserLogicalOrExpression elem_parser; protected: - const char * getName() const { return "expression with ternary operator"; } + const char * getName() const override { return "expression with ternary operator"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -317,9 +317,9 @@ private: ParserTernaryOperatorExpression elem_parser; protected: - const char * getName() const { return "lambda expression"; } + const char * getName() const override { return "lambda expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -333,9 +333,9 @@ public: protected: ParserPtr impl; - const char * getName() const { return "expression with optional alias"; } + const char * getName() const override { return "expression with optional alias"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { return impl->parse(pos, node, expected); } @@ -352,8 +352,8 @@ public: protected: bool allow_alias_without_as_keyword; - const char * getName() const { return "list of expressions"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "list of expressions"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -365,16 +365,16 @@ public: private: ParserExpressionList nested_parser; protected: - const char * getName() const { return "not empty list of expressions"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "not empty list of expressions"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; class ParserOrderByExpressionList : public IParserBase { protected: - const char * getName() const { return "order by expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "order by expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -399,8 +399,8 @@ protected: class ParserTTLExpressionList : public IParserBase { protected: - const char * getName() const { return "ttl expression"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "ttl expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/IParser.h b/dbms/src/Parsers/IParser.h index 7b86d066393..925140bd25e 100644 --- a/dbms/src/Parsers/IParser.h +++ b/dbms/src/Parsers/IParser.h @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include @@ -57,13 +59,15 @@ public: using TokenIterator::TokenIterator; uint32_t depth = 0; - uint32_t max_depth = 1000; + uint32_t max_depth = 0; + + Pos(Tokens & tokens_, uint32_t max_depth_) : TokenIterator(tokens_), max_depth(max_depth_) {} void increaseDepth() { ++depth; - if (depth > max_depth) - throw Exception("Maximum parse depth exceeded", ErrorCodes::TOO_DEEP_RECURSION); + if (max_depth > 0 && depth > max_depth) + throw Exception("Maximum parse depth (" + toString(max_depth) + ") exceeded. Consider rising max_parser_depth parameter.", ErrorCodes::TOO_DEEP_RECURSION); } void decreaseDepth() diff --git a/dbms/src/Parsers/IParserBase.h b/dbms/src/Parsers/IParserBase.h index 95951d5acb8..100450263a4 100644 --- a/dbms/src/Parsers/IParserBase.h +++ b/dbms/src/Parsers/IParserBase.h @@ -35,7 +35,7 @@ public: return res; } - bool parse(Pos & pos, ASTPtr & node, Expected & expected); + bool parse(Pos & pos, ASTPtr & node, Expected & expected) override; protected: virtual bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) = 0; diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 6fb0865d652..61bf4d2075d 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -69,6 +69,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_to_disk("TO DISK"); ParserKeyword s_to_volume("TO VOLUME"); + ParserKeyword s_to_table("TO TABLE"); ParserKeyword s_delete_where("DELETE WHERE"); ParserKeyword s_update("UPDATE"); @@ -240,14 +241,23 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->move_destination_type = PartDestinationType::DISK; else if (s_to_volume.ignore(pos)) command->move_destination_type = PartDestinationType::VOLUME; + else if (s_to_table.ignore(pos)) + { + if (!parseDatabaseAndTableName(pos, expected, command->to_database, command->to_table)) + return false; + command->move_destination_type = PartDestinationType::TABLE; + } else return false; - ASTPtr ast_space_name; - if (!parser_string_literal.parse(pos, ast_space_name, expected)) - return false; + if (command->move_destination_type != PartDestinationType::TABLE) + { + ASTPtr ast_space_name; + if (!parser_string_literal.parse(pos, ast_space_name, expected)) + return false; - command->move_destination_name = ast_space_name->as().value.get(); + command->move_destination_name = ast_space_name->as().value.get(); + } } else if (s_move_partition.ignore(pos, expected)) { @@ -260,14 +270,23 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->move_destination_type = PartDestinationType::DISK; else if (s_to_volume.ignore(pos)) command->move_destination_type = PartDestinationType::VOLUME; + else if (s_to_table.ignore(pos)) + { + if (!parseDatabaseAndTableName(pos, expected, command->to_database, command->to_table)) + return false; + command->move_destination_type = PartDestinationType::TABLE; + } else return false; - ASTPtr ast_space_name; - if (!parser_string_literal.parse(pos, ast_space_name, expected)) - return false; + if (command->move_destination_type != PartDestinationType::TABLE) + { + ASTPtr ast_space_name; + if (!parser_string_literal.parse(pos, ast_space_name, expected)) + return false; - command->move_destination_name = ast_space_name->as().value.get(); + command->move_destination_name = ast_space_name->as().value.get(); + } } else if (s_add_constraint.ignore(pos, expected)) { diff --git a/dbms/src/Parsers/ParserAlterQuery.h b/dbms/src/Parsers/ParserAlterQuery.h index cff4bb49d8e..61a25b9b387 100644 --- a/dbms/src/Parsers/ParserAlterQuery.h +++ b/dbms/src/Parsers/ParserAlterQuery.h @@ -27,16 +27,16 @@ namespace DB class ParserAlterQuery : public IParserBase { protected: - const char * getName() const { return "ALTER query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "ALTER query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; class ParserAlterCommandList : public IParserBase { protected: - const char * getName() const { return "a list of ALTER commands"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "a list of ALTER commands"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; public: bool is_live_view; @@ -48,8 +48,8 @@ public: class ParserAlterCommand : public IParserBase { protected: - const char * getName() const { return "ALTER command"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "ALTER command"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; public: bool is_live_view; @@ -62,8 +62,8 @@ public: class ParserAssignment : public IParserBase { protected: - const char * getName() const { return "column assignment"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "column assignment"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserCheckQuery.h b/dbms/src/Parsers/ParserCheckQuery.h index 0ee587ac36d..fb0c390fa0d 100644 --- a/dbms/src/Parsers/ParserCheckQuery.h +++ b/dbms/src/Parsers/ParserCheckQuery.h @@ -10,8 +10,8 @@ namespace DB class ParserCheckQuery : public IParserBase { protected: - const char * getName() const { return "ALTER query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "ALTER query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index 2ff9d206f6d..6ec55e42ec8 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -19,8 +19,8 @@ namespace DB class ParserNestedTable : public IParserBase { protected: - const char * getName() const { return "nested table"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "nested table"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -33,16 +33,16 @@ protected: class ParserIdentifierWithParameters : public IParserBase { protected: - const char * getName() const { return "identifier with parameters"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "identifier with parameters"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; template class IParserNameTypePair : public IParserBase { protected: - const char * getName() const { return "name and type pair"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "name and type pair"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** The name and type are separated by a space. For example, URL String. */ @@ -75,16 +75,16 @@ bool IParserNameTypePair::parseImpl(Pos & pos, ASTPtr & node, Expect class ParserNameTypePairList : public IParserBase { protected: - const char * getName() const { return "name and type pair list"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "name and type pair list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** List of table names. */ class ParserNameList : public IParserBase { protected: - const char * getName() const { return "name list"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "name list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -99,9 +99,9 @@ public: protected: using ASTDeclarePtr = std::shared_ptr; - const char * getName() const { return "column declaration"; } + const char * getName() const override{ return "column declaration"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool require_type = true; }; @@ -224,8 +224,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E class ParserColumnDeclarationList : public IParserBase { protected: - const char * getName() const { return "column declaration list"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "column declaration list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -284,8 +284,8 @@ protected: class ParserStorage : public IParserBase { protected: - const char * getName() const { return "storage definition"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "storage definition"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /** Query like this: @@ -308,32 +308,32 @@ protected: class ParserCreateTableQuery : public IParserBase { protected: - const char * getName() const { return "CREATE TABLE or ATTACH TABLE query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "CREATE TABLE or ATTACH TABLE query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /// CREATE|ATTACH LIVE VIEW [IF NOT EXISTS] [db.]name [TO [db.]name] AS SELECT ... class ParserCreateLiveViewQuery : public IParserBase { protected: - const char * getName() const { return "CREATE LIVE VIEW query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "CREATE LIVE VIEW query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /// CREATE|ATTACH DATABASE db [ENGINE = engine] class ParserCreateDatabaseQuery : public IParserBase { protected: - const char * getName() const { return "CREATE DATABASE query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "CREATE DATABASE query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /// CREATE[OR REPLACE]|ATTACH [[MATERIALIZED] VIEW] | [VIEW]] [IF NOT EXISTS] [db.]name [TO [db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... class ParserCreateViewQuery : public IParserBase { protected: - const char * getName() const { return "CREATE VIEW query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "CREATE VIEW query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; /// Parses complete dictionary create query. Uses ParserDictionary and @@ -372,8 +372,8 @@ protected: class ParserCreateQuery : public IParserBase { protected: - const char * getName() const { return "CREATE TABLE or ATTACH TABLE query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "CREATE TABLE or ATTACH TABLE query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserDescribeTableQuery.h b/dbms/src/Parsers/ParserDescribeTableQuery.h index a00e2790195..caf3590ed32 100644 --- a/dbms/src/Parsers/ParserDescribeTableQuery.h +++ b/dbms/src/Parsers/ParserDescribeTableQuery.h @@ -13,8 +13,8 @@ namespace DB class ParserDescribeTableQuery : public IParserBase { protected: - const char * getName() const { return "DESCRIBE query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "DESCRIBE query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserDictionaryAttributeDeclaration.h b/dbms/src/Parsers/ParserDictionaryAttributeDeclaration.h index 725fb7ea4c7..ff4b145aa03 100644 --- a/dbms/src/Parsers/ParserDictionaryAttributeDeclaration.h +++ b/dbms/src/Parsers/ParserDictionaryAttributeDeclaration.h @@ -25,8 +25,8 @@ protected: class ParserDictionaryAttributeDeclarationList : public IParserBase { protected: - const char * getName() const { return "attribute declaration list"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "attribute declaration list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserDropQuery.h b/dbms/src/Parsers/ParserDropQuery.h index 04db03b4c3b..704f93de1cf 100644 --- a/dbms/src/Parsers/ParserDropQuery.h +++ b/dbms/src/Parsers/ParserDropQuery.h @@ -19,8 +19,8 @@ namespace DB class ParserDropQuery : public IParserBase { protected: - const char * getName() const { return "DROP query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "DROP query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool parseDropQuery(Pos & pos, ASTPtr & node, Expected & expected); bool parseDetachQuery(Pos & pos, ASTPtr & node, Expected & expected); diff --git a/dbms/src/Parsers/ParserOptimizeQuery.h b/dbms/src/Parsers/ParserOptimizeQuery.h index f81db083136..860b0b2927a 100644 --- a/dbms/src/Parsers/ParserOptimizeQuery.h +++ b/dbms/src/Parsers/ParserOptimizeQuery.h @@ -12,8 +12,8 @@ namespace DB class ParserOptimizeQuery : public IParserBase { protected: - const char * getName() const { return "OPTIMIZE query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "OPTIMIZE query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserRenameQuery.h b/dbms/src/Parsers/ParserRenameQuery.h index 8c1099f18eb..c95bc893b51 100644 --- a/dbms/src/Parsers/ParserRenameQuery.h +++ b/dbms/src/Parsers/ParserRenameQuery.h @@ -14,8 +14,8 @@ namespace DB class ParserRenameQuery : public IParserBase { protected: - const char * getName() const { return "RENAME query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "RENAME query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserRoleList.h b/dbms/src/Parsers/ParserRoleList.h index eca205a748c..2913a4953c8 100644 --- a/dbms/src/Parsers/ParserRoleList.h +++ b/dbms/src/Parsers/ParserRoleList.h @@ -11,8 +11,8 @@ namespace DB class ParserRoleList : public IParserBase { protected: - const char * getName() const { return "RoleList"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "RoleList"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserSampleRatio.h b/dbms/src/Parsers/ParserSampleRatio.h index b820e9f2e9e..b1888271d3a 100644 --- a/dbms/src/Parsers/ParserSampleRatio.h +++ b/dbms/src/Parsers/ParserSampleRatio.h @@ -12,8 +12,8 @@ namespace DB class ParserSampleRatio : public IParserBase { protected: - const char * getName() const { return "Sample ratio or offset"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "Sample ratio or offset"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserShowProcesslistQuery.h b/dbms/src/Parsers/ParserShowProcesslistQuery.h index 5d33b3f50fa..de08894e059 100644 --- a/dbms/src/Parsers/ParserShowProcesslistQuery.h +++ b/dbms/src/Parsers/ParserShowProcesslistQuery.h @@ -14,9 +14,9 @@ namespace DB class ParserShowProcesslistQuery : public IParserBase { protected: - const char * getName() const { return "SHOW PROCESSLIST query"; } + const char * getName() const override { return "SHOW PROCESSLIST query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { auto query = std::make_shared(); diff --git a/dbms/src/Parsers/ParserShowTablesQuery.h b/dbms/src/Parsers/ParserShowTablesQuery.h index 29b6d56159b..1bbd3cb4ef6 100644 --- a/dbms/src/Parsers/ParserShowTablesQuery.h +++ b/dbms/src/Parsers/ParserShowTablesQuery.h @@ -14,8 +14,8 @@ namespace DB class ParserShowTablesQuery : public IParserBase { protected: - const char * getName() const { return "SHOW [TEMPORARY] TABLES|DATABASES [[NOT] LIKE 'str'] [LIMIT expr]"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "SHOW [TEMPORARY] TABLES|DATABASES [[NOT] LIKE 'str'] [LIMIT expr]"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserTablesInSelectQuery.h b/dbms/src/Parsers/ParserTablesInSelectQuery.h index 9d46fc40fa1..82e4c3c171c 100644 --- a/dbms/src/Parsers/ParserTablesInSelectQuery.h +++ b/dbms/src/Parsers/ParserTablesInSelectQuery.h @@ -11,8 +11,8 @@ namespace DB class ParserTablesInSelectQuery : public IParserBase { protected: - const char * getName() const { return "table, table function, subquery or list of joined tables"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "table, table function, subquery or list of joined tables"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -22,8 +22,8 @@ public: ParserTablesInSelectQueryElement(bool is_first_) : is_first(is_first_) {} protected: - const char * getName() const { return "table, table function, subquery or list of joined tables"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "table, table function, subquery or list of joined tables"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; private: bool is_first; @@ -33,16 +33,16 @@ private: class ParserTableExpression : public IParserBase { protected: - const char * getName() const { return "table or subquery or table function"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "table or subquery or table function"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; class ParserArrayJoin : public IParserBase { protected: - const char * getName() const { return "array join"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "array join"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/dbms/src/Parsers/ParserUseQuery.h b/dbms/src/Parsers/ParserUseQuery.h index 9d6812c5505..f5b0be7a86c 100644 --- a/dbms/src/Parsers/ParserUseQuery.h +++ b/dbms/src/Parsers/ParserUseQuery.h @@ -11,8 +11,8 @@ namespace DB class ParserUseQuery : public IParserBase { protected: - const char * getName() const { return "USE query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override{ return "USE query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/ParserWatchQuery.h b/dbms/src/Parsers/ParserWatchQuery.h index 330f0a432df..e32043f7690 100644 --- a/dbms/src/Parsers/ParserWatchQuery.h +++ b/dbms/src/Parsers/ParserWatchQuery.h @@ -23,8 +23,8 @@ namespace DB class ParserWatchQuery : public IParserBase { protected: - const char * getName() const { return "WATCH query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); + const char * getName() const override { return "WATCH query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; } diff --git a/dbms/src/Parsers/parseQuery.cpp b/dbms/src/Parsers/parseQuery.cpp index da779ae83ba..b8ab940f2dd 100644 --- a/dbms/src/Parsers/parseQuery.cpp +++ b/dbms/src/Parsers/parseQuery.cpp @@ -215,10 +215,11 @@ ASTPtr tryParseQuery( bool hilite, const std::string & query_description, bool allow_multi_statements, - size_t max_query_size) + size_t max_query_size, + size_t max_parser_depth) { Tokens tokens(pos, end, max_query_size); - IParser::Pos token_iterator(tokens); + IParser::Pos token_iterator(tokens, max_parser_depth); if (token_iterator->isEnd() || token_iterator->type == TokenType::Semicolon) @@ -297,10 +298,11 @@ ASTPtr parseQueryAndMovePosition( const char * end, const std::string & query_description, bool allow_multi_statements, - size_t max_query_size) + size_t max_query_size, + size_t max_parser_depth) { std::string error_message; - ASTPtr res = tryParseQuery(parser, pos, end, error_message, false, query_description, allow_multi_statements, max_query_size); + ASTPtr res = tryParseQuery(parser, pos, end, error_message, false, query_description, allow_multi_statements, max_query_size, max_parser_depth); if (res) return res; @@ -314,10 +316,11 @@ ASTPtr parseQuery( const char * begin, const char * end, const std::string & query_description, - size_t max_query_size) + size_t max_query_size, + size_t max_parser_depth) { auto pos = begin; - return parseQueryAndMovePosition(parser, pos, end, query_description, false, max_query_size); + return parseQueryAndMovePosition(parser, pos, end, query_description, false, max_query_size, max_parser_depth); } diff --git a/dbms/src/Parsers/parseQuery.h b/dbms/src/Parsers/parseQuery.h index 85851eedacf..feea204181e 100644 --- a/dbms/src/Parsers/parseQuery.h +++ b/dbms/src/Parsers/parseQuery.h @@ -15,8 +15,9 @@ ASTPtr tryParseQuery( bool hilite, const std::string & description, bool allow_multi_statements, /// If false, check for non-space characters after semicolon and set error message if any. - size_t max_query_size); /// If (end - pos) > max_query_size and query is longer than max_query_size then throws "Max query size exceeded". + size_t max_query_size, /// If (end - pos) > max_query_size and query is longer than max_query_size then throws "Max query size exceeded". /// Disabled if zero. Is used in order to check query size if buffer can contains data for INSERT query. + size_t max_parser_depth = 0); /// Parse query or throw an exception with error message. @@ -26,15 +27,16 @@ ASTPtr parseQueryAndMovePosition( const char * end, const std::string & description, bool allow_multi_statements, - size_t max_query_size); - + size_t max_query_size = 0, + size_t max_parser_depth = 0); ASTPtr parseQuery( IParser & parser, const char * begin, const char * end, const std::string & description, - size_t max_query_size); + size_t max_query_size, + size_t max_parser_depth = 0); ASTPtr parseQuery( IParser & parser, diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index de32a07e240..05d4398d3c9 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -11,9 +11,6 @@ #include #include -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#include -#endif namespace DB { @@ -143,8 +140,7 @@ static void executeJob(IProcessor * processor) catch (Exception & exception) { if (checkCanAddAdditionalInfoToException(exception)) - exception.addMessage("While executing " + processor->getName() + " (" - + toString(reinterpret_cast(processor)) + ") "); + exception.addMessage("While executing " + processor->getName()); throw; } } @@ -259,14 +255,15 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue /// In this method we have ownership on node. auto & node = graph[pid]; - bool need_traverse = false; bool need_expand_pipeline = false; std::vector updated_back_edges; std::vector updated_direct_edges; { - /// Stopwatch watch; +#ifndef N_DEBUG + Stopwatch watch; +#endif std::unique_lock lock(std::move(node_lock)); @@ -280,7 +277,9 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue return false; } - /// node.execution_state->preparation_time_ns += watch.elapsed(); +#ifndef N_DEBUG + node.execution_state->preparation_time_ns += watch.elapsed(); +#endif node.updated_input_ports.clear(); node.updated_output_ports.clear(); @@ -290,13 +289,11 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue case IProcessor::Status::NeedData: case IProcessor::Status::PortFull: { - need_traverse = true; node.status = ExecStatus::Idle; break; } case IProcessor::Status::Finished: { - need_traverse = true; node.status = ExecStatus::Finished; break; } @@ -325,7 +322,6 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue } } - if (need_traverse) { for (auto & edge_id : node.post_updated_input_ports) { @@ -346,7 +342,6 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue } } - if (need_traverse) { for (auto & edge : updated_direct_edges) { @@ -469,11 +464,11 @@ void PipelineExecutor::execute(size_t num_threads) if (node.execution_state->exception) std::rethrow_exception(node.execution_state->exception); } - catch (Exception & exception) + catch (...) { - if (checkCanAddAdditionalInfoToException(exception)) - exception.addMessage("\nCurrent state:\n" + dumpPipeline()); - +#ifndef N_DEBUG + LOG_TRACE(log, "Exception while executing query. Current state:\n" << dumpPipeline()); +#endif throw; } @@ -491,28 +486,15 @@ void PipelineExecutor::execute(size_t num_threads) void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads) { -#if !defined(__APPLE__) && !defined(__FreeBSD__) - /// Specify CPU core for thread if can. - /// It may reduce the number of context swithches. - /* - if (num_threads > 1) - { - cpu_set_t cpu_set; - CPU_ZERO(&cpu_set); - CPU_SET(thread_num, &cpu_set); +#ifndef N_DEBUG + UInt64 total_time_ns = 0; + UInt64 execution_time_ns = 0; + UInt64 processing_time_ns = 0; + UInt64 wait_time_ns = 0; - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set) == -1) - LOG_TRACE(log, "Cannot set affinity for thread " << num_threads); - } - */ + Stopwatch total_time_watch; #endif -// UInt64 total_time_ns = 0; -// UInt64 execution_time_ns = 0; -// UInt64 processing_time_ns = 0; -// UInt64 wait_time_ns = 0; - -// Stopwatch total_time_watch; ExecutionState * state = nullptr; auto prepare_processor = [&](UInt64 pid, Queue & queue) @@ -543,7 +525,13 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads if (!task_queue.empty() && !threads_queue.empty() /*&& task_queue.quota() > threads_queue.size()*/) { - auto thread_to_wake = threads_queue.pop_any(); + auto thread_to_wake = task_queue.getAnyThreadWithTasks(thread_num + 1 == num_threads ? 0 : (thread_num + 1)); + + if (threads_queue.has(thread_to_wake)) + threads_queue.pop(thread_to_wake); + else + thread_to_wake = threads_queue.pop_any(); + lock.unlock(); wake_up_executor(thread_to_wake); } @@ -584,9 +572,15 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads addJob(state); { - // Stopwatch execution_time_watch; +#ifndef N_DEBUG + Stopwatch execution_time_watch; +#endif + state->job(); - // execution_time_ns += execution_time_watch.elapsed(); + +#ifndef N_DEBUG + execution_time_ns += execution_time_watch.elapsed(); +#endif } if (state->exception) @@ -595,7 +589,9 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads if (finished) break; - // Stopwatch processing_time_watch; +#ifndef N_DEBUG + Stopwatch processing_time_watch; +#endif /// Try to execute neighbour processor. { @@ -627,9 +623,15 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads queue.pop(); } - if (!threads_queue.empty() /* && task_queue.quota() > threads_queue.size()*/) + if (!threads_queue.empty() && !finished /* && task_queue.quota() > threads_queue.size()*/) { - auto thread_to_wake = threads_queue.pop_any(); + auto thread_to_wake = task_queue.getAnyThreadWithTasks(thread_num + 1 == num_threads ? 0 : (thread_num + 1)); + + if (threads_queue.has(thread_to_wake)) + threads_queue.pop(thread_to_wake); + else + thread_to_wake = threads_queue.pop_any(); + lock.unlock(); wake_up_executor(thread_to_wake); @@ -641,19 +643,22 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads doExpandPipeline(task, false); } - // processing_time_ns += processing_time_watch.elapsed(); +#ifndef N_DEBUG + processing_time_ns += processing_time_watch.elapsed(); +#endif } } -// total_time_ns = total_time_watch.elapsed(); -// wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; -/* +#ifndef N_DEBUG + total_time_ns = total_time_watch.elapsed(); + wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; + LOG_TRACE(log, "Thread finished." << " Total time: " << (total_time_ns / 1e9) << " sec." << " Execution time: " << (execution_time_ns / 1e9) << " sec." << " Processing time: " << (processing_time_ns / 1e9) << " sec." << " Wait time: " << (wait_time_ns / 1e9) << "sec."); -*/ +#endif } void PipelineExecutor::executeImpl(size_t num_threads) @@ -755,10 +760,18 @@ String PipelineExecutor::dumpPipeline() const for (auto & node : graph) { if (node.execution_state) - node.processor->setDescription( - "(" + std::to_string(node.execution_state->num_executed_jobs) + " jobs, execution time: " - + std::to_string(node.execution_state->execution_time_ns / 1e9) + " sec., preparation time: " - + std::to_string(node.execution_state->preparation_time_ns / 1e9) + " sec.)"); + { + WriteBufferFromOwnString buffer; + buffer << "(" << node.execution_state->num_executed_jobs << " jobs"; + +#ifndef N_DEBUG + buffer << ", execution time: " << node.execution_state->execution_time_ns / 1e9 << " sec."; + buffer << ", preparation time: " << node.execution_state->preparation_time_ns / 1e9 << " sec."; +#endif + + buffer << ")"; + node.processor->setDescription(buffer.str()); + } } std::vector statuses; diff --git a/dbms/src/Processors/Executors/PipelineExecutor.h b/dbms/src/Processors/Executors/PipelineExecutor.h index 2231c19284b..e8abc5788e3 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.h +++ b/dbms/src/Processors/Executors/PipelineExecutor.h @@ -149,32 +149,37 @@ private: ++quota_; } - ExecutionState * pop(size_t thread_num) + size_t getAnyThreadWithTasks(size_t from_thread = 0) { if (size_ == 0) - throw Exception("TaskQueue is not empty.", ErrorCodes::LOGICAL_ERROR); + throw Exception("TaskQueue is empty.", ErrorCodes::LOGICAL_ERROR); for (size_t i = 0; i < queues.size(); ++i) { - if (!queues[thread_num].empty()) - { - ExecutionState * state = queues[thread_num].front(); - queues[thread_num].pop(); + if (!queues[from_thread].empty()) + return from_thread; - --size_; - - if (state->has_quota) - ++quota_; - - return state; - } - - ++thread_num; - if (thread_num >= queues.size()) - thread_num = 0; + ++from_thread; + if (from_thread >= queues.size()) + from_thread = 0; } - throw Exception("TaskQueue is not empty.", ErrorCodes::LOGICAL_ERROR); + throw Exception("TaskQueue is empty.", ErrorCodes::LOGICAL_ERROR); + } + + ExecutionState * pop(size_t thread_num) + { + auto thread_with_tasks = getAnyThreadWithTasks(thread_num); + + ExecutionState * state = queues[thread_with_tasks].front(); + queues[thread_with_tasks].pop(); + + --size_; + + if (state->has_quota) + ++quota_; + + return state; } size_t size() const { return size_; } diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp new file mode 100644 index 00000000000..acbd892eb48 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -0,0 +1,671 @@ +#include "AvroRowInputFormat.h" +#if USE_AVRO + +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int THERE_IS_NO_COLUMN; + extern const int INCORRECT_DATA; + extern const int ILLEGAL_COLUMN; + extern const int TYPE_MISMATCH; +} + +class InputStreamReadBufferAdapter : public avro::InputStream +{ +public: + InputStreamReadBufferAdapter(ReadBuffer & in_) : in(in_) {} + + bool next(const uint8_t ** data, size_t * len) override + { + if (in.eof()) + { + *len = 0; + return false; + } + + *data = reinterpret_cast(in.position()); + *len = in.available(); + + in.position() += in.available(); + return true; + } + + void backup(size_t len) override { in.position() -= len; } + + void skip(size_t len) override { in.tryIgnore(len); } + + size_t byteCount() const override { return in.count(); } + +private: + ReadBuffer & in; +}; + +static void deserializeNoop(IColumn &, avro::Decoder &) +{ +} + +/// Insert value with conversion to the column of target type. +template +static void insertNumber(IColumn & column, WhichDataType type, T value) +{ + switch (type.idx) + { + case TypeIndex::UInt8: + assert_cast(column).insertValue(value); + break; + case TypeIndex::Date: [[fallthrough]]; + case TypeIndex::UInt16: + assert_cast(column).insertValue(value); + break; + case TypeIndex::DateTime: [[fallthrough]]; + case TypeIndex::UInt32: + assert_cast(column).insertValue(value); + break; + case TypeIndex::DateTime64: [[fallthrough]]; + case TypeIndex::UInt64: + assert_cast(column).insertValue(value); + break; + case TypeIndex::Int8: + assert_cast(column).insertValue(value); + break; + case TypeIndex::Int16: + assert_cast(column).insertValue(value); + break; + case TypeIndex::Int32: + assert_cast(column).insertValue(value); + break; + case TypeIndex::Int64: + assert_cast(column).insertValue(value); + break; + case TypeIndex::Float32: + assert_cast(column).insertValue(value); + break; + case TypeIndex::Float64: + assert_cast(column).insertValue(value); + break; + default: + throw Exception("Type is not compatible with Avro", ErrorCodes::ILLEGAL_COLUMN); + } +} + + +AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type) +{ + WhichDataType target(target_type); + switch (root_node->type()) + { + case avro::AVRO_STRING: [[fallthrough]]; + case avro::AVRO_BYTES: + if (target.isString() || target.isFixedString()) + { + return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable + { + decoder.decodeString(tmp); + column.insertData(tmp.c_str(), tmp.length()); + }; + } + break; + case avro::AVRO_INT: + return [target](IColumn & column, avro::Decoder & decoder) + { + insertNumber(column, target, decoder.decodeInt()); + }; + case avro::AVRO_LONG: + if (target.isDateTime64()) + { + auto date_time_scale = assert_cast(*target_type).getScale(); + auto logical_type = root_node->logicalType().type(); + if ((logical_type == avro::LogicalType::TIMESTAMP_MILLIS && date_time_scale == 3) + || (logical_type == avro::LogicalType::TIMESTAMP_MICROS && date_time_scale == 6)) + { + return [](IColumn & column, avro::Decoder & decoder) + { + assert_cast(column).insertValue(decoder.decodeLong()); + }; + } + } + else + { + return [target](IColumn & column, avro::Decoder & decoder) + { + insertNumber(column, target, decoder.decodeLong()); + }; + } + break; + case avro::AVRO_FLOAT: + return [target](IColumn & column, avro::Decoder & decoder) + { + insertNumber(column, target, decoder.decodeFloat()); + }; + case avro::AVRO_DOUBLE: + return [target](IColumn & column, avro::Decoder & decoder) + { + insertNumber(column, target, decoder.decodeDouble()); + }; + case avro::AVRO_BOOL: + return [target](IColumn & column, avro::Decoder & decoder) + { + insertNumber(column, target, decoder.decodeBool()); + }; + case avro::AVRO_ARRAY: + if (target.isArray()) + { + auto nested_source_type = root_node->leafAt(0); + auto nested_target_type = assert_cast(*target_type).getNestedType(); + auto nested_deserialize = createDeserializeFn(nested_source_type, nested_target_type); + return [nested_deserialize](IColumn & column, avro::Decoder & decoder) + { + ColumnArray & column_array = assert_cast(column); + ColumnArray::Offsets & offsets = column_array.getOffsets(); + IColumn & nested_column = column_array.getData(); + size_t total = 0; + for (size_t n = decoder.arrayStart(); n != 0; n = decoder.arrayNext()) + { + total += n; + for (size_t i = 0; i < n; i++) + { + nested_deserialize(nested_column, decoder); + } + } + offsets.push_back(offsets.back() + total); + }; + } + break; + case avro::AVRO_UNION: + { + auto nullable_deserializer = [root_node, target_type](size_t non_null_union_index) + { + auto nested_deserialize = createDeserializeFn(root_node->leafAt(non_null_union_index), removeNullable(target_type)); + return [non_null_union_index, nested_deserialize](IColumn & column, avro::Decoder & decoder) + { + ColumnNullable & col = assert_cast(column); + size_t union_index = decoder.decodeUnionIndex(); + if (union_index == non_null_union_index) + { + nested_deserialize(col.getNestedColumn(), decoder); + col.getNullMapData().push_back(0); + } + else + { + col.insertDefault(); + } + }; + }; + if (root_node->leaves() == 2 && target.isNullable()) + { + if (root_node->leafAt(0)->type() == avro::AVRO_NULL) + return nullable_deserializer(1); + if (root_node->leafAt(1)->type() == avro::AVRO_NULL) + return nullable_deserializer(0); + } + break; + } + case avro::AVRO_NULL: + if (target.isNullable()) + { + auto nested_type = removeNullable(target_type); + if (nested_type->getTypeId() == TypeIndex::Nothing) + { + return [](IColumn &, avro::Decoder & decoder) + { + decoder.decodeNull(); + }; + } + else + { + return [](IColumn & column, avro::Decoder & decoder) + { + ColumnNullable & col = assert_cast(column); + decoder.decodeNull(); + col.insertDefault(); + }; + } + } + break; + case avro::AVRO_ENUM: + if (target.isString()) + { + std::vector symbols; + for (size_t i = 0; i < root_node->names(); i++) + { + symbols.push_back(root_node->nameAt(i)); + } + return [symbols](IColumn & column, avro::Decoder & decoder) + { + size_t enum_index = decoder.decodeEnum(); + const auto & enum_symbol = symbols[enum_index]; + column.insertData(enum_symbol.c_str(), enum_symbol.length()); + }; + } + if (target.isEnum()) + { + const auto & enum_type = dynamic_cast(*target_type); + std::vector symbol_mapping; + for (size_t i = 0; i < root_node->names(); i++) + { + symbol_mapping.push_back(enum_type.castToValue(root_node->nameAt(i))); + } + return [symbol_mapping](IColumn & column, avro::Decoder & decoder) + { + size_t enum_index = decoder.decodeEnum(); + column.insert(symbol_mapping[enum_index]); + }; + } + break; + case avro::AVRO_FIXED: + { + size_t fixed_size = root_node->fixedSize(); + if (target.isFixedString() && target_type->getSizeOfValueInMemory() == fixed_size) + { + return [tmp_fixed = std::vector(fixed_size)](IColumn & column, avro::Decoder & decoder) mutable + { + decoder.decodeFixed(tmp_fixed.size(), tmp_fixed); + column.insertData(reinterpret_cast(tmp_fixed.data()), tmp_fixed.size()); + }; + } + break; + } + case avro::AVRO_MAP: [[fallthrough]]; + case avro::AVRO_RECORD: [[fallthrough]]; + default: + break; + } + + throw Exception( + "Type " + target_type->getName() + " is not compatible with Avro " + avro::ValidSchema(root_node).toJson(false), + ErrorCodes::ILLEGAL_COLUMN); +} + +AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) +{ + switch (root_node->type()) + { + case avro::AVRO_STRING: + return [](avro::Decoder & decoder) { decoder.skipString(); }; + case avro::AVRO_BYTES: + return [](avro::Decoder & decoder) { decoder.skipBytes(); }; + case avro::AVRO_INT: + return [](avro::Decoder & decoder) { decoder.decodeInt(); }; + case avro::AVRO_LONG: + return [](avro::Decoder & decoder) { decoder.decodeLong(); }; + case avro::AVRO_FLOAT: + return [](avro::Decoder & decoder) { decoder.decodeFloat(); }; + case avro::AVRO_DOUBLE: + return [](avro::Decoder & decoder) { decoder.decodeDouble(); }; + case avro::AVRO_BOOL: + return [](avro::Decoder & decoder) { decoder.decodeBool(); }; + case avro::AVRO_ARRAY: + { + auto nested_skip_fn = createSkipFn(root_node->leafAt(0)); + return [nested_skip_fn](avro::Decoder & decoder) + { + for (size_t n = decoder.arrayStart(); n != 0; n = decoder.arrayNext()) + { + for (size_t i = 0; i < n; ++i) + { + nested_skip_fn(decoder); + } + } + }; + } + case avro::AVRO_UNION: + { + std::vector union_skip_fns; + for (size_t i = 0; i < root_node->leaves(); i++) + { + union_skip_fns.push_back(createSkipFn(root_node->leafAt(i))); + } + return [union_skip_fns](avro::Decoder & decoder) { union_skip_fns[decoder.decodeUnionIndex()](decoder); }; + } + case avro::AVRO_NULL: + return [](avro::Decoder & decoder) { decoder.decodeNull(); }; + case avro::AVRO_ENUM: + return [](avro::Decoder & decoder) { decoder.decodeEnum(); }; + case avro::AVRO_FIXED: + { + auto fixed_size = root_node->fixedSize(); + return [fixed_size](avro::Decoder & decoder) { decoder.skipFixed(fixed_size); }; + } + case avro::AVRO_MAP: + { + auto value_skip_fn = createSkipFn(root_node->leafAt(1)); + return [value_skip_fn](avro::Decoder & decoder) + { + for (size_t n = decoder.mapStart(); n != 0; n = decoder.mapNext()) + { + for (size_t i = 0; i < n; ++i) + { + decoder.skipString(); + value_skip_fn(decoder); + } + } + }; + } + case avro::AVRO_RECORD: + { + std::vector field_skip_fns; + for (size_t i = 0; i < root_node->leaves(); i++) + { + field_skip_fns.push_back(createSkipFn(root_node->leafAt(i))); + } + return [field_skip_fns](avro::Decoder & decoder) + { + for (auto & skip_fn : field_skip_fns) + skip_fn(decoder); + }; + } + default: + throw Exception("Unsupported Avro type " + root_node->name().fullname() + " (" + toString(int(root_node->type())) + ")", ErrorCodes::ILLEGAL_COLUMN); + } +} + + +AvroDeserializer::AvroDeserializer(const ColumnsWithTypeAndName & columns, avro::ValidSchema schema) +{ + auto schema_root = schema.root(); + if (schema_root->type() != avro::AVRO_RECORD) + { + throw Exception("Root schema must be a record", ErrorCodes::TYPE_MISMATCH); + } + + field_mapping.resize(schema_root->leaves(), -1); + + for (size_t i = 0; i < schema_root->leaves(); ++i) + { + skip_fns.push_back(createSkipFn(schema_root->leafAt(i))); + deserialize_fns.push_back(&deserializeNoop); + } + + for (size_t i = 0; i < columns.size(); ++i) + { + const auto & column = columns[i]; + size_t field_index = 0; + if (!schema_root->nameIndex(column.name, field_index)) + { + throw Exception("Field " + column.name + " not found in Avro schema", ErrorCodes::THERE_IS_NO_COLUMN); + } + auto field_schema = schema_root->leafAt(field_index); + try + { + deserialize_fns[field_index] = createDeserializeFn(field_schema, column.type); + } + catch (Exception & e) + { + e.addMessage("column " + column.name); + throw; + } + field_mapping[field_index] = i; + } +} + +void AvroDeserializer::deserializeRow(MutableColumns & columns, avro::Decoder & decoder) +{ + for (size_t i = 0; i < field_mapping.size(); i++) + { + if (field_mapping[i] >= 0) + { + deserialize_fns[i](*columns[field_mapping[i]], decoder); + } + else + { + skip_fns[i](decoder); + } + } +} + + +AvroRowInputFormat::AvroRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) + : IRowInputFormat(header_, in_, params_) + , file_reader(std::make_unique(in_)) + , deserializer(header_.getColumnsWithTypeAndName(), file_reader.dataSchema()) +{ + file_reader.init(); +} + +bool AvroRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (file_reader.hasMore()) + { + file_reader.decr(); + deserializer.deserializeRow(columns, file_reader.decoder()); + return true; + } + return false; +} + +#if USE_POCO_JSON +class AvroConfluentRowInputFormat::SchemaRegistry +{ +public: + SchemaRegistry(const std::string & base_url_) + { + if (base_url_.empty()) + { + throw Exception("Empty Schema Registry URL", ErrorCodes::BAD_ARGUMENTS); + } + try + { + base_url = base_url_; + } + catch (const Poco::SyntaxException & e) + { + throw Exception("Invalid Schema Registry URL: " + e.displayText(), ErrorCodes::BAD_ARGUMENTS); + } + } + + avro::ValidSchema getSchema(uint32_t id) const + { + try + { + try + { + /// TODO Host checking to prevent SSRF + + Poco::URI url(base_url, "/schemas/ids/" + std::to_string(id)); + + /// One second for connect/send/receive. Just in case. + ConnectionTimeouts timeouts({1, 0}, {1, 0}, {1, 0}); + + Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, url.getPathAndQuery()); + + auto session = makePooledHTTPSession(url, timeouts, 1); + session->sendRequest(request); + + Poco::Net::HTTPResponse response; + auto & response_body = session->receiveResponse(response); + + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + { + throw Exception("HTTP code " + std::to_string(response.getStatus()), ErrorCodes::INCORRECT_DATA); + } + + Poco::JSON::Parser parser; + auto json_body = parser.parse(response_body).extract(); + auto schema = json_body->getValue("schema"); + return avro::compileJsonSchemaFromString(schema); + } + catch (const Exception &) + { + throw; + } + catch (const Poco::Exception & e) + { + throw Exception(Exception::CreateFromPoco, e); + } + catch (const avro::Exception & e) + { + throw Exception(e.what(), ErrorCodes::INCORRECT_DATA); + } + } + catch (Exception & e) + { + e.addMessage("while fetching schema id = " + std::to_string(id)); + throw; + } + } + +private: + Poco::URI base_url; +}; + +static uint32_t readConfluentSchemaId(ReadBuffer & in) +{ + uint8_t magic; + uint32_t schema_id; + + readBinaryBigEndian(magic, in); + readBinaryBigEndian(schema_id, in); + + if (magic != 0x00) + { + throw Exception("Invalid magic byte before AvroConfluent schema identifier." + " Must be zero byte, found " + std::to_string(int(magic)) + " instead", ErrorCodes::INCORRECT_DATA); + } + + return schema_id; +} + +AvroConfluentRowInputFormat::AvroConfluentRowInputFormat( + const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) + : IRowInputFormat(header_.cloneEmpty(), in_, params_) + , header_columns(header_.getColumnsWithTypeAndName()) + , schema_registry(std::make_unique(format_settings_.avro.schema_registry_url)) + , input_stream(std::make_unique(in)) + , decoder(avro::binaryDecoder()) + +{ + decoder->init(*input_stream); +} + +bool AvroConfluentRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (in.eof()) + { + return false; + } + SchemaId schema_id = readConfluentSchemaId(in); + auto & deserializer = getOrCreateDeserializer(schema_id); + deserializer.deserializeRow(columns, *decoder); + decoder->drain(); + return true; +} + +AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(SchemaId schema_id) +{ + auto it = deserializer_cache.find(schema_id); + if (it == deserializer_cache.end()) + { + auto schema = schema_registry->getSchema(schema_id); + AvroDeserializer deserializer(header_columns, schema); + it = deserializer_cache.emplace(schema_id, deserializer).first; + } + return it->second; +} +#endif + +void registerInputFormatProcessorAvro(FormatFactory & factory) +{ + factory.registerInputFormatProcessor("Avro", []( + ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & params, + const FormatSettings &) + { + return std::make_shared(sample, buf, params); + }); + +#if USE_POCO_JSON + + /// AvroConfluent format is disabled for the following reasons: + /// 1. There is no test for it. + /// 2. RemoteHostFilter is not used to prevent CSRF attacks. + +#if 0 + factory.registerInputFormatProcessor("AvroConfluent",[]( + ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & params, + const FormatSettings & settings) + { + return std::make_shared(sample, buf, params, settings); + }); +#endif + +#endif + +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerInputFormatProcessorAvro(FormatFactory &) +{ +} +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h new file mode 100644 index 00000000000..0fb979b4f4e --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -0,0 +1,79 @@ +#pragma once +#include "config_formats.h" +#include "config_core.h" +#if USE_AVRO + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ +class AvroDeserializer +{ +public: + AvroDeserializer(const ColumnsWithTypeAndName & columns, avro::ValidSchema schema); + void deserializeRow(MutableColumns & columns, avro::Decoder & decoder); + +private: + using DeserializeFn = std::function; + using SkipFn = std::function; + static DeserializeFn createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type); + static SkipFn createSkipFn(avro::NodePtr root_node); + + /// Map from field index in Avro schema to column number in block header. Or -1 if there is no corresponding column. + std::vector field_mapping; + + /// How to skip the corresponding field in Avro schema. + std::vector skip_fns; + + /// How to deserialize the corresponding field in Avro schema. + std::vector deserialize_fns; +}; + +class AvroRowInputFormat : public IRowInputFormat +{ +public: + AvroRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); + virtual bool readRow(MutableColumns & columns, RowReadExtension & ext) override; + String getName() const override { return "AvroRowInputFormat"; } + +private: + avro::DataFileReaderBase file_reader; + AvroDeserializer deserializer; +}; + +#if USE_POCO_JSON +class AvroConfluentRowInputFormat : public IRowInputFormat +{ +public: + AvroConfluentRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_); + virtual bool readRow(MutableColumns & columns, RowReadExtension & ext) override; + String getName() const override { return "AvroConfluentRowInputFormat"; } + +private: + const ColumnsWithTypeAndName header_columns; + + class SchemaRegistry; + std::unique_ptr schema_registry; + + using SchemaId = uint32_t; + std::unordered_map deserializer_cache; + AvroDeserializer & getOrCreateDeserializer(SchemaId schema_id); + + avro::InputStreamPtr input_stream; + avro::DecoderPtr decoder; +}; +#endif + +} +#endif diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp new file mode 100644 index 00000000000..26b427dfa31 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -0,0 +1,396 @@ +#include "AvroRowOutputFormat.h" +#if USE_AVRO + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; + extern const int BAD_ARGUMENTS; + extern const int THERE_IS_NO_COLUMN; + extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; + extern const int CANNOT_READ_ALL_DATA; +} + +class OutputStreamWriteBufferAdapter : public avro::OutputStream +{ +public: + OutputStreamWriteBufferAdapter(WriteBuffer & out_) : out(out_) {} + + virtual bool next(uint8_t ** data, size_t * len) override + { + out.nextIfAtEnd(); + *data = reinterpret_cast(out.position()); + *len = out.available(); + out.position() += out.available(); + + return true; + } + + virtual void backup(size_t len) override { out.position() -= len; } + + virtual uint64_t byteCount() const override { return out.count(); } + virtual void flush() override { out.next(); } + +private: + WriteBuffer & out; +}; + + +AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeFn(DataTypePtr data_type, size_t & type_name_increment) +{ + ++type_name_increment; + + switch (data_type->getTypeId()) + { + case TypeIndex::UInt8: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int8: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::UInt16: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int16: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::UInt32: [[fallthrough]]; + case TypeIndex::DateTime: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int32: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::UInt64: + return {avro::LongSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeLong(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int64: + return {avro::LongSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeLong(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Float32: + return {avro::FloatSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeFloat(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Float64: + return {avro::DoubleSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeDouble(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Date: + { + auto schema = avro::IntSchema(); + schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::DATE)); + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + UInt16 date = assert_cast(column).getElement(row_num); + encoder.encodeInt(date); + }}; + } + case TypeIndex::DateTime64: + { + auto schema = avro::LongSchema(); + const auto & provided_type = assert_cast(*data_type); + + if (provided_type.getScale() == 3) + schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::TIMESTAMP_MILLIS)); + else if (provided_type.getScale() == 6) + schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::TIMESTAMP_MICROS)); + else + break; + + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const auto & col = assert_cast(column); + encoder.encodeLong(col.getElement(row_num)); + }}; + } + case TypeIndex::String: + return {avro::BytesSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const StringRef & s = assert_cast(column).getDataAt(row_num); + encoder.encodeBytes(reinterpret_cast(s.data), s.size); + }}; + case TypeIndex::FixedString: + { + auto size = data_type->getSizeOfValueInMemory(); + auto schema = avro::FixedSchema(size, "fixed_" + toString(type_name_increment)); + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const StringRef & s = assert_cast(column).getDataAt(row_num); + encoder.encodeFixed(reinterpret_cast(s.data), s.size); + }}; + } + case TypeIndex::Enum8: + { + auto schema = avro::EnumSchema("enum8_" + toString(type_name_increment)); /// type names must be different for different types. + std::unordered_map enum_mapping; + const auto & enum_values = assert_cast(*data_type).getValues(); + for (size_t i = 0; i < enum_values.size(); ++i) + { + schema.addSymbol(enum_values[i].first); + enum_mapping.emplace(enum_values[i].second, i); + } + return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + auto enum_value = assert_cast(column).getElement(row_num); + encoder.encodeEnum(enum_mapping.at(enum_value)); + }}; + } + case TypeIndex::Enum16: + { + auto schema = avro::EnumSchema("enum16" + toString(type_name_increment)); + std::unordered_map enum_mapping; + const auto & enum_values = assert_cast(*data_type).getValues(); + for (size_t i = 0; i < enum_values.size(); ++i) + { + schema.addSymbol(enum_values[i].first); + enum_mapping.emplace(enum_values[i].second, i); + } + return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + auto enum_value = assert_cast(column).getElement(row_num); + encoder.encodeEnum(enum_mapping.at(enum_value)); + }}; + } + case TypeIndex::Array: + { + const auto & array_type = assert_cast(*data_type); + auto nested_mapping = createSchemaWithSerializeFn(array_type.getNestedType(), type_name_increment); + auto schema = avro::ArraySchema(nested_mapping.schema); + return {schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const ColumnArray & column_array = assert_cast(column); + const ColumnArray::Offsets & offsets = column_array.getOffsets(); + size_t offset = offsets[row_num - 1]; + size_t next_offset = offsets[row_num]; + size_t row_count = next_offset - offset; + const IColumn & nested_column = column_array.getData(); + + encoder.arrayStart(); + if (row_count > 0) + { + encoder.setItemCount(row_count); + } + for (size_t i = offset; i < next_offset; ++i) + { + nested_mapping.serialize(nested_column, i, encoder); + } + encoder.arrayEnd(); + }}; + } + case TypeIndex::Nullable: + { + auto nested_type = removeNullable(data_type); + auto nested_mapping = createSchemaWithSerializeFn(nested_type, type_name_increment); + if (nested_type->getTypeId() == TypeIndex::Nothing) + { + return nested_mapping; + } + else + { + avro::UnionSchema union_schema; + union_schema.addType(avro::NullSchema()); + union_schema.addType(nested_mapping.schema); + return {union_schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const ColumnNullable & col = assert_cast(column); + if (!col.isNullAt(row_num)) + { + encoder.encodeUnionIndex(1); + nested_mapping.serialize(col.getNestedColumn(), row_num, encoder); + } + else + { + encoder.encodeUnionIndex(0); + encoder.encodeNull(); + } + }}; + } + } + case TypeIndex::LowCardinality: + { + const auto & nested_type = removeLowCardinality(data_type); + auto nested_mapping = createSchemaWithSerializeFn(nested_type, type_name_increment); + return {nested_mapping.schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const auto & col = assert_cast(column); + nested_mapping.serialize(*col.getDictionary().getNestedColumn(), col.getIndexAt(row_num), encoder); + }}; + } + case TypeIndex::Nothing: + return {avro::NullSchema(), [](const IColumn &, size_t, avro::Encoder & encoder) { encoder.encodeNull(); }}; + default: + break; + } + throw Exception("Type " + data_type->getName() + " is not supported for Avro output", ErrorCodes::ILLEGAL_COLUMN); +} + + +AvroSerializer::AvroSerializer(const ColumnsWithTypeAndName & columns) +{ + avro::RecordSchema record_schema("row"); + + size_t type_name_increment = 0; + for (auto & column : columns) + { + try + { + auto field_mapping = createSchemaWithSerializeFn(column.type, type_name_increment); + serialize_fns.push_back(field_mapping.serialize); + //TODO: verify name starts with A-Za-z_ + record_schema.addField(column.name, field_mapping.schema); + } + catch (Exception & e) + { + e.addMessage("column " + column.name); + throw; + } + } + schema.setSchema(record_schema); +} + +void AvroSerializer::serializeRow(const Columns & columns, size_t row_num, avro::Encoder & encoder) +{ + size_t num_columns = columns.size(); + for (size_t i = 0; i < num_columns; ++i) + { + serialize_fns[i](*columns[i], row_num, encoder); + } +} + +static avro::Codec getCodec(const std::string & codec_name) +{ + if (codec_name == "") + { +#ifdef SNAPPY_CODEC_AVAILABLE + return avro::Codec::SNAPPY_CODEC; +#else + return avro::Codec::DEFLATE_CODEC; +#endif + } + + if (codec_name == "null") return avro::Codec::NULL_CODEC; + if (codec_name == "deflate") return avro::Codec::DEFLATE_CODEC; +#ifdef SNAPPY_CODEC_AVAILABLE + if (codec_name == "snappy") return avro::Codec::SNAPPY_CODEC; +#endif + + throw Exception("Avro codec " + codec_name + " is not available", ErrorCodes::BAD_ARGUMENTS); +} + +AvroRowOutputFormat::AvroRowOutputFormat( + WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_) + : IRowOutputFormat(header_, out_, callback) + , settings(settings_) + , serializer(header_.getColumnsWithTypeAndName()) + , file_writer( + std::make_unique(out_), + serializer.getSchema(), + settings.avro.output_sync_interval, + getCodec(settings.avro.output_codec)) +{ +} + +AvroRowOutputFormat::~AvroRowOutputFormat() = default; + +void AvroRowOutputFormat::writePrefix() +{ + file_writer.syncIfNeeded(); +} + +void AvroRowOutputFormat::write(const Columns & columns, size_t row_num) +{ + file_writer.syncIfNeeded(); + serializer.serializeRow(columns, row_num, file_writer.encoder()); + file_writer.incr(); +} + +void AvroRowOutputFormat::writeSuffix() +{ + file_writer.close(); +} + +void registerOutputFormatProcessorAvro(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor("Avro", []( + WriteBuffer & buf, + const Block & sample, + FormatFactory::WriteCallback callback, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, callback, settings); + }); +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerOutputFormatProcessorAvro(FormatFactory &) +{ +} +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.h new file mode 100644 index 00000000000..4d404337d74 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.h @@ -0,0 +1,62 @@ +#pragma once +#include "config_formats.h" +#if USE_AVRO +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace DB +{ +class WriteBuffer; + +class AvroSerializer +{ +public: + AvroSerializer(const ColumnsWithTypeAndName & columns); + const avro::ValidSchema & getSchema() const { return schema; } + void serializeRow(const Columns & columns, size_t row_num, avro::Encoder & encoder); + +private: + using SerializeFn = std::function; + struct SchemaWithSerializeFn + { + avro::Schema schema; + SerializeFn serialize; + }; + + /// Type names for different complex types (e.g. enums, fixed strings) must be unique. We use simple incremental number to give them different names. + static SchemaWithSerializeFn createSchemaWithSerializeFn(DataTypePtr data_type, size_t & type_name_increment); + + std::vector serialize_fns; + avro::ValidSchema schema; +}; + +class AvroRowOutputFormat : public IRowOutputFormat +{ +public: + AvroRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_); + virtual ~AvroRowOutputFormat() override; + + String getName() const override { return "AvroRowOutputFormat"; } + void write(const Columns & columns, size_t row_num) override; + void writeField(const IColumn &, const IDataType &, size_t) override {} + virtual void writePrefix() override; + virtual void writeSuffix() override; + +private: + FormatSettings settings; + AvroSerializer serializer; + avro::DataFileWriterBase file_writer; +}; + +} +#endif diff --git a/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index 0c7f06f6022..3bc131df07d 100644 --- a/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -328,12 +328,12 @@ ConstantExpressionTemplate::Cache::getFromCacheOrConstruct(const DataTypePtr & r return res; } -bool ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const FormatSettings & settings) +bool ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const FormatSettings & format_settings, const Settings & settings) { size_t cur_column = 0; try { - if (tryParseExpression(istr, settings, cur_column)) + if (tryParseExpression(istr, format_settings, cur_column, settings)) { ++rows_count; return true; @@ -355,7 +355,7 @@ bool ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const Format return false; } -bool ConstantExpressionTemplate::tryParseExpression(ReadBuffer & istr, const FormatSettings & settings, size_t & cur_column) +bool ConstantExpressionTemplate::tryParseExpression(ReadBuffer & istr, const FormatSettings & format_settings, size_t & cur_column, const Settings & settings) { size_t cur_token = 0; size_t num_columns = structure->literals.columns(); @@ -372,13 +372,13 @@ bool ConstantExpressionTemplate::tryParseExpression(ReadBuffer & istr, const For skipWhitespaceIfAny(istr); const DataTypePtr & type = structure->literals.getByPosition(cur_column).type; - if (settings.values.accurate_types_of_literals && !structure->special_parser[cur_column].useDefaultParser()) + if (format_settings.values.accurate_types_of_literals && !structure->special_parser[cur_column].useDefaultParser()) { - if (!parseLiteralAndAssertType(istr, type.get(), cur_column)) + if (!parseLiteralAndAssertType(istr, type.get(), cur_column, settings)) return false; } else - type->deserializeAsTextQuoted(*columns[cur_column], istr, settings); + type->deserializeAsTextQuoted(*columns[cur_column], istr, format_settings); ++cur_column; } @@ -392,7 +392,7 @@ bool ConstantExpressionTemplate::tryParseExpression(ReadBuffer & istr, const For return true; } -bool ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, const IDataType * complex_type, size_t column_idx) +bool ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, const IDataType * complex_type, size_t column_idx, const Settings & settings) { using Type = Field::Types::Which; @@ -410,7 +410,7 @@ bool ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, co /// TODO faster way to check types without using Parsers ParserArrayOfLiterals parser_array; Tokens tokens_number(istr.position(), istr.buffer().end()); - IParser::Pos iterator(tokens_number); + IParser::Pos iterator(tokens_number, settings.max_parser_depth); Expected expected; ASTPtr ast; diff --git a/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.h b/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.h index 96e1c653c03..bbac2e2a999 100644 --- a/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.h +++ b/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.h @@ -66,7 +66,7 @@ public: /// Read expression from istr, assert it has the same structure and the same types of literals (template matches) /// and parse literals into temporary columns - bool parseExpression(ReadBuffer & istr, const FormatSettings & settings); + bool parseExpression(ReadBuffer & istr, const FormatSettings & format_settings, const Settings & settings); /// Evaluate batch of expressions were parsed using template. /// If template was deduced with null_as_default == true, set bits in nulls for NULL values in column_idx, starting from offset. @@ -75,8 +75,8 @@ public: size_t rowsCount() const { return rows_count; } private: - bool tryParseExpression(ReadBuffer & istr, const FormatSettings & settings, size_t & cur_column); - bool parseLiteralAndAssertType(ReadBuffer & istr, const IDataType * type, size_t column_idx); + bool tryParseExpression(ReadBuffer & istr, const FormatSettings & format_settings, size_t & cur_column, const Settings & settings); + bool parseLiteralAndAssertType(ReadBuffer & istr, const IDataType * type, size_t column_idx, const Settings & settings); private: TemplateStructurePtr structure; diff --git a/dbms/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/dbms/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index b324719527a..9ce3d62a244 100644 --- a/dbms/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -129,7 +129,8 @@ void ValuesBlockInputFormat::readRow(MutableColumns & columns, size_t row_num) bool ValuesBlockInputFormat::tryParseExpressionUsingTemplate(MutableColumnPtr & column, size_t column_idx) { /// Try to parse expression using template if one was successfully deduced while parsing the first row - if (templates[column_idx]->parseExpression(buf, format_settings)) + auto settings = context->getSettingsRef(); + if (templates[column_idx]->parseExpression(buf, format_settings, settings)) { ++rows_parsed_using_template[column_idx]; return true; @@ -187,6 +188,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx { const Block & header = getPort().getHeader(); const IDataType & type = *header.getByPosition(column_idx).type; + auto settings = context->getSettingsRef(); /// We need continuous memory containing the expression to use Lexer skipToNextRow(0, 1); @@ -195,7 +197,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx Expected expected; Tokens tokens(buf.position(), buf.buffer().end()); - IParser::Pos token_iterator(tokens); + IParser::Pos token_iterator(tokens, settings.max_parser_depth); ASTPtr ast; bool parsed = parser.parse(token_iterator, ast, expected); @@ -265,7 +267,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx ++attempts_to_deduce_template[column_idx]; buf.rollbackToCheckpoint(); - if (templates[column_idx]->parseExpression(buf, format_settings)) + if (templates[column_idx]->parseExpression(buf, format_settings, settings)) { ++rows_parsed_using_template[column_idx]; parser_type_for_column[column_idx] = ParserType::BatchTemplate; diff --git a/dbms/src/Processors/Formats/LazyOutputFormat.h b/dbms/src/Processors/Formats/LazyOutputFormat.h index 56aaf249480..59cdbc4e81e 100644 --- a/dbms/src/Processors/Formats/LazyOutputFormat.h +++ b/dbms/src/Processors/Formats/LazyOutputFormat.h @@ -7,6 +7,9 @@ namespace DB { +/// LazyOutputFormat is used to retrieve ready data from executing pipeline. +/// You can periodically call `getBlock` from separate thread. +/// Used in TCPHandler. class LazyOutputFormat : public IOutputFormat { @@ -20,14 +23,18 @@ public: Block getTotals(); Block getExtremes(); - bool isFinished() { return finished_processing; } + bool isFinished() { return finished_processing && queue.size() == 0; } BlockStreamProfileInfo & getProfileInfo() { return info; } void setRowsBeforeLimit(size_t rows_before_limit) override; - void finish() { finished_processing = true; } - void clearQueue() { queue.clear(); } + void finish() + { + finished_processing = true; + /// Clear queue in case if somebody is waiting lazy_format to push. + queue.clear(); + } protected: void consume(Chunk chunk) override diff --git a/dbms/src/Processors/Formats/OutputStreamToOutputFormat.h b/dbms/src/Processors/Formats/OutputStreamToOutputFormat.h index 350486f9b41..a85de12b49d 100644 --- a/dbms/src/Processors/Formats/OutputStreamToOutputFormat.h +++ b/dbms/src/Processors/Formats/OutputStreamToOutputFormat.h @@ -9,6 +9,7 @@ class IOutputFormat; using OutputFormatPtr = std::shared_ptr; +/// Wrapper. Implements IBlockOutputStream interface using IOutputFormat object. class OutputStreamToOutputFormat : public IBlockOutputStream { public: diff --git a/dbms/src/Processors/IInflatingTransform.h b/dbms/src/Processors/IInflatingTransform.h index 45edf5302e5..0ad12f6cd65 100644 --- a/dbms/src/Processors/IInflatingTransform.h +++ b/dbms/src/Processors/IInflatingTransform.h @@ -4,6 +4,20 @@ namespace DB { +/// Transform which can generate several chunks on every consumed. +/// It can be assumed that class is used in following way: +/// +/// for (chunk : input_chunks) +/// { +/// transform.consume(chunk); +/// +/// while (transform.canGenerate()) +/// { +/// transformed_chunk = transform.generate(); +/// ... (process transformed chunk) +/// } +/// } +/// class IInflatingTransform : public IProcessor { protected: diff --git a/dbms/src/Processors/ISimpleTransform.cpp b/dbms/src/Processors/ISimpleTransform.cpp index 292ea24b0ef..fc8f1ba30f4 100644 --- a/dbms/src/Processors/ISimpleTransform.cpp +++ b/dbms/src/Processors/ISimpleTransform.cpp @@ -33,6 +33,10 @@ ISimpleTransform::Status ISimpleTransform::prepare() { output.pushData(std::move(current_data)); transformed = false; + + if (!no_more_data_needed) + return Status::PortFull; + } /// Stop if don't need more data. @@ -52,12 +56,13 @@ ISimpleTransform::Status ISimpleTransform::prepare() return Status::Finished; } - input.setNeeded(); - if (!input.hasData()) + { + input.setNeeded(); return Status::NeedData; + } - current_data = input.pullData(); + current_data = input.pullData(true); has_input = true; if (current_data.exception) @@ -95,7 +100,7 @@ void ISimpleTransform::work() return; } - has_input = false; + has_input = !needInputData(); if (!skip_empty_chunks || current_data.chunk) transformed = true; diff --git a/dbms/src/Processors/ISimpleTransform.h b/dbms/src/Processors/ISimpleTransform.h index 0a952840964..1f0c5a5b040 100644 --- a/dbms/src/Processors/ISimpleTransform.h +++ b/dbms/src/Processors/ISimpleTransform.h @@ -27,6 +27,7 @@ protected: bool set_input_not_needed_after_read = false; virtual void transform(Chunk & chunk) = 0; + virtual bool needInputData() const { return true; } void stopReading() { no_more_data_needed = true; } public: diff --git a/dbms/src/Processors/Port.h b/dbms/src/Processors/Port.h index ff5d1d8dee0..c5b3c5979d1 100644 --- a/dbms/src/Processors/Port.h +++ b/dbms/src/Processors/Port.h @@ -161,12 +161,17 @@ protected: throw Exception("Cannot push block to port which already has data.", ErrorCodes::LOGICAL_ERROR); } - void ALWAYS_INLINE pull(DataPtr & data_, std::uintptr_t & flags) + void ALWAYS_INLINE pull(DataPtr & data_, std::uintptr_t & flags, bool set_not_needed = false) { - flags = data_.swap(data, 0, HAS_DATA); + uintptr_t mask = HAS_DATA; + + if (set_not_needed) + mask |= IS_NEEDED; + + flags = data_.swap(data, 0, mask); /// It's ok to check because this flag can be changed only by pulling thread. - if (unlikely((flags & IS_NEEDED) == 0)) + if (unlikely((flags & IS_NEEDED) == 0) && !set_not_needed) throw Exception("Cannot pull block from port which is not needed.", ErrorCodes::LOGICAL_ERROR); if (unlikely((flags & HAS_DATA) == 0)) @@ -266,14 +271,15 @@ private: public: using Port::Port; - Data ALWAYS_INLINE pullData() + Data ALWAYS_INLINE pullData(bool set_not_needed = false) { - updateVersion(); + if (!set_not_needed) + updateVersion(); assumeConnected(); std::uintptr_t flags = 0; - state->pull(data, flags); + state->pull(data, flags, set_not_needed); is_finished = flags & State::IS_FINISHED; @@ -293,9 +299,9 @@ public: return std::move(*data); } - Chunk ALWAYS_INLINE pull() + Chunk ALWAYS_INLINE pull(bool set_not_needed = false) { - auto data_ = pullData(); + auto data_ = pullData(set_not_needed); if (data_.exception) std::rethrow_exception(data_.exception); diff --git a/dbms/src/Processors/QueryPipeline.cpp b/dbms/src/Processors/QueryPipeline.cpp index 25abeb6c6d3..ed62e2a1e58 100644 --- a/dbms/src/Processors/QueryPipeline.cpp +++ b/dbms/src/Processors/QueryPipeline.cpp @@ -234,7 +234,7 @@ void QueryPipeline::addDelayedStream(ProcessorPtr source) addPipe({ std::move(processor) }); } -void QueryPipeline::resize(size_t num_streams, bool force) +void QueryPipeline::resize(size_t num_streams, bool force, bool strict) { checkInitialized(); @@ -243,7 +243,13 @@ void QueryPipeline::resize(size_t num_streams, bool force) has_resize = true; - auto resize = std::make_shared(current_header, getNumStreams(), num_streams); + ProcessorPtr resize; + + if (strict) + resize = std::make_shared(current_header, getNumStreams(), num_streams); + else + resize = std::make_shared(current_header, getNumStreams(), num_streams); + auto stream = streams.begin(); for (auto & input : resize->getInputs()) connect(**(stream++), input); diff --git a/dbms/src/Processors/QueryPipeline.h b/dbms/src/Processors/QueryPipeline.h index 29ebaf22955..d8320bb51d6 100644 --- a/dbms/src/Processors/QueryPipeline.h +++ b/dbms/src/Processors/QueryPipeline.h @@ -29,11 +29,14 @@ public: void init(Pipe pipe); /// Simple init for single pipe bool initialized() { return !processors.empty(); } + /// Type of logical data stream for simple transform. + /// Sometimes it's important to know which part of pipeline we are working for. + /// Example: ExpressionTransform need special logic for totals. enum class StreamType { - Main = 0, - Totals, - Extremes, + Main = 0, /// Stream for query data. There may be several streams of this type. + Totals, /// Stream for totals. No more then one. + Extremes, /// Stream for extremes. No more then one. }; using ProcessorGetter = std::function; @@ -61,7 +64,7 @@ public: /// Check if resize transform was used. (In that case another distinct transform will be added). bool hasMixedStreams() const { return has_resize || hasMoreThanOneStream(); } - void resize(size_t num_streams, bool force = false); + void resize(size_t num_streams, bool force = false, bool strict = false); void enableQuotaForCurrentStreams(); diff --git a/dbms/src/Processors/ResizeProcessor.cpp b/dbms/src/Processors/ResizeProcessor.cpp index 59d1f0db75e..26cae466727 100644 --- a/dbms/src/Processors/ResizeProcessor.cpp +++ b/dbms/src/Processors/ResizeProcessor.cpp @@ -1,5 +1,5 @@ #include - +#include namespace DB { @@ -257,5 +257,143 @@ IProcessor::Status ResizeProcessor::prepare(const PortNumbers & updated_inputs, return Status::PortFull; } +IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_inputs, const PortNumbers & updated_outputs) +{ + if (!initialized) + { + initialized = true; + + for (auto & input : inputs) + input_ports.push_back({.port = &input, .status = InputStatus::NotActive, .waiting_output = -1}); + + for (UInt64 i = 0; i < input_ports.size(); ++i) + disabled_input_ports.push(i); + + for (auto & output : outputs) + output_ports.push_back({.port = &output, .status = OutputStatus::NotActive}); + } + + for (auto & output_number : updated_outputs) + { + auto & output = output_ports[output_number]; + if (output.port->isFinished()) + { + if (output.status != OutputStatus::Finished) + { + ++num_finished_outputs; + output.status = OutputStatus::Finished; + } + + continue; + } + + if (output.port->canPush()) + { + if (output.status != OutputStatus::NeedData) + { + output.status = OutputStatus::NeedData; + waiting_outputs.push(output_number); + } + } + } + + if (num_finished_outputs == outputs.size()) + { + for (auto & input : inputs) + input.close(); + + return Status::Finished; + } + + std::queue inputs_with_data; + + for (auto & input_number : updated_inputs) + { + auto & input = input_ports[input_number]; + if (input.port->isFinished()) + { + if (input.status != InputStatus::Finished) + { + input.status = InputStatus::Finished; + ++num_finished_inputs; + + waiting_outputs.push(input.waiting_output); + } + continue; + } + + if (input.port->hasData()) + { + if (input.status != InputStatus::NotActive) + { + input.status = InputStatus::NotActive; + inputs_with_data.push(input_number); + } + } + } + + while (!inputs_with_data.empty()) + { + auto input_number = inputs_with_data.front(); + auto & input_with_data = input_ports[input_number]; + inputs_with_data.pop(); + + if (input_with_data.waiting_output == -1) + throw Exception("No associated output for input with data.", ErrorCodes::LOGICAL_ERROR); + + auto & waiting_output = output_ports[input_with_data.waiting_output]; + + if (waiting_output.status != OutputStatus::NeedData) + throw Exception("Invalid status for associated output.", ErrorCodes::LOGICAL_ERROR); + + waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_deeded = */ true)); + waiting_output.status = OutputStatus::NotActive; + + if (input_with_data.port->isFinished()) + { + input_with_data.status = InputStatus::Finished; + ++num_finished_inputs; + } + else + disabled_input_ports.push(input_number); + } + + if (num_finished_inputs == inputs.size()) + { + for (auto & output : outputs) + output.finish(); + + return Status::Finished; + } + + /// Enable more inputs if needed. + while (!disabled_input_ports.empty() && !waiting_outputs.empty()) + { + auto & input = input_ports[disabled_input_ports.front()]; + disabled_input_ports.pop(); + + input.port->setNeeded(); + input.status = InputStatus::NeedData; + input.waiting_output = waiting_outputs.front(); + + waiting_outputs.pop(); + } + + while (!waiting_outputs.empty()) + { + auto & output = output_ports[waiting_outputs.front()]; + waiting_outputs.pop(); + + output.status = OutputStatus::Finished; + output.port->finish(); + ++num_finished_outputs; + } + + if (disabled_input_ports.empty()) + return Status::NeedData; + + return Status::PortFull; +} + } diff --git a/dbms/src/Processors/ResizeProcessor.h b/dbms/src/Processors/ResizeProcessor.h index 3a9c906ecbd..1f364ffcf15 100644 --- a/dbms/src/Processors/ResizeProcessor.h +++ b/dbms/src/Processors/ResizeProcessor.h @@ -74,4 +74,60 @@ private: std::vector output_ports; }; +class StrictResizeProcessor : public IProcessor +{ +public: + /// TODO Check that there is non zero number of inputs and outputs. + StrictResizeProcessor(const Block & header, size_t num_inputs, size_t num_outputs) + : IProcessor(InputPorts(num_inputs, header), OutputPorts(num_outputs, header)) + , current_input(inputs.begin()) + , current_output(outputs.begin()) + { + } + + String getName() const override { return "StrictResize"; } + + Status prepare(const PortNumbers &, const PortNumbers &) override; + +private: + InputPorts::iterator current_input; + OutputPorts::iterator current_output; + + size_t num_finished_inputs = 0; + size_t num_finished_outputs = 0; + std::queue disabled_input_ports; + std::queue waiting_outputs; + bool initialized = false; + + enum class OutputStatus + { + NotActive, + NeedData, + Finished, + }; + + enum class InputStatus + { + NotActive, + NeedData, + Finished, + }; + + struct InputPortWithStatus + { + InputPort * port; + InputStatus status; + ssize_t waiting_output; + }; + + struct OutputPortWithStatus + { + OutputPort * port; + OutputStatus status; + }; + + std::vector input_ports; + std::vector output_ports; +}; + } diff --git a/dbms/src/Processors/Transforms/AggregatingTransform.cpp b/dbms/src/Processors/Transforms/AggregatingTransform.cpp index 7a5bf77da68..5e13813d2a9 100644 --- a/dbms/src/Processors/Transforms/AggregatingTransform.cpp +++ b/dbms/src/Processors/Transforms/AggregatingTransform.cpp @@ -6,6 +6,7 @@ #include #include + namespace ProfileEvents { extern const Event ExternalAggregationMerge; @@ -14,23 +15,36 @@ namespace ProfileEvents namespace DB { -/// Convert block to chunk. -/// Adds additional info about aggregation. -static Chunk convertToChunk(const Block & block) -{ - auto info = std::make_shared(); - info->bucket_num = block.info.bucket_num; - info->is_overflows = block.info.is_overflows; - - UInt64 num_rows = block.rows(); - Chunk chunk(block.getColumns(), num_rows); - chunk.setChunkInfo(std::move(info)); - - return chunk; -} - namespace { + /// Convert block to chunk. + /// Adds additional info about aggregation. + Chunk convertToChunk(const Block & block) + { + auto info = std::make_shared(); + info->bucket_num = block.info.bucket_num; + info->is_overflows = block.info.is_overflows; + + UInt64 num_rows = block.rows(); + Chunk chunk(block.getColumns(), num_rows); + chunk.setChunkInfo(std::move(info)); + + return chunk; + } + + const AggregatedChunkInfo * getInfoFromChunk(const Chunk & chunk) + { + auto & info = chunk.getChunkInfo(); + if (!info) + throw Exception("Chunk info was not set for chunk.", ErrorCodes::LOGICAL_ERROR); + + auto * agg_info = typeid_cast(info.get()); + if (!agg_info) + throw Exception("Chunk should have AggregatedChunkInfo.", ErrorCodes::LOGICAL_ERROR); + + return agg_info; + } + /// Reads chunks from file in native format. Provide chunks with aggregation info. class SourceFromNativeStream : public ISource { @@ -77,12 +91,13 @@ public: struct SharedData { std::atomic next_bucket_to_merge = 0; - std::array, NUM_BUCKETS> source_for_bucket; + std::array, NUM_BUCKETS> is_bucket_processed; + std::atomic is_cancelled = false; SharedData() { - for (auto & source : source_for_bucket) - source = -1; + for (auto & flag : is_bucket_processed) + flag = false; } }; @@ -92,13 +107,11 @@ public: AggregatingTransformParamsPtr params_, ManyAggregatedDataVariantsPtr data_, SharedDataPtr shared_data_, - Int32 source_number_, Arena * arena_) : ISource(params_->getHeader()) , params(std::move(params_)) , data(std::move(data_)) , shared_data(std::move(shared_data_)) - , source_number(source_number_) , arena(arena_) {} @@ -112,10 +125,10 @@ protected: if (bucket_num >= NUM_BUCKETS) return {}; - Block block = params->aggregator.mergeAndConvertOneBucketToBlock(*data, arena, params->final, bucket_num); + Block block = params->aggregator.mergeAndConvertOneBucketToBlock(*data, arena, params->final, bucket_num, &shared_data->is_cancelled); Chunk chunk = convertToChunk(block); - shared_data->source_for_bucket[bucket_num] = source_number; + shared_data->is_bucket_processed[bucket_num] = true; return chunk; } @@ -124,7 +137,6 @@ private: AggregatingTransformParamsPtr params; ManyAggregatedDataVariantsPtr data; SharedDataPtr shared_data; - Int32 source_number; Arena * arena; }; @@ -201,6 +213,9 @@ public: for (auto & input : inputs) input.close(); + if (shared_data) + shared_data->is_cancelled.store(true); + return Status::Finished; } @@ -245,16 +260,23 @@ private: { auto & output = outputs.front(); - Int32 next_input_num = shared_data->source_for_bucket[current_bucket_num]; - if (next_input_num < 0) + for (auto & input : inputs) + { + if (!input.isFinished() && input.hasData()) + { + auto chunk = input.pull(); + auto bucket = getInfoFromChunk(chunk)->bucket_num; + chunks[bucket] = std::move(chunk); + } + } + + if (!shared_data->is_bucket_processed[current_bucket_num]) return Status::NeedData; - auto next_input = std::next(inputs.begin(), next_input_num); - /// next_input can't be finished till data was not pulled. - if (!next_input->hasData()) + if (!chunks[current_bucket_num]) return Status::NeedData; - output.push(next_input->pull()); + output.push(std::move(chunks[current_bucket_num])); ++current_bucket_num; if (current_bucket_num == NUM_BUCKETS) @@ -282,6 +304,7 @@ private: UInt32 current_bucket_num = 0; static constexpr Int32 NUM_BUCKETS = 256; + std::array chunks; Processors processors; @@ -355,7 +378,7 @@ private: { Arena * arena = first->aggregates_pools.at(thread).get(); auto source = std::make_shared( - params, data, shared_data, thread, arena); + params, data, shared_data, arena); processors.emplace_back(std::move(source)); } @@ -429,11 +452,16 @@ IProcessor::Status AggregatingTransform::prepare() } } - input.setNeeded(); if (!input.hasData()) + { + input.setNeeded(); return Status::NeedData; + } - current_chunk = input.pull(); + if (is_consume_finished) + input.setNeeded(); + + current_chunk = input.pull(/*set_not_needed = */ !is_consume_finished); read_current_chunk = true; if (is_consume_finished) diff --git a/dbms/src/Processors/Transforms/ExpressionTransform.cpp b/dbms/src/Processors/Transforms/ExpressionTransform.cpp index a5755ae072b..9bd4ba89db6 100644 --- a/dbms/src/Processors/Transforms/ExpressionTransform.cpp +++ b/dbms/src/Processors/Transforms/ExpressionTransform.cpp @@ -37,6 +37,8 @@ void ExpressionTransform::transform(Chunk & chunk) if (on_totals) { + /// Drop totals if both out stream and joined stream doesn't have ones. + /// See comment in ExpressionTransform.h if (default_totals && !expression->hasTotalsInJoin()) return; diff --git a/dbms/src/Processors/Transforms/ExpressionTransform.h b/dbms/src/Processors/Transforms/ExpressionTransform.h index 5a5d60bfacf..87f2c01ea1d 100644 --- a/dbms/src/Processors/Transforms/ExpressionTransform.h +++ b/dbms/src/Processors/Transforms/ExpressionTransform.h @@ -10,7 +10,11 @@ using ExpressionActionsPtr = std::shared_ptr; class ExpressionTransform : public ISimpleTransform { public: - ExpressionTransform(const Block & header_, ExpressionActionsPtr expression_, bool on_totals_ = false, bool default_totals_ = false); + ExpressionTransform( + const Block & header_, + ExpressionActionsPtr expression_, + bool on_totals_ = false, + bool default_totals_ = false); String getName() const override { return "ExpressionTransform"; } @@ -20,6 +24,9 @@ protected: private: ExpressionActionsPtr expression; bool on_totals; + /// This flag means that we have manually added totals to our pipeline. + /// It may happen in case if joined subquery has totals, but out string doesn't. + /// We need to join default values with subquery totals if we have them, or return empty chunk is haven't. bool default_totals; bool initialized = false; }; diff --git a/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp new file mode 100644 index 00000000000..6653aa0c5c7 --- /dev/null +++ b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp @@ -0,0 +1,68 @@ +#include +#include + +namespace DB +{ + +static Block transformHeader(Block header, const ExpressionActionsPtr & expression) +{ + expression->execute(header, true); + return header; +} + +InflatingExpressionTransform::InflatingExpressionTransform(Block input_header, ExpressionActionsPtr expression_, + bool on_totals_, bool default_totals_) + : ISimpleTransform(input_header, transformHeader(input_header, expression_), on_totals_) + , expression(std::move(expression_)) + , on_totals(on_totals_) + , default_totals(default_totals_) +{} + +void InflatingExpressionTransform::transform(Chunk & chunk) +{ + if (!initialized) + { + initialized = true; + + if (expression->resultIsAlwaysEmpty()) + { + stopReading(); + chunk = Chunk(getOutputPort().getHeader().getColumns(), 0); + return; + } + } + + Block block; + if (on_totals) + { + /// We have to make chunk empty before return + block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); + if (default_totals && !expression->hasTotalsInJoin()) + return; + expression->executeOnTotals(block); + } + else + block = readExecute(chunk); + + auto num_rows = block.rows(); + chunk.setColumns(block.getColumns(), num_rows); +} + +Block InflatingExpressionTransform::readExecute(Chunk & chunk) +{ + Block res; + if (likely(!not_processed)) + { + res = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); + if (res) + expression->execute(res, not_processed, action_number); + } + else + { + res = std::move(not_processed->block); + expression->execute(res, not_processed, action_number); + } + return res; +} + +} diff --git a/dbms/src/Processors/Transforms/InflatingExpressionTransform.h b/dbms/src/Processors/Transforms/InflatingExpressionTransform.h new file mode 100644 index 00000000000..b490d0699ad --- /dev/null +++ b/dbms/src/Processors/Transforms/InflatingExpressionTransform.h @@ -0,0 +1,35 @@ +#pragma once +#include + + +namespace DB +{ + +class ExpressionActions; +using ExpressionActionsPtr = std::shared_ptr; + +class InflatingExpressionTransform : public ISimpleTransform +{ +public: + InflatingExpressionTransform(Block input_header, ExpressionActionsPtr expression_, + bool on_totals_ = false, bool default_totals_ = false); + + String getName() const override { return "InflatingExpressionTransform"; } + +protected: + void transform(Chunk & chunk) override; + bool needInputData() const override { return !not_processed; } + +private: + ExpressionActionsPtr expression; + bool on_totals; + bool default_totals; + bool initialized = false; + + ExtraBlockPtr not_processed; + size_t action_number = 0; + + Block readExecute(Chunk & chunk); +}; + +} diff --git a/dbms/src/Processors/Transforms/MergeSortingTransform.cpp b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp index 39da24ba149..060d860b0b5 100644 --- a/dbms/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace ProfileEvents @@ -95,11 +96,11 @@ MergeSortingTransform::MergeSortingTransform( const SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_, + size_t max_bytes_before_external_sort_, VolumePtr tmp_volume_, size_t min_free_disk_space_) : SortingTransform(header, description_, max_merged_block_size_, limit_) , max_bytes_before_remerge(max_bytes_before_remerge_) - , max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_) + , max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_volume(tmp_volume_) , min_free_disk_space(min_free_disk_space_) {} Processors MergeSortingTransform::expandPipeline() @@ -172,10 +173,14 @@ void MergeSortingTransform::consume(Chunk chunk) */ if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) { - if (!enoughSpaceInDirectory(tmp_path, sum_bytes_in_blocks + min_free_disk_space)) - throw Exception("Not enough space for external sort in " + tmp_path, ErrorCodes::NOT_ENOUGH_SPACE); + size_t size = sum_bytes_in_blocks + min_free_disk_space; + auto reservation = tmp_volume->reserve(size); + if (!reservation) + throw Exception("Not enough space for external sort in temporary storage", ErrorCodes::NOT_ENOUGH_SPACE); + const std::string tmp_path(reservation->getDisk()->getPath()); temporary_files.emplace_back(createTemporaryFile(tmp_path)); + const std::string & path = temporary_files.back()->path(); merge_sorter = std::make_unique(std::move(chunks), description, max_merged_block_size, limit); auto current_processor = std::make_shared(header_without_constants, log, path); diff --git a/dbms/src/Processors/Transforms/MergeSortingTransform.h b/dbms/src/Processors/Transforms/MergeSortingTransform.h index ecfaeb4f272..09c2b182fc7 100644 --- a/dbms/src/Processors/Transforms/MergeSortingTransform.h +++ b/dbms/src/Processors/Transforms/MergeSortingTransform.h @@ -9,6 +9,9 @@ namespace DB { +class Volume; +using VolumePtr = std::shared_ptr; + class MergeSortingTransform : public SortingTransform { public: @@ -17,7 +20,7 @@ public: const SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_, + size_t max_bytes_before_external_sort_, VolumePtr tmp_volume_, size_t min_free_disk_space_); String getName() const override { return "MergeSortingTransform"; } @@ -32,7 +35,7 @@ protected: private: size_t max_bytes_before_remerge; size_t max_bytes_before_external_sort; - const std::string tmp_path; + VolumePtr tmp_volume; size_t min_free_disk_space; Logger * log = &Logger::get("MergeSortingTransform"); diff --git a/dbms/src/Processors/Transforms/MergingSortedTransform.cpp b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp index ddbd91b38d1..a77465ef88d 100644 --- a/dbms/src/Processors/Transforms/MergingSortedTransform.cpp +++ b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp @@ -74,16 +74,8 @@ IProcessor::Status MergingSortedTransform::prepare() return Status::Finished; } - if (!output.isNeeded()) - { - for (auto & in : inputs) - in.setNotNeeded(); - - return Status::PortFull; - } - - if (output.hasData()) - return Status::PortFull; + /// Do not disable inputs, so it will work in the same way as with AsynchronousBlockInputStream, like before. + bool is_port_full = !output.canPush(); /// Special case for single input. if (inputs.size() == 1) @@ -96,14 +88,20 @@ IProcessor::Status MergingSortedTransform::prepare() } input.setNeeded(); + if (input.hasData()) - output.push(input.pull()); + { + if (!is_port_full) + output.push(input.pull()); + + return Status::PortFull; + } return Status::NeedData; } /// Push if has data. - if (merged_data.mergedRows()) + if (merged_data.mergedRows() && !is_port_full) output.push(merged_data.pull()); if (!is_initialized) @@ -119,7 +117,7 @@ IProcessor::Status MergingSortedTransform::prepare() if (!cursors[i].empty()) { - input.setNotNeeded(); + // input.setNotNeeded(); continue; } @@ -159,6 +157,10 @@ IProcessor::Status MergingSortedTransform::prepare() { if (is_finished) { + + if (is_port_full) + return Status::PortFull; + for (auto & input : inputs) input.close(); @@ -192,6 +194,9 @@ IProcessor::Status MergingSortedTransform::prepare() need_data = false; } + if (is_port_full) + return Status::PortFull; + return Status::Ready; } } diff --git a/dbms/src/Processors/Transforms/SortingTransform.cpp b/dbms/src/Processors/Transforms/SortingTransform.cpp index 30f53742ec0..45160793071 100644 --- a/dbms/src/Processors/Transforms/SortingTransform.cpp +++ b/dbms/src/Processors/Transforms/SortingTransform.cpp @@ -241,12 +241,13 @@ IProcessor::Status SortingTransform::prepareConsume() if (input.isFinished()) return Status::Finished; - input.setNeeded(); - if (!input.hasData()) + { + input.setNeeded(); return Status::NeedData; + } - current_chunk = input.pull(); + current_chunk = input.pull(true); } /// Now consume. diff --git a/dbms/src/Processors/tests/processors_test_aggregation.cpp b/dbms/src/Processors/tests/processors_test_aggregation.cpp index ccf31d953ac..903633c18ec 100644 --- a/dbms/src/Processors/tests/processors_test_aggregation.cpp +++ b/dbms/src/Processors/tests/processors_test_aggregation.cpp @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -187,6 +189,8 @@ try auto & factory = AggregateFunctionFactory::instance(); auto cur_path = Poco::Path().absolute().toString(); + auto disk = std::make_shared("tmp", cur_path, 0); + auto tmp_volume = std::make_shared("tmp", std::vector{disk}, 0); auto execute_one_stream = [&](String msg, size_t num_threads, bool two_level, bool external) { @@ -228,7 +232,7 @@ try group_by_two_level_threshold_bytes, max_bytes_before_external_group_by, false, /// empty_result_for_aggregation_by_empty_set - cur_path, /// tmp_path + tmp_volume, 1, /// max_threads 0 ); @@ -301,7 +305,7 @@ try group_by_two_level_threshold_bytes, max_bytes_before_external_group_by, false, /// empty_result_for_aggregation_by_empty_set - cur_path, /// tmp_path + tmp_volume, 1, /// max_threads 0 ); diff --git a/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp b/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp index 8e6b4655127..f0b20959e87 100644 --- a/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp +++ b/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include @@ -116,7 +118,10 @@ try Logger::root().setChannel(channel); Logger::root().setLevel("trace"); - auto execute_chain = []( + auto disk = std::make_shared("tmp", ".", 0); + auto tmp_volume = std::make_shared("tmp", std::vector{disk}, 0); + + auto execute_chain = [tmp_volume]( String msg, UInt64 source_block_size, UInt64 blocks_count, @@ -133,7 +138,9 @@ try SortDescription description = {{0, 1, 1}}; auto transform = std::make_shared( source->getPort().getHeader(), description, - max_merged_block_size, limit, max_bytes_before_remerge, max_bytes_before_external_sort, ".", 0); + max_merged_block_size, limit, + max_bytes_before_remerge, max_bytes_before_external_sort, + tmp_volume, 0); auto sink = std::make_shared(); connect(source->getPort(), transform->getInputs().front()); diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index b3caeaa767b..8b491679138 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -208,6 +208,12 @@ void ColumnsDescription::flattenNested() continue; } + if (!type_tuple->haveExplicitNames()) + { + ++it; + continue; + } + ColumnDescription column = std::move(*it); it = columns.get<0>().erase(it); diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index e33e7c2b001..43a595cde6f 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -80,12 +80,11 @@ namespace StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( - StorageDistributed & storage_, std::string name_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_) + StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_) /// It's important to initialize members before `thread` to avoid race. : storage(storage_) , pool(std::move(pool_)) - , name(std::move(name_)) - , path{storage.path + name + '/'} + , path{path_ + '/'} , should_batch_inserts(storage.global_context.getSettingsRef().distributed_directory_monitor_batch_inserts) , min_batched_block_size_rows(storage.global_context.getSettingsRef().min_insert_block_size_rows) , min_batched_block_size_bytes(storage.global_context.getSettingsRef().min_insert_block_size_bytes) @@ -689,13 +688,13 @@ bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & f std::string StorageDistributedDirectoryMonitor::getLoggerName() const { - return storage.table_name + '.' + storage.getName() + ".DirectoryMonitor"; + return storage.getStorageID().getFullTableName() + ".DirectoryMonitor"; } -void StorageDistributedDirectoryMonitor::updatePath() +void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_path) { std::lock_guard lock{mutex}; - path = storage.path + name + '/'; + path = new_path; current_batch_file_path = path + "current_batch.txt"; } diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.h b/dbms/src/Storages/Distributed/DirectoryMonitor.h index 4ee77072ee3..de7dcffa92e 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.h +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.h @@ -20,13 +20,13 @@ class StorageDistributedDirectoryMonitor { public: StorageDistributedDirectoryMonitor( - StorageDistributed & storage_, std::string name_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_); + StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_); ~StorageDistributedDirectoryMonitor(); static ConnectionPoolPtr createPool(const std::string & name, const StorageDistributed & storage); - void updatePath(); + void updatePath(const std::string & new_path); void flushAllData(); @@ -47,7 +47,6 @@ private: StorageDistributed & storage; const ConnectionPoolPtr pool; - const std::string name; std::string path; const bool should_batch_inserts = false; diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 24ead5b5832..348cb741123 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -90,7 +91,7 @@ void DistributedBlockOutputStream::write(const Block & block) if (ordinary_block.has(col.name)) { ordinary_block.erase(col.name); - LOG_DEBUG(log, storage.getTableName() + LOG_DEBUG(log, storage.getStorageID().getNameForLogs() << ": column " + col.name + " will be removed, " << "because it is MATERIALIZED"); } @@ -515,7 +516,7 @@ void DistributedBlockOutputStream::writeAsyncImpl(const Block & block, const siz else { if (shard_info.dir_name_for_internal_replication.empty()) - throw Exception("Directory name for async inserts is empty, table " + storage.getTableName(), ErrorCodes::LOGICAL_ERROR); + throw Exception("Directory name for async inserts is empty, table " + storage.getStorageID().getNameForLogs(), ErrorCodes::LOGICAL_ERROR); writeToShard(block, {shard_info.dir_name_for_internal_replication}); } @@ -563,11 +564,12 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// write first file, hardlink the others for (const auto & dir_name : dir_names) { - const auto & path = storage.getPath() + dir_name + '/'; + const auto & [disk, data_path] = storage.getPath(); + const std::string path(disk + data_path + dir_name + '/'); /// ensure shard subdirectory creation and notify storage if (Poco::File(path).createDirectory()) - storage.requireDirectoryMonitor(dir_name); + storage.requireDirectoryMonitor(disk, dir_name); const auto & file_name = toString(storage.file_names_increment.get()) + ".bin"; const auto & block_file_path = path + file_name; diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index e48e9896597..332ecbc8681 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -30,7 +30,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } -IStorage::IStorage(ColumnsDescription virtuals_) : virtuals(std::move(virtuals_)) +IStorage::IStorage(StorageID storage_id_, ColumnsDescription virtuals_) : storage_id(std::move(storage_id_)), virtuals(std::move(virtuals_)) { } @@ -177,7 +177,7 @@ void IStorage::check(const Names & column_names, bool include_virtuals) const { if (columns_map.end() == columns_map.find(name)) throw Exception( - "There is no column with name " + backQuote(name) + " in table " + getTableName() + ". There are columns: " + list_of_columns, + "There is no column with name " + backQuote(name) + " in table " + getStorageID().getNameForLogs() + ". There are columns: " + list_of_columns, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); if (unique_names.end() != unique_names.find(name)) @@ -339,7 +339,7 @@ TableStructureWriteLockHolder IStorage::lockAlterIntention(const String & query_ void IStorage::lockNewDataStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id) { if (!lock_holder.alter_intention_lock) - throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Alter intention lock for table " + getStorageID().getNameForLogs() + " was not taken. This is a bug.", ErrorCodes::LOGICAL_ERROR); lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); } @@ -347,7 +347,7 @@ void IStorage::lockNewDataStructureExclusively(TableStructureWriteLockHolder & l void IStorage::lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id) { if (!lock_holder.alter_intention_lock) - throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Alter intention lock for table " + getStorageID().getNameForLogs() + " was not taken. This is a bug.", ErrorCodes::LOGICAL_ERROR); if (!lock_holder.new_data_structure_lock) lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); @@ -383,14 +383,11 @@ void IStorage::alter( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - const String database_name = getDatabaseName(); - const String table_name = getTableName(); - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - + auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(database_name)->alterTable(context, table_name, metadata); + context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } @@ -425,4 +422,17 @@ BlockInputStreams IStorage::read( return res; } +StorageID IStorage::getStorageID() const +{ + std::lock_guard lock(id_mutex); + return storage_id; +} + +void IStorage::renameInMemory(const String & new_database_name, const String & new_table_name) +{ + std::lock_guard lock(id_mutex); + storage_id.database_name = new_database_name; + storage_id.table_name = new_table_name; +} + } diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 69bbca86879..d5499e0e9d7 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -75,8 +76,9 @@ struct ColumnSize class IStorage : public std::enable_shared_from_this, public TypePromotion { public: - IStorage() = default; - explicit IStorage(ColumnsDescription virtuals_); + IStorage() = delete; + explicit IStorage(StorageID storage_id_) : storage_id(std::move(storage_id_)) {} + IStorage(StorageID id_, ColumnsDescription virtuals_); virtual ~IStorage() = default; IStorage(const IStorage &) = delete; @@ -86,8 +88,7 @@ public: virtual std::string getName() const = 0; /// The name of the table. - virtual std::string getTableName() const = 0; - virtual std::string getDatabaseName() const { return {}; } + StorageID getStorageID() const; /// Returns true if the storage receives data from a remote server or servers. virtual bool isRemote() const { return false; } @@ -104,6 +105,9 @@ public: /// Returns true if the storage replicates SELECT, INSERT and ALTER commands among replicas. virtual bool supportsReplication() const { return false; } + /// Returns true if the storage supports parallel insert. + virtual bool supportsParallelInsert() const { return false; } + /// Returns true if the storage supports deduplication of inserted data blocks. virtual bool supportsDeduplication() const { return false; } @@ -113,6 +117,8 @@ public: /// Returns true if the blocks shouldn't be pushed to associated views on insert. virtual bool noPushingToViews() const { return false; } + virtual bool hasEvenlyDistributedRead() const { return false; } + /// Optional size information of each physical column. /// Currently it's only used by the MergeTree family for query optimizations. using ColumnSizeByName = std::unordered_map; @@ -165,6 +171,8 @@ protected: /// still thread-unsafe part. private: + StorageID storage_id; + mutable std::mutex id_mutex; ColumnsDescription columns; /// combined real and virtual columns const ColumnsDescription virtuals = {}; IndicesDescription indices; @@ -303,12 +311,18 @@ public: * In this function, you need to rename the directory with the data, if any. * Called when the table structure is locked for write. */ - virtual void rename(const String & /*new_path_to_table_data*/, const String & /*new_database_name*/, const String & /*new_table_name*/, + virtual void rename(const String & /*new_path_to_table_data*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { - throw Exception("Method rename is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); + renameInMemory(new_database_name, new_table_name); } + /** + * Just updates names of database and table without moving any data on disk + * Can be called directly only from DatabaseAtomic. + */ + virtual void renameInMemory(const String & new_database_name, const String & new_table_name); + /** ALTER tables in the form of column changes that do not affect the change to Storage or its parameters. * This method must fully execute the ALTER query, taking care of the locks itself. * To update the table metadata on disk, this method should call InterpreterAlterQuery::updateMetadata. diff --git a/dbms/src/Storages/IStorage_fwd.h b/dbms/src/Storages/IStorage_fwd.h index e80fa2a0eb6..4983a734d21 100644 --- a/dbms/src/Storages/IStorage_fwd.h +++ b/dbms/src/Storages/IStorage_fwd.h @@ -11,7 +11,6 @@ namespace DB class IStorage; using StoragePtr = std::shared_ptr; -using StorageWeakPtr = std::weak_ptr; using Tables = std::map; } diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index c1fc156ca37..1980637f4b6 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -75,8 +75,7 @@ namespace } StorageKafka::StorageKafka( - const std::string & table_name_, - const std::string & database_name_, + const StorageID & table_id_, Context & context_, const ColumnsDescription & columns_, const String & brokers_, @@ -89,14 +88,12 @@ StorageKafka::StorageKafka( UInt64 max_block_size_, size_t skip_broken_, bool intermediate_commit_) - : IStorage( + : IStorage(table_id_, ColumnsDescription({{"_topic", std::make_shared()}, {"_key", std::make_shared()}, {"_offset", std::make_shared()}, {"_partition", std::make_shared()}, {"_timestamp", std::make_shared(std::make_shared())}}, true)) - , table_name(table_name_) - , database_name(database_name_) , global_context(context_.getGlobalContext()) , kafka_context(Context(global_context)) , topics(global_context.getMacros()->expand(topics_)) @@ -107,7 +104,7 @@ StorageKafka::StorageKafka( , schema_name(global_context.getMacros()->expand(schema_name_)) , num_consumers(num_consumers_) , max_block_size(max_block_size_) - , log(&Logger::get("StorageKafka (" + table_name_ + ")")) + , log(&Logger::get("StorageKafka (" + table_id_.table_name + ")")) , semaphore(0, num_consumers_) , skip_broken(skip_broken_) , intermediate_commit(intermediate_commit_) @@ -195,14 +192,6 @@ void StorageKafka::shutdown() task->deactivate(); } - -void StorageKafka::rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) -{ - table_name = new_table_name; - database_name = new_database_name; -} - - void StorageKafka::updateDependencies() { task->activateAndSchedule(); @@ -303,17 +292,17 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) } } -bool StorageKafka::checkDependencies(const String & current_database_name, const String & current_table_name) +bool StorageKafka::checkDependencies(const StorageID & table_id) { // Check if all dependencies are attached - auto dependencies = global_context.getDependencies(current_database_name, current_table_name); + auto dependencies = global_context.getDependencies(table_id); if (dependencies.size() == 0) return true; // Check the dependencies are ready? for (const auto & db_tab : dependencies) { - auto table = global_context.tryGetTable(db_tab.first, db_tab.second); + auto table = global_context.tryGetTable(db_tab); if (!table) return false; @@ -323,7 +312,7 @@ bool StorageKafka::checkDependencies(const String & current_database_name, const return false; // Check all its dependencies - if (!checkDependencies(db_tab.first, db_tab.second)) + if (!checkDependencies(db_tab)) return false; } @@ -334,13 +323,14 @@ void StorageKafka::threadFunc() { try { + auto table_id = getStorageID(); // Check if at least one direct dependency is attached - auto dependencies = global_context.getDependencies(database_name, table_name); + auto dependencies = global_context.getDependencies(table_id); // Keep streaming as long as there are attached views and streaming is not cancelled while (!stream_cancelled && num_created_consumers > 0 && dependencies.size() > 0) { - if (!checkDependencies(database_name, table_name)) + if (!checkDependencies(table_id)) break; LOG_DEBUG(log, "Started streaming to " << dependencies.size() << " attached views"); @@ -363,14 +353,15 @@ void StorageKafka::threadFunc() bool StorageKafka::streamToViews() { - auto table = global_context.getTable(database_name, table_name); + auto table_id = getStorageID(); + auto table = global_context.getTable(table_id); if (!table) - throw Exception("Engine table " + backQuote(database_name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); // Create an INSERT query for streaming data auto insert = std::make_shared(); - insert->database = database_name; - insert->table = table_name; + insert->database = table_id.database_name; + insert->table = table_id.table_name; const Settings & settings = global_context.getSettingsRef(); size_t block_size = max_block_size; @@ -420,7 +411,7 @@ bool StorageKafka::streamToViews() void registerStorageKafka(StorageFactory & factory) { - factory.registerStorage("Kafka", [](const StorageFactory::Arguments & args) + auto creator_fn = [](const StorageFactory::Arguments & args) { ASTs & engine_args = args.engine_args; size_t args_count = engine_args.size(); @@ -643,9 +634,11 @@ void registerStorageKafka(StorageFactory & factory) } return StorageKafka::create( - args.table_name, args.database_name, args.context, args.columns, + args.table_id, args.context, args.columns, brokers, group, topics, format, row_delimiter, schema, num_consumers, max_block_size, skip_broken, intermediate_commit); - }); + }; + + factory.registerStorage("Kafka", creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, }); } diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index 7c1ba219245..bf710f58202 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -28,8 +28,6 @@ class StorageKafka : public ext::shared_ptr_helper, public IStorag friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Kafka"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } bool supportsSettings() const override { return true; } bool noPushingToViews() const override { return true; } @@ -49,8 +47,6 @@ public: const ASTPtr & query, const Context & context) override; - void rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; - void updateDependencies() override; void pushReadBuffer(ConsumerBufferPtr buf); @@ -66,19 +62,22 @@ public: protected: StorageKafka( - const std::string & table_name_, - const std::string & database_name_, + const StorageID & table_id_, Context & context_, const ColumnsDescription & columns_, - const String & brokers_, const String & group_, const Names & topics_, - const String & format_name_, char row_delimiter_, const String & schema_name_, - size_t num_consumers_, UInt64 max_block_size_, size_t skip_broken, + const String & brokers_, + const String & group_, + const Names & topics_, + const String & format_name_, + char row_delimiter_, + const String & schema_name_, + size_t num_consumers_, + UInt64 max_block_size_, + size_t skip_broken, bool intermediate_commit_); private: // Configuration and state - String table_name; - String database_name; Context global_context; Context kafka_context; Names topics; @@ -116,7 +115,7 @@ private: void threadFunc(); bool streamToViews(); - bool checkDependencies(const String & database_name, const String & table_name); + bool checkDependencies(const StorageID & table_id); }; } diff --git a/dbms/src/Storages/LiveView/StorageBlocks.h b/dbms/src/Storages/LiveView/StorageBlocks.h index bd60e6f0b97..f02306a0f58 100644 --- a/dbms/src/Storages/LiveView/StorageBlocks.h +++ b/dbms/src/Storages/LiveView/StorageBlocks.h @@ -12,21 +12,19 @@ class StorageBlocks : public IStorage * Used by Live Views to complete stored query based on the mergeable blocks. */ public: - StorageBlocks(const std::string & database_name_, const std::string & table_name_, + StorageBlocks(const StorageID & table_id_, const ColumnsDescription & columns_, BlockInputStreams streams_, QueryProcessingStage::Enum to_stage_) - : database_name(database_name_), table_name(table_name_), streams(streams_), to_stage(to_stage_) + : IStorage(table_id_), streams(streams_), to_stage(to_stage_) { setColumns(columns_); } - static StoragePtr createStorage(const std::string & database_name, const std::string & table_name, + static StoragePtr createStorage(const StorageID & table_id, const ColumnsDescription & columns, BlockInputStreams streams, QueryProcessingStage::Enum to_stage) { - return std::make_shared(database_name, table_name, columns, streams, to_stage); + return std::make_shared(table_id, columns, streams, to_stage); } std::string getName() const override { return "Blocks"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } QueryProcessingStage::Enum getQueryProcessingStage(const Context & /*context*/) const override { return to_stage; } BlockInputStreams read( @@ -41,8 +39,6 @@ public: } private: - std::string database_name; - std::string table_name; Block res_block; BlockInputStreams streams; QueryProcessingStage::Enum to_stage; diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index eae8eaa1d3c..9331eb386b0 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -13,7 +13,6 @@ limitations under the License. */ #include #include #include -#include #include #include #include @@ -54,20 +53,15 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } -static void extractDependentTable(ASTPtr & query, String & select_database_name, String & select_table_name, const String & table_name, ASTPtr & inner_subquery) + +static StorageID extractDependentTable(ASTPtr & query, Context & context, const String & table_name, ASTPtr & inner_subquery) { ASTSelectQuery & select_query = typeid_cast(*query); - auto db_and_table = getDatabaseAndTable(select_query, 0); - ASTPtr subquery = extractTableExpression(select_query, 0); - if (!db_and_table && !subquery) + if (auto db_and_table = getDatabaseAndTable(select_query, 0)) { - return; - } - - if (db_and_table) - { - select_table_name = db_and_table->table; + String select_database_name = context.getCurrentDatabase(); + String select_table_name = db_and_table->table; if (db_and_table->database.empty()) { @@ -79,20 +73,27 @@ static void extractDependentTable(ASTPtr & query, String & select_database_name, select_database_name = db_and_table->database; select_query.replaceDatabaseAndTable("", table_name + "_blocks"); + return StorageID(select_database_name, select_table_name); } - else if (auto * ast_select = subquery->as()) + else if (auto subquery = extractTableExpression(select_query, 0)) { + auto * ast_select = subquery->as(); + if (!ast_select) + throw Exception("Logical error while creating StorageLiveView." + " Could not retrieve table name from select query.", + DB::ErrorCodes::LOGICAL_ERROR); if (ast_select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW); inner_subquery = ast_select->list_of_selects->children.at(0)->clone(); - extractDependentTable(ast_select->list_of_selects->children.at(0), select_database_name, select_table_name, table_name, inner_subquery); + return extractDependentTable(ast_select->list_of_selects->children.at(0), context, table_name, inner_subquery); } else - throw Exception("Logical error while creating StorageLiveView." - " Could not retrieve table name from select query.", - DB::ErrorCodes::LOGICAL_ERROR); + { + /// If the table is not specified - use the table `system.one` + return StorageID("system", "one"); + } } MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(const Context & context) @@ -138,12 +139,13 @@ BlockInputStreamPtr StorageLiveView::completeQuery(BlockInputStreams from) auto block_context = std::make_unique(global_context); block_context->makeQueryContext(); - auto blocks_storage = StorageBlocks::createStorage(database_name, table_name, parent_storage->getColumns(), + auto blocks_storage_id = getBlocksStorageID(); + auto blocks_storage = StorageBlocks::createStorage(blocks_storage_id, getParentStorage()->getColumns(), std::move(from), QueryProcessingStage::WithMergeableState); - block_context->addExternalTable(table_name + "_blocks", blocks_storage); + block_context->addExternalTable(blocks_storage_id.table_name, blocks_storage); - InterpreterSelectQuery select(inner_blocks_query->clone(), *block_context, StoragePtr(), SelectQueryOptions(QueryProcessingStage::Complete)); + InterpreterSelectQuery select(getInnerBlocksQuery(), *block_context, StoragePtr(), SelectQueryOptions(QueryProcessingStage::Complete)); BlockInputStreamPtr data = std::make_shared(select.execute().in); /// Squashing is needed here because the view query can generate a lot of blocks @@ -193,6 +195,8 @@ void StorageLiveView::writeIntoLiveView( } } + auto blocks_storage_id = live_view.getBlocksStorageID(); + if (!is_block_processed) { ASTPtr mergeable_query = live_view.getInnerQuery(); @@ -202,7 +206,7 @@ void StorageLiveView::writeIntoLiveView( BlockInputStreams streams = {std::make_shared(block)}; - auto blocks_storage = StorageBlocks::createStorage(live_view.database_name, live_view.table_name, + auto blocks_storage = StorageBlocks::createStorage(blocks_storage_id, live_view.getParentStorage()->getColumns(), std::move(streams), QueryProcessingStage::FetchColumns); InterpreterSelectQuery select_block(mergeable_query, context, blocks_storage, @@ -232,13 +236,11 @@ void StorageLiveView::writeIntoLiveView( StorageLiveView::StorageLiveView( - const String & table_name_, - const String & database_name_, + const StorageID & table_id_, Context & local_context, const ASTCreateQuery & query, const ColumnsDescription & columns_) - : table_name(table_name_), - database_name(database_name_), global_context(local_context.getGlobalContext()) + : IStorage(table_id_), global_context(local_context.getGlobalContext()) { live_view_context = std::make_unique(global_context); live_view_context->makeQueryContext(); @@ -249,29 +251,15 @@ StorageLiveView::StorageLiveView( throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); /// Default value, if only table name exist in the query - select_database_name = local_context.getCurrentDatabase(); if (query.select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW); inner_query = query.select->list_of_selects->children.at(0); - inner_blocks_query = inner_query->clone(); - InterpreterSelectQuery(inner_blocks_query, *live_view_context, SelectQueryOptions().modify().analyze()); + auto inner_query_tmp = inner_query->clone(); + select_table_id = extractDependentTable(inner_query_tmp, global_context, table_id_.table_name, inner_subquery); - extractDependentTable(inner_blocks_query, select_database_name, select_table_name, table_name, inner_subquery); - - /// If the table is not specified - use the table `system.one` - if (select_table_name.empty()) - { - select_database_name = "system"; - select_table_name = "one"; - } - - global_context.addDependency( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); - - parent_storage = local_context.getTable(select_database_name, select_table_name); + global_context.addDependency(select_table_id, table_id_); is_temporary = query.temporary; temporary_live_view_timeout = local_context.getSettingsRef().temporary_live_view_timeout.totalSeconds(); @@ -318,6 +306,22 @@ Block StorageLiveView::getHeader() const return sample_block; } +ASTPtr StorageLiveView::getInnerBlocksQuery() +{ + std::lock_guard lock(sample_block_lock); + if (!inner_blocks_query) + { + inner_blocks_query = inner_query->clone(); + /// Rewrite inner query with right aliases for JOIN. + /// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table, + /// which is not loaded yet during server startup, so we do it lazily + InterpreterSelectQuery(inner_blocks_query, *live_view_context, SelectQueryOptions().modify().analyze()); + auto table_id = getStorageID(); + extractDependentTable(inner_blocks_query, global_context, table_id.table_name, inner_subquery); + } + return inner_blocks_query->clone(); +} + bool StorageLiveView::getNewBlocks() { SipHash hash; @@ -366,11 +370,12 @@ bool StorageLiveView::getNewBlocks() void StorageLiveView::checkTableCanBeDropped() const { - Dependencies dependencies = global_context.getDependencies(database_name, table_name); + auto table_id = getStorageID(); + Dependencies dependencies = global_context.getDependencies(table_id); if (!dependencies.empty()) { - DatabaseAndTableName database_and_table_name = dependencies.front(); - throw Exception("Table has dependency " + database_and_table_name.first + "." + database_and_table_name.second, ErrorCodes::TABLE_WAS_NOT_DROPPED); + StorageID dependent_table_id = dependencies.front(); + throw Exception("Table has dependency " + dependent_table_id.getNameForLogs(), ErrorCodes::TABLE_WAS_NOT_DROPPED); } } @@ -381,6 +386,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co if (storage->shutdown_called) return; + auto table_id = storage->getStorageID(); { while (1) { @@ -392,7 +398,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co return; if (storage->hasUsers()) return; - if (!storage->global_context.getDependencies(storage->database_name, storage->table_name).empty()) + if (!storage->global_context.getDependencies(table_id).empty()) continue; drop_table = true; } @@ -402,14 +408,14 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co if (drop_table) { - if (storage->global_context.tryGetTable(storage->database_name, storage->table_name)) + if (storage->global_context.tryGetTable(table_id)) { try { /// We create and execute `drop` query for this table auto drop_query = std::make_shared(); - drop_query->database = storage->database_name; - drop_query->table = storage->table_name; + drop_query->database = table_id.database_name; + drop_query->table = table_id.table_name; drop_query->kind = ASTDropQuery::Kind::Drop; ASTPtr ast_drop_query = drop_query; InterpreterDropQuery drop_interpreter(ast_drop_query, storage->global_context); @@ -417,6 +423,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co } catch (...) { + tryLogCurrentException(__PRETTY_FUNCTION__); } } } @@ -463,6 +470,7 @@ void StorageLiveView::startup() void StorageLiveView::shutdown() { + global_context.removeDependency(select_table_id, getStorageID()); bool expected = false; if (!shutdown_called.compare_exchange_strong(expected, true)) return; @@ -493,9 +501,8 @@ StorageLiveView::~StorageLiveView() void StorageLiveView::drop(TableStructureWriteLockHolder &) { - global_context.removeDependency( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); + auto table_id = getStorageID(); + global_context.removeDependency(select_table_id, table_id); std::lock_guard lock(mutex); is_dropped = true; @@ -623,7 +630,7 @@ void registerStorageLiveView(StorageFactory & factory) if (!args.attach && !args.local_context.getSettingsRef().allow_experimental_live_view) throw Exception("Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')", ErrorCodes::SUPPORT_IS_DISABLED); - return StorageLiveView::create(args.table_name, args.database_name, args.local_context, args.query, args.columns); + return StorageLiveView::create(args.table_id, args.local_context, args.query, args.columns); }); } diff --git a/dbms/src/Storages/LiveView/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h index 916406a1dbd..0af1a297391 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -48,11 +48,12 @@ friend class LiveViewBlockOutputStream; public: ~StorageLiveView() override; String getName() const override { return "LiveView"; } - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } - String getSelectDatabaseName() const { return select_database_name; } - String getSelectTableName() const { return select_table_name; } - StoragePtr getParentStorage() const { return parent_storage; } + StorageID getSelectTableID() const { return select_table_id; } + StorageID getBlocksStorageID() const + { + return StorageID("", getStorageID().table_name + "_blocks"); + } + StoragePtr getParentStorage() const { return global_context.getTable(select_table_id); } NameAndTypePair getColumn(const String & column_name) const override; bool hasColumn(const String & column_name) const override; @@ -64,12 +65,7 @@ public: return inner_subquery->clone(); return nullptr; } - ASTPtr getInnerBlocksQuery() const - { - if (inner_blocks_query) - return inner_blocks_query->clone(); - return nullptr; - } + ASTPtr getInnerBlocksQuery(); /// It is passed inside the query and solved at its level. bool supportsSampling() const override { return true; } @@ -170,19 +166,15 @@ public: const Context & context); private: - String select_database_name; - String select_table_name; - String table_name; - String database_name; + StorageID select_table_id = StorageID::createEmpty(); /// Will be initialized in constructor ASTPtr inner_query; /// stored query : SELECT * FROM ( SELECT a FROM A) ASTPtr inner_subquery; /// stored query's innermost subquery if any ASTPtr inner_blocks_query; /// query over the mergeable blocks to produce final result Context & global_context; std::unique_ptr live_view_context; - StoragePtr parent_storage; bool is_temporary = false; - /// Mutex to protect access to sample block + /// Mutex to protect access to sample block and inner_blocks_query mutable std::mutex sample_block_lock; mutable Block sample_block; @@ -210,8 +202,7 @@ private: UInt64 temporary_live_view_timeout; StorageLiveView( - const String & table_name_, - const String & database_name_, + const StorageID & table_id_, Context & local_context, const ASTCreateQuery & query, const ColumnsDescription & columns diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 12137b4f023..e459de6fa58 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -1,8 +1,6 @@ #include -#include #include #include -#include #include #include #include @@ -53,9 +51,6 @@ std::string Service::getId(const std::string & node_id) const void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*body*/, WriteBuffer & out, Poco::Net::HTTPServerResponse & response) { - if (blocker.isCancelled()) - throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); - String client_protocol_version = params.get("client_protocol_version", REPLICATION_PROTOCOL_VERSION_WITHOUT_PARTS_SIZE); @@ -88,15 +83,11 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo ++data.current_table_sends; SCOPE_EXIT({--data.current_table_sends;}); - StoragePtr owned_storage = storage.lock(); - if (!owned_storage) - throw Exception("The table was already dropped", ErrorCodes::UNKNOWN_TABLE); - LOG_TRACE(log, "Sending part " << part_name); try { - auto storage_lock = owned_storage->lockStructureForShare(false, RWLockImpl::NO_QUERY); + auto storage_lock = data.lockStructureForShare(false, RWLockImpl::NO_QUERY); MergeTreeData::DataPartPtr part = findPart(part_name); diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.h b/dbms/src/Storages/MergeTree/DataPartsExchange.h index 00d46870866..c0e8c0d2331 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.h +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.h @@ -20,8 +20,8 @@ namespace DataPartsExchange class Service final : public InterserverIOEndpoint { public: - Service(MergeTreeData & data_, StoragePtr & storage_) : data(data_), - storage(storage_), log(&Logger::get(data.getLogName() + " (Replicated PartsService)")) {} + Service(MergeTreeData & data_) + : data(data_), log(&Logger::get(data.getLogName() + " (Replicated PartsService)")) {} Service(const Service &) = delete; Service & operator=(const Service &) = delete; @@ -33,8 +33,9 @@ private: MergeTreeData::DataPartPtr findPart(const String & name); private: + /// StorageReplicatedMergeTree::shutdown() waits for all parts exchange handlers to finish, + /// so Service will never access dangling reference to storage MergeTreeData & data; - StorageWeakPtr storage; Logger * log; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 96728178101..146440b11c3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -101,6 +101,7 @@ namespace ErrorCodes extern const int UNKNOWN_SETTING; extern const int READONLY_SETTING; extern const int ABORTED; + extern const int UNEXPECTED_AST_STRUCTURE; } @@ -111,8 +112,7 @@ namespace MergeTreeData::MergeTreeData( - const String & database_, - const String & table_, + const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata, Context & context_, @@ -122,17 +122,16 @@ MergeTreeData::MergeTreeData( bool require_part_metadata_, bool attach, BrokenPartCallback broken_part_callback_) - : global_context(context_) + : IStorage(table_id_) + , global_context(context_) , merging_params(merging_params_) , partition_by_ast(metadata.partition_by_ast) , sample_by_ast(metadata.sample_by_ast) , settings_ast(metadata.settings_ast) , require_part_metadata(require_part_metadata_) - , database_name(database_) - , table_name(table_) , relative_data_path(relative_data_path_) , broken_part_callback(broken_part_callback_) - , log_name(database_name + "." + table_name) + , log_name(table_id_.getNameForLogs()) , log(&Logger::get(log_name)) , storage_settings(std::move(storage_settings_)) , storage_policy(context_.getStoragePolicy(getSettings()->storage_policy)) @@ -140,6 +139,9 @@ MergeTreeData::MergeTreeData( , data_parts_by_state_and_info(data_parts_indexes.get()) , parts_mover(this) { + if (relative_data_path.empty()) + throw Exception("MergeTree storages require data path", ErrorCodes::INCORRECT_FILE_NAME); + const auto settings = getSettings(); setProperties(metadata); @@ -628,40 +630,43 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription::ColumnTTLs & new { auto new_ttl_entry = create_ttl_entry(ast); if (!only_check) - column_ttl_entries_by_name.emplace(name, new_ttl_entry); + column_ttl_entries_by_name[name] = new_ttl_entry; } } } if (new_ttl_table_ast) { + std::vector update_move_ttl_entries; + TTLEntry update_rows_ttl_entry; + bool seen_delete_ttl = false; for (auto ttl_element_ptr : new_ttl_table_ast->children) { - ASTTTLElement & ttl_element = static_cast(*ttl_element_ptr); - if (ttl_element.destination_type == PartDestinationType::DELETE) + const auto * ttl_element = ttl_element_ptr->as(); + if (!ttl_element) + throw Exception("Unexpected AST element in TTL expression", ErrorCodes::UNEXPECTED_AST_STRUCTURE); + + if (ttl_element->destination_type == PartDestinationType::DELETE) { if (seen_delete_ttl) { throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION); } - auto new_ttl_table_entry = create_ttl_entry(ttl_element.children[0]); + auto new_rows_ttl_entry = create_ttl_entry(ttl_element->children[0]); if (!only_check) - { - ttl_table_ast = ttl_element.children[0]; - ttl_table_entry = new_ttl_table_entry; - } + update_rows_ttl_entry = new_rows_ttl_entry; seen_delete_ttl = true; } else { - auto new_ttl_entry = create_ttl_entry(ttl_element.children[0]); + auto new_ttl_entry = create_ttl_entry(ttl_element->children[0]); new_ttl_entry.entry_ast = ttl_element_ptr; - new_ttl_entry.destination_type = ttl_element.destination_type; - new_ttl_entry.destination_name = ttl_element.destination_name; + new_ttl_entry.destination_type = ttl_element->destination_type; + new_ttl_entry.destination_name = ttl_element->destination_name; if (!new_ttl_entry.getDestination(getStoragePolicy())) { String message; @@ -673,11 +678,18 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription::ColumnTTLs & new } if (!only_check) - { - move_ttl_entries.emplace_back(std::move(new_ttl_entry)); - } + update_move_ttl_entries.emplace_back(std::move(new_ttl_entry)); } } + + if (!only_check) + { + rows_ttl_entry = update_rows_ttl_entry; + ttl_table_ast = new_ttl_table_ast; + + auto move_ttl_entries_lock = std::lock_guard(move_ttl_entries_mutex); + move_ttl_entries = update_move_ttl_entries; + } } } @@ -1257,7 +1269,9 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa /// Data parts is still alive (since DataPartsVector holds shared_ptrs) and contain useful metainformation for logging /// NOTE: There is no need to log parts deletion somewhere else, all deleting parts pass through this function and pass away - if (auto part_log = global_context.getPartLog(database_name)) + + auto table_id = getStorageID(); + if (auto part_log = global_context.getPartLog(table_id.database_name)) { PartLogElement part_log_elem; @@ -1265,8 +1279,8 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa part_log_elem.event_time = time(nullptr); part_log_elem.duration_ms = 0; - part_log_elem.database_name = database_name; - part_log_elem.table_name = table_name; + part_log_elem.database_name = table_id.database_name; + part_log_elem.table_name = table_id.table_name; for (auto & part : parts) { @@ -1341,8 +1355,7 @@ void MergeTreeData::rename( global_context.dropCaches(); relative_data_path = new_table_path; - table_name = new_table_name; - database_name = new_database_name; + renameInMemory(new_database_name, new_table_name); } void MergeTreeData::dropAllData() @@ -3294,7 +3307,7 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ ReservationPtr reservation; auto ttl_entry = selectTTLEntryForTTLInfos(ttl_infos, time_of_move); - if (ttl_entry != nullptr) + if (ttl_entry) { SpacePtr destination_ptr = ttl_entry->getDestination(storage_policy); if (!destination_ptr) @@ -3353,27 +3366,28 @@ bool MergeTreeData::TTLEntry::isPartInDestination(const StoragePolicyPtr & polic return false; } -const MergeTreeData::TTLEntry * MergeTreeData::selectTTLEntryForTTLInfos( +std::optional MergeTreeData::selectTTLEntryForTTLInfos( const MergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const { - const MergeTreeData::TTLEntry * result = nullptr; - /// Prefer TTL rule which went into action last. time_t max_max_ttl = 0; + std::vector::const_iterator best_entry_it; - for (const auto & ttl_entry : move_ttl_entries) + auto lock = std::lock_guard(move_ttl_entries_mutex); + for (auto ttl_entry_it = move_ttl_entries.begin(); ttl_entry_it != move_ttl_entries.end(); ++ttl_entry_it) { - auto ttl_info_it = ttl_infos.moves_ttl.find(ttl_entry.result_column); + auto ttl_info_it = ttl_infos.moves_ttl.find(ttl_entry_it->result_column); + /// Prefer TTL rule which went into action last. if (ttl_info_it != ttl_infos.moves_ttl.end() && ttl_info_it->second.max <= time_of_move && max_max_ttl <= ttl_info_it->second.max) { - result = &ttl_entry; + best_entry_it = ttl_entry_it; max_max_ttl = ttl_info_it->second.max; } } - return result; + return max_max_ttl ? *best_entry_it : std::optional(); } MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const @@ -3521,11 +3535,12 @@ bool MergeTreeData::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, con } } -MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(const StoragePtr & source_table) const +MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage * source_table) const { - MergeTreeData * src_data = dynamic_cast(source_table.get()); + MergeTreeData * src_data = dynamic_cast(source_table); if (!src_data) - throw Exception("Table " + table_name + " supports attachPartitionFrom only for MergeTree family of table engines." + throw Exception("Table " + source_table->getStorageID().getNameForLogs() + + " supports attachPartitionFrom only for MergeTree family of table engines." " Got " + source_table->getName(), ErrorCodes::NOT_IMPLEMENTED); if (getColumns().getAllPhysical().sizeOfDifference(src_data->getColumns().getAllPhysical())) @@ -3548,6 +3563,11 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(const StoragePt return *src_data; } +MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(const StoragePtr & source_table) const +{ + return checkStructureAndGetMergeTreeData(source_table.get()); +} + MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk(const MergeTreeData::DataPartPtr & src_part, const String & tmp_part_prefix, const MergeTreePartInfo & dst_part_info) @@ -3686,7 +3706,8 @@ void MergeTreeData::writePartLog( const MergeListEntry * merge_entry) try { - auto part_log = global_context.getPartLog(database_name); + auto table_id = getStorageID(); + auto part_log = global_context.getPartLog(table_id.database_name); if (!part_log) return; @@ -3701,8 +3722,8 @@ try /// TODO: Stop stopwatch in outer code to exclude ZK timings and so on part_log_elem.duration_ms = elapsed_ns / 1000000; - part_log_elem.database_name = database_name; - part_log_elem.table_name = table_name; + part_log_elem.database_name = table_id.database_name; + part_log_elem.table_name = table_id.table_name; part_log_elem.partition_id = MergeTreePartInfo::fromPartName(new_part_name, format_version).partition_id; part_log_elem.part_name = new_part_name; @@ -3767,7 +3788,15 @@ bool MergeTreeData::selectPartsAndMove() bool MergeTreeData::areBackgroundMovesNeeded() const { - return storage_policy->getVolumes().size() > 1; + auto policy = storage_policy; + + if (policy->getVolumes().size() > 1) + return true; + + if (policy->getVolumes().size() == 1 && policy->getVolumes()[0]->disks.size() > 1 && move_ttl_entries.size() > 0) + return true; + + return false; } bool MergeTreeData::movePartsToSpace(const DataPartsVector & parts, SpacePtr space) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index be6ba8f0022..ab5644749ee 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -330,7 +330,7 @@ public: /// /// require_part_metadata - should checksums.txt and columns.txt exist in the part directory. /// attach - whether the existing table is attached or the new table is created. - MergeTreeData(const String & database_, const String & table_, + MergeTreeData(const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata, Context & context_, @@ -396,9 +396,6 @@ public: || column_name == "_sample_factor"; } - String getDatabaseName() const override { return database_name; } - String getTableName() const override { return table_name; } - /// Load the set of data parts from disk. Call once - immediately after the object is created. void loadDataParts(bool skip_sanity_checks); @@ -579,8 +576,10 @@ public: bool hasSortingKey() const { return !sorting_key_columns.empty(); } bool hasPrimaryKey() const { return !primary_key_columns.empty(); } bool hasSkipIndices() const { return !skip_indices.empty(); } - bool hasTableTTL() const { return ttl_table_ast != nullptr; } + bool hasAnyColumnTTL() const { return !column_ttl_entries_by_name.empty(); } + bool hasAnyMoveTTL() const { return !move_ttl_entries.empty(); } + bool hasRowsTTL() const { return !rows_ttl_entry.isEmpty(); } /// Check that the part is not broken and calculate the checksums for it if they are not present. MutableDataPartPtr loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path); @@ -627,6 +626,7 @@ public: /// and checks that their structure suitable for ALTER TABLE ATTACH PARTITION FROM /// Tables structure should be locked. MergeTreeData & checkStructureAndGetMergeTreeData(const StoragePtr & source_table) const; + MergeTreeData & checkStructureAndGetMergeTreeData(IStorage * source_table) const; MergeTreeData::MutableDataPartPtr cloneAndLoadDataPartOnSameDisk( const MergeTreeData::DataPartPtr & src_part, const String & tmp_part_prefix, const MergeTreePartInfo & dst_part_info); @@ -737,14 +737,21 @@ public: /// Checks if given part already belongs destination disk or volume for this rule. bool isPartInDestination(const StoragePolicyPtr & policy, const MergeTreeDataPart & part) const; + + bool isEmpty() const { return expression == nullptr; } }; - const TTLEntry * selectTTLEntryForTTLInfos(const MergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const; + std::optional selectTTLEntryForTTLInfos(const MergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const; using TTLEntriesByName = std::unordered_map; TTLEntriesByName column_ttl_entries_by_name; - TTLEntry ttl_table_entry; + TTLEntry rows_ttl_entry; + + /// This mutex is required for background move operations which do not obtain global locks. + mutable std::mutex move_ttl_entries_mutex; + + /// Vector rw operations have to be done under "move_ttl_entries_mutex". std::vector move_ttl_entries; String sampling_expr_column_name; @@ -786,8 +793,6 @@ protected: bool require_part_metadata; - String database_name; - String table_name; String relative_data_path; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 8ec47777dee..9079e2ee174 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -345,7 +345,7 @@ MergeTreeDataPart::~MergeTreeDataPart() } } - dir.remove(true); + remove(); if (state == State::DeleteOnDestroy) { @@ -394,7 +394,6 @@ void MergeTreeDataPart::remove() const String to = full_path + "delete_tmp_" + name; // TODO directory delete_tmp_ is never removed if server crashes before returning from this function - Poco::File from_dir{from}; Poco::File to_dir{to}; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 1c8e2e5621a..549345de8d1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -278,8 +278,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterBlocksAlreadySorted); } - if (data.hasTableTTL()) - updateTTL(data.ttl_table_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.table_ttl, block, true); + if (data.hasRowsTTL()) + updateTTL(data.rows_ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.table_ttl, block, true); for (const auto & [name, ttl_entry] : data.column_ttl_entries_by_name) updateTTL(ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.columns_ttl[name], block, true); diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp index 551a0de1338..2085f7b4fec 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -127,14 +127,14 @@ bool MergeTreePartsMover::selectPartsForMove( if (!can_move(part, &reason)) continue; - const MergeTreeData::TTLEntry * ttl_entry_ptr = part->storage.selectTTLEntryForTTLInfos(part->ttl_infos, time_of_move); + auto ttl_entry = part->storage.selectTTLEntryForTTLInfos(part->ttl_infos, time_of_move); auto to_insert = need_to_move.find(part->disk); ReservationPtr reservation; - if (ttl_entry_ptr) + if (ttl_entry) { - auto destination = ttl_entry_ptr->getDestination(policy); - if (destination && !ttl_entry_ptr->isPartInDestination(policy, *part)) - reservation = part->storage.tryReserveSpace(part->bytes_on_disk, ttl_entry_ptr->getDestination(policy)); + auto destination = ttl_entry->getDestination(policy); + if (destination && !ttl_entry->isPartInDestination(policy, *part)) + reservation = part->storage.tryReserveSpace(part->bytes_on_disk, ttl_entry->getDestination(policy)); } if (reservation) /// Found reservation by TTL rule. @@ -221,7 +221,9 @@ void MergeTreePartsMover::swapClonedPart(const MergeTreeData::DataPartPtr & clon return; } - cloned_part->renameTo(active_part->name); + /// Don't remove new directory but throw an error because it may contain part which is currently in use. + cloned_part->renameTo(active_part->name, false); + /// TODO what happen if server goes down here? data->swapActivePart(cloned_part); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index d308667a67b..a70dfc2d78c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -217,7 +217,7 @@ std::vector MergeTreeReadPool::fillPerPartInfo( per_part_sum_marks.push_back(sum_marks); - per_part_columns_lock.emplace_back(part.data_part->columns_lock); + per_part_columns_lock.emplace_back(part.data_part, part.data_part->columns_lock); auto [required_columns, required_pre_columns, should_reorder] = getReadTaskColumns(data, part.data_part, column_names, prewhere_info, check_columns); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h index 2e9cb76f0cd..4151b781d6e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -93,7 +94,7 @@ private: const size_t threads, const size_t sum_marks, std::vector per_part_sum_marks, RangesInDataParts & parts, const size_t min_marks_for_concurrent_read); - std::vector> per_part_columns_lock; + std::vector>> per_part_columns_lock; const MergeTreeData & data; Names column_names; bool do_not_steal_tasks; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 395d7cf8cca..666cff599d2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -86,8 +86,8 @@ struct MergeTreeSettings : public SettingsCollection M(SettingBool, ttl_only_drop_parts, false, "Only drop altogether the expired parts and not partially prune them.", 0) \ M(SettingBool, write_final_mark, 1, "Write final mark after end of column (0 - disabled, do nothing if index_granularity_bytes=0)", 0) \ M(SettingBool, enable_mixed_granularity_parts, 0, "Enable parts with adaptive and non adaptive granularity", 0) \ - M(SettingMaxThreads, max_part_loading_threads, 0, "The number of theads to load data parts at startup.", 0) \ - M(SettingMaxThreads, max_part_removal_threads, 0, "The number of theads for concurrent removal of inactive data parts. One is usually enough, but in 'Google Compute Environment SSD Persistent Disks' file removal (unlink) operation is extraordinarily slow and you probably have to increase this number (recommended is up to 16).", 0) \ + M(SettingMaxThreads, max_part_loading_threads, 0, "The number of threads to load data parts at startup.", 0) \ + M(SettingMaxThreads, max_part_removal_threads, 0, "The number of threads for concurrent removal of inactive data parts. One is usually enough, but in 'Google Compute Environment SSD Persistent Disks' file removal (unlink) operation is extraordinarily slow and you probably have to increase this number (recommended is up to 16).", 0) \ M(SettingUInt64, concurrent_part_removal_threshold, 100, "Activate concurrent part removal (see 'max_part_removal_threads') only if the number of inactive data parts is at least this.", 0) \ M(SettingString, storage_policy, "default", "Name of storage disk policy", 0) diff --git a/dbms/src/Storages/MergeTree/PartDestinationType.h b/dbms/src/Storages/MergeTree/PartDestinationType.h index 0d63051acca..8f3e44537a6 100644 --- a/dbms/src/Storages/MergeTree/PartDestinationType.h +++ b/dbms/src/Storages/MergeTree/PartDestinationType.h @@ -8,6 +8,7 @@ enum class PartDestinationType { DISK, VOLUME, + TABLE, DELETE, }; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index c639275158f..d1a201df1be 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -25,7 +25,7 @@ static const auto ALTER_ERROR_SLEEP_MS = 10 * 1000; ReplicatedMergeTreeAlterThread::ReplicatedMergeTreeAlterThread(StorageReplicatedMergeTree & storage_) : storage(storage_) , zk_node_cache([&] { return storage.getZooKeeper(); }) - , log_name(storage.database_name + "." + storage.table_name + " (ReplicatedMergeTreeAlterThread)") + , log_name(storage.getStorageID().getFullTableName() + " (ReplicatedMergeTreeAlterThread)") , log(&Logger::get(log_name)) { task = storage_.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); @@ -99,8 +99,8 @@ void ReplicatedMergeTreeAlterThread::run() /// Temporarily cancel parts sending ActionLock data_parts_exchange_blocker; - if (storage.data_parts_exchange_endpoint_holder) - data_parts_exchange_blocker = storage.data_parts_exchange_endpoint_holder->getBlocker().cancel(); + if (storage.data_parts_exchange_endpoint) + data_parts_exchange_blocker = storage.data_parts_exchange_endpoint->blocker.cancel(); /// Temporarily cancel part fetches auto fetches_blocker = storage.fetcher.blocker.cancel(); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index b89fd9629c0..c4d55f2e26f 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -19,7 +19,7 @@ namespace ErrorCodes ReplicatedMergeTreeCleanupThread::ReplicatedMergeTreeCleanupThread(StorageReplicatedMergeTree & storage_) : storage(storage_) - , log_name(storage.database_name + "." + storage.table_name + " (ReplicatedMergeTreeCleanupThread)") + , log_name(storage.getStorageID().getFullTableName() + " (ReplicatedMergeTreeCleanupThread)") , log(&Logger::get(log_name)) { task = storage.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index abb7e8e88ce..557aae3e01a 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -24,7 +24,7 @@ static const auto PART_CHECK_ERROR_SLEEP_MS = 5 * 1000; ReplicatedMergeTreePartCheckThread::ReplicatedMergeTreePartCheckThread(StorageReplicatedMergeTree & storage_) : storage(storage_) - , log_name(storage.database_name + "." + storage.table_name + " (ReplicatedMergeTreePartCheckThread)") + , log_name(storage.getStorageID().getFullTableName() + " (ReplicatedMergeTreePartCheckThread)") , log(&Logger::get(log_name)) { task = storage.global_context.getSchedulePool().createTask(log_name, [this] { run(); }); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index fce4479c16f..fad1196a4ba 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -40,7 +40,7 @@ static String generateActiveNodeIdentifier() ReplicatedMergeTreeRestartingThread::ReplicatedMergeTreeRestartingThread(StorageReplicatedMergeTree & storage_) : storage(storage_) - , log_name(storage.database_name + "." + storage.table_name + " (ReplicatedMergeTreeRestartingThread)") + , log_name(storage.getStorageID().getFullTableName() + " (ReplicatedMergeTreeRestartingThread)") , log(&Logger::get(log_name)) , active_node_identifier(generateActiveNodeIdentifier()) { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 703659bb4ea..4ebb51f0b41 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -55,15 +55,6 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr ttl_table = formattedAST(data.ttl_table_ast); - std::ostringstream ttl_move_stream; - for (const auto & ttl_entry : data.move_ttl_entries) - { - if (ttl_move_stream.tellp() > 0) - ttl_move_stream << ", "; - ttl_move_stream << formattedAST(ttl_entry.entry_ast); - } - ttl_move = ttl_move_stream.str(); - skip_indices = data.getIndices().toString(); if (data.canUseAdaptiveGranularity()) index_granularity_bytes = data_settings->index_granularity_bytes; @@ -95,9 +86,6 @@ void ReplicatedMergeTreeTableMetadata::write(WriteBuffer & out) const if (!ttl_table.empty()) out << "ttl: " << ttl_table << "\n"; - if (!ttl_move.empty()) - out << "move ttl: " << ttl_move << "\n"; - if (!skip_indices.empty()) out << "indices: " << skip_indices << "\n"; @@ -139,9 +127,6 @@ void ReplicatedMergeTreeTableMetadata::read(ReadBuffer & in) if (checkString("ttl: ", in)) in >> ttl_table >> "\n"; - if (checkString("move ttl: ", in)) - in >> ttl_move >> "\n"; - if (checkString("indices: ", in)) in >> skip_indices >> "\n"; @@ -252,21 +237,6 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl ErrorCodes::METADATA_MISMATCH); } - if (ttl_move != from_zk.ttl_move) - { - if (allow_alter) - { - diff.ttl_move_changed = true; - diff.new_ttl_move = from_zk.ttl_move; - } - else - throw Exception( - "Existing table metadata in ZooKeeper differs in move TTL." - " Stored in ZooKeeper: " + from_zk.ttl_move + - ", local: " + ttl_move, - ErrorCodes::METADATA_MISMATCH); - } - if (skip_indices != from_zk.skip_indices) { if (allow_alter) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h index 23fc4f6a024..d8af3c2087a 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h @@ -28,7 +28,6 @@ struct ReplicatedMergeTreeTableMetadata String skip_indices; String constraints; String ttl_table; - String ttl_move; UInt64 index_granularity_bytes; ReplicatedMergeTreeTableMetadata() = default; @@ -54,12 +53,9 @@ struct ReplicatedMergeTreeTableMetadata bool ttl_table_changed = false; String new_ttl_table; - bool ttl_move_changed = false; - String new_ttl_move; - bool empty() const { - return !sorting_key_changed && !skip_indices_changed && !ttl_table_changed && !constraints_changed && !ttl_move_changed; + return !sorting_key_changed && !skip_indices_changed && !ttl_table_changed && !constraints_changed; } }; diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 6865cc956fa..1326c61b1f6 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -18,8 +18,6 @@ class StorageFromMergeTreeDataPart : public ext::shared_ptr_helper; public: String getName() const override { return "FromMergeTreeDataPart"; } - String getTableName() const override { return part->storage.getTableName() + " (part " + part->name + ")"; } - String getDatabaseName() const override { return part->storage.getDatabaseName(); } BlockInputStreams read( const Names & column_names, @@ -51,7 +49,8 @@ public: protected: StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) - : IStorage(part_->storage.getVirtuals()), part(part_) + : IStorage(getIDFromPart(part_), part_->storage.getVirtuals()) + , part(part_) { setColumns(part_->storage.getColumns()); setIndices(part_->storage.getIndices()); @@ -59,6 +58,12 @@ protected: private: MergeTreeData::DataPartPtr part; + + static StorageID getIDFromPart(const MergeTreeData::DataPartPtr & part_) + { + auto table_id = part_->storage.getStorageID(); + return StorageID(table_id.database_name, table_id.table_name + " (part " + part_->name + ")"); + } }; } diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 858a49f015f..0c4359b0f65 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -656,12 +656,12 @@ static StoragePtr create(const StorageFactory::Arguments & args) }; if (replicated) return StorageReplicatedMergeTree::create( - zookeeper_path, replica_name, args.attach, args.database_name, args.table_name, args.relative_data_path, + zookeeper_path, replica_name, args.attach, args.table_id, args.relative_data_path, metadata, args.context, date_column_name, merging_params, std::move(storage_settings), args.has_force_restore_data_flag); else return StorageMergeTree::create( - args.database_name, args.table_name, args.relative_data_path, metadata, args.attach, args.context, + args.table_id, args.relative_data_path, metadata, args.attach, args.context, date_column_name, merging_params, std::move(storage_settings), args.has_force_restore_data_flag); } @@ -669,21 +669,31 @@ static StoragePtr create(const StorageFactory::Arguments & args) void registerStorageMergeTree(StorageFactory & factory) { - factory.registerStorage("MergeTree", create); - factory.registerStorage("CollapsingMergeTree", create); - factory.registerStorage("ReplacingMergeTree", create); - factory.registerStorage("AggregatingMergeTree", create); - factory.registerStorage("SummingMergeTree", create); - factory.registerStorage("GraphiteMergeTree", create); - factory.registerStorage("VersionedCollapsingMergeTree", create); + StorageFactory::StorageFeatures features{ + .supports_settings = true, + .supports_skipping_indices = true, + .supports_sort_order = true, + .supports_ttl = true, + }; - factory.registerStorage("ReplicatedMergeTree", create); - factory.registerStorage("ReplicatedCollapsingMergeTree", create); - factory.registerStorage("ReplicatedReplacingMergeTree", create); - factory.registerStorage("ReplicatedAggregatingMergeTree", create); - factory.registerStorage("ReplicatedSummingMergeTree", create); - factory.registerStorage("ReplicatedGraphiteMergeTree", create); - factory.registerStorage("ReplicatedVersionedCollapsingMergeTree", create); + factory.registerStorage("MergeTree", create, features); + factory.registerStorage("CollapsingMergeTree", create, features); + factory.registerStorage("ReplacingMergeTree", create, features); + factory.registerStorage("AggregatingMergeTree", create, features); + factory.registerStorage("SummingMergeTree", create, features); + factory.registerStorage("GraphiteMergeTree", create, features); + factory.registerStorage("VersionedCollapsingMergeTree", create, features); + + features.supports_replication = true; + features.supports_deduplication = true; + + factory.registerStorage("ReplicatedMergeTree", create, features); + factory.registerStorage("ReplicatedCollapsingMergeTree", create, features); + factory.registerStorage("ReplicatedReplacingMergeTree", create, features); + factory.registerStorage("ReplicatedAggregatingMergeTree", create, features); + factory.registerStorage("ReplicatedSummingMergeTree", create, features); + factory.registerStorage("ReplicatedGraphiteMergeTree", create, features); + factory.registerStorage("ReplicatedVersionedCollapsingMergeTree", create, features); } } diff --git a/dbms/src/Storages/PartitionCommands.cpp b/dbms/src/Storages/PartitionCommands.cpp index 885c6ddaec7..0da9f6d5ba4 100644 --- a/dbms/src/Storages/PartitionCommands.cpp +++ b/dbms/src/Storages/PartitionCommands.cpp @@ -54,10 +54,16 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * case PartDestinationType::VOLUME: res.move_destination_type = PartitionCommand::MoveDestinationType::VOLUME; break; + case PartDestinationType::TABLE: + res.move_destination_type = PartitionCommand::MoveDestinationType::TABLE; + res.to_database = command_ast->to_database; + res.to_table = command_ast->to_table; + break; default: break; } - res.move_destination_name = command_ast->move_destination_name; + if (res.move_destination_type != PartitionCommand::MoveDestinationType::TABLE) + res.move_destination_name = command_ast->move_destination_name; return res; } else if (command_ast->type == ASTAlterCommand::REPLACE_PARTITION) diff --git a/dbms/src/Storages/PartitionCommands.h b/dbms/src/Storages/PartitionCommands.h index d937d64a986..423287453ec 100644 --- a/dbms/src/Storages/PartitionCommands.h +++ b/dbms/src/Storages/PartitionCommands.h @@ -27,7 +27,7 @@ struct PartitionCommand FETCH_PARTITION, FREEZE_ALL_PARTITIONS, FREEZE_PARTITION, - REPLACE_PARTITION + REPLACE_PARTITION, }; Type type; @@ -47,6 +47,10 @@ struct PartitionCommand String from_table; bool replace = true; + /// For MOVE PARTITION + String to_database; + String to_table; + /// For FETCH PARTITION - path in ZK to the shard, from which to download the partition. String from_zookeeper_path; @@ -57,6 +61,7 @@ struct PartitionCommand { DISK, VOLUME, + TABLE, }; MoveDestinationType move_destination_type; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 53685d8694d..1178058a313 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -56,18 +56,27 @@ namespace ErrorCodes } -StorageBuffer::StorageBuffer(const std::string & database_name_, const std::string & table_name_, - const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, +StorageBuffer::StorageBuffer( + const StorageID & table_id_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, Context & context_, - size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, - const String & destination_database_, const String & destination_table_, bool allow_materialized_) - : - table_name(table_name_), database_name(database_name_), global_context(context_), - num_shards(num_shards_), buffers(num_shards_), - min_thresholds(min_thresholds_), max_thresholds(max_thresholds_), - destination_database(destination_database_), destination_table(destination_table_), - no_destination(destination_database.empty() && destination_table.empty()), - allow_materialized(allow_materialized_), log(&Logger::get("StorageBuffer (" + table_name + ")")) + size_t num_shards_, + const Thresholds & min_thresholds_, + const Thresholds & max_thresholds_, + const String & destination_database_, + const String & destination_table_, + bool allow_materialized_) + : IStorage(table_id_) + , global_context(context_) + , num_shards(num_shards_), buffers(num_shards_) + , min_thresholds(min_thresholds_) + , max_thresholds(max_thresholds_) + , destination_database(destination_database_) + , destination_table(destination_table_) + , no_destination(destination_database.empty() && destination_table.empty()) + , allow_materialized(allow_materialized_) + , log(&Logger::get("StorageBuffer (" + table_id_.getFullTableName() + ")")) { setColumns(columns_); setConstraints(constraints_); @@ -716,15 +725,15 @@ void StorageBuffer::alter(const AlterCommands & params, const Context & context, { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - const String database_name_ = getDatabaseName(); - const String table_name_ = getTableName(); + auto table_id = getStorageID(); + checkAlterIsPossible(params, context.getSettingsRef()); /// So that no blocks of the old structure remain. optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(database_name_)->alterTable(context, table_name_, metadata); + context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } @@ -763,8 +772,9 @@ void registerStorageBuffer(StorageFactory & factory) UInt64 max_bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[8]->as().value); return StorageBuffer::create( - args.database_name, - args.table_name, args.columns, args.constraints, + args.table_id, + args.columns, + args.constraints, args.context, num_buckets, StorageBuffer::Thresholds{min_time, min_rows, min_bytes}, diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index e7bdbc947f5..4436ead13d5 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -53,8 +53,6 @@ public: }; std::string getName() const override { return "Buffer"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } QueryProcessingStage::Enum getQueryProcessingStage(const Context & context) const override; @@ -73,12 +71,6 @@ public: void shutdown() override; bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override - { - table_name = new_table_name; - database_name = new_database_name; - } - bool supportsSampling() const override { return true; } bool supportsPrewhere() const override { @@ -102,9 +94,6 @@ public: ~StorageBuffer() override; private: - String table_name; - String database_name; - Context global_context; struct Buffer @@ -147,11 +136,17 @@ protected: /** num_shards - the level of internal parallelism (the number of independent buffers) * The buffer is flushed if all minimum thresholds or at least one of the maximum thresholds are exceeded. */ - StorageBuffer(const std::string & database_name_, const std::string & table_name_, - const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + StorageBuffer( + const StorageID & table_id_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, Context & context_, - size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, - const String & destination_database_, const String & destination_table_, bool allow_materialized_); + size_t num_shards_, + const Thresholds & min_thresholds_, + const Thresholds & max_thresholds_, + const String & destination_database_, + const String & destination_table_, + bool allow_materialized_); }; } diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index 6fb2a774812..51914b53ae6 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -24,16 +24,14 @@ namespace ErrorCodes StorageDictionary::StorageDictionary( - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const Context & context, bool attach, const String & dictionary_name_) - : table_name(table_name_), - database_name(database_name_), - dictionary_name(dictionary_name_), - logger(&Poco::Logger::get("StorageDictionary")) + : IStorage(table_id_) + , dictionary_name(dictionary_name_) + , logger(&Poco::Logger::get("StorageDictionary")) { setColumns(columns_); @@ -108,7 +106,7 @@ void registerStorageDictionary(StorageFactory & factory) String dictionary_name = args.engine_args[0]->as().value.safeGet(); return StorageDictionary::create( - args.database_name, args.table_name, args.columns, args.context, args.attach, dictionary_name); + args.table_id, args.columns, args.context, args.attach, dictionary_name); }); } diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 9539240e75d..1e741a05094 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -24,8 +24,6 @@ class StorageDictionary : public ext::shared_ptr_helper, publ friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Dictionary"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, @@ -59,8 +57,6 @@ public: private: using Ptr = MultiVersion::Version; - String table_name; - String database_name; String dictionary_name; Poco::Logger * logger; @@ -68,8 +64,7 @@ private: protected: StorageDictionary( - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const Context & context, bool attach, diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 86ef945d49f..ddf727c21de 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include @@ -49,6 +51,12 @@ #include +namespace +{ +static const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY = 1; +static const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_ALWAYS = 2; +} + namespace DB { @@ -63,6 +71,7 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int TOO_MANY_ROWS; + extern const int UNABLE_TO_SKIP_UNUSED_SHARDS; } namespace ActionLocks @@ -146,12 +155,6 @@ UInt64 getMaximumFileNumber(const std::string & dir_path) return res; } -void initializeFileNamesIncrement(const std::string & path, SimpleIncrement & increment) -{ - if (!path.empty()) - increment.set(getMaximumFileNumber(path)); -} - /// the same as DistributedBlockOutputStream::createSelector, should it be static? IColumn::Selector createSelector(const ClusterPtr cluster, const ColumnWithTypeAndName & result) { @@ -204,9 +207,9 @@ static ExpressionActionsPtr buildShardingKeyExpression(const ASTPtr & sharding_k return ExpressionAnalyzer(query, syntax_result, context).getActions(project); } + StorageDistributed::StorageDistributed( - const String & database_name_, - const String & table_name_, + const StorageID & id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & remote_database_, @@ -214,15 +217,22 @@ StorageDistributed::StorageDistributed( const String & cluster_name_, const Context & context_, const ASTPtr & sharding_key_, + const String & storage_policy_, const String & relative_data_path_, bool attach_) - : IStorage(ColumnsDescription({ - {"_shard_num", std::make_shared()}, - }, true)), - table_name(table_name_), database_name(database_name_), - remote_database(remote_database_), remote_table(remote_table_), - global_context(context_), cluster_name(global_context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_), - path(relative_data_path_.empty() ? "" : (context_.getPath() + relative_data_path_)) + : IStorage(id_, + ColumnsDescription( + { + {"_shard_num", std::make_shared()}, + }, + true)) + , remote_database(remote_database_) + , remote_table(remote_table_) + , global_context(context_) + , cluster_name(global_context.getMacros()->expand(cluster_name_)) + , has_sharding_key(sharding_key_) + , storage_policy(storage_policy_) + , relative_data_path(relative_data_path_) { setColumns(columns_); setConstraints(constraints_); @@ -233,55 +243,78 @@ StorageDistributed::StorageDistributed( sharding_key_column_name = sharding_key_->getColumnName(); } + if (!relative_data_path.empty()) + createStorage(); + /// Sanity check. Skip check if the table is already created to allow the server to start. if (!attach_ && !cluster_name.empty()) { size_t num_local_shards = global_context.getCluster(cluster_name)->getLocalShardCount(); - if (num_local_shards && remote_database == database_name && remote_table == table_name) - throw Exception("Distributed table " + table_name + " looks at itself", ErrorCodes::INFINITE_LOOP); + if (num_local_shards && remote_database == id_.database_name && remote_table == id_.table_name) + throw Exception("Distributed table " + id_.table_name + " looks at itself", ErrorCodes::INFINITE_LOOP); } } StorageDistributed::StorageDistributed( - const String & database_name_, - const String & table_name_, + const StorageID & id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, ASTPtr remote_table_function_ptr_, const String & cluster_name_, const Context & context_, const ASTPtr & sharding_key_, + const String & storage_policy_, const String & relative_data_path_, bool attach) - : StorageDistributed(database_name_, table_name_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, relative_data_path_, attach) + : StorageDistributed(id_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, storage_policy_, relative_data_path_, attach) { - remote_table_function_ptr = remote_table_function_ptr_; + remote_table_function_ptr = std::move(remote_table_function_ptr_); } +void StorageDistributed::createStorage() +{ + /// Create default policy with the relative_data_path_ + if (storage_policy.empty()) + { + std::string path(global_context.getPath()); + /// Disk must ends with '/' + if (!path.ends_with('/')) + path += '/'; + auto disk = std::make_shared("default", path, 0); + volume = std::make_shared("default", std::vector{disk}, 0); + } + else + { + auto policy = global_context.getStoragePolicySelector()[storage_policy]; + if (policy->getVolumes().size() != 1) + throw Exception("Policy for Distributed table, should have exactly one volume", ErrorCodes::BAD_ARGUMENTS); + volume = policy->getVolume(0); + } +} StoragePtr StorageDistributed::createWithOwnCluster( - const std::string & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const String & remote_database_, /// database on remote servers. const String & remote_table_, /// The name of the table on the remote servers. ClusterPtr owned_cluster_, const Context & context_) { - auto res = create(String{}, table_name_, columns_, ConstraintsDescription{}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), false); - res->owned_cluster = owned_cluster_; + auto res = create(table_id_, columns_, ConstraintsDescription{}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), String(), false); + res->owned_cluster = std::move(owned_cluster_); return res; } StoragePtr StorageDistributed::createWithOwnCluster( - const std::string & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, ASTPtr & remote_table_function_ptr_, ClusterPtr & owned_cluster_, const Context & context_) { - auto res = create(String{}, table_name_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), false); + auto res = create(table_id_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), String(), false); res->owned_cluster = owned_cluster_; return res; } @@ -337,23 +370,44 @@ BlockInputStreams StorageDistributed::read( : ClusterProxy::SelectStreamFactory( header, processed_stage, QualifiedTableName{remote_database, remote_table}, scalars, has_virtual_shard_num_column, context.getExternalTables()); + UInt64 force = settings.force_optimize_skip_unused_shards; if (settings.optimize_skip_unused_shards) { + ClusterPtr smaller_cluster; + auto table_id = getStorageID(); + if (has_sharding_key) { - auto smaller_cluster = skipUnusedShards(cluster, query_info); + smaller_cluster = skipUnusedShards(cluster, query_info); if (smaller_cluster) { cluster = smaller_cluster; - LOG_DEBUG(log, "Reading from " << database_name << "." << table_name << ": " + LOG_DEBUG(log, "Reading from " << table_id.getNameForLogs() << ": " "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): " " " << makeFormattedListOfShards(cluster)); } - else + } + + if (!smaller_cluster) + { + LOG_DEBUG(log, "Reading from " << table_id.getNameForLogs() << + (has_sharding_key ? "" : "(no sharding key)") << ": " + "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - " + "the query will be sent to all shards of the cluster"); + + if (force) { - LOG_DEBUG(log, "Reading from " << database_name << "." << table_name << ": " - "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster"); + std::stringstream exception_message; + if (has_sharding_key) + exception_message << "No sharding key"; + else + exception_message << "Sharding key " << sharding_key_column_name << " is not used"; + + if (force == FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_ALWAYS) + throw Exception(exception_message.str(), ErrorCodes::UNABLE_TO_SKIP_UNUSED_SHARDS); + if (force == FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY && has_sharding_key) + throw Exception(exception_message.str(), ErrorCodes::UNABLE_TO_SKIP_UNUSED_SHARDS); } } } @@ -369,7 +423,7 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c const auto & settings = context.getSettingsRef(); /// Ban an attempt to make async insert into the table belonging to DatabaseMemory - if (path.empty() && !owned_cluster && !settings.insert_distributed_sync) + if (!volume && !owned_cluster && !settings.insert_distributed_sync) { throw Exception("Storage " + getName() + " must has own data directory to enable asynchronous inserts", ErrorCodes::BAD_ARGUMENTS); @@ -410,21 +464,31 @@ void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, co void StorageDistributed::alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + auto table_id = getStorageID(); - const String current_database_name = getDatabaseName(); - const String current_table_name = getTableName(); - + checkAlterIsPossible(params, context.getSettingsRef()); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, metadata); + context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } void StorageDistributed::startup() { - createDirectoryMonitors(); - initializeFileNamesIncrement(path, file_names_increment); + if (!volume) + return; + + for (const DiskPtr & disk : volume->disks) + createDirectoryMonitors(disk->getPath()); + + for (const String & path : getDataPaths()) + { + UInt64 inc = getMaximumFileNumber(path); + if (inc > file_names_increment.value) + file_names_increment.value.store(inc); + } + LOG_DEBUG(log, "Auto-increment is " << file_names_increment.value); } @@ -433,6 +497,18 @@ void StorageDistributed::shutdown() cluster_nodes_data.clear(); } +Strings StorageDistributed::getDataPaths() const +{ + Strings paths; + + if (relative_data_path.empty()) + return paths; + + for (const DiskPtr & disk : volume->disks) + paths.push_back(disk->getPath() + relative_data_path); + + return paths; +} void StorageDistributed::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { @@ -478,33 +554,28 @@ bool StorageDistributed::hasColumn(const String & column_name) const return virtual_columns.count(column_name) || getColumns().hasPhysical(column_name); } -void StorageDistributed::createDirectoryMonitors() +void StorageDistributed::createDirectoryMonitors(const std::string & disk) { - if (path.empty()) - return; - + const std::string path(disk + relative_data_path); Poco::File{path}.createDirectories(); std::filesystem::directory_iterator begin(path); std::filesystem::directory_iterator end; for (auto it = begin; it != end; ++it) if (std::filesystem::is_directory(*it)) - requireDirectoryMonitor(it->path().filename().string()); + requireDirectoryMonitor(disk, it->path().filename().string()); } -void StorageDistributed::requireDirectoryMonitor(const std::string & name) +void StorageDistributed::requireDirectoryMonitor(const std::string & disk, const std::string & name) { - std::lock_guard lock(cluster_nodes_mutex); - cluster_nodes_data[name].requireDirectoryMonitor(name, *this, monitors_blocker); -} + const std::string path(disk + relative_data_path + name); + const std::string key(disk + name); -ConnectionPoolPtr StorageDistributed::requireConnectionPool(const std::string & name) -{ std::lock_guard lock(cluster_nodes_mutex); - auto & node_data = cluster_nodes_data[name]; - node_data.requireConnectionPool(name, *this); - return node_data.conneciton_pool; + auto & node_data = cluster_nodes_data[key]; + node_data.conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); + node_data.directory_monitor = std::make_unique(*this, path, node_data.conneciton_pool, monitors_blocker); } size_t StorageDistributed::getShardCount() const @@ -512,25 +583,16 @@ size_t StorageDistributed::getShardCount() const return getCluster()->getShardCount(); } +std::pair StorageDistributed::getPath() +{ + return {volume->getNextDisk()->getPath(), relative_data_path}; +} + ClusterPtr StorageDistributed::getCluster() const { return owned_cluster ? owned_cluster : global_context.getCluster(cluster_name); } -void StorageDistributed::ClusterNodeData::requireConnectionPool(const std::string & name, const StorageDistributed & storage) -{ - if (!conneciton_pool) - conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, storage); -} - -void StorageDistributed::ClusterNodeData::requireDirectoryMonitor( - const std::string & name, StorageDistributed & storage, ActionBlocker & monitor_blocker) -{ - requireConnectionPool(name, storage); - if (!directory_monitor) - directory_monitor = std::make_unique(storage, name, conneciton_pool, monitor_blocker); -} - void StorageDistributed::ClusterNodeData::flushAllData() { directory_monitor->flushAllData(); @@ -545,11 +607,6 @@ void StorageDistributed::ClusterNodeData::shutdownAndDropAllData() /// using constraints from "PREWHERE" and "WHERE" conditions, otherwise returns `nullptr` ClusterPtr StorageDistributed::skipUnusedShards(ClusterPtr cluster, const SelectQueryInfo & query_info) { - if (!has_sharding_key) - { - throw Exception("Internal error: cannot determine shards of a distributed table if no sharding expression is supplied", ErrorCodes::LOGICAL_ERROR); - } - const auto & select = query_info.query->as(); if (!select.prewhere() && !select.where()) @@ -610,17 +667,26 @@ void StorageDistributed::flushClusterNodesAllData() void StorageDistributed::rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { - table_name = new_table_name; - database_name = new_database_name; - if (!path.empty()) + if (!relative_data_path.empty()) + renameOnDisk(new_path_to_table_data); + renameInMemory(new_database_name, new_table_name); +} +void StorageDistributed::renameOnDisk(const String & new_path_to_table_data) +{ + for (const DiskPtr & disk : volume->disks) { - auto new_path = global_context.getPath() + new_path_to_table_data; - Poco::File(path).renameTo(new_path); - path = new_path; + const String path(disk->getPath()); + auto new_path = path + new_path_to_table_data; + Poco::File(path + relative_data_path).renameTo(new_path); + + LOG_DEBUG(log, "Updating path to " << new_path); + std::lock_guard lock(cluster_nodes_mutex); for (auto & node : cluster_nodes_data) - node.second.directory_monitor->updatePath(); + node.second.directory_monitor->updatePath(new_path); } + + relative_data_path = new_path_to_table_data; } @@ -632,6 +698,7 @@ void registerStorageDistributed(StorageFactory & factory) * - name of cluster in configuration; * - name of remote database; * - name of remote table; + * - policy to store data in; * * Remote database may be specified in following form: * - identifier; @@ -642,10 +709,15 @@ void registerStorageDistributed(StorageFactory & factory) ASTs & engine_args = args.engine_args; - if (!(engine_args.size() == 3 || engine_args.size() == 4)) - throw Exception("Storage Distributed requires 3 or 4 parameters" - " - name of configuration section with list of remote servers, name of remote database, name of remote table," - " sharding key expression (optional).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + if (engine_args.size() < 3 || engine_args.size() > 5) + throw Exception( + "Storage Distributed requires from 3 to 5 parameters - " + "name of configuration section with list of remote servers, " + "name of remote database, " + "name of remote table, " + "sharding key expression (optional), " + "policy to store data in (optional).", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); String cluster_name = getClusterName(*engine_args[0]); @@ -655,7 +727,8 @@ void registerStorageDistributed(StorageFactory & factory) String remote_database = engine_args[1]->as().value.safeGet(); String remote_table = engine_args[2]->as().value.safeGet(); - const auto & sharding_key = engine_args.size() == 4 ? engine_args[3] : nullptr; + const auto & sharding_key = engine_args.size() >= 4 ? engine_args[3] : nullptr; + const auto & storage_policy = engine_args.size() >= 5 ? engine_args[4]->as().value.safeGet() : ""; /// Check that sharding_key exists in the table and has numeric type. if (sharding_key) @@ -674,9 +747,12 @@ void registerStorageDistributed(StorageFactory & factory) } return StorageDistributed::create( - args.database_name, args.table_name, args.columns, args.constraints, + args.table_id, args.columns, args.constraints, remote_database, remote_table, cluster_name, - args.context, sharding_key, args.relative_data_path, + args.context, + sharding_key, + storage_policy, + args.relative_data_path, args.attach); }); } diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index e29587691d4..a3fd6f3c6e1 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -20,6 +20,9 @@ namespace DB class Context; class StorageDistributedDirectoryMonitor; +class Volume; +using VolumePtr = std::shared_ptr; + /** A distributed table that resides on multiple servers. * Uses data from the specified database and tables on each server. @@ -37,7 +40,7 @@ public: ~StorageDistributed() override; static StoragePtr createWithOwnCluster( - const std::string & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const String & remote_database_, /// database on remote servers. const String & remote_table_, /// The name of the table on the remote servers. @@ -45,15 +48,13 @@ public: const Context & context_); static StoragePtr createWithOwnCluster( - const std::string & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, ASTPtr & remote_table_function_ptr_, /// Table function ptr. ClusterPtr & owned_cluster_, const Context & context_); std::string getName() const override { return "Distributed"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } bool supportsSampling() const override { return true; } bool supportsFinal() const override { return true; } @@ -83,6 +84,7 @@ public: void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; void rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; + void renameOnDisk(const String & new_path_to_table_data); void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; @@ -94,22 +96,20 @@ public: void startup() override; void shutdown() override; - Strings getDataPaths() const override { return {path}; } + Strings getDataPaths() const override; const ExpressionActionsPtr & getShardingKeyExpr() const { return sharding_key_expr; } const String & getShardingKeyColumnName() const { return sharding_key_column_name; } size_t getShardCount() const; - const String & getPath() const { return path; } + std::pair getPath(); std::string getRemoteDatabaseName() const { return remote_database; } std::string getRemoteTableName() const { return remote_table; } std::string getClusterName() const { return cluster_name; } /// Returns empty string if tables is used by TableFunctionRemote /// create directory monitors for each existing subdirectory - void createDirectoryMonitors(); - /// ensure directory monitor thread creation by subdirectory name - void requireDirectoryMonitor(const std::string & name); - /// ensure connection pool creation and return it - ConnectionPoolPtr requireConnectionPool(const std::string & name); + void createDirectoryMonitors(const std::string & disk); + /// ensure directory monitor thread and connectoin pool creation by disk and subdirectory name + void requireDirectoryMonitor(const std::string & disk, const std::string & name); void flushClusterNodesAllData(); @@ -117,8 +117,6 @@ public: ActionLock getActionLock(StorageActionBlockType type) override; - String table_name; - String database_name; String remote_database; String remote_table; ASTPtr remote_table_function_ptr; @@ -135,24 +133,6 @@ public: bool has_sharding_key; ExpressionActionsPtr sharding_key_expr; String sharding_key_column_name; - String path; /// Can be empty if data_path_ is empty. In this case, a directory for the data to be sent is not created. - - struct ClusterNodeData - { - std::unique_ptr directory_monitor; - ConnectionPoolPtr conneciton_pool; - - /// Creates connection_pool if not exists. - void requireConnectionPool(const std::string & name, const StorageDistributed & storage); - /// Creates directory_monitor if not exists. - void requireDirectoryMonitor(const std::string & name, StorageDistributed & storage, ActionBlocker & monitor_blocker); - - void flushAllData(); - - void shutdownAndDropAllData(); - }; - std::unordered_map cluster_nodes_data; - std::mutex cluster_nodes_mutex; /// Used for global monotonic ordering of files to send. SimpleIncrement file_names_increment; @@ -161,8 +141,7 @@ public: protected: StorageDistributed( - const String & database_name_, - const String & table_name_, + const StorageID & id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & remote_database_, @@ -170,22 +149,42 @@ protected: const String & cluster_name_, const Context & context_, const ASTPtr & sharding_key_, + const String & storage_policy_, const String & relative_data_path_, bool attach_); StorageDistributed( - const String & database_name, - const String & table_name_, + const StorageID & id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, ASTPtr remote_table_function_ptr_, const String & cluster_name_, const Context & context_, const ASTPtr & sharding_key_, + const String & storage_policy_, const String & relative_data_path_, bool attach); ClusterPtr skipUnusedShards(ClusterPtr cluster, const SelectQueryInfo & query_info); + + void createStorage(); + + String storage_policy; + String relative_data_path; + /// Can be empty if relative_data_path is empty. In this case, a directory for the data to be sent is not created. + VolumePtr volume; + + struct ClusterNodeData + { + std::unique_ptr directory_monitor; + ConnectionPoolPtr conneciton_pool; + + void flushAllData(); + void shutdownAndDropAllData(); + }; + std::unordered_map cluster_nodes_data; + std::mutex cluster_nodes_mutex; + }; } diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 23efcd8aa39..d61d2a04fdf 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -30,9 +31,9 @@ static void checkAllTypesAreAllowedInTable(const NamesAndTypesList & names_and_t } -void StorageFactory::registerStorage(const std::string & name, Creator creator) +void StorageFactory::registerStorage(const std::string & name, CreatorFn creator_fn, StorageFeatures features) { - if (!storages.emplace(name, std::move(creator)).second) + if (!storages.emplace(name, Creator{std::move(creator_fn), features}).second) throw Exception("TableFunctionFactory: the table function name '" + name + "' is not unique", ErrorCodes::LOGICAL_ERROR); } @@ -41,13 +42,10 @@ void StorageFactory::registerStorage(const std::string & name, Creator creator) StoragePtr StorageFactory::get( const ASTCreateQuery & query, const String & relative_data_path, - const String & table_name, - const String & database_name, Context & local_context, Context & context, const ColumnsDescription & columns, const ConstraintsDescription & constraints, - bool attach, bool has_force_restore_data_flag) const { String name; @@ -95,24 +93,6 @@ StoragePtr StorageFactory::get( name = engine_def.name; - if (storage_def->settings && !endsWith(name, "MergeTree") && name != "Kafka" && name != "Join") - { - throw Exception( - "Engine " + name + " doesn't support SETTINGS clause. " - "Currently only the MergeTree family of engines, Kafka engine and Join engine support it", - ErrorCodes::BAD_ARGUMENTS); - } - - if ((storage_def->partition_by || storage_def->primary_key || storage_def->order_by || storage_def->sample_by || - storage_def->ttl_table || !columns.getColumnTTLs().empty() || - (query.columns_list && query.columns_list->indices && !query.columns_list->indices->children.empty())) - && !endsWith(name, "MergeTree")) - { - throw Exception( - "Engine " + name + " doesn't support PARTITION BY, PRIMARY KEY, ORDER BY, TTL or SAMPLE BY clauses and skipping indices. " - "Currently only the MergeTree family of engines supports them", ErrorCodes::BAD_ARGUMENTS); - } - if (name == "View") { throw Exception( @@ -131,17 +111,55 @@ StoragePtr StorageFactory::get( "Direct creation of tables with ENGINE LiveView is not supported, use CREATE LIVE VIEW statement", ErrorCodes::INCORRECT_QUERY); } - } - } - auto it = storages.find(name); - if (it == storages.end()) - { - auto hints = getHints(name); - if (!hints.empty()) - throw Exception("Unknown table engine " + name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_STORAGE); - else - throw Exception("Unknown table engine " + name, ErrorCodes::UNKNOWN_STORAGE); + auto it = storages.find(name); + if (it == storages.end()) + { + auto hints = getHints(name); + if (!hints.empty()) + throw Exception("Unknown table engine " + name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_STORAGE); + else + throw Exception("Unknown table engine " + name, ErrorCodes::UNKNOWN_STORAGE); + } + + auto checkFeature = [&](String feature_description, FeatureMatcherFn feature_matcher_fn) + { + if (!feature_matcher_fn(it->second.features)) + { + String msg = "Engine " + name + " doesn't support " + feature_description + ". " + "Currently only the following engines have support for the feature: ["; + auto supporting_engines = getAllRegisteredNamesByFeatureMatcherFn(feature_matcher_fn); + for (size_t index = 0; index < supporting_engines.size(); ++index) + { + if (index) + msg += ", "; + msg += supporting_engines[index]; + } + msg += "]"; + throw Exception(msg, ErrorCodes::BAD_ARGUMENTS); + } + }; + + if (storage_def->settings) + checkFeature( + "SETTINGS clause", + [](StorageFeatures features) { return features.supports_settings; }); + + if (storage_def->partition_by || storage_def->primary_key || storage_def->order_by || storage_def->sample_by) + checkFeature( + "PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses", + [](StorageFeatures features) { return features.supports_sort_order; }); + + if (storage_def->ttl_table || !columns.getColumnTTLs().empty()) + checkFeature( + "TTL clause", + [](StorageFeatures features) { return features.supports_ttl; }); + + if (query.columns_list && query.columns_list->indices && !query.columns_list->indices->children.empty()) + checkFeature( + "skipping indices", + [](StorageFeatures features) { return features.supports_skipping_indices; }); + } } Arguments arguments @@ -151,17 +169,16 @@ StoragePtr StorageFactory::get( .storage_def = storage_def, .query = query, .relative_data_path = relative_data_path, - .table_name = table_name, - .database_name = database_name, + .table_id = StorageID(query.database, query.table, query.uuid), .local_context = local_context, .context = context, .columns = columns, .constraints = constraints, - .attach = attach, + .attach = query.attach, .has_force_restore_data_flag = has_force_restore_data_flag }; - return it->second(arguments); + return storages.at(name).creator_fn(arguments); } StorageFactory & StorageFactory::instance() diff --git a/dbms/src/Storages/StorageFactory.h b/dbms/src/Storages/StorageFactory.h index 205096147c8..e64d8647dd8 100644 --- a/dbms/src/Storages/StorageFactory.h +++ b/dbms/src/Storages/StorageFactory.h @@ -15,6 +15,7 @@ namespace DB class Context; class ASTCreateQuery; class ASTStorage; +struct StorageID; /** Allows to create a table by the name and parameters of the engine. @@ -36,8 +37,7 @@ public: /// Path to table data. /// Relative to from server config (possibly of some of some for *MergeTree) const String & relative_data_path; - const String & table_name; - const String & database_name; + const StorageID & table_id; Context & local_context; Context & context; const ColumnsDescription & columns; @@ -46,25 +46,46 @@ public: bool has_force_restore_data_flag; }; - using Creator = std::function; + struct StorageFeatures + { + bool supports_settings = false; + bool supports_skipping_indices = false; + bool supports_sort_order = false; + bool supports_ttl = false; + bool supports_replication = false; + bool supports_deduplication = false; + }; + + using CreatorFn = std::function; + struct Creator + { + CreatorFn creator_fn; + StorageFeatures features; + }; + + using Storages = std::unordered_map; StoragePtr get( const ASTCreateQuery & query, const String & relative_data_path, - const String & table_name, - const String & database_name, Context & local_context, Context & context, const ColumnsDescription & columns, const ConstraintsDescription & constraints, - bool attach, bool has_force_restore_data_flag) const; /// Register a table engine by its name. /// No locking, you must register all engines before usage of get. - void registerStorage(const std::string & name, Creator creator); + void registerStorage(const std::string & name, CreatorFn creator_fn, StorageFeatures features = StorageFeatures{ + .supports_settings = false, + .supports_skipping_indices = false, + .supports_sort_order = false, + .supports_ttl = false, + .supports_replication = false, + .supports_deduplication = false, + }); - const auto & getAllStorages() const + const Storages & getAllStorages() const { return storages; } @@ -77,8 +98,18 @@ public: return result; } + using FeatureMatcherFn = std::function; + std::vector getAllRegisteredNamesByFeatureMatcherFn(FeatureMatcherFn feature_matcher_fn) const + { + std::vector result; + for (const auto& pair : storages) + if (feature_matcher_fn(pair.second.features)) + result.push_back(pair.first); + return result; + } + + private: - using Storages = std::unordered_map; Storages storages; }; diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 8df0385d55f..5e7177b3e25 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -181,8 +182,17 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu } StorageFile::StorageFile(CommonArguments args) - : table_name(args.table_name), database_name(args.database_name), format_name(args.format_name) - , compression_method(args.compression_method), base_path(args.context.getPath()) + : IStorage(args.table_id, + ColumnsDescription({ + {"_path", std::make_shared()}, + {"_file", std::make_shared()} + }, + true /// all_virtuals + ) + ) + , format_name(args.format_name) + , compression_method(args.compression_method) + , base_path(args.context.getPath()) { if (args.format_name != "Distributed") setColumns(args.columns); @@ -194,11 +204,12 @@ class StorageFileBlockInputStream : public IBlockInputStream { public: StorageFileBlockInputStream(std::shared_ptr storage_, - const Context & context, UInt64 max_block_size, - std::string file_path, - const CompressionMethod compression_method, + const Context & context_, UInt64 max_block_size_, + std::string file_path_, bool need_path, bool need_file, + const CompressionMethod compression_method_, BlockInputStreamPtr prepared_reader = nullptr) - : storage(std::move(storage_)), reader(std::move(prepared_reader)) + : storage(std::move(storage_)), reader(std::move(prepared_reader)), + context(context_), max_block_size(max_block_size_), compression_method(compression_method_) { if (storage->use_table_fd) { @@ -218,16 +229,14 @@ public: } storage->table_fd_was_used = true; - read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(storage->table_fd), compression_method); } else { shared_lock = std::shared_lock(storage->rwlock); - read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(file_path), compression_method); + file_path = std::make_optional(file_path_); + with_file_column = need_file; + with_path_column = need_path; } - - if (!reader) - reader = FormatFactory::instance().getInput(storage->format_name, *read_buf, storage->getSampleBlock(), context, max_block_size); } String getName() const override @@ -237,34 +246,81 @@ public: Block readImpl() override { - return reader->read(); + /// Open file lazily on first read. This is needed to avoid too many open files from different streams. + if (!reader) + { + read_buf = wrapReadBufferWithCompressionMethod(storage->use_table_fd + ? std::make_unique(storage->table_fd) + : std::make_unique(file_path.value()), + compression_method); + + reader = FormatFactory::instance().getInput(storage->format_name, *read_buf, storage->getSampleBlock(), context, max_block_size); + reader->readPrefix(); + } + + auto res = reader->read(); + + /// Enrich with virtual columns. + if (res && file_path) + { + if (with_path_column) + res.insert({DataTypeString().createColumnConst(res.rows(), file_path.value())->convertToFullColumnIfConst(), + std::make_shared(), "_path"}); /// construction with const is for probably generating less code + if (with_file_column) + { + size_t last_slash_pos = file_path.value().find_last_of('/'); + res.insert({DataTypeString().createColumnConst(res.rows(), file_path.value().substr( + last_slash_pos + 1))->convertToFullColumnIfConst(), + std::make_shared(), "_file"}); + } + } + + /// Close file prematurally if stream was ended. + if (!res) + { + reader->readSuffix(); + reader.reset(); + read_buf.reset(); + } + + return res; } - Block getHeader() const override { return reader->getHeader(); } - - void readPrefixImpl() override + Block getHeader() const override { - reader->readPrefix(); - } + auto res = storage->getSampleBlock(); - void readSuffixImpl() override - { - reader->readSuffix(); + if (res && file_path) + { + if (with_path_column) + res.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + if (with_file_column) + res.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + } + + return res; } private: std::shared_ptr storage; + std::optional file_path; + bool with_path_column = false; + bool with_file_column = false; Block sample_block; std::unique_ptr read_buf; BlockInputStreamPtr reader; + const Context & context; /// TODO Untangle potential issues with context lifetime. + UInt64 max_block_size; + const CompressionMethod compression_method; + std::shared_lock shared_lock; std::unique_lock unique_lock; }; BlockInputStreams StorageFile::read( - const Names & /*column_names*/, + const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, QueryProcessingStage::Enum /*processed_stage*/, @@ -274,14 +330,23 @@ BlockInputStreams StorageFile::read( const ColumnsDescription & columns_ = getColumns(); auto column_defaults = columns_.getDefaults(); BlockInputStreams blocks_input; + if (use_table_fd) /// need to call ctr BlockInputStream paths = {""}; /// when use fd, paths are empty else - { if (paths.size() == 1 && !Poco::File(paths[0]).exists()) throw Exception("File " + paths[0] + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); - } + blocks_input.reserve(paths.size()); + bool need_path_column = false; + bool need_file_column = false; + for (const auto & column : column_names) + { + if (column == "_path") + need_path_column = true; + if (column == "_file") + need_file_column = true; + } for (const auto & file_path : paths) { BlockInputStreamPtr cur_block; @@ -290,7 +355,7 @@ BlockInputStreams StorageFile::read( cur_block = StorageDistributedDirectoryMonitor::createStreamFromFile(file_path); else cur_block = std::make_shared( - std::static_pointer_cast(shared_from_this()), context, max_block_size, file_path, chooseCompressionMethod(file_path, compression_method)); + std::static_pointer_cast(shared_from_this()), context, max_block_size, file_path, need_path_column, need_file_column, chooseCompressionMethod(file_path, compression_method)); blocks_input.push_back(column_defaults.empty() ? cur_block : std::make_shared(cur_block, column_defaults, context)); } @@ -318,7 +383,7 @@ public: else { if (storage.paths.size() != 1) - throw Exception("Table '" + storage.table_name + "' is in readonly mode because of globs in filepath", ErrorCodes::DATABASE_ACCESS_DENIED); + throw Exception("Table '" + storage.getStorageID().getNameForLogs() + "' is in readonly mode because of globs in filepath", ErrorCodes::DATABASE_ACCESS_DENIED); write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(storage.paths[0], DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT), compression_method, 3); @@ -370,17 +435,17 @@ BlockOutputStreamPtr StorageFile::write( Strings StorageFile::getDataPaths() const { if (paths.empty()) - throw Exception("Table '" + table_name + "' is in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); + throw Exception("Table '" + getStorageID().getNameForLogs() + "' is in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); return paths; } void StorageFile::rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { if (!is_db_table) - throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED); + throw Exception("Can't rename table " + getStorageID().getNameForLogs() + " binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED); if (paths.size() != 1) - throw Exception("Can't rename table '" + table_name + "' in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); + throw Exception("Can't rename table " + getStorageID().getNameForLogs() + " in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); std::unique_lock lock(rwlock); @@ -389,14 +454,13 @@ void StorageFile::rename(const String & new_path_to_table_data, const String & n Poco::File(paths[0]).renameTo(path_new); paths[0] = std::move(path_new); - table_name = new_table_name; - database_name = new_database_name; + renameInMemory(new_database_name, new_table_name); } void StorageFile::truncate(const ASTPtr & /*query*/, const Context & /* context */, TableStructureWriteLockHolder &) { if (paths.size() != 1) - throw Exception("Can't truncate table '" + table_name + "' in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); + throw Exception("Can't truncate table '" + getStorageID().getNameForLogs() + "' in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); std::unique_lock lock(rwlock); @@ -431,7 +495,7 @@ void registerStorageFile(StorageFactory & factory) String format_name = engine_args[0]->as().value.safeGet(); String compression_method; - StorageFile::CommonArguments common_args{args.database_name, args.table_name, format_name, compression_method, + StorageFile::CommonArguments common_args{args.table_id, format_name, compression_method, args.columns, args.constraints, args.context}; if (engine_args.size() == 1) /// Table in database diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index 23a6d6e7ff5..46c3f787db7 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -23,8 +23,6 @@ class StorageFile : public ext::shared_ptr_helper, public IStorage friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "File"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -46,8 +44,7 @@ public: struct CommonArguments { - const std::string & database_name; - const std::string & table_name; + const StorageID & table_id; const std::string & format_name; const std::string & compression_method; const ColumnsDescription & columns; @@ -71,8 +68,6 @@ protected: private: explicit StorageFile(CommonArguments args); - std::string table_name; - std::string database_name; std::string format_name; int table_fd = -1; diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp index 8e5db910092..391119c3a30 100644 --- a/dbms/src/Storages/StorageHDFS.cpp +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -35,17 +36,22 @@ namespace ErrorCodes } StorageHDFS::StorageHDFS(const String & uri_, - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, const String & format_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, Context & context_, const String & compression_method_ = "") - : uri(uri_) + : IStorage(table_id_, + ColumnsDescription({ + {"_path", std::make_shared()}, + {"_file", std::make_shared()} + }, + true /// all_virtuals + ) + ) + , uri(uri_) , format_name(format_name_) - , table_name(table_name_) - , database_name(database_name_) , context(context_) , compression_method(compression_method_) { @@ -61,6 +67,8 @@ class HDFSBlockInputStream : public IBlockInputStream { public: HDFSBlockInputStream(const String & uri, + bool need_path, + bool need_file, const String & format, const Block & sample_block, const Context & context, @@ -68,7 +76,9 @@ public: const CompressionMethod compression_method) { auto read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(uri), compression_method); - + file_path = uri; + with_file_column = need_file; + with_path_column = need_path; auto input_stream = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); reader = std::make_shared>(input_stream, std::move(read_buf)); } @@ -80,12 +90,34 @@ public: Block readImpl() override { - return reader->read(); + auto res = reader->read(); + if (res) + { + if (with_path_column) + res.insert({DataTypeString().createColumnConst(res.rows(), file_path)->convertToFullColumnIfConst(), std::make_shared(), + "_path"}); /// construction with const is for probably generating less code + if (with_file_column) + { + size_t last_slash_pos = file_path.find_last_of('/'); + res.insert({DataTypeString().createColumnConst(res.rows(), file_path.substr( + last_slash_pos + 1))->convertToFullColumnIfConst(), std::make_shared(), + "_file"}); + } + } + return res; } Block getHeader() const override { - return reader->getHeader(); + auto res = reader->getHeader(); + if (res) + { + if (with_path_column) + res.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + if (with_file_column) + res.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + } + return res; } void readPrefixImpl() override @@ -100,6 +132,9 @@ public: private: BlockInputStreamPtr reader; + String file_path; + bool with_path_column = false; + bool with_file_column = false; }; class HDFSBlockOutputStream : public IBlockOutputStream @@ -194,7 +229,7 @@ Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, c BlockInputStreams StorageHDFS::read( - const Names & /*column_names*/, + const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context_, QueryProcessingStage::Enum /*processed_stage*/, @@ -210,21 +245,24 @@ BlockInputStreams StorageHDFS::read( const Strings res_paths = LSWithRegexpMatching("/", fs, path_from_uri); BlockInputStreams result; + bool need_path_column = false; + bool need_file_column = false; + for (const auto & column : column_names) + { + if (column == "_path") + need_path_column = true; + if (column == "_file") + need_file_column = true; + } for (const auto & res_path : res_paths) { - result.push_back(std::make_shared(uri_without_path + res_path, format_name, getSampleBlock(), context_, + result.push_back(std::make_shared(uri_without_path + res_path, need_path_column, need_file_column, format_name, getSampleBlock(), context_, max_block_size, chooseCompressionMethod(res_path, compression_method))); } return narrowBlockInputStreams(result, num_streams); } -void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) -{ - table_name = new_table_name; - database_name = new_database_name; -} - BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const Context & /*context*/) { return std::make_shared(uri, @@ -259,7 +297,7 @@ void registerStorageHDFS(StorageFactory & factory) compression_method = engine_args[2]->as().value.safeGet(); } else compression_method = "auto"; - return StorageHDFS::create(url, args.database_name, args.table_name, format_name, args.columns, args.constraints, args.context, compression_method); + return StorageHDFS::create(url, args.table_id, format_name, args.columns, args.constraints, args.context, compression_method); }); } diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h index 5c02793f781..42f1891597e 100644 --- a/dbms/src/Storages/StorageHDFS.h +++ b/dbms/src/Storages/StorageHDFS.h @@ -18,8 +18,6 @@ class StorageHDFS : public ext::shared_ptr_helper, public IStorage friend struct ext::shared_ptr_helper; public: String getName() const override { return "HDFS"; } - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, @@ -30,12 +28,9 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; - protected: StorageHDFS(const String & uri_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const String & format_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, @@ -45,8 +40,6 @@ protected: private: String uri; String format_name; - String table_name; - String database_name; Context & context; String compression_method; diff --git a/dbms/src/Storages/StorageID.h b/dbms/src/Storages/StorageID.h new file mode 100644 index 00000000000..2cc19818de5 --- /dev/null +++ b/dbms/src/Storages/StorageID.h @@ -0,0 +1,98 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_"; + +struct StorageID +{ + String database_name; + String table_name; + UUID uuid = UUID{UInt128(0, 0)}; + + + StorageID(const String & database, const String & table, UUID uuid_ = UUID{UInt128(0, 0)}) + : database_name(database), table_name(table), uuid(uuid_) + { + assertNotEmpty(); + } + + String getDatabaseName() const + { + assertNotEmpty(); + return database_name; + } + + String getTableName() const + { + assertNotEmpty(); + return table_name; + } + + String getFullTableName() const + { + assertNotEmpty(); + return (database_name.empty() ? "" : database_name + ".") + table_name; + } + + String getNameForLogs() const + { + assertNotEmpty(); + return (database_name.empty() ? "" : backQuoteIfNeed(database_name) + ".") + backQuoteIfNeed(table_name) + + (hasUUID() ? " (UUID " + toString(uuid) + ")" : ""); + } + + bool empty() const + { + return table_name.empty() && !hasUUID(); + } + + bool hasUUID() const + { + return uuid != UUID{UInt128(0, 0)}; + } + + bool operator<(const StorageID & rhs) const + { + assertNotEmpty(); + /// It's needed for ViewDependencies + if (!hasUUID() && !rhs.hasUUID()) + /// If both IDs don't have UUID, compare them like pair of strings + return std::tie(database_name, table_name) < std::tie(rhs.database_name, rhs.table_name); + else if (hasUUID() && rhs.hasUUID()) + /// If both IDs have UUID, compare UUIDs and ignore database and table name + return uuid < rhs.uuid; + else + /// All IDs without UUID are less, then all IDs with UUID + return !hasUUID(); + } + + void assertNotEmpty() const + { + if (empty()) + throw Exception("Both table name and UUID are empty", ErrorCodes::LOGICAL_ERROR); + if (table_name == TABLE_WITH_UUID_NAME_PLACEHOLDER && !hasUUID()) + throw Exception("Table name was replaced with placeholder, but UUID is Nil", ErrorCodes::LOGICAL_ERROR); + if (table_name.empty() && !database_name.empty()) + throw Exception("Table name is empty, but database name is not", ErrorCodes::LOGICAL_ERROR); + } + + /// Avoid implicit construction of empty StorageID. However, it's needed for deferred initialization. + static StorageID createEmpty() { return {}; } + +private: + StorageID() = default; +}; + +} diff --git a/dbms/src/Storages/StorageInput.cpp b/dbms/src/Storages/StorageInput.cpp index 38424f22632..68b9dfe8810 100644 --- a/dbms/src/Storages/StorageInput.cpp +++ b/dbms/src/Storages/StorageInput.cpp @@ -15,8 +15,8 @@ namespace ErrorCodes extern const int INVALID_USAGE_OF_INPUT; } -StorageInput::StorageInput(const String &table_name_, const ColumnsDescription & columns_) - : IStorage(columns_), table_name(table_name_) +StorageInput::StorageInput(const String & table_name_, const ColumnsDescription & columns_) + : IStorage({"", table_name_}) { setColumns(columns_); } diff --git a/dbms/src/Storages/StorageInput.h b/dbms/src/Storages/StorageInput.h index bfc70633c48..706bb5bbfdf 100644 --- a/dbms/src/Storages/StorageInput.h +++ b/dbms/src/Storages/StorageInput.h @@ -13,7 +13,6 @@ class StorageInput : public ext::shared_ptr_helper, public IStorag friend struct ext::shared_ptr_helper; public: String getName() const override { return "Input"; } - String getTableName() const override { return table_name; } /// A table will read from this stream. void setInputStream(BlockInputStreamPtr input_stream_); @@ -27,7 +26,6 @@ public: unsigned num_streams) override; private: - String table_name; BlockInputStreamPtr input_stream; protected: diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index 871bd595f38..f18859ae90a 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -30,8 +30,7 @@ namespace ErrorCodes StorageJoin::StorageJoin( const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const Names & key_names_, bool use_nulls_, SizeLimits limits_, @@ -41,7 +40,7 @@ StorageJoin::StorageJoin( const ConstraintsDescription & constraints_, bool overwrite, const Context & context_) - : StorageSetOrJoinBase{relative_path_, database_name_, table_name_, columns_, constraints_, context_} + : StorageSetOrJoinBase{relative_path_, table_id_, columns_, constraints_, context_} , key_names(key_names_) , use_nulls(use_nulls_) , limits(limits_) @@ -72,11 +71,11 @@ void StorageJoin::truncate(const ASTPtr &, const Context &, TableStructureWriteL HashJoinPtr StorageJoin::getJoin(std::shared_ptr analyzed_join) const { if (kind != analyzed_join->kind() || strictness != analyzed_join->strictness()) - throw Exception("Table " + backQuote(table_name) + " has incompatible type of JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); + throw Exception("Table " + getStorageID().getNameForLogs() + " has incompatible type of JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); if ((analyzed_join->forceNullableRight() && !use_nulls) || (!analyzed_join->forceNullableRight() && isLeftOrFull(analyzed_join->kind()) && use_nulls)) - throw Exception("Table " + backQuote(table_name) + " needs the same join_use_nulls setting as present in LEFT or FULL JOIN.", + throw Exception("Table " + getStorageID().getNameForLogs() + " needs the same join_use_nulls setting as present in LEFT or FULL JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); /// TODO: check key columns @@ -96,7 +95,7 @@ size_t StorageJoin::getSize() const { return join->getTotalRowCount(); } void registerStorageJoin(StorageFactory & factory) { - factory.registerStorage("Join", [](const StorageFactory::Arguments & args) + auto creator_fn = [](const StorageFactory::Arguments & args) { /// Join(ANY, LEFT, k1, k2, ...) @@ -200,8 +199,7 @@ void registerStorageJoin(StorageFactory & factory) return StorageJoin::create( args.relative_data_path, - args.database_name, - args.table_name, + args.table_id, key_names, join_use_nulls, SizeLimits{max_rows_in_join, max_bytes_in_join, join_overflow_mode}, @@ -211,7 +209,9 @@ void registerStorageJoin(StorageFactory & factory) args.constraints, join_any_take_last_row, args.context); - }); + }; + + factory.registerStorage("Join", creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, }); } template diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index ebd0c5ff0d5..22500a23baa 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -62,8 +62,7 @@ private: protected: StorageJoin( const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const Names & key_names_, bool use_nulls_, SizeLimits limits_, diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 82e90060cd1..44c3af28267 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -413,14 +413,15 @@ void LogBlockOutputStream::writeMarks(MarksForColumns && marks) StorageLog::StorageLog( DiskPtr disk_, const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, size_t max_compress_block_size_) - : disk(std::move(disk_)), table_path(relative_path_), database_name(database_name_), table_name(table_name_), - max_compress_block_size(max_compress_block_size_), - file_checker(disk, table_path + "sizes.json") + : IStorage(table_id_) + , disk(std::move(disk_)) + , table_path(relative_path_) + , max_compress_block_size(max_compress_block_size_) + , file_checker(disk, table_path + "sizes.json") { setColumns(columns_); setConstraints(constraints_); @@ -512,14 +513,13 @@ void StorageLog::rename(const String & new_path_to_table_data, const String & ne disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; - database_name = new_database_name; - table_name = new_table_name; file_checker.setPath(table_path + "sizes.json"); for (auto & file : files) file.second.data_file_path = table_path + fileName(file.second.data_file_path); marks_file_path = table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME; + renameInMemory(new_database_name, new_table_name); } void StorageLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) @@ -633,7 +633,7 @@ void registerStorageLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageLog::create( - args.context.getDefaultDisk(), args.relative_data_path, args.database_name, args.table_name, args.columns, args.constraints, + args.context.getDefaultDisk(), args.relative_data_path, args.table_id, args.columns, args.constraints, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index dcde2f33805..696196e4069 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -23,8 +23,6 @@ class StorageLog : public ext::shared_ptr_helper, public IStorage public: String getName() const override { return "Log"; } - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -55,9 +53,8 @@ protected: */ StorageLog( DiskPtr disk_, - const String & relative_path_, - const String & database_name_, - const String & table_name_, + const std::string & relative_path_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, size_t max_compress_block_size_); @@ -87,8 +84,6 @@ private: DiskPtr disk; String table_path; - String database_name; - String table_name; mutable std::shared_mutex rwlock; @@ -113,9 +108,6 @@ private: /// You can not call with a write locked `rwlock`. void loadMarks(); - /// The order of adding files should not change: it corresponds to the order of the columns in the marks file. - void addFile(const String & column_name, const IDataType & type, size_t level = 0); - /** For normal columns, the number of rows in the block is specified in the marks. * For array columns and nested structures, there are more than one group of marks that correspond to different files * - for elements (file name.bin) - the total number of array elements in the block is specified, diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index b94cdba682d..6725fd6458e 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int INCORRECT_QUERY; extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW; + extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW; } static inline String generateInnerTableName(const String & table_name) @@ -35,40 +36,35 @@ static inline String generateInnerTableName(const String & table_name) return ".inner." + table_name; } -static void extractDependentTable(ASTSelectQuery & query, String & select_database_name, String & select_table_name) +static StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, Context & context, bool add_default_db = true) { - auto db_and_table = getDatabaseAndTable(query, 0); - ASTPtr subquery = extractTableExpression(query, 0); - - if (!db_and_table && !subquery) - return; - - if (db_and_table) + if (add_default_db) { - select_table_name = db_and_table->table; - - if (db_and_table->database.empty()) - { - db_and_table->database = select_database_name; - AddDefaultDatabaseVisitor visitor(select_database_name); - visitor.visit(query); - } - else - select_database_name = db_and_table->database; + AddDefaultDatabaseVisitor visitor(context.getCurrentDatabase(), nullptr); + visitor.visit(query); } - else if (auto * ast_select = subquery->as()) + + if (auto db_and_table = getDatabaseAndTable(query, 0)) { + return StorageID(db_and_table->database, db_and_table->table/*, db_and_table->uuid*/); + } + else if (auto subquery = extractTableExpression(query, 0)) + { + auto * ast_select = subquery->as(); + if (!ast_select) + throw Exception("Logical error while creating StorageMaterializedView. " + "Could not retrieve table name from select query.", + DB::ErrorCodes::LOGICAL_ERROR); if (ast_select->list_of_selects->children.size() != 1) - throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); + throw Exception("UNION is not supported for MATERIALIZED VIEW", + ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); auto & inner_query = ast_select->list_of_selects->children.at(0); - extractDependentTable(inner_query->as(), select_database_name, select_table_name); + return extractDependentTableFromSelectQuery(inner_query->as(), context, false); } else - throw Exception("Logical error while creating StorageMaterializedView." - " Could not retrieve table name from select query.", - DB::ErrorCodes::LOGICAL_ERROR); + return StorageID::createEmpty(); } @@ -94,61 +90,47 @@ static void checkAllowedQueries(const ASTSelectQuery & query) StorageMaterializedView::StorageMaterializedView( - const String & table_name_, - const String & database_name_, + const StorageID & table_id_, Context & local_context, const ASTCreateQuery & query, const ColumnsDescription & columns_, bool attach_) - : table_name(table_name_), - database_name(database_name_), global_context(local_context.getGlobalContext()) + : IStorage(table_id_), global_context(local_context.getGlobalContext()) { setColumns(columns_); if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); - if (!query.storage && query.to_table.empty()) + /// If the destination table is not set, use inner table + has_inner_table = query.to_table.empty(); + if (has_inner_table && !query.storage) throw Exception( "You must specify where to save results of a MaterializedView query: either ENGINE or an existing table in a TO clause", ErrorCodes::INCORRECT_QUERY); - /// Default value, if only table name exist in the query - select_database_name = local_context.getCurrentDatabase(); if (query.select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); inner_query = query.select->list_of_selects->children.at(0); auto & select_query = inner_query->as(); - extractDependentTable(select_query, select_database_name, select_table_name); + select_table_id = extractDependentTableFromSelectQuery(select_query, local_context); checkAllowedQueries(select_query); - if (!select_table_name.empty()) - global_context.addDependency( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); - - // If the destination table is not set, use inner table - if (!query.to_table.empty()) + if (!has_inner_table) + target_table_id = StorageID(query.to_database, query.to_table); + else if (attach_) { - target_database_name = query.to_database; - target_table_name = query.to_table; + /// If there is an ATTACH request, then the internal table must already be created. + target_table_id = StorageID(table_id_.database_name, generateInnerTableName(table_id_.table_name)); } else - { - target_database_name = database_name; - target_table_name = generateInnerTableName(table_name); - has_inner_table = true; - } - - /// If there is an ATTACH request, then the internal table must already be connected. - if (!attach_ && has_inner_table) { /// We will create a query to create an internal table. auto manual_create_query = std::make_shared(); - manual_create_query->database = target_database_name; - manual_create_query->table = target_table_name; + manual_create_query->database = table_id_.database_name; + manual_create_query->table = generateInnerTableName(table_id_.table_name); auto new_columns_list = std::make_shared(); new_columns_list->set(new_columns_list->columns, query.columns_list->columns->ptr()); @@ -156,24 +138,15 @@ StorageMaterializedView::StorageMaterializedView( manual_create_query->set(manual_create_query->columns_list, new_columns_list); manual_create_query->set(manual_create_query->storage, query.storage->ptr()); - /// Execute the query. - try - { - InterpreterCreateQuery create_interpreter(manual_create_query, local_context); - create_interpreter.setInternal(true); - create_interpreter.execute(); - } - catch (...) - { - /// In case of any error we should remove dependency to the view. - if (!select_table_name.empty()) - global_context.removeDependency( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); + InterpreterCreateQuery create_interpreter(manual_create_query, local_context); + create_interpreter.setInternal(true); + create_interpreter.execute(); - throw; - } + target_table_id = global_context.getTable(manual_create_query->database, manual_create_query->table)->getStorageID(); } + + if (!select_table_id.empty()) + global_context.addDependency(select_table_id, table_id_); } NameAndTypePair StorageMaterializedView::getColumn(const String & column_name) const @@ -220,14 +193,14 @@ BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const } -static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const String & target_database_name, const String & target_table_name) +static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const StorageID & target_table_id) { - if (global_context.tryGetTable(target_database_name, target_table_name)) + if (global_context.tryGetTable(target_table_id)) { /// We create and execute `drop` query for internal table. auto drop_query = std::make_shared(); - drop_query->database = target_database_name; - drop_query->table = target_table_name; + drop_query->database = target_table_id.database_name; + drop_query->table = target_table_id.table_name; drop_query->kind = kind; ASTPtr ast_drop_query = drop_query; InterpreterDropQuery drop_interpreter(ast_drop_query, global_context); @@ -238,25 +211,25 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, void StorageMaterializedView::drop(TableStructureWriteLockHolder &) { - global_context.removeDependency( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); + auto table_id = getStorageID(); + if (!select_table_id.empty()) + global_context.removeDependency(select_table_id, table_id); if (has_inner_table && tryGetTargetTable()) - executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name); + executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_table_id); } void StorageMaterializedView::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { if (has_inner_table) - executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_database_name, target_table_name); + executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_table_id); } void StorageMaterializedView::checkStatementCanBeForwarded() const { if (!has_inner_table) throw Exception( - "MATERIALIZED VIEW targets existing table " + target_database_name + "." + target_table_name + ". " + "MATERIALIZED VIEW targets existing table " + target_table_id.getNameForLogs() + ". " + "Execute the statement directly on it.", ErrorCodes::INCORRECT_QUERY); } @@ -278,71 +251,54 @@ void StorageMaterializedView::mutate(const MutationCommands & commands, const Co getTargetTable()->mutate(commands, context); } -static void executeRenameQuery(Context & global_context, const String & database_name, const String & table_original_name, const String & new_table_name) -{ - if (global_context.tryGetTable(database_name, table_original_name)) - { - auto rename = std::make_shared(); - - ASTRenameQuery::Table from; - from.database = database_name; - from.table = table_original_name; - - ASTRenameQuery::Table to; - to.database = database_name; - to.table = new_table_name; - - ASTRenameQuery::Element elem; - elem.from = from; - elem.to = to; - - rename->elements.emplace_back(elem); - - InterpreterRenameQuery(rename, global_context).execute(); - } -} - - void StorageMaterializedView::rename( const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { if (has_inner_table && tryGetTargetTable()) { - String new_target_table_name = generateInnerTableName(new_table_name); - executeRenameQuery(global_context, target_database_name, target_table_name, new_target_table_name); - target_table_name = new_target_table_name; + auto new_target_table_name = generateInnerTableName(new_table_name); + auto rename = std::make_shared(); + + ASTRenameQuery::Table from; + from.database = target_table_id.database_name; + from.table = target_table_id.table_name; + + ASTRenameQuery::Table to; + to.database = target_table_id.database_name; + to.table = new_target_table_name; + + ASTRenameQuery::Element elem; + elem.from = from; + elem.to = to; + rename->elements.emplace_back(elem); + + InterpreterRenameQuery(rename, global_context).execute(); + target_table_id.table_name = new_target_table_name; } auto lock = global_context.getLock(); - - global_context.removeDependencyUnsafe( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); - - table_name = new_table_name; - database_name = new_database_name; - - global_context.addDependencyUnsafe( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); + if (!select_table_id.empty()) + global_context.removeDependencyUnsafe(select_table_id, getStorageID()); + IStorage::renameInMemory(new_database_name, new_table_name); + if (!select_table_id.empty()) + global_context.addDependencyUnsafe(select_table_id, getStorageID()); } void StorageMaterializedView::shutdown() { /// Make sure the dependency is removed after DETACH TABLE - global_context.removeDependency( - DatabaseAndTableName(select_database_name, select_table_name), - DatabaseAndTableName(database_name, table_name)); + if (!select_table_id.empty()) + global_context.removeDependency(select_table_id, getStorageID()); } StoragePtr StorageMaterializedView::getTargetTable() const { - return global_context.getTable(target_database_name, target_table_name); + return global_context.getTable(target_table_id); } StoragePtr StorageMaterializedView::tryGetTargetTable() const { - return global_context.tryGetTable(target_database_name, target_table_name); + return global_context.tryGetTable(target_table_id); } Strings StorageMaterializedView::getDataPaths() const @@ -389,7 +345,7 @@ void registerStorageMaterializedView(StorageFactory & factory) { /// Pass local_context here to convey setting for inner table return StorageMaterializedView::create( - args.table_name, args.database_name, args.local_context, args.query, + args.table_id, args.local_context, args.query, args.columns, args.attach); }); } diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 4ac608d96d2..b811d5a6e2f 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -14,8 +14,6 @@ class StorageMaterializedView : public ext::shared_ptr_helper; public: std::string getName() const override { return "MaterializedView"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } ASTPtr getInnerQuery() const { return inner_query->clone(); } @@ -68,12 +66,11 @@ public: Strings getDataPaths() const override; private: - String select_database_name; - String select_table_name; - String target_database_name; - String target_table_name; - String table_name; - String database_name; + /// Can be empty if SELECT query doesn't contain table + StorageID select_table_id = StorageID::createEmpty(); + /// Will be initialized in constructor + StorageID target_table_id = StorageID::createEmpty(); + ASTPtr inner_query; Context & global_context; bool has_inner_table = false; @@ -82,8 +79,7 @@ private: protected: StorageMaterializedView( - const String & table_name_, - const String & database_name_, + const StorageID & table_id_, Context & local_context, const ASTCreateQuery & query, const ColumnsDescription & columns_, diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index 1b820e55c5e..b2cb5682f6d 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -74,8 +74,8 @@ private: }; -StorageMemory::StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_, ConstraintsDescription constraints_) - : database_name(std::move(database_name_)), table_name(std::move(table_name_)) +StorageMemory::StorageMemory(const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_) + : IStorage(table_id_) { setColumns(std::move(columns_description_)); setConstraints(std::move(constraints_)); @@ -145,7 +145,7 @@ void registerStorageMemory(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageMemory::create(args.database_name, args.table_name, args.columns, args.constraints); + return StorageMemory::create(args.table_id, args.columns, args.constraints); }); } diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index eb2d6ff2e21..1e66b17606b 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -25,8 +25,6 @@ friend struct ext::shared_ptr_helper; public: String getName() const override { return "Memory"; } - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } size_t getSize() const { return data.size(); } @@ -44,23 +42,14 @@ public: void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override - { - table_name = new_table_name; - database_name = new_database_name; - } - private: - String database_name; - String table_name; - /// The data itself. `list` - so that when inserted to the end, the existing iterators are not invalidated. BlocksList data; std::mutex mutex; protected: - StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_, ConstraintsDescription constraints_); + StorageMemory(const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_); }; } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 5be6353514e..1455556f366 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -7,19 +7,16 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include #include #include #include -#include #include #include #include @@ -27,7 +24,6 @@ #include #include #include -#include #include @@ -37,7 +33,6 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_PREWHERE; - extern const int INCOMPATIBLE_SOURCE_TABLES; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int BLOCKS_HAVE_DIFFERENT_STRUCTURE; @@ -46,15 +41,12 @@ namespace ErrorCodes StorageMerge::StorageMerge( - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const String & source_database_, const String & table_name_regexp_, const Context & context_) - : IStorage(ColumnsDescription({{"_table", std::make_shared()}}, true)) - , table_name(table_name_) - , database_name(database_name_) + : IStorage(table_id_, ColumnsDescription({{"_table", std::make_shared()}}, true)) , source_database(source_database_) , table_name_regexp(table_name_regexp_) , global_context(context_) @@ -125,7 +117,7 @@ bool StorageMerge::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, cons size_t i = 0; for (const auto & table : selected_tables) { - if (table.first->mayBenefitFromIndexForIn(left_in_operand, query_context)) + if (std::get<0>(table)->mayBenefitFromIndexForIn(left_in_operand, query_context)) return true; ++i; @@ -197,12 +189,12 @@ BlockInputStreams StorageMerge::read( * This is necessary to correctly pass the recommended number of threads to each table. */ StorageListWithLocks selected_tables = getSelectedTables( - query_info.query, has_table_virtual_column, true, context.getCurrentQueryId()); + query_info.query, has_table_virtual_column, context.getCurrentQueryId()); if (selected_tables.empty()) /// FIXME: do we support sampling in this case? return createSourceStreams( - query_info, processed_stage, max_block_size, header, {}, {}, real_column_names, modified_context, 0, has_table_virtual_column); + query_info, processed_stage, max_block_size, header, {}, real_column_names, modified_context, 0, has_table_virtual_column); size_t tables_count = selected_tables.size(); Float64 num_streams_multiplier = std::min(unsigned(tables_count), std::max(1U, unsigned(context.getSettingsRef().max_streams_multiplier_for_merge_tables))); @@ -214,7 +206,7 @@ BlockInputStreams StorageMerge::read( { for (auto it = selected_tables.begin(); it != selected_tables.end(); ++it) { - auto current_info = query_info.order_by_optimizer->getInputOrder(it->first); + auto current_info = query_info.order_by_optimizer->getInputOrder(std::get<0>(*it)); if (it == selected_tables.begin()) input_sorting_info = current_info; else if (!current_info || (input_sorting_info && *current_info != *input_sorting_info)) @@ -234,8 +226,7 @@ BlockInputStreams StorageMerge::read( remaining_streams -= current_streams; current_streams = std::max(size_t(1), current_streams); - StoragePtr storage = it->first; - TableStructureReadLockHolder struct_lock = it->second; + auto & storage = std::get<0>(*it); /// If sampling requested, then check that table supports it. if (query_info.query->as()->sample_size() && !storage->supportsSampling()) @@ -246,8 +237,8 @@ BlockInputStreams StorageMerge::read( if (current_streams) { source_streams = createSourceStreams( - query_info, processed_stage, max_block_size, header, storage, - struct_lock, real_column_names, modified_context, current_streams, has_table_virtual_column); + query_info, processed_stage, max_block_size, header, *it, real_column_names, modified_context, + current_streams, has_table_virtual_column); } else { @@ -255,7 +246,7 @@ BlockInputStreams StorageMerge::read( header, [=, this]() mutable -> BlockInputStreamPtr { BlockInputStreams streams = createSourceStreams(query_info, processed_stage, max_block_size, - header, storage, struct_lock, real_column_names, + header, *it, real_column_names, modified_context, current_streams, has_table_virtual_column, true); if (!streams.empty() && streams.size() != 1) @@ -276,15 +267,16 @@ BlockInputStreams StorageMerge::read( } BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, - const UInt64 max_block_size, const Block & header, const StoragePtr & storage, - const TableStructureReadLockHolder & struct_lock, Names & real_column_names, + const UInt64 max_block_size, const Block & header, const StorageWithLockAndName & storage_with_lock, + Names & real_column_names, Context & modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams) { + auto & [storage, struct_lock, table_name] = storage_with_lock; SelectQueryInfo modified_query_info = query_info; modified_query_info.query = query_info.query->clone(); - VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", storage ? storage->getTableName() : ""); + VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", table_name); if (!storage) return BlockInputStreams{ @@ -296,7 +288,7 @@ BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & quer if (processed_stage <= storage->getQueryProcessingStage(modified_context)) { /// If there are only virtual columns in query, you must request at least one other column. - if (real_column_names.size() ==0) + if (real_column_names.empty()) real_column_names.push_back(ExpressionActions::getSmallestColumn(storage->getColumns().getAllPhysical())); source_streams = storage->read(real_column_names, modified_query_info, modified_context, processed_stage, max_block_size, @@ -304,7 +296,7 @@ BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & quer } else if (processed_stage > storage->getQueryProcessingStage(modified_context)) { - modified_query_info.query->as()->replaceDatabaseAndTable(source_database, storage->getTableName()); + modified_query_info.query->as()->replaceDatabaseAndTable(source_database, table_name); /// Maximum permissible parallelism is streams_num modified_context.getSettingsRef().max_threads = UInt64(streams_num); @@ -335,7 +327,7 @@ BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & quer { if (has_table_virtual_column) source_stream = std::make_shared>( - source_stream, std::make_shared(), storage->getTableName(), "_table"); + source_stream, std::make_shared(), table_name, "_table"); /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. @@ -358,7 +350,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String { auto & table = iterator->table(); if (table.get() != this) - selected_tables.emplace_back(table, table->lockStructureForShare(false, query_id)); + selected_tables.emplace_back(table, table->lockStructureForShare(false, query_id), iterator->name()); iterator->next(); } @@ -367,7 +359,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String } -StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr & query, bool has_virtual_column, bool get_lock, const String & query_id) const +StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr & query, bool has_virtual_column, const String & query_id) const { StorageListWithLocks selected_tables; DatabaseTablesIteratorPtr iterator = getDatabaseIterator(global_context); @@ -383,8 +375,8 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr if (storage.get() != this) { - selected_tables.emplace_back(storage, get_lock ? storage->lockStructureForShare(false, query_id) : TableStructureReadLockHolder{}); - virtual_column->insert(storage->getTableName()); + selected_tables.emplace_back(storage, storage->lockStructureForShare(false, query_id), iterator->name()); + virtual_column->insert(iterator->name()); } iterator->next(); @@ -397,7 +389,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr auto values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_table"); /// Remove unused tables from the list - selected_tables.remove_if([&] (const auto & elem) { return values.find(elem.first->getTableName()) == values.end(); }); + selected_tables.remove_if([&] (const auto & elem) { return values.find(std::get<2>(elem)) == values.end(); }); } return selected_tables; @@ -429,10 +421,11 @@ void StorageMerge::alter( const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + auto table_id = getStorageID(); StorageInMemoryMetadata storage_metadata = getInMemoryMetadata(); params.apply(storage_metadata); - context.getDatabase(database_name)->alterTable(context, table_name, storage_metadata); + context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, storage_metadata); setColumns(storage_metadata.columns); } @@ -523,7 +516,7 @@ void registerStorageMerge(StorageFactory & factory) String table_name_regexp = engine_args[1]->as().value.safeGet(); return StorageMerge::create( - args.database_name, args.table_name, args.columns, + args.table_id, args.columns, source_database, table_name_regexp, args.context); }); } diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 70bed6498f1..c773ff3ae15 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -17,8 +17,6 @@ class StorageMerge : public ext::shared_ptr_helper, public IStorag friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Merge"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } bool isRemote() const override; @@ -42,13 +40,6 @@ public: size_t max_block_size, unsigned num_streams) override; - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override - { - table_name = new_table_name; - database_name = new_database_name; - } - - void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; /// you need to add and remove columns in the sub-tables manually @@ -58,17 +49,16 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override; private: - String table_name; - String database_name; String source_database; OptimizedRegularExpression table_name_regexp; Context global_context; - using StorageListWithLocks = std::list>; + using StorageWithLockAndName = std::tuple; + using StorageListWithLocks = std::list; StorageListWithLocks getSelectedTables(const String & query_id) const; - StorageMerge::StorageListWithLocks getSelectedTables(const ASTPtr & query, bool has_virtual_column, bool get_lock, const String & query_id) const; + StorageMerge::StorageListWithLocks getSelectedTables(const ASTPtr & query, bool has_virtual_column, const String & query_id) const; template StoragePtr getFirstTable(F && predicate) const; @@ -77,8 +67,7 @@ private: protected: StorageMerge( - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const String & source_database_, const String & table_name_regexp_, @@ -88,8 +77,8 @@ protected: const Context & context, QueryProcessingStage::Enum processed_stage); BlockInputStreams createSourceStreams(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, - const UInt64 max_block_size, const Block & header, const StoragePtr & storage, - const TableStructureReadLockHolder & struct_lock, Names & real_column_names, + const UInt64 max_block_size, const Block & header, const StorageWithLockAndName & storage_with_lock, + Names & real_column_names, Context & modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams = false); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 314625de5f1..d8b25627a7e 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -53,8 +53,7 @@ namespace ActionLocks StorageMergeTree::StorageMergeTree( - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata, bool attach, @@ -64,15 +63,14 @@ StorageMergeTree::StorageMergeTree( std::unique_ptr storage_settings_, bool has_force_restore_data_flag) : MergeTreeData( - database_name_, - table_name_, + table_id_, relative_data_path_, metadata, context_, date_column_name, merging_params_, std::move(storage_settings_), - false, + false, /// require_part_metadata attach) , reader(*this) , writer(*this) @@ -150,12 +148,14 @@ BlockOutputStreamPtr StorageMergeTree::write(const ASTPtr & /*query*/, const Con void StorageMergeTree::checkTableCanBeDropped() const { + auto table_id = getStorageID(); const_cast(*this).recalculateColumnSizes(); - global_context.checkTableCanBeDropped(database_name, table_name, getTotalActiveSizeInBytes()); + global_context.checkTableCanBeDropped(table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes()); } void StorageMergeTree::checkPartitionCanBeDropped(const ASTPtr & partition) { + auto table_id = getStorageID(); const_cast(*this).recalculateColumnSizes(); const String partition_id = getPartitionIDFromQuery(partition, global_context); @@ -167,7 +167,7 @@ void StorageMergeTree::checkPartitionCanBeDropped(const ASTPtr & partition) { partition_size += part->bytes_on_disk; } - global_context.checkPartitionCanBeDropped(database_name, table_name, partition_size); + global_context.checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size); } void StorageMergeTree::drop(TableStructureWriteLockHolder &) @@ -248,8 +248,7 @@ void StorageMergeTree::alter( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - const String current_database_name = getDatabaseName(); - const String current_table_name = getTableName(); + auto table_id = getStorageID(); lockNewDataStructureExclusively(table_lock_holder, context.getCurrentQueryId()); @@ -273,7 +272,7 @@ void StorageMergeTree::alter( { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, metadata); + context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); update_metadata(); } @@ -290,7 +289,7 @@ void StorageMergeTree::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, metadata); + context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); update_metadata(); @@ -630,7 +629,8 @@ bool StorageMergeTree::merge( merging_tagger.emplace(future_part, MergeTreeDataMergerMutator::estimateNeededDiskSpace(future_part.parts), *this, false); } - MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(database_name, table_name, future_part); + auto table_id = getStorageID(); + MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(table_id.database_name, table_id.table_name, future_part); /// Logging Stopwatch stopwatch; @@ -652,7 +652,7 @@ bool StorageMergeTree::merge( { /// Force filter by TTL in 'OPTIMIZE ... FINAL' query to remove expired values from old parts /// without TTL infos or with outdated TTL infos, e.g. after 'ALTER ... MODIFY TTL' query. - bool force_ttl = (final && (hasTableTTL() || hasAnyColumnTTL())); + bool force_ttl = (final && (hasRowsTTL() || hasAnyColumnTTL())); new_part = merger_mutator.mergePartsToTemporaryPart( future_part, *merge_entry, table_lock_holder, time(nullptr), @@ -754,7 +754,8 @@ bool StorageMergeTree::tryMutatePart() if (!tagger) return false; - MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(database_name, table_name, future_part); + auto table_id = getStorageID(); + MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(table_id.database_name, table_id.table_name, future_part); Stopwatch stopwatch; MutableDataPartPtr new_part; @@ -1030,6 +1031,12 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma case PartitionCommand::MoveDestinationType::VOLUME: movePartitionToVolume(command.partition, command.move_destination_name, command.part, context); break; + case PartitionCommand::MoveDestinationType::TABLE: + checkPartitionCanBeDropped(command.partition); + String dest_database = command.to_database.empty() ? context.getCurrentDatabase() : command.to_database; + auto dest_storage = context.getTable(dest_database, command.to_table); + movePartitionToTable(dest_storage, command.partition, context); + break; } } @@ -1206,6 +1213,86 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } } +void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & context) +{ + auto lock1 = lockStructureForShare(false, context.getCurrentQueryId()); + auto lock2 = dest_table->lockStructureForShare(false, context.getCurrentQueryId()); + + auto dest_table_storage = std::dynamic_pointer_cast(dest_table); + if (!dest_table_storage) + throw Exception("Table " + getStorageID().getNameForLogs() + " supports attachPartitionFrom only for MergeTree family of table engines." + " Got " + dest_table->getName(), ErrorCodes::NOT_IMPLEMENTED); + if (dest_table_storage->getStoragePolicy() != this->getStoragePolicy()) + throw Exception("Destination table " + dest_table_storage->getStorageID().getNameForLogs() + + " should have the same storage policy of source table " + getStorageID().getNameForLogs() + ". " + + getStorageID().getNameForLogs() + ": " + this->getStoragePolicy()->getName() + ", " + + dest_table_storage->getStorageID().getNameForLogs() + ": " + dest_table_storage->getStoragePolicy()->getName(), ErrorCodes::LOGICAL_ERROR); + Stopwatch watch; + + MergeTreeData & src_data = dest_table_storage->checkStructureAndGetMergeTreeData(this); + String partition_id = getPartitionIDFromQuery(partition, context); + + DataPartsVector src_parts = src_data.getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + MutableDataPartsVector dst_parts; + + static const String TMP_PREFIX = "tmp_replace_from_"; + + for (const DataPartPtr & src_part : src_parts) + { + if (!dest_table_storage->canReplacePartition(src_part)) + throw Exception( + "Cannot replace partition '" + partition_id + "' because part '" + src_part->name + "' has inconsistent granularity with table", + ErrorCodes::LOGICAL_ERROR); + + /// This will generate unique name in scope of current server process. + Int64 temp_index = insert_increment.get(); + MergeTreePartInfo dst_part_info(partition_id, temp_index, temp_index, src_part->info.level); + + std::shared_lock part_lock(src_part->columns_lock); + dst_parts.emplace_back(dest_table_storage->cloneAndLoadDataPartOnSameDisk(src_part, TMP_PREFIX, dst_part_info)); + } + + /// ATTACH empty part set + if (dst_parts.empty()) + return; + + MergeTreePartInfo drop_range; + + drop_range.partition_id = partition_id; + drop_range.min_block = 0; + drop_range.max_block = increment.get(); // there will be a "hole" in block numbers + drop_range.level = std::numeric_limits::max(); + + /// Atomically add new parts and remove old ones + try + { + { + Transaction transaction(*dest_table_storage); + + auto src_data_parts_lock = lockParts(); + auto dest_data_parts_lock = dest_table_storage->lockParts(); + + std::mutex mutex; + DataPartsLock lock(mutex); + + for (MutableDataPartPtr & part : dst_parts) + dest_table_storage->renameTempPartAndReplace(part, &increment, &transaction, lock); + + removePartsFromWorkingSet(src_parts, true, lock); + transaction.commit(&lock); + } + + clearOldMutations(true); + clearOldPartsFromFilesystem(); + + PartLog::addNewParts(global_context, dst_parts, watch.elapsed()); + } + catch (...) + { + PartLog::addNewParts(global_context, dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + throw; + } +} ActionLock StorageMergeTree::getActionLock(StorageActionBlockType action_type) { diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 2a30c5ffe25..96e5d3cf2ed 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -32,8 +32,8 @@ public: ~StorageMergeTree() override; std::string getName() const override { return merging_params.getModeName() + "MergeTree"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } + + bool supportsParallelInsert() const override { return true; } bool supportsIndexForIn() const override { return true; } @@ -140,6 +140,7 @@ private: void clearColumnOrIndexInPartition(const ASTPtr & partition, const AlterCommand & alter_command, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); + void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & context); bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; /// Just checks versions of each active data part @@ -158,8 +159,7 @@ protected: * See MergeTreeData constructor for comments on parameters. */ StorageMergeTree( - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata, bool attach, diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index 388c38f118c..6f3c6d5b5ae 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -37,8 +37,7 @@ static String backQuoteMySQL(const String & x) } StorageMySQL::StorageMySQL( - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, mysqlxx::Pool && pool_, const std::string & remote_database_name_, const std::string & remote_table_name_, @@ -47,8 +46,7 @@ StorageMySQL::StorageMySQL( const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const Context & context_) - : table_name(table_name_) - , database_name(database_name_) + : IStorage(table_id_) , remote_database_name(remote_database_name_) , remote_table_name(remote_table_name_) , replace_query{replace_query_} @@ -235,8 +233,7 @@ void registerStorageMySQL(StorageFactory & factory) ErrorCodes::BAD_ARGUMENTS); return StorageMySQL::create( - args.database_name, - args.table_name, + args.table_id, std::move(pool), remote_database, remote_table, diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 5f369da4d25..574221377dc 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -21,8 +21,7 @@ class StorageMySQL : public ext::shared_ptr_helper, public IStorag friend struct ext::shared_ptr_helper; public: StorageMySQL( - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, mysqlxx::Pool && pool_, const std::string & remote_database_name_, const std::string & remote_table_name_, @@ -33,8 +32,6 @@ public: const Context & context_); std::string getName() const override { return "MySQL"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -48,8 +45,6 @@ public: private: friend class StorageMySQLBlockOutputStream; - std::string table_name; - std::string database_name; std::string remote_database_name; std::string remote_table_name; diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 342f786bf4f..bbb620132c8 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -26,7 +26,7 @@ void registerStorageNull(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageNull::create(args.database_name, args.table_name, args.columns, args.constraints); + return StorageNull::create(args.table_id, args.columns, args.constraints); }); } @@ -47,13 +47,11 @@ void StorageNull::alter( const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - - const String current_database_name = getDatabaseName(); - const String current_table_name = getTableName(); + auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, metadata); + context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index 0122632fb69..347f1501ee1 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -19,8 +19,6 @@ class StorageNull : public ext::shared_ptr_helper, public IStorage friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Null"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -38,23 +36,15 @@ public: return std::make_shared(getSampleBlock()); } - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override - { - table_name = new_table_name; - database_name = new_database_name; - } - void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; private: - String table_name; - String database_name; protected: - StorageNull(String database_name_, String table_name_, ColumnsDescription columns_description_, ConstraintsDescription constraints_) - : table_name(std::move(table_name_)), database_name(std::move(database_name_)) + StorageNull(const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_) + : IStorage(table_id_) { setColumns(std::move(columns_description_)); setConstraints(std::move(constraints_)); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index c65b05ef67b..db113624f68 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -192,8 +192,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const String & zookeeper_path_, const String & replica_name_, bool attach, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata, Context & context_, @@ -201,15 +200,27 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const MergingParams & merging_params_, std::unique_ptr settings_, bool has_force_restore_data_flag) - : MergeTreeData(database_name_, table_name_, relative_data_path_, metadata, - context_, date_column_name, merging_params_, std::move(settings_), true, attach, - [this] (const std::string & name) { enqueuePartForCheck(name); }), - - zookeeper_path(global_context.getMacros()->expand(zookeeper_path_, database_name_, table_name_)), - replica_name(global_context.getMacros()->expand(replica_name_, database_name_, table_name_)), - reader(*this), writer(*this), merger_mutator(*this, global_context.getBackgroundPool().getNumberOfThreads()), - queue(*this), fetcher(*this), cleanup_thread(*this), alter_thread(*this), - part_check_thread(*this), restarting_thread(*this) + : MergeTreeData(table_id_, + relative_data_path_, + metadata, + context_, + date_column_name, + merging_params_, + std::move(settings_), + true, /// require_part_metadata + attach, + [this] (const std::string & name) { enqueuePartForCheck(name); }) + , zookeeper_path(global_context.getMacros()->expand(zookeeper_path_, table_id_.database_name, table_id_.table_name)) + , replica_name(global_context.getMacros()->expand(replica_name_, table_id_.database_name, table_id_.table_name)) + , reader(*this) + , writer(*this) + , merger_mutator(*this, global_context.getBackgroundPool().getNumberOfThreads()) + , queue(*this) + , fetcher(*this) + , cleanup_thread(*this) + , alter_thread(*this) + , part_check_thread(*this) + , restarting_thread(*this) { if (!zookeeper_path.empty() && zookeeper_path.back() == '/') zookeeper_path.resize(zookeeper_path.size() - 1); @@ -218,15 +229,15 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( zookeeper_path = "/" + zookeeper_path; replica_path = zookeeper_path + "/replicas/" + replica_name; - queue_updating_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::queueUpdatingTask)", [this]{ queueUpdatingTask(); }); + queue_updating_task = global_context.getSchedulePool().createTask(getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::queueUpdatingTask)", [this]{ queueUpdatingTask(); }); - mutations_updating_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mutationsUpdatingTask)", [this]{ mutationsUpdatingTask(); }); + mutations_updating_task = global_context.getSchedulePool().createTask(getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::mutationsUpdatingTask)", [this]{ mutationsUpdatingTask(); }); - merge_selecting_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mergeSelectingTask)", [this] { mergeSelectingTask(); }); + merge_selecting_task = global_context.getSchedulePool().createTask(getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::mergeSelectingTask)", [this] { mergeSelectingTask(); }); /// Will be activated if we win leader election. merge_selecting_task->deactivate(); - mutations_finalizing_task = global_context.getSchedulePool().createTask(database_name + "." + table_name + " (StorageReplicatedMergeTree::mutationsFinalizingTask)", [this] { mutationsFinalizingTask(); }); + mutations_finalizing_task = global_context.getSchedulePool().createTask(getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::mutationsFinalizingTask)", [this] { mutationsFinalizingTask(); }); if (global_context.hasZooKeeper()) current_zookeeper = global_context.getZooKeeper(); @@ -529,7 +540,8 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column } } - global_context.getDatabase(database_name)->alterTable(global_context, table_name, metadata); + auto table_id = getStorageID(); + global_context.getDatabase(table_id.database_name)->alterTable(global_context, table_id.table_name, metadata); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. @@ -687,7 +699,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) if (insane && !skip_sanity_checks) { std::stringstream why; - why << "The local set of parts of table " << database_name << "." << table_name << " doesn't look like the set of parts " + why << "The local set of parts of table " << getStorageID().getNameForLogs() << " doesn't look like the set of parts " << "in ZooKeeper: " << formatReadableQuantity(unexpected_parts_rows) << " rows of " << formatReadableQuantity(total_rows_on_filesystem) << " total rows in filesystem are suspicious."; @@ -1072,7 +1084,8 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } future_merged_part.updatePath(*this, reserved_space); - MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(database_name, table_name, future_merged_part); + auto table_id = getStorageID(); + MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(table_id.database_name, table_id.table_name, future_merged_part); Transaction transaction(*this); MutableDataPartPtr part; @@ -1208,8 +1221,9 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM future_mutated_part.name = entry.new_part_name; future_mutated_part.updatePath(*this, reserved_space); + auto table_id = getStorageID(); MergeList::EntryPtr merge_entry = global_context.getMergeList().insert( - database_name, table_name, future_mutated_part); + table_id.database_name, table_id.table_name, future_mutated_part); Stopwatch stopwatch; @@ -2914,13 +2928,11 @@ void StorageReplicatedMergeTree::startup() queue.initialize( zookeeper_path, replica_path, - database_name + "." + table_name + " (ReplicatedMergeTreeQueue)", + getStorageID().getFullTableName() + " (ReplicatedMergeTreeQueue)", getDataParts()); - StoragePtr ptr = shared_from_this(); - InterserverIOEndpointPtr data_parts_exchange_endpoint = std::make_shared(*this, ptr); - data_parts_exchange_endpoint_holder = std::make_shared( - data_parts_exchange_endpoint->getId(replica_path), data_parts_exchange_endpoint, global_context.getInterserverIOHandler()); + data_parts_exchange_endpoint = std::make_shared(*this); + global_context.getInterserverIOHandler().addEndpoint(data_parts_exchange_endpoint->getId(replica_path), data_parts_exchange_endpoint); queue_task_handle = global_context.getBackgroundPool().addTask([this] { return queueTask(); }); if (areBackgroundMovesNeeded()) @@ -2952,11 +2964,15 @@ void StorageReplicatedMergeTree::shutdown() global_context.getBackgroundMovePool().removeTask(move_parts_task_handle); move_parts_task_handle.reset(); - if (data_parts_exchange_endpoint_holder) + if (data_parts_exchange_endpoint) { - data_parts_exchange_endpoint_holder->getBlocker().cancelForever(); - data_parts_exchange_endpoint_holder = nullptr; + global_context.getInterserverIOHandler().removeEndpointIfExists(data_parts_exchange_endpoint->getId(replica_path)); + /// Ask all parts exchange handlers to finish asap. New ones will fail to start + data_parts_exchange_endpoint->blocker.cancelForever(); + /// Wait for all of them + std::unique_lock lock(data_parts_exchange_endpoint->rwlock); } + data_parts_exchange_endpoint.reset(); } @@ -3116,7 +3132,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p return false; }; - bool force_ttl = (final && (hasTableTTL() || hasAnyColumnTTL())); + bool force_ttl = (final && (hasRowsTTL() || hasAnyColumnTTL())); const auto storage_settings_ptr = getSettings(); if (!partition && final) @@ -3198,8 +3214,7 @@ void StorageReplicatedMergeTree::alter( LOG_DEBUG(log, "Doing ALTER"); - const String current_database_name = getDatabaseName(); - const String current_table_name = getTableName(); + auto table_id = getStorageID(); /// We cannot check this alter commands with method isModifyingData() /// because ReplicatedMergeTree stores both columns and metadata for @@ -3215,7 +3230,7 @@ void StorageReplicatedMergeTree::alter( changeSettings(metadata.settings_ast, table_lock_holder); - global_context.getDatabase(current_database_name)->alterTable(query_context, current_table_name, metadata); + global_context.getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, metadata); return; } @@ -3293,7 +3308,7 @@ void StorageReplicatedMergeTree::alter( auto old_metadata = getInMemoryMetadata(); old_metadata.settings_ast = metadata.settings_ast; changeSettings(metadata.settings_ast, table_lock_holder); - global_context.getDatabase(current_database_name)->alterTable(query_context, current_table_name, old_metadata); + global_context.getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, old_metadata); /// Modify shared metadata nodes in ZooKeeper. Coordination::Requests ops; @@ -3536,6 +3551,12 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part case PartitionCommand::MoveDestinationType::VOLUME: movePartitionToVolume(command.partition, command.move_destination_name, command.part, query_context); break; + case PartitionCommand::MoveDestinationType::TABLE: + checkPartitionCanBeDropped(command.partition); + String dest_database = command.to_database.empty() ? query_context.getCurrentDatabase() : command.to_database; + auto dest_storage = query_context.getTable(dest_database, command.to_table); + movePartitionToTable(dest_storage, command.partition, query_context); + break; } } break; @@ -3755,7 +3776,8 @@ void StorageReplicatedMergeTree::checkTableCanBeDropped() const { /// Consider only synchronized data const_cast(*this).recalculateColumnSizes(); - global_context.checkTableCanBeDropped(database_name, table_name, getTotalActiveSizeInBytes()); + auto table_id = getStorageID(); + global_context.checkTableCanBeDropped(table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes()); } @@ -3771,7 +3793,8 @@ void StorageReplicatedMergeTree::checkPartitionCanBeDropped(const ASTPtr & parti for (const auto & part : parts_to_remove) partition_size += part->bytes_on_disk; - global_context.checkPartitionCanBeDropped(database_name, table_name, partition_size); + auto table_id = getStorageID(); + global_context.checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size); } @@ -4985,6 +5008,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ ReplicatedMergeTreeLogEntryData entry; { + auto src_table_id = src_data.getStorageID(); entry.type = ReplicatedMergeTreeLogEntryData::REPLACE_RANGE; entry.source_replica = replica_name; entry.create_time = time(nullptr); @@ -4992,8 +5016,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ auto & entry_replace = *entry.replace_range_entry; entry_replace.drop_range_part_name = drop_range_fake_part_name; - entry_replace.from_database = src_data.database_name; - entry_replace.from_table = src_data.table_name; + entry_replace.from_database = src_table_id.database_name; + entry_replace.from_table = src_table_id.table_name; for (const auto & part : src_parts) entry_replace.src_part_names.emplace_back(part->name); for (const auto & part : dst_parts) @@ -5079,6 +5103,200 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ } } +void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & context) +{ + auto lock1 = lockStructureForShare(false, context.getCurrentQueryId()); + auto lock2 = dest_table->lockStructureForShare(false, context.getCurrentQueryId()); + + auto dest_table_storage = std::dynamic_pointer_cast(dest_table); + if (!dest_table_storage) + throw Exception("Table " + getStorageID().getNameForLogs() + " supports attachPartitionFrom only for ReplicatedMergeTree family of table engines." + " Got " + dest_table->getName(), ErrorCodes::NOT_IMPLEMENTED); + if (dest_table_storage->getStoragePolicy() != this->getStoragePolicy()) + throw Exception("Destination table " + dest_table_storage->getStorageID().getNameForLogs() + + " should have the same storage policy of source table " + getStorageID().getNameForLogs() + ". " + + getStorageID().getNameForLogs() + ": " + this->getStoragePolicy()->getName() + ", " + + getStorageID().getNameForLogs() + ": " + dest_table_storage->getStoragePolicy()->getName(), ErrorCodes::LOGICAL_ERROR); + + Stopwatch watch; + MergeTreeData & src_data = dest_table_storage->checkStructureAndGetMergeTreeData(this); + auto src_data_id = src_data.getStorageID(); + String partition_id = getPartitionIDFromQuery(partition, context); + + DataPartsVector src_all_parts = src_data.getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + DataPartsVector src_parts; + MutableDataPartsVector dst_parts; + Strings block_id_paths; + Strings part_checksums; + std::vector ephemeral_locks; + + LOG_DEBUG(log, "Cloning " << src_all_parts.size() << " parts"); + + static const String TMP_PREFIX = "tmp_replace_from_"; + auto zookeeper = getZooKeeper(); + + MergeTreePartInfo drop_range; + drop_range.partition_id = partition_id; + drop_range.max_block = allocateBlockNumber(partition_id, zookeeper)->getNumber(); + drop_range.min_block = 0; + drop_range.level = std::numeric_limits::max(); + + String drop_range_fake_part_name = getPartNamePossiblyFake(format_version, drop_range); + + if (drop_range.getBlocksCount() > 1) + { + std::lock_guard merge_selecting_lock(merge_selecting_mutex); + queue.disableMergesInBlockRange(drop_range_fake_part_name); + } + + for (size_t i = 0; i < src_all_parts.size(); ++i) + { + auto & src_part = src_all_parts[i]; + + if (!dest_table_storage->canReplacePartition(src_part)) + throw Exception( + "Cannot replace partition '" + partition_id + "' because part '" + src_part->name + "' has inconsistent granularity with table", + ErrorCodes::LOGICAL_ERROR); + + String hash_hex = src_part->checksums.getTotalChecksumHex(); + String block_id_path = ""; + + auto lock = dest_table_storage->allocateBlockNumber(partition_id, zookeeper, block_id_path); + if (!lock) + { + LOG_INFO(log, "Part " << src_part->name << " (hash " << hash_hex << ") has been already attached"); + continue; + } + + UInt64 index = lock->getNumber(); + MergeTreePartInfo dst_part_info(partition_id, index, index, src_part->info.level); + auto dst_part = dest_table_storage->cloneAndLoadDataPartOnSameDisk(src_part, TMP_PREFIX, dst_part_info); + + src_parts.emplace_back(src_part); + dst_parts.emplace_back(dst_part); + ephemeral_locks.emplace_back(std::move(*lock)); + block_id_paths.emplace_back(block_id_path); + part_checksums.emplace_back(hash_hex); + } + + ReplicatedMergeTreeLogEntryData entry_delete; + { + entry_delete.type = LogEntry::DROP_RANGE; + entry_delete.source_replica = replica_name; + entry_delete.new_part_name = drop_range_fake_part_name; + entry_delete.detach = false; + entry_delete.create_time = time(nullptr); + } + + ReplicatedMergeTreeLogEntryData entry; + { + MergeTreePartInfo drop_range_dest; + drop_range_dest.partition_id = drop_range.partition_id; + drop_range_dest.max_block = drop_range.max_block; + drop_range_dest.min_block = drop_range.max_block; + drop_range_dest.level = drop_range.level; + + entry.type = ReplicatedMergeTreeLogEntryData::REPLACE_RANGE; + entry.source_replica = dest_table_storage->replica_name; + entry.create_time = time(nullptr); + entry.replace_range_entry = std::make_shared(); + + auto & entry_replace = *entry.replace_range_entry; + entry_replace.drop_range_part_name = getPartNamePossiblyFake(format_version, drop_range_dest); + entry_replace.from_database = src_data_id.database_name; + entry_replace.from_table = src_data_id.table_name; + for (const auto & part : src_parts) + entry_replace.src_part_names.emplace_back(part->name); + for (const auto & part : dst_parts) + entry_replace.new_part_names.emplace_back(part->name); + for (const String & checksum : part_checksums) + entry_replace.part_names_checksums.emplace_back(checksum); + entry_replace.columns_version = columns_version; + } + + queue.removePartProducingOpsInRange(zookeeper, drop_range, entry); + + clearBlocksInPartition(*zookeeper, drop_range.partition_id, drop_range.max_block, drop_range.max_block); + + DataPartsVector parts_to_remove; + Coordination::Responses op_results; + + try + { + Coordination::Requests ops; + for (size_t i = 0; i < dst_parts.size(); ++i) + { + dest_table_storage->getCommitPartOps(ops, dst_parts[i], block_id_paths[i]); + ephemeral_locks[i].getUnlockOps(ops); + + if (ops.size() > zkutil::MULTI_BATCH_SIZE) + { + zookeeper->multi(ops); + ops.clear(); + } + } + + ops.emplace_back(zkutil::makeCreateRequest(dest_table_storage->zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + + { + Transaction transaction(*dest_table_storage); + + auto src_data_parts_lock = lockParts(); + auto dest_data_parts_lock = dest_table_storage->lockParts(); + + std::mutex mutex; + DataPartsLock lock(mutex); + + for (MutableDataPartPtr & part : dst_parts) + dest_table_storage->renameTempPartAndReplace(part, nullptr, &transaction, lock); + + op_results = zookeeper->multi(ops); + + parts_to_remove = removePartsInRangeFromWorkingSet(drop_range, true, false, lock); + transaction.commit(&lock); + } + + PartLog::addNewParts(global_context, dst_parts, watch.elapsed()); + } + catch (...) + { + PartLog::addNewParts(global_context, dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + throw; + } + + String log_znode_path = dynamic_cast(*op_results.back()).path_created; + entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + for (auto & lock : ephemeral_locks) + lock.assumeUnlocked(); + + tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); + + parts_to_remove.clear(); + cleanup_thread.wakeup(); + + if (context.getSettingsRef().replication_alter_partitions_sync > 1) + { + lock2.release(); + dest_table_storage->waitForAllReplicasToProcessLogEntry(entry); + } + + Coordination::Requests ops_dest; + + ops_dest.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); + + op_results = zookeeper->multi(ops_dest); + + log_znode_path = dynamic_cast(*op_results.back()).path_created; + entry_delete.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + if (context.getSettingsRef().replication_alter_partitions_sync > 1) + { + lock1.release(); + waitForAllReplicasToProcessLogEntry(entry_delete); + } +} + void StorageReplicatedMergeTree::getCommitPartOps( Coordination::Requests & ops, MutableDataPartPtr & part, @@ -5183,13 +5401,14 @@ void StorageReplicatedMergeTree::updatePartHeaderInZooKeeperAndCommit( ReplicatedMergeTreeAddress StorageReplicatedMergeTree::getReplicatedMergeTreeAddress() const { auto host_port = global_context.getInterserverIOAddress(); + auto table_id = getStorageID(); ReplicatedMergeTreeAddress res; res.host = host_port.first; res.replication_port = host_port.second; res.queries_port = global_context.getTCPPort(); - res.database = database_name; - res.table = table_name; + res.database = table_id.database_name; + res.table = table_id.table_name; res.scheme = global_context.getInterserverScheme(); return res; } @@ -5206,7 +5425,7 @@ ActionLock StorageReplicatedMergeTree::getActionLock(StorageActionBlockType acti return fetcher.blocker.cancel(); if (action_type == ActionLocks::PartsSend) - return data_parts_exchange_endpoint_holder ? data_parts_exchange_endpoint_holder->getBlocker().cancel() : ActionLock(); + return data_parts_exchange_endpoint ? data_parts_exchange_endpoint->blocker.cancel() : ActionLock(); if (action_type == ActionLocks::ReplicationQueue) return queue.actions_blocker.cancel(); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 60c2ea0b870..aadbdd1b558 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -83,9 +83,8 @@ public: ~StorageReplicatedMergeTree() override; std::string getName() const override { return "Replicated" + merging_params.getModeName() + "MergeTree"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } + bool supportsParallelInsert() const override { return true; } bool supportsReplication() const override { return true; } bool supportsDeduplication() const override { return true; } @@ -233,7 +232,7 @@ private: std::atomic is_leader {false}; zkutil::LeaderElectionPtr leader_election; - InterserverIOEndpointHolderPtr data_parts_exchange_endpoint_holder; + InterserverIOEndpointPtr data_parts_exchange_endpoint; MergeTreeDataSelectExecutor reader; MergeTreeDataWriter writer; @@ -526,6 +525,7 @@ private: void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & query_context); void attachPartition(const ASTPtr & partition, bool part, const Context & query_context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & query_context); + void movePartitionToTable(const StoragePtr & source_table, const ASTPtr & partition, const Context & query_context); void fetchPartition(const ASTPtr & partition, const String & from, const Context & query_context); /// Check granularity of already existing replicated table in zookeeper if it exists @@ -543,7 +543,7 @@ protected: const String & zookeeper_path_, const String & replica_name_, bool attach, - const String & database_name_, const String & name_, + const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata, Context & context_, diff --git a/dbms/src/Storages/StorageS3.cpp b/dbms/src/Storages/StorageS3.cpp index 14732a291b1..0798820104b 100644 --- a/dbms/src/Storages/StorageS3.cpp +++ b/dbms/src/Storages/StorageS3.cpp @@ -133,23 +133,21 @@ namespace } -StorageS3::StorageS3(const S3::URI & uri_, +StorageS3::StorageS3( + const S3::URI & uri_, const String & access_key_id_, const String & secret_access_key_, - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, const String & format_name_, UInt64 min_upload_part_size_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, Context & context_, const String & compression_method_ = "") - : IStorage(columns_) + : IStorage(table_id_, columns_) , uri(uri_) , context_global(context_) , format_name(format_name_) - , database_name(database_name_) - , table_name(table_name_) , min_upload_part_size(min_upload_part_size_) , compression_method(compression_method_) , client(S3::ClientFactory::instance().create(uri_.endpoint, access_key_id_, secret_access_key_)) @@ -185,12 +183,6 @@ BlockInputStreams StorageS3::read( return {std::make_shared(block_input, column_defaults, context)}; } -void StorageS3::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) -{ - table_name = new_table_name; - database_name = new_database_name; -} - BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const Context & /*context*/) { return std::make_shared( @@ -234,7 +226,7 @@ void registerStorageS3(StorageFactory & factory) else compression_method = "auto"; - return StorageS3::create(s3_uri, access_key_id, secret_access_key, args.database_name, args.table_name, format_name, min_upload_part_size, args.columns, args.constraints, args.context); + return StorageS3::create(s3_uri, access_key_id, secret_access_key, args.table_id, format_name, min_upload_part_size, args.columns, args.constraints, args.context); }); } diff --git a/dbms/src/Storages/StorageS3.h b/dbms/src/Storages/StorageS3.h index 90494a9d851..9854f70a733 100644 --- a/dbms/src/Storages/StorageS3.h +++ b/dbms/src/Storages/StorageS3.h @@ -28,8 +28,7 @@ public: StorageS3(const S3::URI & uri, const String & access_key_id, const String & secret_access_key, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const String & format_name_, UInt64 min_upload_part_size_, const ColumnsDescription & columns_, @@ -47,11 +46,6 @@ public: return getSampleBlock(); } - String getTableName() const override - { - return table_name; - } - BlockInputStreams read( const Names & column_names, const SelectQueryInfo & query_info, @@ -62,15 +56,11 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; - private: S3::URI uri; const Context & context_global; String format_name; - String database_name; - String table_name; UInt64 min_upload_part_size; String compression_method; std::shared_ptr client; diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp index 4b870060d1c..30f20b7df57 100644 --- a/dbms/src/Storages/StorageSet.cpp +++ b/dbms/src/Storages/StorageSet.cpp @@ -89,12 +89,11 @@ BlockOutputStreamPtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const StorageSetOrJoinBase::StorageSetOrJoinBase( const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const Context & context_) - : table_name(table_name_), database_name(database_name_) + : IStorage(table_id_) { setColumns(columns_); setConstraints(constraints_); @@ -110,12 +109,11 @@ StorageSetOrJoinBase::StorageSetOrJoinBase( StorageSet::StorageSet( const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const Context & context_) - : StorageSetOrJoinBase{relative_path_, database_name_, table_name_, columns_, constraints_, context_}, + : StorageSetOrJoinBase{relative_path_, table_id_, columns_, constraints_, context_}, set(std::make_shared(SizeLimits(), false)) { Block header = getSampleBlock(); @@ -209,8 +207,7 @@ void StorageSetOrJoinBase::rename( Poco::File(path).renameTo(new_path); path = new_path; - table_name = new_table_name; - database_name = new_database_name; + renameInMemory(new_database_name, new_table_name); } @@ -223,7 +220,7 @@ void registerStorageSet(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageSet::create(args.relative_data_path, args.database_name, args.table_name, args.columns, args.constraints, args.context); + return StorageSet::create(args.relative_data_path, args.table_id, args.columns, args.constraints, args.context); }); } diff --git a/dbms/src/Storages/StorageSet.h b/dbms/src/Storages/StorageSet.h index 9a14fc08ac7..41e2e9ccdd6 100644 --- a/dbms/src/Storages/StorageSet.h +++ b/dbms/src/Storages/StorageSet.h @@ -19,9 +19,6 @@ class StorageSetOrJoinBase : public IStorage friend class SetOrJoinBlockOutputStream; public: - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } - void rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; @@ -31,16 +28,13 @@ public: protected: StorageSetOrJoinBase( const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const Context & context_); String base_path; String path; - String table_name; - String database_name; std::atomic increment = 0; /// For the backup file names. @@ -85,8 +79,7 @@ private: protected: StorageSet( const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const Context & context_); diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index c9f04987b17..5aa3b52f969 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -197,16 +197,17 @@ private: StorageStripeLog::StorageStripeLog( DiskPtr disk_, const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, bool attach, size_t max_compress_block_size_) - : disk(std::move(disk_)), table_path(relative_path_), database_name(database_name_), table_name(table_name_), - max_compress_block_size(max_compress_block_size_), - file_checker(disk, table_path + "sizes.json"), - log(&Logger::get("StorageStripeLog")) + : IStorage(table_id_) + , disk(std::move(disk_)) + , table_path(relative_path_) + , max_compress_block_size(max_compress_block_size_) + , file_checker(disk, table_path + "sizes.json") + , log(&Logger::get("StorageStripeLog")) { setColumns(columns_); setConstraints(constraints_); @@ -229,9 +230,8 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Strin disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; - database_name = new_database_name; - table_name = new_table_name; file_checker.setPath(table_path + "sizes.json"); + renameInMemory(new_database_name, new_table_name); } @@ -295,9 +295,6 @@ CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Conte void StorageStripeLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { - if (table_name.empty()) - throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); - std::shared_lock lock(rwlock); disk->clearDirectory(table_path); @@ -316,7 +313,7 @@ void registerStorageStripeLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageStripeLog::create( - args.context.getDefaultDisk(), args.relative_data_path, args.database_name, args.table_name, args.columns, args.constraints, + args.context.getDefaultDisk(), args.relative_data_path, args.table_id, args.columns, args.constraints, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index 6f2e6b2a1f5..d87e3f9170a 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -24,8 +24,6 @@ class StorageStripeLog : public ext::shared_ptr_helper, public public: String getName() const override { return "StripeLog"; } - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -53,8 +51,7 @@ protected: StorageStripeLog( DiskPtr disk_, const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, bool attach, @@ -69,8 +66,6 @@ private: DiskPtr disk; String table_path; - String database_name; - String table_name; size_t max_compress_block_size; diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 5a447139599..aeb90399816 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -317,16 +317,17 @@ void TinyLogBlockOutputStream::write(const Block & block) StorageTinyLog::StorageTinyLog( DiskPtr disk_, const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, bool attach, size_t max_compress_block_size_) - : disk(std::move(disk_)), table_path(relative_path_), database_name(database_name_), table_name(table_name_), - max_compress_block_size(max_compress_block_size_), - file_checker(disk, table_path + "sizes.json"), - log(&Logger::get("StorageTinyLog")) + : IStorage(table_id_) + , disk(std::move(disk_)) + , table_path(relative_path_) + , max_compress_block_size(max_compress_block_size_) + , file_checker(disk, table_path + "sizes.json") + , log(&Logger::get("StorageTinyLog")) { setColumns(columns_); setConstraints(constraints_); @@ -374,12 +375,11 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const String disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; - database_name = new_database_name; - table_name = new_table_name; file_checker.setPath(table_path + "sizes.json"); for (auto & file : files) file.second.data_file_path = table_path + fileName(file.second.data_file_path); + renameInMemory(new_database_name, new_table_name); } @@ -414,9 +414,6 @@ CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context void StorageTinyLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { - if (table_name.empty()) - throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); - std::unique_lock lock(rwlock); disk->clearDirectory(table_path); @@ -428,6 +425,12 @@ void StorageTinyLog::truncate(const ASTPtr &, const Context &, TableStructureWri addFiles(column.name, *column.type); } +void StorageTinyLog::drop(TableStructureWriteLockHolder &) +{ + std::unique_lock lock(rwlock); + disk->removeRecursive(table_path); + files.clear(); +} void registerStorageTinyLog(StorageFactory & factory) { @@ -439,7 +442,7 @@ void registerStorageTinyLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageTinyLog::create( - args.context.getDefaultDisk(), args.relative_data_path, args.database_name, args.table_name, args.columns, args.constraints, + args.context.getDefaultDisk(), args.relative_data_path, args.table_id, args.columns, args.constraints, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index b4131277ed8..b9a45a9f271 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -23,8 +23,6 @@ class StorageTinyLog : public ext::shared_ptr_helper, public ISt public: String getName() const override { return "TinyLog"; } - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -48,12 +46,13 @@ public: void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; + void drop(TableStructureWriteLockHolder &) override; + protected: StorageTinyLog( DiskPtr disk_, const String & relative_path_, - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, bool attach, @@ -68,8 +67,6 @@ private: DiskPtr disk; String table_path; - String database_name; - String table_name; size_t max_compress_block_size; @@ -80,7 +77,6 @@ private: Logger * log; - void addFile(const String & column_name, const IDataType & type, size_t level = 0); void addFiles(const String & column_name, const IDataType & type); }; diff --git a/dbms/src/Storages/StorageURL.cpp b/dbms/src/Storages/StorageURL.cpp index efe15dc1928..b7eb61c991a 100644 --- a/dbms/src/Storages/StorageURL.cpp +++ b/dbms/src/Storages/StorageURL.cpp @@ -30,13 +30,16 @@ namespace ErrorCodes IStorageURLBase::IStorageURLBase( const Poco::URI & uri_, const Context & context_, - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, const String & format_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & compression_method_) - : uri(uri_), context_global(context_), compression_method(compression_method_), format_name(format_name_), table_name(table_name_), database_name(database_name_) + : IStorage(table_id_) + , uri(uri_) + , context_global(context_) + , compression_method(compression_method_) + , format_name(format_name_) { context_global.getRemoteHostFilter().checkURL(uri); setColumns(columns_); @@ -207,12 +210,6 @@ BlockInputStreams IStorageURLBase::read(const Names & column_names, return {std::make_shared(block_input, column_defaults, context)}; } -void IStorageURLBase::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) -{ - table_name = new_table_name; - database_name = new_database_name; -} - BlockOutputStreamPtr IStorageURLBase::write(const ASTPtr & /*query*/, const Context & /*context*/) { return std::make_shared( @@ -249,7 +246,7 @@ void registerStorageURL(StorageFactory & factory) return StorageURL::create( uri, - args.database_name, args.table_name, + args.table_id, format_name, args.columns, args.constraints, args.context, compression_method); diff --git a/dbms/src/Storages/StorageURL.h b/dbms/src/Storages/StorageURL.h index a518aed71f7..b0fec1527c1 100644 --- a/dbms/src/Storages/StorageURL.h +++ b/dbms/src/Storages/StorageURL.h @@ -16,9 +16,6 @@ namespace DB class IStorageURLBase : public IStorage { public: - String getTableName() const override { return table_name; } - String getDatabaseName() const override { return database_name; } - BlockInputStreams read( const Names & column_names, const SelectQueryInfo & query_info, @@ -29,14 +26,11 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; - protected: IStorageURLBase( const Poco::URI & uri_, const Context & context_, - const std::string & database_name_, - const std::string & table_name_, + const StorageID & id_, const String & format_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, @@ -48,8 +42,6 @@ protected: private: String format_name; - String table_name; - String database_name; virtual std::string getReadMethod() const; @@ -77,14 +69,13 @@ class StorageURL : public ext::shared_ptr_helper, public IStorageURL public: StorageURL( const Poco::URI & uri_, - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, const String & format_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, Context & context_, const String & compression_method_) - : IStorageURLBase(uri_, context_, database_name_, table_name_, format_name_, columns_, constraints_, compression_method_) + : IStorageURLBase(uri_, context_, table_id_, format_name_, columns_, constraints_, compression_method_) { } diff --git a/dbms/src/Storages/StorageValues.cpp b/dbms/src/Storages/StorageValues.cpp index 452d815e5ea..080343a0956 100644 --- a/dbms/src/Storages/StorageValues.cpp +++ b/dbms/src/Storages/StorageValues.cpp @@ -7,8 +7,8 @@ namespace DB { -StorageValues::StorageValues(const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, const Block & res_block_) - : database_name(database_name_), table_name(table_name_), res_block(res_block_) +StorageValues::StorageValues(const StorageID & table_id_, const ColumnsDescription & columns_, const Block & res_block_) + : IStorage(table_id_), res_block(res_block_) { setColumns(columns_); } diff --git a/dbms/src/Storages/StorageValues.h b/dbms/src/Storages/StorageValues.h index c07cf7cbb63..14f8e2cb4e3 100644 --- a/dbms/src/Storages/StorageValues.h +++ b/dbms/src/Storages/StorageValues.h @@ -14,8 +14,6 @@ class StorageValues : public ext::shared_ptr_helper, public IStor friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Values"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } BlockInputStreams read( const Names & column_names, @@ -26,12 +24,10 @@ public: unsigned num_streams) override; private: - std::string database_name; - std::string table_name; Block res_block; protected: - StorageValues(const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, const Block & res_block_); + StorageValues(const StorageID & table_id_, const ColumnsDescription & columns_, const Block & res_block_); }; } diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 5c8543bbb33..5f49cdb1263 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -29,11 +29,10 @@ namespace ErrorCodes StorageView::StorageView( - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ASTCreateQuery & query, const ColumnsDescription & columns_) - : table_name(table_name_), database_name(database_name_) + : IStorage(table_id_) { setColumns(columns_); @@ -123,7 +122,7 @@ void registerStorageView(StorageFactory & factory) if (args.query.storage) throw Exception("Specifying ENGINE is not allowed for a View", ErrorCodes::INCORRECT_QUERY); - return StorageView::create(args.database_name, args.table_name, args.query, args.columns); + return StorageView::create(args.table_id, args.query, args.columns); }); } diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index 6d2e1d04e6f..3a78c474456 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -15,8 +15,6 @@ class StorageView : public ext::shared_ptr_helper, public IStorage friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "View"; } - std::string getTableName() const override { return table_name; } - std::string getDatabaseName() const override { return database_name; } /// It is passed inside the query and solved at its level. bool supportsSampling() const override { return true; } @@ -30,23 +28,14 @@ public: size_t max_block_size, unsigned num_streams) override; - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override - { - table_name = new_table_name; - database_name = new_database_name; - } - private: - String table_name; - String database_name; ASTPtr inner_query; void replaceTableNameWithSubquery(ASTSelectQuery * select_query, ASTPtr & subquery); protected: StorageView( - const String & database_name_, - const String & table_name_, + const StorageID & table_id_, const ASTCreateQuery & query, const ColumnsDescription & columns_); }; diff --git a/dbms/src/Storages/StorageXDBC.cpp b/dbms/src/Storages/StorageXDBC.cpp index 0dcbf372b28..5206ae780de 100644 --- a/dbms/src/Storages/StorageXDBC.cpp +++ b/dbms/src/Storages/StorageXDBC.cpp @@ -24,15 +24,20 @@ namespace ErrorCodes StorageXDBC::StorageXDBC( - const std::string & database_name_, - const std::string & table_name_, + const StorageID & table_id_, const std::string & remote_database_name_, const std::string & remote_table_name_, const ColumnsDescription & columns_, const Context & context_, const BridgeHelperPtr bridge_helper_) /// Please add support for constraints as soon as StorageODBC or JDBC will support insertion. - : IStorageURLBase(Poco::URI(), context_, database_name_, table_name_, IXDBCBridgeHelper::DEFAULT_FORMAT, columns_, ConstraintsDescription{}, "" /* CompressionMethod */) + : IStorageURLBase(Poco::URI(), + context_, + table_id_, + IXDBCBridgeHelper::DEFAULT_FORMAT, + columns_, + ConstraintsDescription{}, + "" /* CompressionMethod */) , bridge_helper(bridge_helper_) , remote_database_name(remote_database_name_) , remote_table_name(remote_table_name_) @@ -120,7 +125,7 @@ namespace BridgeHelperPtr bridge_helper = std::make_shared>(args.context, args.context.getSettingsRef().http_receive_timeout.value, engine_args[0]->as().value.safeGet()); - return std::make_shared(args.database_name, args.table_name, + return std::make_shared(args.table_id, engine_args[1]->as().value.safeGet(), engine_args[2]->as().value.safeGet(), args.columns, diff --git a/dbms/src/Storages/StorageXDBC.h b/dbms/src/Storages/StorageXDBC.h index c658fc5c8a1..e7cf2932785 100644 --- a/dbms/src/Storages/StorageXDBC.h +++ b/dbms/src/Storages/StorageXDBC.h @@ -24,8 +24,7 @@ public: unsigned num_streams) override; - StorageXDBC(const std::string & database_name_, - const std::string & table_name_, + StorageXDBC(const StorageID & table_id_, const std::string & remote_database_name, const std::string & remote_table_name, const ColumnsDescription & columns_, diff --git a/dbms/src/Storages/System/IStorageSystemOneBlock.h b/dbms/src/Storages/System/IStorageSystemOneBlock.h index b5c8c2ad78f..68a42511061 100644 --- a/dbms/src/Storages/System/IStorageSystemOneBlock.h +++ b/dbms/src/Storages/System/IStorageSystemOneBlock.h @@ -20,14 +20,11 @@ protected: virtual void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const = 0; public: - IStorageSystemOneBlock(const String & name_) : name(name_) + IStorageSystemOneBlock(const String & name_) : IStorage({"system", name_}) { setColumns(ColumnsDescription(Self::getNamesAndTypes())); } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } - BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -43,9 +40,6 @@ public: return BlockInputStreams(1, std::make_shared(sample_block.cloneWithColumns(std::move(res_columns)))); } - -private: - const String name; }; } diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 25e7086c1a6..65c4f19b7cb 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -36,7 +36,6 @@ const char * auto_config_build[] "USE_INTERNAL_MEMCPY", "@USE_INTERNAL_MEMCPY@", "USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@", "USE_JEMALLOC", "@USE_JEMALLOC@", - "USE_TCMALLOC", "@USE_TCMALLOC@", "USE_MIMALLOC", "@USE_MIMALLOC@", "USE_UNWIND", "@USE_UNWIND@", "USE_ICU", "@USE_ICU@", diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 31c7b1c45c3..a370625ef1d 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -22,7 +22,7 @@ namespace ErrorCodes } StorageSystemColumns::StorageSystemColumns(const std::string & name_) - : name(name_) + : IStorage({"system", name_}) { setColumns(ColumnsDescription( { diff --git a/dbms/src/Storages/System/StorageSystemColumns.h b/dbms/src/Storages/System/StorageSystemColumns.h index b569c2bafff..4c7457d7f8d 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.h +++ b/dbms/src/Storages/System/StorageSystemColumns.h @@ -16,8 +16,6 @@ class StorageSystemColumns : public ext::shared_ptr_helper friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemColumns"; } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, @@ -27,9 +25,6 @@ public: size_t max_block_size, unsigned num_streams) override; -private: - const std::string name; - protected: StorageSystemColumns(const std::string & name_); }; diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index acda98203db..583df8aefa5 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -24,11 +24,10 @@ class StorageSystemDetachedParts : friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemDetachedParts"; } - std::string getTableName() const override { return "detached_parts"; } - std::string getDatabaseName() const override { return "system"; } protected: explicit StorageSystemDetachedParts() + : IStorage({"system", "detached_parts"}) { setColumns(ColumnsDescription{{ {"database", std::make_shared()}, diff --git a/dbms/src/Storages/System/StorageSystemDisks.cpp b/dbms/src/Storages/System/StorageSystemDisks.cpp index 3deb9b2399f..3e40a6f1fc2 100644 --- a/dbms/src/Storages/System/StorageSystemDisks.cpp +++ b/dbms/src/Storages/System/StorageSystemDisks.cpp @@ -10,7 +10,7 @@ namespace ErrorCodes StorageSystemDisks::StorageSystemDisks(const std::string & name_) - : name(name_) + : IStorage({"system", name_}) { setColumns(ColumnsDescription( { diff --git a/dbms/src/Storages/System/StorageSystemDisks.h b/dbms/src/Storages/System/StorageSystemDisks.h index e54adbc1da8..8b472177ac0 100644 --- a/dbms/src/Storages/System/StorageSystemDisks.h +++ b/dbms/src/Storages/System/StorageSystemDisks.h @@ -19,8 +19,6 @@ class StorageSystemDisks : public ext::shared_ptr_helper, pu friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemDisks"; } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, @@ -30,9 +28,6 @@ public: size_t max_block_size, unsigned num_streams) override; -private: - const std::string name; - protected: StorageSystemDisks(const std::string & name_); }; diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index 36839e06196..b58762b3860 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -48,20 +48,21 @@ StorageSystemGraphite::Configs StorageSystemGraphite::getConfigs(const Context & { const String & config_name = table_data->merging_params.graphite_params.config_name; + auto table_id = table_data->getStorageID(); if (!graphite_configs.count(config_name)) { Config new_config = { table_data->merging_params.graphite_params, - { table_data->getDatabaseName() }, - { table_data->getTableName() }, + { table_id.database_name }, + { table_id.table_name }, }; graphite_configs.emplace(config_name, new_config); } else { - graphite_configs[config_name].databases.emplace_back(table_data->getDatabaseName()); - graphite_configs[config_name].tables.emplace_back(table_data->getTableName()); + graphite_configs[config_name].databases.emplace_back(table_id.database_name); + graphite_configs[config_name].tables.emplace_back(table_id.table_name); } } } diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 2f155e22a11..8e3e362332f 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -104,7 +104,7 @@ private: StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, std::optional limit_, UInt64 offset_, bool even_distribution_) - : name(name_), multithreaded(multithreaded_), even_distribution(even_distribution_), limit(limit_), offset(offset_) + : IStorage({"system", name_}), multithreaded(multithreaded_), even_distribution(even_distribution_), limit(limit_), offset(offset_) { setColumns(ColumnsDescription({{"number", std::make_shared()}})); } diff --git a/dbms/src/Storages/System/StorageSystemNumbers.h b/dbms/src/Storages/System/StorageSystemNumbers.h index 76070839012..ce417200ebe 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.h +++ b/dbms/src/Storages/System/StorageSystemNumbers.h @@ -28,8 +28,6 @@ class StorageSystemNumbers : public ext::shared_ptr_helper friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemNumbers"; } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, @@ -39,8 +37,9 @@ public: size_t max_block_size, unsigned num_streams) override; + bool hasEvenlyDistributedRead() const override { return true; } + private: - const std::string name; bool multithreaded; bool even_distribution; std::optional limit; diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp index 827ee5ca6a1..10375aaf028 100644 --- a/dbms/src/Storages/System/StorageSystemOne.cpp +++ b/dbms/src/Storages/System/StorageSystemOne.cpp @@ -11,7 +11,7 @@ namespace DB StorageSystemOne::StorageSystemOne(const std::string & name_) - : name(name_) + : IStorage({"system", name_}) { setColumns(ColumnsDescription({{"dummy", std::make_shared()}})); } diff --git a/dbms/src/Storages/System/StorageSystemOne.h b/dbms/src/Storages/System/StorageSystemOne.h index 3e35fdb3477..a64c25e36eb 100644 --- a/dbms/src/Storages/System/StorageSystemOne.h +++ b/dbms/src/Storages/System/StorageSystemOne.h @@ -20,8 +20,6 @@ class StorageSystemOne : public ext::shared_ptr_helper, public friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemOne"; } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, @@ -31,9 +29,6 @@ public: size_t max_block_size, unsigned num_streams) override; -private: - const std::string name; - protected: StorageSystemOne(const std::string & name_); }; diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index 2e4f5dee01f..9be407913f1 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -257,7 +257,7 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const } StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_) - : name(std::move(name_)) + : IStorage({"system", name_}) { ColumnsDescription tmp_columns(std::move(columns_)); diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.h b/dbms/src/Storages/System/StorageSystemPartsBase.h index 080153a2a91..1357160c50b 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.h +++ b/dbms/src/Storages/System/StorageSystemPartsBase.h @@ -53,9 +53,6 @@ private: class StorageSystemPartsBase : public IStorage { public: - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } - NameAndTypePair getColumn(const String & column_name) const override; bool hasColumn(const String & column_name) const override; @@ -69,8 +66,6 @@ public: unsigned num_streams) override; private: - const std::string name; - bool hasStateColumn(const Names & column_names) const; protected: diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index 3e319e19bd7..08f49ba97ee 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -15,7 +15,7 @@ namespace DB StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) - : name(name_) + : IStorage({"system", name_}) { setColumns(ColumnsDescription({ { "database", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemReplicas.h b/dbms/src/Storages/System/StorageSystemReplicas.h index fb006f4cbde..d6cf8480eaf 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.h +++ b/dbms/src/Storages/System/StorageSystemReplicas.h @@ -17,8 +17,6 @@ class StorageSystemReplicas : public ext::shared_ptr_helper; public: std::string getName() const override { return "SystemReplicas"; } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, @@ -28,9 +26,6 @@ public: size_t max_block_size, unsigned num_streams) override; -private: - const std::string name; - protected: StorageSystemReplicas(const std::string & name_); }; diff --git a/dbms/src/Storages/System/StorageSystemStoragePolicies.cpp b/dbms/src/Storages/System/StorageSystemStoragePolicies.cpp index 5f4e0a63e39..a134d4fa379 100644 --- a/dbms/src/Storages/System/StorageSystemStoragePolicies.cpp +++ b/dbms/src/Storages/System/StorageSystemStoragePolicies.cpp @@ -13,7 +13,7 @@ namespace ErrorCodes StorageSystemStoragePolicies::StorageSystemStoragePolicies(const std::string & name_) - : name(name_) + : IStorage({"system", name_}) { setColumns( ColumnsDescription({ diff --git a/dbms/src/Storages/System/StorageSystemStoragePolicies.h b/dbms/src/Storages/System/StorageSystemStoragePolicies.h index 3848052f99e..6d596b5f0c9 100644 --- a/dbms/src/Storages/System/StorageSystemStoragePolicies.h +++ b/dbms/src/Storages/System/StorageSystemStoragePolicies.h @@ -19,8 +19,6 @@ class StorageSystemStoragePolicies : public ext::shared_ptr_helper; public: std::string getName() const override { return "SystemStoragePolicies"; } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, @@ -30,9 +28,6 @@ public: size_t max_block_size, unsigned num_streams) override; -private: - const std::string name; - protected: StorageSystemStoragePolicies(const std::string & name_); }; diff --git a/dbms/src/Storages/System/StorageSystemTableEngines.cpp b/dbms/src/Storages/System/StorageSystemTableEngines.cpp index d40fc6fa49e..e63923f69b6 100644 --- a/dbms/src/Storages/System/StorageSystemTableEngines.cpp +++ b/dbms/src/Storages/System/StorageSystemTableEngines.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -7,15 +8,26 @@ namespace DB NamesAndTypesList StorageSystemTableEngines::getNamesAndTypes() { - return {{"name", std::make_shared()}}; + return {{"name", std::make_shared()}, + {"supports_settings", std::make_shared()}, + {"supports_skipping_indices", std::make_shared()}, + {"supports_sort_order", std::make_shared()}, + {"supports_ttl", std::make_shared()}, + {"supports_replication", std::make_shared()}, + {"supports_deduplication", std::make_shared()}}; } void StorageSystemTableEngines::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const { - const auto & storages = StorageFactory::instance().getAllStorages(); - for (const auto & pair : storages) + for (const auto & pair : StorageFactory::instance().getAllStorages()) { res_columns[0]->insert(pair.first); + res_columns[1]->insert(pair.second.features.supports_settings); + res_columns[2]->insert(pair.second.features.supports_skipping_indices); + res_columns[3]->insert(pair.second.features.supports_sort_order); + res_columns[4]->insert(pair.second.features.supports_ttl); + res_columns[5]->insert(pair.second.features.supports_replication); + res_columns[6]->insert(pair.second.features.supports_deduplication); } } diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 791b6bfb99a..a3ae21eb167 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -27,7 +27,7 @@ namespace ErrorCodes StorageSystemTables::StorageSystemTables(const std::string & name_) - : name(name_) + : IStorage({"system", name_}) { setColumns(ColumnsDescription( { @@ -266,14 +266,14 @@ protected: Array dependencies_database_name_array; if (columns_mask[src_index] || columns_mask[src_index + 1]) { - const auto dependencies = context.getDependencies(database_name, table_name); + const auto dependencies = context.getDependencies(StorageID(database_name, table_name)); dependencies_table_name_array.reserve(dependencies.size()); dependencies_database_name_array.reserve(dependencies.size()); for (const auto & dependency : dependencies) { - dependencies_table_name_array.push_back(dependency.second); - dependencies_database_name_array.push_back(dependency.first); + dependencies_table_name_array.push_back(dependency.table_name); + dependencies_database_name_array.push_back(dependency.database_name); } } diff --git a/dbms/src/Storages/System/StorageSystemTables.h b/dbms/src/Storages/System/StorageSystemTables.h index fc5a82e9a52..05c96cd14bc 100644 --- a/dbms/src/Storages/System/StorageSystemTables.h +++ b/dbms/src/Storages/System/StorageSystemTables.h @@ -17,8 +17,6 @@ class StorageSystemTables : public ext::shared_ptr_helper, friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemTables"; } - std::string getTableName() const override { return name; } - std::string getDatabaseName() const override { return "system"; } BlockInputStreams read( const Names & column_names, @@ -28,9 +26,6 @@ public: size_t max_block_size, unsigned num_streams) override; -private: - const std::string name; - protected: StorageSystemTables(const std::string & name_); }; diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index b346bdbb033..21cbecc167e 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -30,7 +30,7 @@ struct State { registerFunctions(); DatabasePtr database = std::make_shared("test"); - database->attachTable("table", StorageMemory::create("test", "table", ColumnsDescription{columns}, ConstraintsDescription{})); + database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); context.makeGlobalContext(); context.addDatabase("test", database); context.setCurrentDatabase("test"); diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp index b634f645769..f2a19ead7a4 100644 --- a/dbms/src/Storages/tests/storage_log.cpp +++ b/dbms/src/Storages/tests/storage_log.cpp @@ -30,7 +30,8 @@ try context.setPath("./"); DiskPtr disk = std::make_unique("default", "./", 0); - StoragePtr table = StorageLog::create(disk, "./", "test", "test", ColumnsDescription{names_and_types}, ConstraintsDescription{}, 1048576); + StoragePtr table = StorageLog::create(disk, "./", StorageID("test", "test"), ColumnsDescription{names_and_types}, ConstraintsDescription{}, 1048576); + table->startup(); /// write into it diff --git a/dbms/src/TableFunctions/ITableFunctionXDBC.cpp b/dbms/src/TableFunctions/ITableFunctionXDBC.cpp index af483886445..29a29eb1312 100644 --- a/dbms/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/dbms/src/TableFunctions/ITableFunctionXDBC.cpp @@ -81,7 +81,7 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co readStringBinary(columns_info, buf); NamesAndTypesList columns = NamesAndTypesList::parse(columns_info); - auto result = std::make_shared(getDatabaseName(), table_name, schema_name, remote_table_name, ColumnsDescription{columns}, context, helper); + auto result = std::make_shared(StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, ColumnsDescription{columns}, context, helper); if (!result) throw Exception("Failed to instantiate storage from table function " + getName(), ErrorCodes::UNKNOWN_EXCEPTION); diff --git a/dbms/src/TableFunctions/TableFunctionFile.cpp b/dbms/src/TableFunctions/TableFunctionFile.cpp index 75c2847732a..d5304392c3b 100644 --- a/dbms/src/TableFunctions/TableFunctionFile.cpp +++ b/dbms/src/TableFunctions/TableFunctionFile.cpp @@ -9,7 +9,7 @@ namespace DB StoragePtr TableFunctionFile::getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const std::string & compression_method) const { - StorageFile::CommonArguments args{getDatabaseName(), table_name, format, compression_method, columns, ConstraintsDescription{}, global_context}; + StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format, compression_method, columns, ConstraintsDescription{}, global_context}; return StorageFile::create(source, global_context.getUserFilesPath(), args); } diff --git a/dbms/src/TableFunctions/TableFunctionHDFS.cpp b/dbms/src/TableFunctions/TableFunctionHDFS.cpp index ae4a220fbbe..740857cd068 100644 --- a/dbms/src/TableFunctions/TableFunctionHDFS.cpp +++ b/dbms/src/TableFunctions/TableFunctionHDFS.cpp @@ -13,8 +13,7 @@ StoragePtr TableFunctionHDFS::getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const { return StorageHDFS::create(source, - getDatabaseName(), - table_name, + StorageID(getDatabaseName(), table_name), format, columns, ConstraintsDescription{}, diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index ad18cd8676c..bb99dee1651 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -68,8 +68,7 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co String table_name_regexp = args[1]->as().value.safeGet(); auto res = StorageMerge::create( - getDatabaseName(), - table_name, + StorageID(getDatabaseName(), table_name), ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)}, source_database, table_name_regexp, diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp index 833d0afc90d..9a0b04d9753 100644 --- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp +++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp @@ -115,8 +115,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co throw Exception("MySQL table " + backQuoteIfNeed(remote_database_name) + "." + backQuoteIfNeed(remote_table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); auto res = StorageMySQL::create( - getDatabaseName(), - table_name, + StorageID(getDatabaseName(), table_name), std::move(pool), remote_database_name, remote_table_name, diff --git a/dbms/src/TableFunctions/TableFunctionRemote.cpp b/dbms/src/TableFunctions/TableFunctionRemote.cpp index 033839009bb..a8f97f9eb71 100644 --- a/dbms/src/TableFunctions/TableFunctionRemote.cpp +++ b/dbms/src/TableFunctions/TableFunctionRemote.cpp @@ -190,13 +190,13 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C StoragePtr res = remote_table_function_ptr ? StorageDistributed::createWithOwnCluster( - table_name, + StorageID("", table_name), structure_remote_table, remote_table_function_ptr, cluster, context) : StorageDistributed::createWithOwnCluster( - table_name, + StorageID("", table_name), structure_remote_table, remote_database, remote_table, diff --git a/dbms/src/TableFunctions/TableFunctionS3.cpp b/dbms/src/TableFunctions/TableFunctionS3.cpp index c739af30a68..42199d83ed6 100644 --- a/dbms/src/TableFunctions/TableFunctionS3.cpp +++ b/dbms/src/TableFunctions/TableFunctionS3.cpp @@ -85,7 +85,7 @@ StoragePtr TableFunctionS3::getStorage( S3::URI s3_uri (uri); UInt64 min_upload_part_size = global_context.getSettingsRef().s3_min_upload_part_size; - return StorageS3::create(s3_uri, access_key_id, secret_access_key, getDatabaseName(), table_name, format, min_upload_part_size, columns, ConstraintsDescription{}, global_context, compression_method); + return StorageS3::create(s3_uri, access_key_id, secret_access_key, StorageID(getDatabaseName(), table_name), format, min_upload_part_size, columns, ConstraintsDescription{}, global_context, compression_method); } void registerTableFunctionS3(TableFunctionFactory & factory) diff --git a/dbms/src/TableFunctions/TableFunctionURL.cpp b/dbms/src/TableFunctions/TableFunctionURL.cpp index a338aad3839..98134ed7539 100644 --- a/dbms/src/TableFunctions/TableFunctionURL.cpp +++ b/dbms/src/TableFunctions/TableFunctionURL.cpp @@ -12,7 +12,7 @@ StoragePtr TableFunctionURL::getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const { Poco::URI uri(source); - return StorageURL::create(uri, getDatabaseName(), table_name, format, columns, ConstraintsDescription{}, global_context, compression_method); + return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), format, columns, ConstraintsDescription{}, global_context, compression_method); } void registerTableFunctionURL(TableFunctionFactory & factory) diff --git a/dbms/src/TableFunctions/TableFunctionValues.cpp b/dbms/src/TableFunctions/TableFunctionValues.cpp index 40237d6c3f3..f72783d6dd4 100644 --- a/dbms/src/TableFunctions/TableFunctionValues.cpp +++ b/dbms/src/TableFunctions/TableFunctionValues.cpp @@ -88,7 +88,7 @@ StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const C Block res_block = sample_block.cloneWithColumns(std::move(res_columns)); - auto res = StorageValues::create(getDatabaseName(), table_name, columns, res_block); + auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); return res; } diff --git a/dbms/src/TableFunctions/parseColumnsListForTableFunction.cpp b/dbms/src/TableFunctions/parseColumnsListForTableFunction.cpp index cdc9063d415..41aaec9d1fb 100644 --- a/dbms/src/TableFunctions/parseColumnsListForTableFunction.cpp +++ b/dbms/src/TableFunctions/parseColumnsListForTableFunction.cpp @@ -18,6 +18,8 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con Tokens tokens(structure.c_str(), structure.c_str() + structure.size()); IParser::Pos token_iterator(tokens); + const Settings & settings = context.getSettingsRef(); + token_iterator.max_depth = settings.max_parser_depth; ParserColumnDeclarationList parser; ASTPtr columns_list_raw; diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index 6f34f9d4f03..acac89003ee 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -99,7 +99,7 @@ def get_processlist(client_cmd): def get_stacktraces(server_pid): - cmd = "gdb -q -ex 'set pagination off' -ex 'backtrace' -ex 'thread apply all backtrace' -ex 'detach' -ex 'quit' --pid {} 2>/dev/null".format(server_pid) + cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid) try: return subprocess.check_output(cmd, shell=True) except Exception as ex: @@ -107,12 +107,11 @@ def get_stacktraces(server_pid): def get_server_pid(server_tcp_port): - cmd = "lsof -i tcp:{port} | fgrep 'TCP *:{port} (LISTEN)'".format(port=server_tcp_port) + cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | gawk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) try: output = subprocess.check_output(cmd, shell=True) if output: - columns = output.strip().split(' ') - return int(columns[1]) + return int(output[1:]) else: return None # server dead except Exception as ex: @@ -182,7 +181,11 @@ def run_tests_array(all_tests_with_params): or 'global' in name): print(MSG_SKIPPED + " - no shard") skipped_total += 1 - elif not args.no_long and 'long' in name: + elif not args.no_long and ('long' in name + # Tests for races and deadlocks usually are runned in loop + # for significant amount of time + or 'deadlock' in name + or 'race' in name): print(MSG_SKIPPED + " - no long") skipped_total += 1 else: @@ -449,12 +452,22 @@ def main(args): if args.hung_check: processlist = get_processlist(args.client_with_database) if processlist: - server_pid = get_server_pid(os.getenv("CLICKHOUSE_PORT_TCP", '9000')) print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"])) print(processlist) + + clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000') + server_pid = get_server_pid(clickhouse_tcp_port) if server_pid: - print("\nStacktraces of all threads:") + print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) + print("\nCollecting stacktraces from all running threads:") print(get_stacktraces(server_pid)) + else: + print( + colored( + "\nUnable to locate ClickHouse server process listening at TCP port {}. " + "It must have crashed or exited prematurely!".format(clickhouse_tcp_port), + args, "red", attrs=["bold"])) + exit_code = 1 else: print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) diff --git a/dbms/tests/instructions/easy_tasks_sorted_ru.md b/dbms/tests/instructions/easy_tasks_sorted_ru.md index 6b9c0e18f9d..cb94fa1885f 100644 --- a/dbms/tests/instructions/easy_tasks_sorted_ru.md +++ b/dbms/tests/instructions/easy_tasks_sorted_ru.md @@ -1,36 +1,14 @@ # Простые задачи -## Пустой параметр --password в клиенте должен быть эквивалентен --ask-password. - -То есть означать предложение ввести пароль в интерактивном режиме. - -`dbms/programs/client/ConnectionParameters.h` - -\* кстати, сейчас функциональность реализована плохо: ввод пароля не поддерживает корректную обработку backspace. - -## Недостатки юзабилити: у clickhouse-client отсутствует сокращённая опция -C, как вариант --config-file; Недостатки юзабилити, если пользователь не может прочитать конфиг клиента. +## Недостатки юзабилити, если пользователь не может прочитать конфиг клиента. `dbms/programs/client/Client.cpp` -Также делаем `chmod 000 /etc/clickhouse-client/config.xml` и смотрим, что получится. +Делаем `chmod 000 /etc/clickhouse-client/config.xml` и смотрим, что получится. -## Оператор NOT BETWEEN. +## Уменьшать max_memory_usage, если на сервере мало оперативки. -`SELECT * FROM system.numbers WHERE number NOT BETWEEN 5 AND 10 LIMIT 10` - -`ExpressionListParsers.cpp`: `ParserBetweenExpression::parseImpl` - -## HTTP заголовок query_id. - -`programs/server/HTTPHandler.cpp` - смотрим метод `executeQuery` - -`src/Interpreters/executeQuery.h` - -`src/Interpreters/executeQuery.cpp` - смотрим колбэк на выставление Content-Type - -## Уменьшать max_memory_usage и размеры кэшей при старте, если на сервере мало оперативки. - -Смотрим, сколько на сервере оперативки. Если `max_memory_usage`, `max_memory_usage_for_all_queries` ограничены, но больше 90% (настройка) от имеющейся оперативки, то уменьшать их и выводить предупреждение в лог. Аналогично для кэшей: `mark_cache`, `uncompressed_cache`. +Смотрим, сколько на сервере оперативки. Если `max_memory_usage`, `max_memory_usage_for_all_queries` ограничены, но больше 90% (настройка) от имеющейся оперативки, то уменьшать их и выводить предупреждение в лог.. `programs/server/Server.cpp` - инициализация сервера, установка размера кэшей @@ -48,17 +26,6 @@ void memoryBitAnd(const char * a, const char * b, char * result, size_t size); ``` Потом используйте их в вашей функции. -## Функция arrayWithConstant. - -`arrayWithConstant(3, 'hello') = ['hello', 'hello', 'hello']` - -Смотрите метод `IColumn::replicate` для размножения значений столбца. - -## Функция flatten для превращения массивов массивов в массив элементов. - -`flatten([[1, 2, 3], [4, 5]]) = [1, 2, 3, 4, 5]` -`ColumnArray` - внимательно изучаем, как устроены массивы в ClickHouse. - ## Добавить generic вариант функций least, greatest. `SELECT least(123, 456)` - работает. @@ -75,66 +42,23 @@ void memoryBitAnd(const char * a, const char * b, char * result, size_t size); В ClickHouse есть возможность указать collation для сортировки строк. Это не работает для `Nullable(String)`. -## Проверить возможность использования pdqsort вместо std::sort для полной comparison-based сортировки. - -В случае, когда есть ORDER BY без LIMIT, это может позволить слегка увеличить производительность. - ## Запретить чтение значений типа AggregateFunction по-умолчанию и добавить настройку. Состояния агрегатных функций могут быть записаны в дамп и считаны из него. Но десериализация состояний агрегатных функций небезопасна. Аккуратно выбранные пользовательские данные могут привести к segfault или порче памяти. Поэтому нужно просто сделать настройку, которая запрещает читать AggregateFunction из пользовательских данных. -## Опции progress и time для clickhouse-local (по аналогии с clickhouse-client). - -Возможность выводить время выполнения запроса, а также красивый прогресс-бар для каждого запроса. - -## Usability: clickhouse-server должен поддерживать --help. - ## В статистику jemalloc добавить информацию по arenas. В `system.asynchronous_metrics` - суммарный размер арен. -## Добавить агрегатную функцию topKWeighted. - -`SELECT topKWeighted(value, weight)` - учитывать каждое значение с весом. - -## Функция isValidUTF8, toValidUTF8. - -`isValidUTF8` возвращает 1, если строка содержит набор байт в кодировке UTF-8. - -`toValidUTF8` - заменяет последовательности байт, не соответствующие кодировке UTF-8, на replacement character. - - # Более сложные задачи -## CREATE TABLE AS table_function() - -Возможность создать таблицу с таким же типом и структурой, как табличная функция. - -`ParserCreateQuery.cpp`, `InterpreterCreateQuery`, `Context::executeTableFunction` - ## Layout внешних словарей "direct". Как cache, но без кэша — всегда прямой запрос в источник. -## Подсказки в фабриках на основе edit distance. - -Всевозможные объекты: функции, агрегатные функции, типы данных, движки таблиц, и т. п. достаются по имени из фабрик. Часто пользователь допускает опечатку. Например, вместо `SELECT count(*)` может быть написано `SELECT cunt(*)`. В случае опечатки, необходимо в текст сообщения добавлять указание на ближайшие варианты. Для реализации можно использовать расстояние Левенштейна и полный перебор, или (лучше) - триграмный индекс. Подсказки выдаём, если указанное имя отличается от существующего на 1..2 буквы. Сортируем возможные варианты в порядке похожести. Для того, чтобы это работало во всех фабриках, может быть, потребуется обобщить их. - -## Учитывать порядок столбцов в заголовке в форматах CSV и TSV. - -В заголовке CSV, TSV могут быть указаны имена столбцов. Сейчас они полностью игнорируются. Надо учитывать, под настройкой. - ## Функции randomFixedString, randomBinaryString, fuzzBits, fuzzBytes. -## Функции для geoHash. - -Geohash - способ преобразования географических координат в строку, так что отображение обладает свойством локальности. https://en.wikipedia.org/wiki/Geohash В качестве библиотеки следует использовать эту: https://github.com/yinqiwen/geohash-int Необходимо добавить функции для перевода в обе стороны, а также для числового и текстового вариантов. - -## Агрегатные функции для статистических тестов (e.g. тест нормальности распределения) и статистик (e.g. энтропия). - -Энтропию следует считать по гистограмме. Пример расчёта гистограммы смотрите в реализации функции `quantileExact`. - -https://github.com/ClickHouse/ClickHouse/issues/3266 +## Агрегатные функции для статистических тестов (e.g. тест нормальности распределения) и статистик. ## Функции создания и обновления состояния агрегатной функции по одному кортежу аргументов. @@ -150,12 +74,6 @@ https://github.com/ClickHouse/ClickHouse/issues/3266 `createAggregationState('argMax', ('hello', 123))` - то же самое для агрегатных функций, принимающих несколько аргументов. -## Корректное сравнение Date и DateTime. - -https://github.com/ClickHouse/ClickHouse/issues/2011 - -Нужно сравнивать Date и DateTime так, как будто Date расширено до DateTime на начало суток в том же часовом поясе. - ## LEFT ONLY JOIN ## Функции makeDate, makeDateTime. @@ -173,16 +91,6 @@ https://github.com/ClickHouse/ClickHouse/issues/2011 Прогресс бар не должен мерцать. Наверное, надо просто вместо очистки строки, перемещать курсор в начало, не очищая её. -## Функция format для вставки значений в строку-шаблон. - -`format('Hello {2} World {1}', x, y)` - -## Добавить поддержку hyperscan. - -https://github.com/intel/hyperscan - -Реализовать на основе этой библиотеки функцию для матчинга сразу большого количества регулярных выражений. - ## Функция rowNumberForKey. Возвращает инкрементальное число для повторно встречающихся значений key. @@ -207,14 +115,6 @@ position с конца строки. Пример: `avgDistinct(x)` - вычислить среднее по всем различным переданным значениям. -## Проверка набора инструкций при старте сервера. - -Если сервер собран с поддержкой SSE 4.2, 4.1, 4, SSSE 3, SSE 3, то как можно ближе к началу работы, запускаем функцию, которая выполняет нужную инструкцию в качестве теста (asm volatile вставка), а до этого ставим обработчик сигнала SIGILL, который в случае невозможности выполнить инструкцию, сделает siglongjmp, позволит нам вывести понятное сообщение в лог и завершить работу. Замечание: /proc/cpuinfo зачастую не содержит актуальную информацию. - -## Добавить сжатие Brotli для HTTP интерфейса. - -`Content-Encoding: br` - ## Метрики количества ошибок. Добавляем счётчики всех ошибок (ErrorCodes) по аналогии с ProfileEvents. Кроме количества запоминаем также время последней ошибки, стек трейс, сообщение. Добавляем системную таблицу system.errors. Отправка в Graphite. @@ -223,7 +123,7 @@ position с конца строки. Экспериментальные алгоритмы сжатия. Сейчас ClickHouse поддерживает только lz4 и zstd. -## Запрос CREATE OR REPLACE TABLE/VIEW. +## Запрос CREATE OR REPLACE TABLE Атомарно (под блокировкой) удаляет таблицу перед созданием новой, если такая была. @@ -233,12 +133,6 @@ position с конца строки. - сейчас не работает. -## Возможность задать смещение для LIMIT BY. - -https://clickhouse.yandex/docs/ru/query_language/select/#limit-n-by - -`LIMIT 100, 10 BY RegionID` - выдать не более 10 строк для каждого RegionID, но пропустив первые 100 строк. - ## Возможность вставки значений типа AggregateFunction в виде кортежа значений аргументов, а не бинарного дампа состояния, под настройкой. Во входных данных в запросе INSERT должна быть возможность передать значение типа AggregateFunction не в виде сериализованного состояния, а в виде аргументов, которые будут агрегированы, для формирования этого состояния. @@ -255,17 +149,9 @@ https://clickhouse.yandex/docs/en/query_language/create/#create-table Запретить модификацию данных в партиции. На партицию ставится флаг, что она заблокирована. В неё нельзя делать INSERT и ALTER. С файлов снимается доступ на запись. -## Поддержка произвольных константных выражений в LIMIT. - -Возможность писать `LIMIT 1 + 2`. То же самое для `LIMIT BY`. - -## Добавить информацию об exp-smoothed количестве ошибок соединений с репликами в таблицу system.clusters. - -У нас есть счётчик ошибок соединения с серверами для failover. Надо сделать его видимым для пользователя. - ## Настройка join_use_nulls: поддержка для LEFT ARRAY JOIN. -## Внешние словари из Redis/Aerospike/Couchbase/Cassandra (на выбор). +## Внешние словари из Aerospike/Couchbase/Cassandra (на выбор). Подключить одну из key-value БД как источник. @@ -273,10 +159,6 @@ https://clickhouse.yandex/docs/en/query_language/create/#create-table Возможность легко импортировать данные из MongoDB. -## Возможность использования нескольких потоков для INSERT при INSERT SELECT. - -При INSERT SELECT, запрос SELECT может выполняться параллельно, но все данные будут передаваться на вставку в INSERT в один поток. Хотя некоторые таблицы (семейства MergeTree) поддерживают параллельную вставку. Необходимо сделать настройку для максимального количества потоков для INSERT. - ## Корректная обработка multiline значений в Pretty форматах. SELECT 'hello\nworld' AS x, 123 AS y ``` @@ -293,12 +175,6 @@ world │ 123 │ └───────┴─────┘ ``` -## Писать логи ClickHouse в ClickHouse. - -Пишем текстовые логи ClickHouse в системную таблицу в структурированном виде. - -См. SystemLog.h, cpp. - ## Работоспособность внешних данных на время сессии. https://clickhouse.yandex/docs/en/operations/table_engines/external_data/ @@ -317,8 +193,6 @@ https://clickhouse.yandex/docs/en/operations/table_engines/external_data/ Читает данные из таблицы в бесконечном цикле. -## Настройка, позволяющая обратиться ко всем репликам кластера, как к разным шардам. - ## Возможность ATTACH партиции с меньшим или большим количеством столбцов. ## Поддержка неконстантного аргумента с тайм-зоной у некоторых функций для работы с датой и временем. @@ -333,10 +207,4 @@ https://clickhouse.yandex/docs/en/operations/table_engines/external_data/ ## Функция serialize для implementation specific non portable non backwards compatible сериализации любого типа данных в набор байт. -## Функция arrayEnumerateUniqDeep - -Как arrayEnumerateUniq, но смотрит на самые глубокие элементы вложенных массивов. - ## Функция bitEquals и оператор <=>. - -## Параллельный ALTER MODIFY COLUMN. diff --git a/dbms/tests/instructions/sanitizers.md b/dbms/tests/instructions/sanitizers.md index c0347a32cad..45e1304e2a1 100644 --- a/dbms/tests/instructions/sanitizers.md +++ b/dbms/tests/instructions/sanitizers.md @@ -67,5 +67,5 @@ sudo -u clickhouse UBSAN_OPTIONS='print_stacktrace=1' ./clickhouse-ubsan server # How to use Memory Sanitizer ``` -CC=clang-8 CXX=clang++-8 cmake -D ENABLE_HDFS=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0 -D ENABLE_EMBEDDED_COMPILER=0 -D USE_INTERNAL_CAPNP_LIBRARY=0 -D USE_SIMDJSON=0 -D ENABLE_READLINE=0 -D SANITIZE=memory .. +CC=clang-8 CXX=clang++-8 cmake -D ENABLE_HDFS=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0 -D ENABLE_EMBEDDED_COMPILER=0 -D USE_INTERNAL_CAPNP_LIBRARY=0 -D USE_SIMDJSON=0 -D SANITIZE=memory .. ``` diff --git a/dbms/tests/integration/helpers/client.py b/dbms/tests/integration/helpers/client.py index fd59166b137..e986a9ef7c8 100644 --- a/dbms/tests/integration/helpers/client.py +++ b/dbms/tests/integration/helpers/client.py @@ -67,7 +67,11 @@ class CommandRequest: #print " ".join(command) - self.process = sp.Popen(command, stdin=stdin_file, stdout=self.stdout_file, stderr=self.stderr_file) + # we suppress stderror on client becase sometimes thread sanitizer + # can print some debug information there + env = {} + env["TSAN_OPTIONS"] = "verbosity=0" + self.process = sp.Popen(command, stdin=stdin_file, stdout=self.stdout_file, stderr=self.stderr_file, env=env) self.timer = None self.process_finished_before_timeout = True diff --git a/dbms/src/Compression/CompressionCodecWriteBuffer.h b/dbms/tests/integration/test_consistant_parts_after_move_partition/__init__.py similarity index 100% rename from dbms/src/Compression/CompressionCodecWriteBuffer.h rename to dbms/tests/integration/test_consistant_parts_after_move_partition/__init__.py diff --git a/dbms/tests/integration/test_consistant_parts_after_move_partition/configs/remote_servers.xml b/dbms/tests/integration/test_consistant_parts_after_move_partition/configs/remote_servers.xml new file mode 100644 index 00000000000..a6e80ce2b08 --- /dev/null +++ b/dbms/tests/integration/test_consistant_parts_after_move_partition/configs/remote_servers.xml @@ -0,0 +1,19 @@ + + + + + true + + shard_0 + node1 + 9000 + + + shard_0 + node2 + 9000 + + + + + diff --git a/dbms/tests/integration/test_consistant_parts_after_move_partition/test.py b/dbms/tests/integration/test_consistant_parts_after_move_partition/test.py new file mode 100644 index 00000000000..312f5dc30af --- /dev/null +++ b/dbms/tests/integration/test_consistant_parts_after_move_partition/test.py @@ -0,0 +1,58 @@ +import os + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry + + +CLICKHOUSE_DATABASE = 'test' + + +def initialize_database(nodes, shard): + for node in nodes: + node.query(''' + CREATE DATABASE {database}; + CREATE TABLE `{database}`.src (p UInt64, d UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/{database}/tables/test_consistent_shard1{shard}/replicated', '{replica}') + ORDER BY d PARTITION BY p + SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + CREATE TABLE `{database}`.dest (p UInt64, d UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/{database}/tables/test_consistent_shard2{shard}/replicated', '{replica}') + ORDER BY d PARTITION BY p + SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name, database=CLICKHOUSE_DATABASE)) + + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + initialize_database([node1, node2], 1) + yield cluster + except Exception as ex: + print ex + finally: + cluster.shutdown() + + +def test_consistent_part_after_move_partition(start_cluster): + # insert into all replicas + for i in range(100): + node1.query('INSERT INTO `{database}`.src VALUES ({value} % 2, {value})'.format(database=CLICKHOUSE_DATABASE, + value=i)) + query_source = 'SELECT COUNT(*) FROM `{database}`.src'.format(database=CLICKHOUSE_DATABASE) + query_dest = 'SELECT COUNT(*) FROM `{database}`.dest'.format(database=CLICKHOUSE_DATABASE) + assert_eq_with_retry(node2, query_source, node1.query(query_source)) + assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) + + node1.query('ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE)) + + assert_eq_with_retry(node2, query_source, node1.query(query_source)) + assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) diff --git a/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index 01f9b15b51f..784276707b4 100644 --- a/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -5,6 +5,7 @@ from helpers.cluster import ClickHouseCluster from dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed from external_sources import SourceMongo, SourceHTTP, SourceHTTPS, SourceRedis +import math SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') @@ -160,10 +161,10 @@ def setup_module(module): for layout in LAYOUTS: for source in SOURCES: if source.compatible_with_layout(layout): - DICTIONARIES.append(get_dict(source, layout, FIELDS[layout.layout_type])) + DICTIONARIES.append(get_dict(source, layout, FIELDS[layout.layout_type])) else: print "Source", source.name, "incompatible with layout", layout.name - + for layout in LAYOUTS: field_keys = list(filter(lambda x: x.is_key, FIELDS[layout.layout_type])) for source in SOURCES_KV: @@ -198,12 +199,23 @@ def started_cluster(): cluster.shutdown() -def test_simple_dictionaries(started_cluster): +def get_dictionaries(fold, total_folds, all_dicts): + chunk_len = int(math.ceil(len(all_dicts) / float(total_folds))) + if chunk_len * fold >= len(all_dicts): + return [] + return all_dicts[fold * chunk_len : (fold + 1) * chunk_len] + + +@pytest.mark.parametrize("fold", list(range(10))) +def test_simple_dictionaries(started_cluster, fold): fields = FIELDS["simple"] values = VALUES["simple"] data = [Row(fields, vals) for vals in values] - simple_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "simple"] + all_simple_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "simple"] + simple_dicts = get_dictionaries(fold, 10, all_simple_dicts) + + print "Length of dicts:", len(simple_dicts) for dct in simple_dicts: dct.load_data(data) @@ -295,12 +307,14 @@ def test_ranged_dictionaries(started_cluster): assert node.query(query) == str(answer) + '\n' -def test_key_value_simple_dictionaries(started_cluster): +@pytest.mark.parametrize("fold", list(range(10))) +def test_key_value_simple_dictionaries(started_cluster, fold): fields = FIELDS["simple"] values = VALUES["simple"] data = [Row(fields, vals) for vals in values] - simple_dicts = [d for d in DICTIONARIES_KV if d.structure.layout.layout_type == "simple"] + all_simple_dicts = [d for d in DICTIONARIES_KV if d.structure.layout.layout_type == "simple"] + simple_dicts = get_dictionaries(fold, 10, all_simple_dicts) for dct in simple_dicts: queries_with_answers = [] @@ -309,7 +323,7 @@ def test_key_value_simple_dictionaries(started_cluster): local_fields = dct.get_fields() local_values = [row.get_value_by_name(field.name) for field in local_fields if row.has_field(field.name)] local_data.append(Row(local_fields, local_values)) - + dct.load_data(local_data) node.query("system reload dictionary {}".format(dct.name)) @@ -368,7 +382,7 @@ def test_key_value_complex_dictionaries(started_cluster): local_fields = dct.get_fields() local_values = [row.get_value_by_name(field.name) for field in local_fields if row.has_field(field.name)] local_data.append(Row(local_fields, local_values)) - + dct.load_data(local_data) node.query("system reload dictionary {}".format(dct.name)) diff --git a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp b/dbms/tests/integration/test_distributed_storage_configuration/__init__.py similarity index 100% rename from dbms/src/DataStreams/tests/filter_stream_hitlog.cpp rename to dbms/tests/integration/test_distributed_storage_configuration/__init__.py diff --git a/dbms/tests/integration/test_distributed_storage_configuration/configs/config.d/storage_configuration.xml b/dbms/tests/integration/test_distributed_storage_configuration/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..f4e59555229 --- /dev/null +++ b/dbms/tests/integration/test_distributed_storage_configuration/configs/config.d/storage_configuration.xml @@ -0,0 +1,23 @@ + + + + + /disk1/ + + + /disk2/ + + + + + + +
+ disk1 + disk2 +
+ + + + + diff --git a/dbms/tests/integration/test_distributed_storage_configuration/test.py b/dbms/tests/integration/test_distributed_storage_configuration/test.py new file mode 100644 index 00000000000..a7c33b059c3 --- /dev/null +++ b/dbms/tests/integration/test_distributed_storage_configuration/test.py @@ -0,0 +1,65 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance('node', + config_dir='configs', + tmpfs=['/disk1:size=100M', '/disk2:size=100M']) + +@pytest.fixture(scope='module') +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def _files_in_dist_mon(node, root, table): + return int(node.exec_in_container([ + 'bash', + '-c', + # `-maxdepth 1` to avoid /tmp/ subdirectory + 'find /{root}/data/default/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f | wc -l'.format(root=root, table=table) + ]).split('\n')[0]) + +def test_different_versions(start_cluster): + node.query('CREATE TABLE foo (key Int) Engine=Memory()') + node.query(""" + CREATE TABLE dist_foo (key Int) + Engine=Distributed( + test_cluster_two_shards, + currentDatabase(), + foo, + key%2, + 'default' + ) + """) + # manual only + node.query('SYSTEM STOP DISTRIBUTED SENDS dist_foo') + + node.query('INSERT INTO dist_foo SELECT * FROM numbers(100)') + assert _files_in_dist_mon(node, 'disk1', 'dist_foo') == 1 + assert _files_in_dist_mon(node, 'disk2', 'dist_foo') == 0 + + assert node.query('SELECT count() FROM dist_foo') == '100\n' + node.query('SYSTEM FLUSH DISTRIBUTED dist_foo') + assert node.query('SELECT count() FROM dist_foo') == '200\n' + + # + # RENAME + # + node.query('RENAME TABLE dist_foo TO dist2_foo') + + node.query('INSERT INTO dist2_foo SELECT * FROM numbers(100)') + assert _files_in_dist_mon(node, 'disk1', 'dist2_foo') == 0 + assert _files_in_dist_mon(node, 'disk2', 'dist2_foo') == 1 + + assert node.query('SELECT count() FROM dist2_foo') == '300\n' + node.query('SYSTEM FLUSH DISTRIBUTED dist2_foo') + assert node.query('SELECT count() FROM dist2_foo') == '400\n' diff --git a/dbms/tests/integration/test_globs_in_filepath/test.py b/dbms/tests/integration/test_globs_in_filepath/test.py index 1ca3d003f1d..70bdb7777fb 100644 --- a/dbms/tests/integration/test_globs_in_filepath/test.py +++ b/dbms/tests/integration/test_globs_in_filepath/test.py @@ -120,10 +120,14 @@ def test_deep_structure(start_cluster): select count(*) from file('{}{}', 'TSV', 'text String, number Float64') '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) -def test_table_function(start_cluster): +def test_table_function_and_virtual_columns(start_cluster): node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)]) node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)]) node.query("insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)") assert node.query("select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000' node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)") assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()== '1' + assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() + assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() + assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() + assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() \ No newline at end of file diff --git a/dbms/tests/integration/test_multiple_disks/test.py b/dbms/tests/integration/test_multiple_disks/test.py index 441b0b6af56..363c85a2dbd 100644 --- a/dbms/tests/integration/test_multiple_disks/test.py +++ b/dbms/tests/integration/test_multiple_disks/test.py @@ -360,6 +360,7 @@ def test_max_data_part_size(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_with_overflow","MergeTree()"), ("replicated_mt_with_overflow","ReplicatedMergeTree('/clickhouse/replicated_mt_with_overflow', '1')",), @@ -454,6 +455,7 @@ def test_background_move(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("stopped_moving_mt","MergeTree()"), ("stopped_moving_replicated_mt","ReplicatedMergeTree('/clickhouse/stopped_moving_replicated_mt', '1')",), @@ -720,6 +722,7 @@ def produce_alter_move(node, name): pass +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("concurrently_altering_mt","MergeTree()"), ("concurrently_altering_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_altering_replicated_mt', '1')",), @@ -766,13 +769,14 @@ def test_concurrent_alter_move(start_cluster, name, engine): tasks.append(p.apply_async(optimize_table, (100,))) for task in tasks: - task.get(timeout=60) + task.get(timeout=120) assert node1.query("SELECT 1") == "1\n" assert node1.query("SELECT COUNT() FROM {}".format(name)) == "500\n" finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("concurrently_dropping_mt","MergeTree()"), ("concurrently_dropping_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_dropping_replicated_mt', '1')",), @@ -789,10 +793,12 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) + values = list({ random.randint(1, 1000000) for _ in range(0, 1000) }) + def insert(num): for i in range(num): day = random.randint(11, 30) - value = random.randint(1, 1000000) + value = values.pop() month = '0' + str(random.choice([3, 4])) node1.query("INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format(name, m=month, d=day, v=value)) @@ -899,6 +905,8 @@ def test_mutate_to_another_disk(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("alter_modifying_mt","MergeTree()"), ("replicated_alter_modifying_mt","ReplicatedMergeTree('/clickhouse/replicated_alter_modifying_mt', '1')",), @@ -915,10 +923,12 @@ def test_concurrent_alter_modify(start_cluster, name, engine): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) + values = list({ random.randint(1, 1000000) for _ in range(0, 1000) }) + def insert(num): for i in range(num): day = random.randint(11, 30) - value = random.randint(1, 1000000) + value = values.pop() month = '0' + str(random.choice([3, 4])) node1.query("INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format(name, m=month, d=day, v=value)) @@ -942,7 +952,7 @@ def test_concurrent_alter_modify(start_cluster, name, engine): tasks.append(p.apply_async(alter_modify, (100,))) for task in tasks: - task.get(timeout=60) + task.get(timeout=120) assert node1.query("SELECT 1") == "1\n" assert node1.query("SELECT COUNT() FROM {}".format(name)) == "100\n" diff --git a/dbms/tests/integration/test_parts_delete_zookeeper/test.py b/dbms/tests/integration/test_parts_delete_zookeeper/test.py index 1c23a4a658a..7e4a8d36741 100644 --- a/dbms/tests/integration/test_parts_delete_zookeeper/test.py +++ b/dbms/tests/integration/test_parts_delete_zookeeper/test.py @@ -40,13 +40,13 @@ def test_merge_doesnt_work_without_zookeeper(start_cluster): assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "2\n" node1.query("OPTIMIZE TABLE test_table FINAL") - assert node1.query("SELECT count(*) from system.parts") == "3\n" + assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "3\n" - assert_eq_with_retry(node1, "SELECT count(*) from system.parts", "1") + assert_eq_with_retry(node1, "SELECT count(*) from system.parts where table = 'test_table' and active = 1", "1") node1.query("TRUNCATE TABLE test_table") - assert node1.query("SELECT count(*) from system.parts") == "0\n" + assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "0\n" node1.query("INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") node1.query("INSERT INTO test_table VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)") @@ -56,6 +56,6 @@ def test_merge_doesnt_work_without_zookeeper(start_cluster): node1.query("OPTIMIZE TABLE test_table FINAL") pm.drop_instance_zk_connections(node1) time.sleep(10) # > old_parts_lifetime - assert node1.query("SELECT count(*) from system.parts") == "3\n" + assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "3\n" - assert_eq_with_retry(node1, "SELECT count(*) from system.parts", "1") + assert_eq_with_retry(node1, "SELECT count(*) from system.parts where table = 'test_table' and active = 1", "1") diff --git a/dbms/tests/integration/test_prometheus_endpoint/test.py b/dbms/tests/integration/test_prometheus_endpoint/test.py index dcd31621cb5..25d83cfb47c 100644 --- a/dbms/tests/integration/test_prometheus_endpoint/test.py +++ b/dbms/tests/integration/test_prometheus_endpoint/test.py @@ -3,6 +3,7 @@ import pytest import re import requests +import time from helpers.cluster import ClickHouseCluster @@ -24,7 +25,7 @@ def parse_response_line(line): "# HELP", "# TYPE", ] - assert any(line.startswith(prefix) for prefix in allowed_prefixes), msg + assert any(line.startswith(prefix) for prefix in allowed_prefixes) if line.startswith("#"): return {} @@ -34,12 +35,23 @@ def parse_response_line(line): return {name: int(val)} -def get_and_check_metrics(): - response = requests.get("http://{host}:{port}/metrics".format( - host=node.ip_address, port=8001), allow_redirects=False) +def get_and_check_metrics(retries): + while True: + try: + response = requests.get("http://{host}:{port}/metrics".format( + host=node.ip_address, port=8001), allow_redirects=False) - if response.status_code != 200: - response.raise_for_status() + if response.status_code != 200: + response.raise_for_status() + + break + except: + if retries >= 0: + retries -= 1 + time.sleep(0.5) + continue + else: + raise assert response.headers['content-type'].startswith('text/plain') @@ -55,13 +67,13 @@ def get_and_check_metrics(): def test_prometheus_endpoint(start_cluster): - metrics_dict = get_and_check_metrics() + metrics_dict = get_and_check_metrics(10) assert metrics_dict['ClickHouseProfileEvents_Query'] >= 0 prev_query_count = metrics_dict['ClickHouseProfileEvents_Query'] - resp = node.query("SELECT 1") - resp = node.query("SELECT 2") - resp = node.query("SELECT 3") + node.query("SELECT 1") + node.query("SELECT 2") + node.query("SELECT 3") - metrics_dict = get_and_check_metrics() + metrics_dict = get_and_check_metrics(10) assert metrics_dict['ClickHouseProfileEvents_Query'] >= prev_query_count + 3 diff --git a/dbms/tests/integration/test_server_initialization/test.py b/dbms/tests/integration/test_server_initialization/test.py index 64ed3181118..22c8b13d392 100644 --- a/dbms/tests/integration/test_server_initialization/test.py +++ b/dbms/tests/integration/test_server_initialization/test.py @@ -1,3 +1,4 @@ +import time import pytest from helpers.cluster import ClickHouseCluster @@ -6,7 +7,7 @@ from helpers.cluster import ClickHouseCluster def started_cluster(): try: cluster = ClickHouseCluster(__file__) - instance = cluster.add_instance('dummy', clickhouse_path_dir='clickhouse_path') + instance = cluster.add_instance('dummy', clickhouse_path_dir='clickhouse_path', stay_alive=True) cluster.start() cluster_fail = ClickHouseCluster(__file__, name='fail') @@ -34,3 +35,14 @@ def test_partially_dropped_tables(started_cluster): "./var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" assert instance.query("SELECT n FROM should_be_restored") == "1\n2\n3\n" assert instance.query("SELECT count() FROM system.tables WHERE name='should_be_dropped'") == "0\n" + + +def test_live_view_dependency(started_cluster): + instance = started_cluster.instances['dummy'] + instance.query("CREATE DATABASE a_load_first") + instance.query("CREATE DATABASE b_load_second") + instance.query("CREATE TABLE b_load_second.mt (a Int32) Engine=MergeTree order by tuple()") + instance.query("CREATE LIVE VIEW a_load_first.lv AS SELECT sum(a) FROM b_load_second.mt", settings={'allow_experimental_live_view': 1}) + instance.restart_clickhouse() + time.sleep(5) + instance.query("SELECT 1") diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/tests/integration/test_settings_constraints_distributed/__init__.py similarity index 100% rename from dbms/src/DataStreams/tests/native_streams.cpp rename to dbms/tests/integration/test_settings_constraints_distributed/__init__.py diff --git a/dbms/tests/integration/test_settings_constraints_distributed/configs/remote_servers.xml b/dbms/tests/integration/test_settings_constraints_distributed/configs/remote_servers.xml new file mode 100644 index 00000000000..5d8df3ac56d --- /dev/null +++ b/dbms/tests/integration/test_settings_constraints_distributed/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + distributed + + + node2 + 9000 + distributed + + + + + diff --git a/dbms/tests/integration/test_settings_constraints_distributed/configs/users_on_cluster.xml b/dbms/tests/integration/test_settings_constraints_distributed/configs/users_on_cluster.xml new file mode 100644 index 00000000000..e1a1122d9d6 --- /dev/null +++ b/dbms/tests/integration/test_settings_constraints_distributed/configs/users_on_cluster.xml @@ -0,0 +1,41 @@ + + + + 10000000000 + 0 + random + + + 1000000 + 0 + random + 2 + + + 1 + 1000000 + + + + + + + + default + default + + + + distributed_profile + default + + ::/0 + + + + + + + + + diff --git a/dbms/tests/integration/test_settings_constraints_distributed/configs/users_on_distributed.xml b/dbms/tests/integration/test_settings_constraints_distributed/configs/users_on_distributed.xml new file mode 100644 index 00000000000..15db7b92b15 --- /dev/null +++ b/dbms/tests/integration/test_settings_constraints_distributed/configs/users_on_distributed.xml @@ -0,0 +1,34 @@ + + + + 1000000 + 0 + random + + + 2000000000 + 0 + random + + + + + + default + default + + + + remote_profile + default + + ::/0 + + + + + + + + + diff --git a/dbms/tests/integration/test_settings_constraints_distributed/test.py b/dbms/tests/integration/test_settings_constraints_distributed/test.py new file mode 100644 index 00000000000..7b73af2d5fe --- /dev/null +++ b/dbms/tests/integration/test_settings_constraints_distributed/test.py @@ -0,0 +1,42 @@ +import time + +import pytest + +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users_on_cluster.xml']) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users_on_cluster.xml']) + +distributed = cluster.add_instance('distributed', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users_on_distributed.xml']) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for node in [node1, node2]: + node.query("CREATE TABLE sometable(date Date, id UInt32, value Int32) ENGINE = MergeTree() ORDER BY id;") + node.query("INSERT INTO sometable VALUES (toDate('2020-01-20'), 1, 1)") + + distributed.query("CREATE TABLE proxy (date Date, id UInt32, value Int32) ENGINE = Distributed(test_cluster, default, sometable);") + + yield cluster + + finally: + cluster.shutdown() + +def test_settings_under_remote(started_cluster): + assert distributed.query("SELECT COUNT() FROM proxy") == '1\n' + + with pytest.raises(QueryRuntimeException): + distributed.query("SELECT COUNT() FROM proxy", user='remote') + + assert distributed.query("SELECT COUNT() FROM proxy", settings={"max_memory_usage": 1000000}, user='remote') == '1\n' + + with pytest.raises(QueryRuntimeException): + distributed.query("SELECT COUNT() FROM proxy", settings={"max_memory_usage": 1000001}, user='remote') diff --git a/dbms/tests/integration/test_storage_hdfs/test.py b/dbms/tests/integration/test_storage_hdfs/test.py index 575b7593ca0..d65b0efc334 100644 --- a/dbms/tests/integration/test_storage_hdfs/test.py +++ b/dbms/tests/integration/test_storage_hdfs/test.py @@ -119,21 +119,24 @@ def test_globs_in_read_table(started_cluster): for filename in files: hdfs_api.write_data(globs_dir + filename, some_data) - test_requests = [("dir{1..5}/dir_dir/file1", 1), - ("*_table_functio?", 1), - ("dir/fil?", 1), - ("table{3..8}_function", 1), - ("table{2..8}_function", 2), - ("dir/*", 1), - ("dir/*?*?*?*?*", 1), - ("dir/*?*?*?*?*?*", 0), - ("some_dir/*/file", 2), - ("some_dir/dir?/*", 2), - ("*/*/*", 3), - ("?", 0)] + test_requests = [("dir{1..5}/dir_dir/file1", 1, 1), + ("*_table_functio?", 1, 1), + ("dir/fil?", 1, 1), + ("table{3..8}_function", 1, 1), + ("table{2..8}_function", 2, 2), + ("dir/*", 1, 1), + ("dir/*?*?*?*?*", 1, 1), + ("dir/*?*?*?*?*?*", 0, 0), + ("some_dir/*/file", 2, 1), + ("some_dir/dir?/*", 2, 1), + ("*/*/*", 3, 2), + ("?", 0, 0)] - for pattern, value in test_requests: - assert node1.query("select * from hdfs('hdfs://hdfs1:9000" + globs_dir + pattern + "', 'TSV', 'id UInt64, text String, number Float64')") == value * some_data + for pattern, paths_amount, files_amount in test_requests: + inside_table_func = "'hdfs://hdfs1:9000" + globs_dir + pattern + "', 'TSV', 'id UInt64, text String, number Float64'" + assert node1.query("select * from hdfs(" + inside_table_func + ")") == paths_amount * some_data + assert node1.query("select count(distinct _path) from hdfs(" + inside_table_func + ")").rstrip() == str(paths_amount) + assert node1.query("select count(distinct _file) from hdfs(" + inside_table_func + ")").rstrip() == str(files_amount) def test_read_write_gzip_table(started_cluster): hdfs_api = HDFSApi("root") diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/tests/integration/test_text_log_level/__init__.py similarity index 100% rename from dbms/src/DataStreams/tests/sorting_stream.cpp rename to dbms/tests/integration/test_text_log_level/__init__.py diff --git a/dbms/tests/integration/test_text_log_level/configs/config.d/text_log.xml b/dbms/tests/integration/test_text_log_level/configs/config.d/text_log.xml new file mode 100644 index 00000000000..3712c5851ae --- /dev/null +++ b/dbms/tests/integration/test_text_log_level/configs/config.d/text_log.xml @@ -0,0 +1,5 @@ + + + information + + diff --git a/dbms/tests/integration/test_text_log_level/test.py b/dbms/tests/integration/test_text_log_level/test.py new file mode 100644 index 00000000000..d7cf72fd9ea --- /dev/null +++ b/dbms/tests/integration/test_text_log_level/test.py @@ -0,0 +1,32 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance('node', config_dir='configs') + +@pytest.fixture(scope='module') +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + +def test_basic(start_cluster): + with pytest.raises(QueryRuntimeException): + # generates log with "Error" level + node.query('SELECT * FROM no_such_table') + + node.query('SYSTEM FLUSH LOGS') + + assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Trace'")) == 0 + assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Debug'")) == 0 + assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Information'")) >= 1 + assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Error'")) >= 1 diff --git a/dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp b/dbms/tests/integration/test_tinylog_s3/__init__.py similarity index 100% rename from dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp rename to dbms/tests/integration/test_tinylog_s3/__init__.py diff --git a/dbms/tests/integration/test_tinylog_s3/configs/config.d/log_conf.xml b/dbms/tests/integration/test_tinylog_s3/configs/config.d/log_conf.xml new file mode 100644 index 00000000000..318a6bca95d --- /dev/null +++ b/dbms/tests/integration/test_tinylog_s3/configs/config.d/log_conf.xml @@ -0,0 +1,12 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/dbms/tests/integration/test_tinylog_s3/configs/config.xml b/dbms/tests/integration/test_tinylog_s3/configs/config.xml new file mode 100644 index 00000000000..63b4d951eb7 --- /dev/null +++ b/dbms/tests/integration/test_tinylog_s3/configs/config.xml @@ -0,0 +1,40 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + diff --git a/dbms/tests/integration/test_tinylog_s3/configs/users.xml b/dbms/tests/integration/test_tinylog_s3/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/dbms/tests/integration/test_tinylog_s3/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_tinylog_s3/test.py b/dbms/tests/integration/test_tinylog_s3/test.py new file mode 100644 index 00000000000..9060508c06c --- /dev/null +++ b/dbms/tests/integration/test_tinylog_s3/test.py @@ -0,0 +1,49 @@ +import logging + +import pytest +from helpers.cluster import ClickHouseCluster + +logging.getLogger().setLevel(logging.INFO) +logging.getLogger().addHandler(logging.StreamHandler()) + + +# Creates S3 bucket for tests and allows anonymous read-write access to it. +def prepare_s3_bucket(cluster): + minio_client = cluster.minio_client + + if minio_client.bucket_exists(cluster.minio_bucket): + minio_client.remove_bucket(cluster.minio_bucket) + + minio_client.make_bucket(cluster.minio_bucket) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance("node", config_dir="configs", with_minio=True) + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + prepare_s3_bucket(cluster) + logging.info("S3 bucket created") + + yield cluster + finally: + cluster.shutdown() + + +def test_tinylog_s3(cluster): + node = cluster.instances["node"] + minio = cluster.minio_client + + node.query("CREATE TABLE s3_test (id UInt64) Engine=TinyLog") + node.query("INSERT INTO s3_test SELECT number FROM numbers(3)") + assert node.query("SELECT * FROM s3_test") == "0\n1\n2\n" + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 2 + node.query("INSERT INTO s3_test SELECT number + 3 FROM numbers(3)") + assert node.query("SELECT * FROM s3_test") == "0\n1\n2\n3\n4\n5\n" + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 2 + node.query("DROP TABLE s3_test") + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 0 diff --git a/dbms/src/Storages/tests/hit_log.cpp b/dbms/tests/integration/test_tmp_policy/__init__.py similarity index 100% rename from dbms/src/Storages/tests/hit_log.cpp rename to dbms/tests/integration/test_tmp_policy/__init__.py diff --git a/dbms/tests/integration/test_tmp_policy/configs/config.d/storage_configuration.xml b/dbms/tests/integration/test_tmp_policy/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..f8574a38208 --- /dev/null +++ b/dbms/tests/integration/test_tmp_policy/configs/config.d/storage_configuration.xml @@ -0,0 +1,25 @@ + + + + + /disk1/ + + + /disk2/ + + + + + + +
+ disk1 + disk2 +
+ + + + + + tmp + diff --git a/dbms/tests/integration/test_tmp_policy/test.py b/dbms/tests/integration/test_tmp_policy/test.py new file mode 100644 index 00000000000..5c5900cc9dc --- /dev/null +++ b/dbms/tests/integration/test_tmp_policy/test.py @@ -0,0 +1,34 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance('node', + config_dir='configs', + tmpfs=['/disk1:size=100M', '/disk2:size=100M']) + +@pytest.fixture(scope='module') +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def test_different_versions(start_cluster): + query = 'SELECT count(ignore(*)) FROM (SELECT * FROM system.numbers LIMIT 1e7) GROUP BY number' + settings = { + 'max_bytes_before_external_group_by': 1<<20, + 'max_bytes_before_external_sort': 1<<20, + } + + assert node.contains_in_log('Setting up /disk1/ to store temporary data in it') + assert node.contains_in_log('Setting up /disk2/ to store temporary data in it') + + node.query(query, settings=settings) + assert node.contains_in_log('Writing part of aggregation data into temporary file /disk1/') + assert node.contains_in_log('Writing part of aggregation data into temporary file /disk2/') diff --git a/dbms/tests/integration/test_ttl_move/test.py b/dbms/tests/integration/test_ttl_move/test.py index 071257d24ca..290fd1005cf 100644 --- a/dbms/tests/integration/test_ttl_move/test.py +++ b/dbms/tests/integration/test_ttl_move/test.py @@ -50,6 +50,7 @@ def get_used_disks_for_table(node, table_name): return node.query("select disk_name from system.parts where table == '{}' and active=1 order by modification_time".format(table_name)).strip().split('\n') +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,alter", [ ("mt_test_rule_with_invalid_destination","MergeTree()",0), ("replicated_mt_test_rule_with_invalid_destination","ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')",0), @@ -74,41 +75,42 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): {expression} SETTINGS storage_policy='{policy}' """.format(expression=x, name=name, engine=engine, policy=policy) - + if alter: node1.query(get_command(None, "small_jbod_with_external")) - + with pytest.raises(QueryRuntimeException): node1.query(get_command("TTL d1 TO DISK 'unknown'", "small_jbod_with_external")) - + node1.query("DROP TABLE IF EXISTS {}".format(name)) - + if alter: node1.query(get_command(None, "small_jbod_with_external")) - + with pytest.raises(QueryRuntimeException): node1.query(get_command("TTL d1 TO VOLUME 'unknown'", "small_jbod_with_external")) - + node1.query("DROP TABLE IF EXISTS {}".format(name)) - + if alter: node1.query(get_command(None, "only_jbod2")) - + with pytest.raises(QueryRuntimeException): node1.query(get_command("TTL d1 TO DISK 'jbod1'", "only_jbod2")) - + node1.query("DROP TABLE IF EXISTS {}".format(name)) - + if alter: node1.query(get_command(None, "only_jbod2")) - + with pytest.raises(QueryRuntimeException): node1.query(get_command("TTL d1 TO VOLUME 'external'", "only_jbod2")) - + finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_inserts_to_disk_do_not_work","MergeTree()",0), ("replicated_mt_test_inserts_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_do_not_work', '1')",0), @@ -141,6 +143,7 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_moves_to_disk_do_not_work","MergeTree()",0), ("replicated_mt_test_moves_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_do_not_work', '1')",0), @@ -187,6 +190,7 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_test_moves_to_volume_work","MergeTree()"), ("replicated_mt_test_moves_to_volume_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_volume_work', '1')"), @@ -233,6 +237,7 @@ def test_moves_to_volume_work(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_inserts_to_volume_do_not_work","MergeTree()",0), ("replicated_mt_test_inserts_to_volume_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_do_not_work', '1')",0), @@ -271,6 +276,7 @@ def test_inserts_to_volume_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_test_moves_to_disk_eventually_work","MergeTree()"), ("replicated_mt_test_moves_to_disk_eventually_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_eventually_work', '1')"), @@ -326,6 +332,7 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_merges_to_disk_do_not_work","MergeTree()",0), ("replicated_mt_test_merges_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_do_not_work', '1')",0), @@ -383,6 +390,7 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine", [ ("mt_test_merges_with_full_disk_work","MergeTree()"), ("replicated_mt_test_merges_with_full_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_with_full_disk_work', '1')"), @@ -449,6 +457,7 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_moves_after_merges_do_not_work","MergeTree()",0), ("replicated_mt_test_moves_after_merges_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')",0), @@ -501,13 +510,18 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) -@pytest.mark.parametrize("name,engine,positive", [ - ("mt_test_moves_after_merges_do_not_work","MergeTree()",0), - ("replicated_mt_test_moves_after_merges_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')",0), - ("mt_test_moves_after_merges_work","MergeTree()",1), - ("replicated_mt_test_moves_after_merges_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_work', '1')",1), +@pytest.mark.skip(reason="Flappy test") +@pytest.mark.parametrize("name,engine,positive,bar", [ + ("mt_test_moves_after_alter_do_not_work","MergeTree()",0,"DELETE"), + ("replicated_mt_test_moves_after_alter_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')",0,"DELETE"), + ("mt_test_moves_after_alter_work","MergeTree()",1,"DELETE"), + ("replicated_mt_test_moves_after_alter_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')",1,"DELETE"), + ("mt_test_moves_after_alter_do_not_work","MergeTree()",0,"TO DISK 'external'"), + ("replicated_mt_test_moves_after_alter_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')",0,"TO DISK 'external'"), + ("mt_test_moves_after_alter_work","MergeTree()",1,"TO DISK 'external'"), + ("replicated_mt_test_moves_after_alter_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')",1,"TO DISK 'external'"), ]) -def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive): +def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive, bar): try: node1.query(""" CREATE TABLE {name} ( @@ -523,8 +537,8 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive): node1.query(""" ALTER TABLE {name} MODIFY TTL - d1 + INTERVAL 15 MINUTE - """.format(name=name)) # That shall disable TTL. + d1 + INTERVAL 15 MINUTE {bar} + """.format(name=name, bar=bar)) # That shall disable TTL. data = [] # 10MB in total for i in range(10): @@ -540,6 +554,7 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive): node1.query("DROP TABLE IF EXISTS {}".format(name)) +@pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_alter_multiple_ttls_positive", "MergeTree()", True), ("mt_replicated_test_alter_multiple_ttls_positive", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True), @@ -578,14 +593,14 @@ limitations under the License.""" ) ENGINE = {engine} ORDER BY tuple() PARTITION BY p1 - TTL d1 + INTERVAL 30 SECOND TO DISK 'jbod2', + TTL d1 + INTERVAL 30 SECOND TO DISK 'jbod2', d1 + INTERVAL 60 SECOND TO VOLUME 'external' SETTINGS storage_policy='jbods_with_external', merge_with_ttl_timeout=0 """.format(name=name, engine=engine)) node1.query(""" ALTER TABLE {name} MODIFY - TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2', + TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2', d1 + INTERVAL 5 SECOND TO VOLUME 'external', d1 + INTERVAL 10 SECOND DELETE """.format(name=name)) @@ -620,3 +635,144 @@ limitations under the License.""" finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + + +@pytest.mark.skip(reason="Flappy test") +@pytest.mark.parametrize("name,engine", [ + ("concurrently_altering_ttl_mt","MergeTree()"), + ("concurrently_altering_ttl_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_altering_ttl_replicated_mt', '1')",), +]) +def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): + try: + node1.query(""" + CREATE TABLE {name} ( + EventDate Date, + number UInt64 + ) ENGINE = {engine} + ORDER BY tuple() + PARTITION BY toYYYYMM(EventDate) + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + values = list({ random.randint(1, 1000000) for _ in range(0, 1000) }) + + def insert(num): + for i in range(num): + day = random.randint(11, 30) + value = values.pop() + month = '0' + str(random.choice([3, 4])) + node1.query("INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format(name, m=month, d=day, v=value)) + + def alter_move(num): + def produce_alter_move(node, name): + move_type = random.choice(["PART", "PARTITION"]) + if move_type == "PART": + for _ in range(10): + try: + parts = node1.query("SELECT name from system.parts where table = '{}' and active = 1".format(name)).strip().split('\n') + break + except QueryRuntimeException: + pass + else: + raise Exception("Cannot select from system.parts") + + move_part = random.choice(["'" + part + "'" for part in parts]) + else: + move_part = random.choice([201903, 201904]) + + move_disk = random.choice(["DISK", "VOLUME"]) + if move_disk == "DISK": + move_volume = random.choice(["'external'", "'jbod1'", "'jbod2'"]) + else: + move_volume = random.choice(["'main'", "'external'"]) + try: + node1.query("ALTER TABLE {} MOVE {mt} {mp} TO {md} {mv}".format( + name, mt=move_type, mp=move_part, md=move_disk, mv=move_volume)) + except QueryRuntimeException as ex: + pass + + for i in range(num): + produce_alter_move(node1, name) + + def alter_update(num): + for i in range(num): + node1.query("ALTER TABLE {} UPDATE number = number + 1 WHERE 1".format(name)) + + def alter_modify_ttl(num): + for i in range(num): + ttls = [] + for j in range(random.randint(1, 10)): + what = random.choice(["TO VOLUME 'main'", "TO VOLUME 'external'", "TO DISK 'jbod1'", "TO DISK 'jbod2'", "TO DISK 'external'"]) + when = "now()+{}".format(random.randint(-1, 5)) + ttls.append("{} {}".format(when, what)) + node1.query("ALTER TABLE {} MODIFY TTL {}".format(name, ", ".join(ttls))) + + def optimize_table(num): + for i in range(num): + node1.query("OPTIMIZE TABLE {} FINAL".format(name)) + + p = Pool(15) + tasks = [] + for i in range(5): + tasks.append(p.apply_async(insert, (100,))) + tasks.append(p.apply_async(alter_move, (100,))) + tasks.append(p.apply_async(alter_update, (100,))) + tasks.append(p.apply_async(alter_modify_ttl, (100,))) + tasks.append(p.apply_async(optimize_table, (100,))) + + for task in tasks: + task.get(timeout=120) + + assert node1.query("SELECT 1") == "1\n" + assert node1.query("SELECT COUNT() FROM {}".format(name)) == "500\n" + finally: + node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + +@pytest.mark.skip(reason="Flappy test") +@pytest.mark.parametrize("name,positive", [ + ("test_double_move_while_select_negative", 0), + ("test_double_move_while_select_positive", 1), +]) +def test_double_move_while_select(started_cluster, name, positive): + try: + node1.query(""" + CREATE TABLE {name} ( + n Int64, + s String + ) ENGINE = MergeTree + ORDER BY tuple() + PARTITION BY n + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name)) + + node1.query("INSERT INTO {name} VALUES (1, '{string}')".format(name=name, string=get_random_string(10 * 1024 * 1024))) + + parts = node1.query("SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)).splitlines() + assert len(parts) == 1 + + node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) + + def long_select(): + if positive: + node1.query("SELECT sleep(3), sleep(2), sleep(1), n FROM {name}".format(name=name)) + + thread = threading.Thread(target=long_select) + thread.start() + + node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'jbod1'".format(name=name, part=parts[0])) + + # Fill jbod1 to force ClickHouse to make move of partition 1 to external. + node1.query("INSERT INTO {name} VALUES (2, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) + node1.query("INSERT INTO {name} VALUES (3, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) + node1.query("INSERT INTO {name} VALUES (4, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) + + # If SELECT locked old part on external, move shall fail. + assert node1.query("SELECT disk_name FROM system.parts WHERE table = '{name}' AND active = 1 AND name = '{part}'" + .format(name=name, part=parts[0])).splitlines() == ["jbod1" if positive else "external"] + + thread.join() + + assert node1.query("SELECT n FROM {name} ORDER BY n".format(name=name)).splitlines() == ["1", "2", "3", "4"] + + finally: + node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) diff --git a/dbms/tests/performance/bitCount.xml b/dbms/tests/performance/bitCount.xml new file mode 100644 index 00000000000..8936f700b51 --- /dev/null +++ b/dbms/tests/performance/bitCount.xml @@ -0,0 +1,27 @@ + + once + + + + + 2000 + 10000 + + + + + + expr + + number + toUInt32(number) + toUInt16(number) + toUInt8(number) + toInt32(number) + toFloat64(number) + + + + + SELECT bitCount({expr}) FROM system.numbers + diff --git a/dbms/tests/performance/math.xml b/dbms/tests/performance/math.xml index 5f4f302a0e8..f4d31713a08 100644 --- a/dbms/tests/performance/math.xml +++ b/dbms/tests/performance/math.xml @@ -1,14 +1,18 @@ - once + + loop + + 5 + 10000 + - 1000 - 10000 + 50 + 60000 - func @@ -37,7 +41,7 @@ - SELECT count() FROM system.numbers WHERE NOT ignore({func}(toFloat64(number))) - SELECT count() FROM system.numbers WHERE NOT ignore({func}(toFloat32(number))) - SELECT count() FROM system.numbers WHERE NOT ignore({func}(number)) + SELECT count() FROM numbers(100000000) WHERE NOT ignore({func}(toFloat64(number))) + SELECT count() FROM numbers(100000000) WHERE NOT ignore({func}(toFloat32(number))) + SELECT count() FROM numbers(100000000) WHERE NOT ignore({func}(number)) diff --git a/dbms/tests/performance/order_by_decimals.xml b/dbms/tests/performance/order_by_decimals.xml index faf2841e993..c6a7e7f72df 100644 --- a/dbms/tests/performance/order_by_decimals.xml +++ b/dbms/tests/performance/order_by_decimals.xml @@ -17,11 +17,11 @@ - SELECT toInt32(number) AS n FROM numbers(1000000) ORDER BY n DESC - SELECT toDecimal32(number, 0) AS n FROM numbers(1000000) ORDER BY n + SELECT toInt32(number) AS n FROM numbers(1000000) ORDER BY n DESC FORMAT Null + SELECT toDecimal32(number, 0) AS n FROM numbers(1000000) ORDER BY n FORMAT Null - SELECT toDecimal32(number, 0) AS n FROM numbers(1000000) ORDER BY n DESC - SELECT toDecimal64(number, 8) AS n FROM numbers(1000000) ORDER BY n DESC - SELECT toDecimal128(number, 10) AS n FROM numbers(1000000) ORDER BY n DESC + SELECT toDecimal32(number, 0) AS n FROM numbers(1000000) ORDER BY n DESC FORMAT Null + SELECT toDecimal64(number, 8) AS n FROM numbers(1000000) ORDER BY n DESC FORMAT Null + SELECT toDecimal128(number, 10) AS n FROM numbers(1000000) ORDER BY n DESC FORMAT Null diff --git a/dbms/tests/performance/parallel_insert.xml b/dbms/tests/performance/parallel_insert.xml new file mode 100644 index 00000000000..44a2964f881 --- /dev/null +++ b/dbms/tests/performance/parallel_insert.xml @@ -0,0 +1,24 @@ + + loop + + + + 2 + + + + + + + + + default.hits_10m_single + + + CREATE TABLE hits2 AS hits_10m_single + set max_insert_threads=8 + + INSERT INTO hits2 SELECT * FROM hits_10m_single + + DROP TABLE IF EXISTS hits2 + diff --git a/dbms/tests/performance/parse_engine_file.xml b/dbms/tests/performance/parse_engine_file.xml index 080acbd53f2..8a0054bdd7f 100644 --- a/dbms/tests/performance/parse_engine_file.xml +++ b/dbms/tests/performance/parse_engine_file.xml @@ -34,6 +34,7 @@ TSKV RowBinary Native + Avro diff --git a/dbms/tests/performance/select_format.xml b/dbms/tests/performance/select_format.xml index 621247fee1e..189b35a2700 100644 --- a/dbms/tests/performance/select_format.xml +++ b/dbms/tests/performance/select_format.xml @@ -44,6 +44,7 @@ XML ODBCDriver2 MySQLWire + Avro diff --git a/dbms/tests/performance/uniq.xml b/dbms/tests/performance/uniq.xml index 307ad6f88ef..9de2ecdf72b 100644 --- a/dbms/tests/performance/uniq.xml +++ b/dbms/tests/performance/uniq.xml @@ -52,5 +52,5 @@ - SELECT {key} AS k, {func}(UserID) FROM hits_100m_single GROUP BY k + SELECT {key} AS k, {func}(UserID) FROM hits_100m_single GROUP BY k FORMAT Null diff --git a/dbms/tests/queries/0_stateless/00265_content_type.sh b/dbms/tests/queries/0_stateless/00265_content_type.sh deleted file mode 100755 index feddb46a6a4..00000000000 --- a/dbms/tests/queries/0_stateless/00265_content_type.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&default_format=JSONCompact" --data-binary @- <<< "SELECT 1" 2>&1 | grep '< Content-Type'; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT JSON" 2>&1 | grep '< Content-Type'; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1" 2>&1 | grep '< Content-Type'; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT TabSeparated" 2>&1 | grep '< Content-Type'; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Vertical" 2>&1 | grep '< Content-Type'; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Native" 2>&1 | grep '< Content-Type'; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT RowBinary" 2>&1 | grep '< Content-Type'; diff --git a/dbms/tests/queries/0_stateless/00265_content_type.reference b/dbms/tests/queries/0_stateless/00265_content_type_and_format.reference similarity index 59% rename from dbms/tests/queries/0_stateless/00265_content_type.reference rename to dbms/tests/queries/0_stateless/00265_content_type_and_format.reference index 0693d1118da..dbe9ebc0f58 100644 --- a/dbms/tests/queries/0_stateless/00265_content_type.reference +++ b/dbms/tests/queries/0_stateless/00265_content_type_and_format.reference @@ -1,7 +1,14 @@ -< Content-Type: application/json; charset=UTF-8 -< Content-Type: application/json; charset=UTF-8 -< Content-Type: text/tab-separated-values; charset=UTF-8 -< Content-Type: text/tab-separated-values; charset=UTF-8 -< Content-Type: text/plain; charset=UTF-8 -< Content-Type: application/octet-stream -< Content-Type: application/octet-stream +< Content-Type: application/json; charset=UTF-8 +< X-ClickHouse-Format: JSONCompact +< Content-Type: application/json; charset=UTF-8 +< X-ClickHouse-Format: JSON +< Content-Type: text/tab-separated-values; charset=UTF-8 +< X-ClickHouse-Format: TabSeparated +< Content-Type: text/tab-separated-values; charset=UTF-8 +< X-ClickHouse-Format: TabSeparated +< Content-Type: text/plain; charset=UTF-8 +< X-ClickHouse-Format: Vertical +< Content-Type: application/octet-stream +< X-ClickHouse-Format: Native +< Content-Type: application/octet-stream +< X-ClickHouse-Format: RowBinary diff --git a/dbms/tests/queries/0_stateless/00265_content_type_and_format.sh b/dbms/tests/queries/0_stateless/00265_content_type_and_format.sh new file mode 100755 index 00000000000..2a36a17c6a1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00265_content_type_and_format.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&default_format=JSONCompact" --data-binary @- <<< "SELECT 1" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT JSON" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT TabSeparated" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Vertical" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Native" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT RowBinary" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; diff --git a/dbms/tests/queries/0_stateless/00501_http_head.sh b/dbms/tests/queries/0_stateless/00501_http_head.sh index e235da3c192..7251fc2cf21 100755 --- a/dbms/tests/queries/0_stateless/00501_http_head.sh +++ b/dbms/tests/queries/0_stateless/00501_http_head.sh @@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}&query=SELECT%201"; - ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}&query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Server-Display-Name:" | grep -v "X-ClickHouse-Query-Id:" + ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}&query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Server-Display-Name:" | grep -v "X-ClickHouse-Query-Id:" | grep -v "X-ClickHouse-Format:" if [[ `${CLICKHOUSE_CURL} -sS -X POST -I "${CLICKHOUSE_URL}&query=SELECT+1" | grep -c '411 Length Required'` -ne 1 ]]; then echo FAIL diff --git a/dbms/tests/queries/0_stateless/00552_logical_functions_simple.reference b/dbms/tests/queries/0_stateless/00552_logical_functions_simple.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00552_logical_functions_simple.reference @@ -0,0 +1 @@ +OK diff --git a/dbms/tests/queries/0_stateless/00552_logical_functions_simple.sql b/dbms/tests/queries/0_stateless/00552_logical_functions_simple.sql new file mode 100644 index 00000000000..2043199b9c2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00552_logical_functions_simple.sql @@ -0,0 +1,56 @@ + +-- Test simple logic over smaller batch of columns +SELECT + -- x1, x2, x3, x4, + xor(x1, x2, x3, x4) AS xor1, + xor(xor(x1, x2), xor(x3, x4)) AS xor2, + + or(x1, x2, x3, x4) AS or1, + or(x1 or x2, x3 or x4) AS or2, + + and(x1, x2, x3, x4) AS and1, + and(x1 and x2, x3 and x4) AS and2 +FROM ( + SELECT + toUInt8(number % 2) AS x1, + toUInt8(number / 2 % 2) AS x2, + toUInt8(number / 4 % 2) AS x3, + toUInt8(number / 8 % 2) AS x4 + FROM numbers(16) +) +WHERE + xor1 != xor2 OR (and1 != and2 OR or1 != or2) +; + +-- Test simple logic over multiple batches of columns (currently batch spans over 10 columns) +SELECT + -- x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, + xor(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11) AS xor1, + xor(x1, xor(xor(xor(x2, x3), xor(x4, x5)), xor(xor(x6, x7), xor(x8, xor(x9, xor(x10, x11)))))) AS xor2, + + or(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11) AS or1, + or(x1, or(or(or(x2, x3), or(x4, x5)), or(or(x6, x7), or(x8, or(x9, or(x10, x11)))))) AS or2, + + and(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11) AS and1, + and(x1, and((x2 and x3) and (x4 and x5), (x6 and x7) and (x8 and (x9 and (x10 and x11))))) AS and2 +FROM ( + SELECT + toUInt8(number % 2) AS x1, + toUInt8(number / 2 % 2) AS x2, + toUInt8(number / 4 % 2) AS x3, + toUInt8(number / 8 % 2) AS x4, + toUInt8(number / 16 % 2) AS x5, + toUInt8(number / 32 % 2) AS x6, + toUInt8(number / 64 % 2) AS x7, + toUInt8(number / 128 % 2) AS x8, + toUInt8(number / 256 % 2) AS x9, + toUInt8(number / 512 % 2) AS x10, + toUInt8(number / 1024 % 2) AS x11 + FROM numbers(2048) +) +WHERE + xor1 != xor2 OR (and1 != and2 OR or1 != or2) +; + + +SELECT 'OK'; diff --git a/dbms/tests/queries/0_stateless/00552_logical_functions_ternary.reference b/dbms/tests/queries/0_stateless/00552_logical_functions_ternary.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00552_logical_functions_ternary.reference @@ -0,0 +1 @@ +OK diff --git a/dbms/tests/queries/0_stateless/00552_logical_functions_ternary.sql b/dbms/tests/queries/0_stateless/00552_logical_functions_ternary.sql new file mode 100644 index 00000000000..b22e5c61f0f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00552_logical_functions_ternary.sql @@ -0,0 +1,59 @@ + +-- Tests codepath for ternary logic +SELECT + -- x1, x2, x3, x4, + xor(x1, x2, x3, x4) AS xor1, + xor(xor(x1, x2), xor(x3, x4)) AS xor2, + + or(x1, x2, x3, x4) AS or1, + or(x1 or x2, x3 or x4) AS or2, + + and(x1, x2, x3, x4) AS and1, + and(x1 and x2, x3 and x4) AS and2 +FROM ( + SELECT + nullIf(toUInt8(number % 3), 2) AS x1, + nullIf(toUInt8(number / 3 % 3), 2) AS x2, + nullIf(toUInt8(number / 9 % 3), 2) AS x3, + nullIf(toUInt8(number / 27 % 3), 2) AS x4 + FROM numbers(81) +) +WHERE + (xor1 != xor2 OR (xor1 is NULL) != (xor2 is NULL)) OR + (or1 != or2 OR (or1 is NULL) != (or2 is NULL) OR (and1 != and2 OR (and1 is NULL) != (and2 is NULL))) +; + + +-- Test ternary logic over multiple batches of columns (currently batch spans over 10 columns) +SELECT + -- x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, + xor(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11) AS xor1, + xor(x1, xor(xor(xor(x2, x3), xor(x4, x5)), xor(xor(x6, x7), xor(x8, xor(x9, xor(x10, x11)))))) AS xor2, + + or(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11) AS or1, + or(x1, or(or(or(x2, x3), or(x4, x5)), or(or(x6, x7), or(x8, or(x9, or(x10, x11)))))) AS or2, + + and(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11) AS and1, + and(x1, and((x2 and x3) and (x4 and x5), (x6 and x7) and (x8 and (x9 and (x10 and x11))))) AS and2 +FROM ( + SELECT + nullIf(toUInt8(number % 3), 2) AS x1, + nullIf(toUInt8(number / 3 % 3), 2) AS x2, + nullIf(toUInt8(number / 9 % 3), 2) AS x3, + nullIf(toUInt8(number / 27 % 3), 2) AS x4, + nullIf(toUInt8(number / 81 % 3), 2) AS x5, + nullIf(toUInt8(number / 243 % 3), 2) AS x6, + nullIf(toUInt8(number / 729 % 3), 2) AS x7, + nullIf(toUInt8(number / 2187 % 3), 2) AS x8, + nullIf(toUInt8(number / 6561 % 3), 2) AS x9, + nullIf(toUInt8(number / 19683 % 3), 2) AS x10, + nullIf(toUInt8(number / 59049 % 3), 2) AS x11 + FROM numbers(177147) +) +WHERE + (xor1 != xor2 OR (xor1 is NULL) != (xor2 is NULL)) OR + (or1 != or2 OR (or1 is NULL) != (or2 is NULL) OR (and1 != and2 OR (and1 is NULL) != (and2 is NULL))) +; + + +SELECT 'OK'; diff --git a/dbms/tests/queries/0_stateless/00975_move_partition_merge_tree.reference b/dbms/tests/queries/0_stateless/00975_move_partition_merge_tree.reference new file mode 100644 index 00000000000..f5c55cd802f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00975_move_partition_merge_tree.reference @@ -0,0 +1,4 @@ +10000000 +0 +5000000 +5000000 diff --git a/dbms/tests/queries/0_stateless/00975_move_partition_merge_tree.sql b/dbms/tests/queries/0_stateless/00975_move_partition_merge_tree.sql new file mode 100644 index 00000000000..9704628f1dd --- /dev/null +++ b/dbms/tests/queries/0_stateless/00975_move_partition_merge_tree.sql @@ -0,0 +1,29 @@ +DROP TABLE IF EXISTS test_move_partition_src; +DROP TABLE IF EXISTS test_move_partition_dest; + +CREATE TABLE IF NOT EXISTS test_move_partition_src ( + pk UInt8, + val UInt32 +) Engine = MergeTree() + PARTITION BY pk + ORDER BY (pk, val); + +CREATE TABLE IF NOT EXISTS test_move_partition_dest ( + pk UInt8, + val UInt32 +) Engine = MergeTree() + PARTITION BY pk + ORDER BY (pk, val); + +INSERT INTO test_move_partition_src SELECT number % 2, number FROM system.numbers LIMIT 10000000; + +SELECT count() FROM test_move_partition_src; +SELECT count() FROM test_move_partition_dest; + +ALTER TABLE test_move_partition_src MOVE PARTITION 1 TO TABLE test_move_partition_dest; + +SELECT count() FROM test_move_partition_src; +SELECT count() FROM test_move_partition_dest; + +DROP TABLE test_move_partition_src; +DROP TABLE test_move_partition_dest; diff --git a/dbms/tests/queries/0_stateless/00975_sample_prewhere.reference b/dbms/tests/queries/0_stateless/00975_sample_prewhere_distributed.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00975_sample_prewhere.reference rename to dbms/tests/queries/0_stateless/00975_sample_prewhere_distributed.reference diff --git a/dbms/tests/queries/0_stateless/00975_sample_prewhere.sql b/dbms/tests/queries/0_stateless/00975_sample_prewhere_distributed.sql similarity index 100% rename from dbms/tests/queries/0_stateless/00975_sample_prewhere.sql rename to dbms/tests/queries/0_stateless/00975_sample_prewhere_distributed.sql diff --git a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference index c1df99e5f94..a7ec77dc030 100644 --- a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference +++ b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference @@ -1,2 +1 @@ default merge_ab x UInt8 0 0 0 0 0 0 0 -default as_kafka x UInt8 0 0 0 0 0 0 0 diff --git a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql index 43c08b71b97..476377b4ddf 100644 --- a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql +++ b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql @@ -1,26 +1,13 @@ DROP TABLE IF EXISTS merge_a; DROP TABLE IF EXISTS merge_b; DROP TABLE IF EXISTS merge_ab; -DROP TABLE IF EXISTS kafka; -DROP TABLE IF EXISTS as_kafka; CREATE TABLE merge_a (x UInt8) ENGINE = StripeLog; CREATE TABLE merge_b (x UInt8) ENGINE = StripeLog; CREATE TABLE merge_ab AS merge(currentDatabase(), '^merge_[ab]$'); -CREATE TABLE kafka (x UInt8) - ENGINE = Kafka - SETTINGS kafka_broker_list = 'kafka', - kafka_topic_list = 'topic', - kafka_group_name = 'group', - kafka_format = 'CSV'; -CREATE TABLE as_kafka AS kafka ENGINE = Memory; - SELECT * FROM system.columns WHERE database = currentDatabase() AND table = 'merge_ab'; -SELECT * FROM system.columns WHERE database = currentDatabase() AND table = 'as_kafka'; DROP TABLE merge_a; DROP TABLE merge_b; DROP TABLE merge_ab; -DROP TABLE kafka; -DROP TABLE as_kafka; diff --git a/dbms/tests/queries/0_stateless/00995_exception_while_insert.sh b/dbms/tests/queries/0_stateless/00995_exception_while_insert.sh index 5b95174d1f8..0173d1aedae 100755 --- a/dbms/tests/queries/0_stateless/00995_exception_while_insert.sh +++ b/dbms/tests/queries/0_stateless/00995_exception_while_insert.sh @@ -7,8 +7,8 @@ CLICKHOUSE_CLIENT=`echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLIC $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS check;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE check (x UInt64, y UInt64 DEFAULT throwIf(x > 10000000)) ENGINE = Memory;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE check (x UInt64, y UInt64 DEFAULT throwIf(x > 1500000)) ENGINE = Memory;" -seq 1 11000000 | $CLICKHOUSE_CLIENT --query="INSERT INTO check(x) FORMAT TSV" 2>&1 | grep -q "Value passed to 'throwIf' function is non zero." && echo 'OK' || echo 'FAIL' ||: +seq 1 2000000 | $CLICKHOUSE_CLIENT --query="INSERT INTO check(x) FORMAT TSV" 2>&1 | grep -q "Value passed to 'throwIf' function is non zero." && echo 'OK' || echo 'FAIL' ||: $CLICKHOUSE_CLIENT --query="DROP TABLE check;" diff --git a/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.reference b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.reference similarity index 100% rename from dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.reference rename to dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.reference diff --git a/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh similarity index 100% rename from dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh rename to dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh diff --git a/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql b/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql index b28b32ff787..fc0af53c5ae 100644 --- a/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql +++ b/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql @@ -35,7 +35,6 @@ ORDER BY n SETTINGS max_bytes_in_join = 10000000; SET partial_merge_join_optimizations = 1; -SET partial_merge_join_rows_in_left_blocks = 100000; SELECT number * 200000 as n, j FROM numbers(5) LEFT JOIN ( diff --git a/dbms/tests/queries/0_stateless/01015_database_bad_tables.reference b/dbms/tests/queries/0_stateless/01015_database_bad_tables.reference index 1957f3a9604..b69cd02f83d 100644 --- a/dbms/tests/queries/0_stateless/01015_database_bad_tables.reference +++ b/dbms/tests/queries/0_stateless/01015_database_bad_tables.reference @@ -1,2 +1,3 @@ 1 1 1 1 +1 1 diff --git a/dbms/tests/queries/0_stateless/01015_database_bad_tables.sql b/dbms/tests/queries/0_stateless/01015_database_bad_tables.sql index 7092f1c5bb1..458cc597a67 100644 --- a/dbms/tests/queries/0_stateless/01015_database_bad_tables.sql +++ b/dbms/tests/queries/0_stateless/01015_database_bad_tables.sql @@ -1,3 +1,5 @@ +DROP DATABASE IF EXISTS testlazy; + CREATE TABLE `таблица_со_странным_названием` (a UInt64, b UInt64) ENGINE = Log; INSERT INTO `таблица_со_странным_названием` VALUES (1, 1); SELECT * FROM `таблица_со_странным_названием`; @@ -8,4 +10,13 @@ CREATE TABLE testlazy.`таблица_со_странным_названием` INSERT INTO testlazy.`таблица_со_странным_названием` VALUES (1, 1); SELECT * FROM testlazy.`таблица_со_странным_названием`; DROP TABLE testlazy.`таблица_со_странным_названием`; -DROP DATABASE testlazy +DROP DATABASE testlazy; + +CREATE DATABASE testlazy ENGINE = Lazy(10); +DROP TABLE IF EXISTS testlazy.test; +CREATE TABLE IF NOT EXISTS testlazy.test (a UInt64, b UInt64) ENGINE = Log; +CREATE TABLE IF NOT EXISTS testlazy.test (a UInt64, b UInt64) ENGINE = Log; +INSERT INTO testlazy.test VALUES (1, 1); +SELECT * FROM testlazy.test; +DROP TABLE IF EXISTS testlazy.test; +DROP DATABASE testlazy; diff --git a/dbms/tests/queries/0_stateless/01023_materialized_view_query_context.sql b/dbms/tests/queries/0_stateless/01023_materialized_view_query_context.sql index 1a7f8c15678..d68d6df6ea3 100644 --- a/dbms/tests/queries/0_stateless/01023_materialized_view_query_context.sql +++ b/dbms/tests/queries/0_stateless/01023_materialized_view_query_context.sql @@ -1,5 +1,6 @@ -- Create dictionary, since dictGet*() uses DB::Context in executeImpl() -- (To cover scope of the Context in DB::PushingToViewsBlockOutputStream::process) +DROP TABLE IF EXISTS mv; DROP DATABASE IF EXISTS dict_in_01023; CREATE DATABASE dict_in_01023; diff --git a/dbms/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.reference b/dbms/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.reference new file mode 100644 index 00000000000..20a8c2214c9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.reference @@ -0,0 +1,12 @@ +Initial +4 4 +4 8 +MOVE simple +2 2 +6 10 +MOVE incompatible schema missing column +4 4 +0 0 +MOVE incompatible schema different order by +4 4 +0 0 diff --git a/dbms/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh b/dbms/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh new file mode 100755 index 00000000000..9e077d673ee --- /dev/null +++ b/dbms/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +function query_with_retry +{ + retry=0 + until [ $retry -ge 5 ] + do + result=`$CLICKHOUSE_CLIENT $2 --query="$1" 2>&1` + if [ "$?" == 0 ]; then + echo -n $result + return + else + retry=$(($retry + 1)) + sleep 3 + fi + done + echo "Query '$1' failed with '$result'" +} + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" + +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (0, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '1', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (2, '0', 1);" + +$CLICKHOUSE_CLIENT --query="SELECT 'Initial';" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.dst VALUES (0, '1', 2);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.dst VALUES (1, '1', 2), (1, '2', 2);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.dst VALUES (2, '1', 2);" + +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" + + +$CLICKHOUSE_CLIENT --query="SELECT 'MOVE simple';" +query_with_retry "ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema missing column';" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY (d, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY (d, p) SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" + +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (0, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '1', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (2, '0', 1);" + +query_with_retry "ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" &>- +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema different order by';" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY (p, k, d);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY (d, k, p);" + + +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (0, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '0', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (1, '1', 1);" +$CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src VALUES (2, '0', 1);" + +query_with_retry "ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" &>- +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.dst;" + diff --git a/dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas.reference b/dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.reference similarity index 100% rename from dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas.reference rename to dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.reference diff --git a/dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas.sql b/dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql similarity index 100% rename from dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas.sql rename to dbms/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql diff --git a/dbms/tests/queries/0_stateless/01034_sample_final.reference b/dbms/tests/queries/0_stateless/01034_sample_final_distributed.reference similarity index 100% rename from dbms/tests/queries/0_stateless/01034_sample_final.reference rename to dbms/tests/queries/0_stateless/01034_sample_final_distributed.reference diff --git a/dbms/tests/queries/0_stateless/01034_sample_final.sql b/dbms/tests/queries/0_stateless/01034_sample_final_distributed.sql similarity index 100% rename from dbms/tests/queries/0_stateless/01034_sample_final.sql rename to dbms/tests/queries/0_stateless/01034_sample_final_distributed.sql diff --git a/dbms/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.reference b/dbms/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.reference new file mode 100644 index 00000000000..85dfb778810 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.reference @@ -0,0 +1 @@ +did not crash diff --git a/dbms/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/dbms/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh new file mode 100755 index 00000000000..32f8cfb2370 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.src;" +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.dst;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.src (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE $CLICKHOUSE_DATABASE.dst (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" + +function thread1() +{ + while true; + do + $CLICKHOUSE_CLIENT --query="ALTER TABLE $CLICKHOUSE_DATABASE.src MOVE PARTITION 1 TO TABLE $CLICKHOUSE_DATABASE.dst;" --query_id=query1 + done +} + +function thread2() +{ + while true; + do + $CLICKHOUSE_CLIENT --query="INSERT INTO $CLICKHOUSE_DATABASE.src SELECT number % 2, toString(number) FROM system.numbers LIMIT 100000" --query_id=query2 + done +} + +function thread3() +{ + while true; + do + $CLICKHOUSE_CLIENT --query="SELECT * FROM $CLICKHOUSE_DATABASE.src" --query_id=query3 1> /dev/null + done +} + +function thread4() +{ + while true; + do + $CLICKHOUSE_CLIENT --query="SELECT * FROM $CLICKHOUSE_DATABASE.dst" --query_id=query4 1> /dev/null + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; + +TIMEOUT=30 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & + +wait + +echo "DROP TABLE $CLICKHOUSE_DATABASE.src" | ${CLICKHOUSE_CLIENT} +echo "DROP TABLE $CLICKHOUSE_DATABASE.dst" | ${CLICKHOUSE_CLIENT} + +# Check for deadlocks +echo "SELECT * FROM system.processes WHERE query_id LIKE 'query%'" | ${CLICKHOUSE_CLIENT} + +echo 'did not crash' diff --git a/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.reference b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.reference new file mode 100644 index 00000000000..0c1eaef5ab3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.reference @@ -0,0 +1,4 @@ +[(1,2),(2,3),(3,4)] +[(1,2)] +[(1,2,3),(2,3,4)] +[(4,3,1)] diff --git a/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql new file mode 100644 index 00000000000..c9805599549 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql @@ -0,0 +1,16 @@ +SET send_logs_level = 'none'; + +DROP TABLE IF EXISTS array_of_tuples; + +CREATE TABLE array_of_tuples +( + f Array(Tuple(Float64, Float64)), + s Array(Tuple(UInt8, UInt16, UInt32)) +) ENGINE = Memory; + +INSERT INTO array_of_tuples values ([(1, 2), (2, 3), (3, 4)], array(tuple(1, 2, 3), tuple(2, 3, 4))), (array((1.0, 2.0)), [tuple(4, 3, 1)]); + +SELECT f from array_of_tuples; +SELECT s from array_of_tuples; + +DROP TABLE array_of_tuples; diff --git a/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference b/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference index 30237035c2c..49233946390 100644 --- a/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference +++ b/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference @@ -1,2 +1,10 @@ -10000000001 -10000000001 +1000000000 +3221225472 +1567000 +125952 +1567000 +125952 +12000000 +32505856 +1000000000000 +1099511627776 diff --git a/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql b/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql index 494e43b001f..6a4eadf6a40 100644 --- a/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql +++ b/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql @@ -1,7 +1,20 @@ -SET max_memory_usage = 10000000001; - +SET max_memory_usage = '1G'; SELECT value FROM system.settings WHERE name = 'max_memory_usage'; - -SET max_memory_usage = '1G'; -- { serverError 27 } - +SET max_memory_usage = '3Gi'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '1567k'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '123ki'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '1567K'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '123Ki'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '12M'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '31Mi'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '1T'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '1Ti'; SELECT value FROM system.settings WHERE name = 'max_memory_usage'; diff --git a/dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover.reference b/dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.reference similarity index 100% rename from dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover.reference rename to dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.reference diff --git a/dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover.sh b/dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh similarity index 100% rename from dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover.sh rename to dbms/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh diff --git a/dbms/tests/queries/0_stateless/01046_trivial_count_query.reference b/dbms/tests/queries/0_stateless/01046_trivial_count_query_distributed.reference similarity index 100% rename from dbms/tests/queries/0_stateless/01046_trivial_count_query.reference rename to dbms/tests/queries/0_stateless/01046_trivial_count_query_distributed.reference diff --git a/dbms/tests/queries/0_stateless/01046_trivial_count_query.sql b/dbms/tests/queries/0_stateless/01046_trivial_count_query_distributed.sql similarity index 100% rename from dbms/tests/queries/0_stateless/01046_trivial_count_query.sql rename to dbms/tests/queries/0_stateless/01046_trivial_count_query_distributed.sql diff --git a/dbms/tests/queries/0_stateless/01049_join_lc_crach.reference b/dbms/tests/queries/0_stateless/01049_join_lc_crach.reference deleted file mode 100644 index e7e860b7ce8..00000000000 --- a/dbms/tests/queries/0_stateless/01049_join_lc_crach.reference +++ /dev/null @@ -1,6 +0,0 @@ -a 1 -b 0 -a 1 2 -b 0 3 -a 1 a 2 - 0 b 3 diff --git a/dbms/tests/queries/0_stateless/01049_join_lc_crach.sql b/dbms/tests/queries/0_stateless/01049_join_lc_crach.sql deleted file mode 100644 index f66b2610d85..00000000000 --- a/dbms/tests/queries/0_stateless/01049_join_lc_crach.sql +++ /dev/null @@ -1,15 +0,0 @@ -DROP TABLE IF EXISTS Alpha; -DROP TABLE IF EXISTS Beta; - -CREATE TABLE Alpha (foo String, bar UInt64) ENGINE = Memory; -CREATE TABLE Beta (foo LowCardinality(String), baz UInt64) ENGINE = Memory; - -INSERT INTO Alpha VALUES ('a', 1); -INSERT INTO Beta VALUES ('a', 2), ('b', 3); - -SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) USING (foo); -SELECT * FROM Alpha FULL JOIN Beta USING (foo); -SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo; - -DROP TABLE Alpha; -DROP TABLE Beta; diff --git a/dbms/tests/queries/0_stateless/01049_join_low_card_crash.reference b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.reference new file mode 100644 index 00000000000..648a5de0a87 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.reference @@ -0,0 +1,12 @@ +a 1 +b 0 +a 1 2 +b 0 3 + 0 b 3 +a 1 a 2 +a 1 +b \N +a 1 2 +b \N 3 +a 1 a 2 +\N \N b 3 diff --git a/dbms/tests/queries/0_stateless/01049_join_low_card_crash.sql b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.sql new file mode 100644 index 00000000000..99bf09c42e1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS Alpha; +DROP TABLE IF EXISTS Beta; + +CREATE TABLE Alpha (foo String, bar UInt64) ENGINE = Memory; +CREATE TABLE Beta (foo LowCardinality(String), baz UInt64) ENGINE = Memory; + +INSERT INTO Alpha VALUES ('a', 1); +INSERT INTO Beta VALUES ('a', 2), ('b', 3); + +SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo ORDER BY foo; + +SET join_use_nulls = 1; + +SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo ORDER BY foo; + +DROP TABLE Alpha; +DROP TABLE Beta; diff --git a/dbms/tests/queries/0_stateless/01060_avro.reference b/dbms/tests/queries/0_stateless/01060_avro.reference new file mode 100644 index 00000000000..21fcc53f081 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01060_avro.reference @@ -0,0 +1,36 @@ +=== input += primitive +1,1,2,3.4,5.6,"b1","s1" +0,-1,9223372036854775807,3.00004,0.00001,"","" +1,2,"s1" +0,9223372036854775807,"" +"s1",2,1 +"",9223372036854775807,0 +"s1" +"" += complex +"A","t","['s1','s2']","[['a1'],['a2']]","s1",\N,"79cd909892d7e7ade1987cc7422628ba" +"C","f","[]","[]",\N,123,"79cd909892d7e7ade1987cc7422628ba" +"79cd909892d7e7ade1987cc7422628ba" +"79cd909892d7e7ade1987cc7422628ba" += logical_types +"2019-12-20","2020-01-10 07:31:56.227","2020-01-10 07:31:56.227000" +18250,1578641516227,1578641516227000 += compression +1000 +1000 += other +0 +1000 +not found +=== output += primitive +1,1,2,3.4,5.6,"b1","s1" += complex +"A","t","['s1','s2']","[['a1'],['a2']]","s1",\N,"79cd909892d7e7ade1987cc7422628ba" += logical_types +"2019-12-20","2020-01-10 07:31:56.227","2020-01-10 07:31:56.227000" += other +0 +1000 +147 diff --git a/dbms/tests/queries/0_stateless/01060_avro.sh b/dbms/tests/queries/0_stateless/01060_avro.sh new file mode 100755 index 00000000000..b57a7ad7a85 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01060_avro.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + +DATA_DIR=$CUR_DIR/data_avro + +# input +echo === input +echo = primitive + +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_bool UInt8, b_int Int32, c_long Int64, d_float Float32, e_double Float64, f_bytes String, g_string String' -q 'select * from table' +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_bool UInt8, c_long Int64, g_string String' -q 'select * from table' +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'g_string String, c_long Int64, a_bool UInt8' -q 'select * from table' +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'g_string String' -q 'select * from table' + +echo = complex +cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "a_enum_to_string String, b_enum_to_enum Enum('t' = 1, 'f' = 0), c_array_string Array(String), d_array_array_string Array(Array(String)), e_union_null_string Nullable(String), f_union_long_null Nullable(Int64), g_fixed FixedString(32)" -q 'select * from table' +cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "g_fixed FixedString(32)" -q 'select * from table' + +echo = logical_types +cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "a_date Date, b_timestamp_millis DateTime64(3, 'UTC'), c_timestamp_micros DateTime64(6, 'UTC')" -q 'select * from table' +cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_date Int32, b_timestamp_millis Int64, c_timestamp_micros Int64' -q 'select * from table' + + + +echo = compression +cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' +cat $DATA_DIR/simple.deflate.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' + +#snappy is optional +#cat $DATA_DIR/simple.snappy.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' + +echo = other +#no data +cat $DATA_DIR/empty.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' +# type mismatch +cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int32' -q 'select count() from table' +# field not found +cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'b Int64' -q 'select count() from table' 2>&1 | grep -i 'not found' -o + + + + + + +# output +echo === output + +echo = primitive +S1="a_bool UInt8, b_int Int32, c_long Int64, d_float Float32, e_double Float64, f_bytes String, g_string String" +echo '1,1,2,3.4,5.6,"b1","s1"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S1" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S1" -q 'select * from table' + +echo = complex +S2="a_enum_to_string String, b_enum_to_enum Enum('t' = 1, 'f' = 0), c_array_string Array(String), d_array_array_string Array(Array(String)), e_union_null_string Nullable(String), f_union_long_null Nullable(Int64), g_fixed FixedString(32)" +echo "\"A\",\"t\",\"['s1','s2']\",\"[['a1'],['a2']]\",\"s1\",\N,\"79cd909892d7e7ade1987cc7422628ba\"" | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S2" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S2" -q 'select * from table' + +echo = logical_types +S3="a_date Date, b_timestamp_millis DateTime64(3, 'UTC'), c_timestamp_micros DateTime64(6, 'UTC')" +echo '"2019-12-20","2020-01-10 07:31:56.227","2020-01-10 07:31:56.227000"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S3" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S3" -q 'select * from table' + +echo = other +S4="a Int64" +${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(0) format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S4" -q 'select count() from table' +${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(1000) format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S4" -q 'select count() from table' + +# type supported via conversion +${CLICKHOUSE_LOCAL} -q "select toInt16(123) as a format Avro" | wc -c \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql b/dbms/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql index 6c8f6c637fc..730263a2b12 100644 --- a/dbms/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql +++ b/dbms/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql @@ -1,4 +1,5 @@ -CREATE TABLE IF NOT EXISTS test Engine = MergeTree ORDER BY number AS SELECT number, toString(rand()) x from numbers(10000000); +DROP TABLE IF EXISTS test; +CREATE TABLE test Engine = MergeTree ORDER BY number AS SELECT number, toString(rand()) x from numbers(10000000); SELECT count() FROM test; diff --git a/dbms/tests/queries/0_stateless/01062_max_parser_depth.reference b/dbms/tests/queries/0_stateless/01062_max_parser_depth.reference new file mode 100644 index 00000000000..590b981971b --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_max_parser_depth.reference @@ -0,0 +1,4 @@ +- +Maximum parse depth (40) exceeded. +- +Maximum parse depth (20) exceeded. diff --git a/dbms/tests/queries/0_stateless/01062_max_parser_depth.sh b/dbms/tests/queries/0_stateless/01062_max_parser_depth.sh new file mode 100755 index 00000000000..17816db4758 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_max_parser_depth.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +echo 'select 1' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}&max_parser_depth=40" -d @- 2>&1 | grep -oP "Maximum parse depth .* exceeded." +echo - +echo 'select (1+1)*(2+1)' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}&max_parser_depth=40" -d @- 2>&1 | grep -oP "Maximum parse depth .* exceeded." +echo - +echo 'select 1' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}&max_parser_depth=20" -d @- 2>&1 | grep -oP "Maximum parse depth .* exceeded." diff --git a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference new file mode 100644 index 00000000000..07f79def4bb --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference @@ -0,0 +1,12 @@ +defaults +10000000 +10000000 +10000000 10 +max_joined_block_size_rows = 2000 +10000000 +10000000 10000 +10000000 10 +max_rows_in_join = 1000 +10000000 +10000000 10000 +10000000 10 diff --git a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql new file mode 100644 index 00000000000..e79dd0ee5a8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql @@ -0,0 +1,68 @@ +SET max_memory_usage = 50000000; +SET partial_merge_join = 1; + +SELECT 'defaults'; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10)) j + USING k); + +-- errors +SET max_joined_block_size_rows = 0; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); -- { serverError 241 } + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); -- { serverError 241 } + +SELECT 'max_joined_block_size_rows = 2000'; +SET max_joined_block_size_rows = 2000; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10)) j + USING k); + +SELECT 'max_rows_in_join = 1000'; +SET max_rows_in_join = 1000; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10)) j + USING k); diff --git a/dbms/tests/queries/0_stateless/01062_pm_multiple_all_join_same_value.reference b/dbms/tests/queries/0_stateless/01062_pm_multiple_all_join_same_value.reference new file mode 100644 index 00000000000..749fce669df --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_pm_multiple_all_join_same_value.reference @@ -0,0 +1 @@ +1000000 diff --git a/dbms/tests/queries/0_stateless/01062_pm_multiple_all_join_same_value.sql b/dbms/tests/queries/0_stateless/01062_pm_multiple_all_join_same_value.sql new file mode 100644 index 00000000000..a2e1d95e2b6 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_pm_multiple_all_join_same_value.sql @@ -0,0 +1,9 @@ +SET max_memory_usage = 50000000; +SET partial_merge_join = 1; + +SELECT count(1) FROM ( + SELECT t2.n FROM numbers(10) t1 + JOIN (SELECT toUInt32(1) AS k, number n FROM numbers(100)) t2 ON toUInt32(t1.number) = t2.k + JOIN (SELECT toUInt32(1) AS k, number n FROM numbers(100)) t3 ON t2.k = t3.k + JOIN (SELECT toUInt32(1) AS k, number n FROM numbers(100)) t4 ON t2.k = t4.k +); diff --git a/dbms/tests/queries/0_stateless/01063_create_column_set.reference b/dbms/tests/queries/0_stateless/01063_create_column_set.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01063_create_column_set.reference @@ -0,0 +1 @@ +0 diff --git a/dbms/tests/queries/0_stateless/01063_create_column_set.sql b/dbms/tests/queries/0_stateless/01063_create_column_set.sql new file mode 100644 index 00000000000..733f90971f9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01063_create_column_set.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS mt; +CREATE TABLE mt (x UInt8, y Date) ENGINE = MergeTree ORDER BY x; + +SELECT count() +FROM mt +ANY LEFT JOIN +( + SELECT 1 AS x +) USING (x) +PREWHERE x IN (1) WHERE y = today(); + +DROP TABLE mt; diff --git a/dbms/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.reference b/dbms/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.reference new file mode 100644 index 00000000000..02010fd6d80 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.reference @@ -0,0 +1,16 @@ +-- unmerged state +1 0000-00-00 00:00:00 0000-00-00 00:00:00 18446744073709551615 0 +1 0000-00-00 00:00:00 2000-01-01 10:00:00 7200 0 +1 0000-00-00 00:00:00 2000-01-01 11:10:00 600 0 +1 2000-01-01 08:00:00 0000-00-00 00:00:00 18446744073709551615 0 +1 2000-01-01 11:00:00 0000-00-00 00:00:00 18446744073709551615 3600 +2 0000-00-00 00:00:00 0000-00-00 00:00:00 18446744073709551615 0 +2 0000-00-00 00:00:00 2000-01-01 10:00:00 7200 0 +2 0000-00-00 00:00:00 2000-01-01 11:10:00 600 0 +2 0000-00-00 00:00:00 2001-01-01 11:10:02 1 0 +2 2000-01-01 08:00:00 0000-00-00 00:00:00 18446744073709551615 0 +2 2000-01-01 11:00:00 0000-00-00 00:00:00 18446744073709551615 3600 +2 2001-01-01 11:10:01 0000-00-00 00:00:00 18446744073709551615 31622401 +-- merged state +1 2000-01-01 11:00:00 2000-01-01 11:10:00 600 3600 +2 2001-01-01 11:10:01 2001-01-01 11:10:02 1 31622401 diff --git a/dbms/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql b/dbms/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql new file mode 100644 index 00000000000..41888e1baf3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql @@ -0,0 +1,127 @@ +SYSTEM STOP MERGES; + +-- incremental streaming usecase +-- that has sense only if data filling order has guarantees of chronological order + +DROP TABLE IF EXISTS target_table; +DROP TABLE IF EXISTS logins; +DROP TABLE IF EXISTS mv_logins2target; +DROP TABLE IF EXISTS checkouts; +DROP TABLE IF EXISTS mv_checkouts2target; + +-- that is the final table, which is filled incrementally from 2 different sources + +CREATE TABLE target_table Engine=SummingMergeTree() ORDER BY id +AS + SELECT + number as id, + maxState( toDateTime(0) ) as latest_login_time, + maxState( toDateTime(0) ) as latest_checkout_time, + minState( toUInt64(-1) ) as fastest_session, + maxState( toUInt64(0) ) as biggest_inactivity_period +FROM numbers(50000) +GROUP BY id; + +-- source table #1 + +CREATE TABLE logins ( + id UInt64, + ts DateTime +) Engine=MergeTree ORDER BY id; + + +-- and mv with something like feedback from target table + +CREATE MATERIALIZED VIEW mv_logins2target TO target_table +AS + SELECT + id, + maxState( ts ) as latest_login_time, + maxState( toDateTime(0) ) as latest_checkout_time, + minState( toUInt64(-1) ) as fastest_session, + if(max(current_latest_checkout_time) > 0, maxState(toUInt64(ts - current_latest_checkout_time)), maxState( toUInt64(0) ) ) as biggest_inactivity_period + FROM logins + LEFT JOIN ( + SELECT + id, + maxMerge(latest_checkout_time) as current_latest_checkout_time + + -- normal MV sees only the incoming block, but we need something like feedback here + -- so we do join with target table, the most important thing here is that + -- we extract from target table only row affected by that MV, referencing src table + -- it second time + FROM target_table + WHERE id IN (SELECT id FROM logins) + GROUP BY id + ) USING (id) + GROUP BY id; + + +-- the same for second pipeline +CREATE TABLE checkouts ( + id UInt64, + ts DateTime +) Engine=MergeTree ORDER BY id; + +CREATE MATERIALIZED VIEW mv_checkouts2target TO target_table +AS + SELECT + id, + maxState( toDateTime(0) ) as latest_login_time, + maxState( ts ) as latest_checkout_time, + if(max(current_latest_login_time) > 0, minState( toUInt64(ts - current_latest_login_time)), minState( toUInt64(-1) ) ) as fastest_session, + maxState( toUInt64(0) ) as biggest_inactivity_period + FROM checkouts + LEFT JOIN (SELECT id, maxMerge(latest_login_time) as current_latest_login_time FROM target_table WHERE id IN (SELECT id FROM checkouts) GROUP BY id) USING (id) + GROUP BY id; + + +-- feed with some initial values +INSERT INTO logins SELECT number as id, '2000-01-01 08:00:00' from numbers(50000); +INSERT INTO checkouts SELECT number as id, '2000-01-01 10:00:00' from numbers(50000); + +-- ensure that we don't read whole target table during join +set max_rows_to_read = 2000; + +INSERT INTO logins SELECT number as id, '2000-01-01 11:00:00' from numbers(1000); +INSERT INTO checkouts SELECT number as id, '2000-01-01 11:10:00' from numbers(1000); + +set max_rows_to_read = 10; + +INSERT INTO logins SELECT number+2 as id, '2001-01-01 11:10:01' from numbers(1); +INSERT INTO checkouts SELECT number+2 as id, '2001-01-01 11:10:02' from numbers(1); + + +set max_rows_to_read = 0; + +select '-- unmerged state'; + +select + id, + finalizeAggregation(latest_login_time) as current_latest_login_time, + finalizeAggregation(latest_checkout_time) as current_latest_checkout_time, + finalizeAggregation(fastest_session) as current_fastest_session, + finalizeAggregation(biggest_inactivity_period) as current_biggest_inactivity_period +from target_table +where id in (1,2) +ORDER BY id, current_latest_login_time, current_latest_checkout_time; + +select '-- merged state'; + +SELECT + id, + maxMerge(latest_login_time) as current_latest_login_time, + maxMerge(latest_checkout_time) as current_latest_checkout_time, + minMerge(fastest_session) as current_fastest_session, + maxMerge(biggest_inactivity_period) as current_biggest_inactivity_period +FROM target_table +where id in (1,2) +GROUP BY id +ORDER BY id; + +DROP TABLE IF EXISTS logins; +DROP TABLE IF EXISTS mv_logins2target; +DROP TABLE IF EXISTS checkouts; +DROP TABLE IF EXISTS mv_checkouts2target; + +SYSTEM START MERGES; \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.reference b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.reference new file mode 100644 index 00000000000..917fac168c2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.reference @@ -0,0 +1,19 @@ +100000 1 +100000 1 +100000 1 +100000 1 +first nullable +100000 1 +100000 1 +100000 1 +100000 1 +second nullable +100000 1 +100000 1 +100000 1 +100000 1 +both nullable +100000 1 +100000 1 +100000 1 +100000 1 diff --git a/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.sql b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.sql new file mode 100644 index 00000000000..631fda510b0 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.sql @@ -0,0 +1,87 @@ +SET partial_merge_join = 1; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT 'first nullable'; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT 'second nullable'; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT 'both nullable'; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); diff --git a/dbms/tests/queries/0_stateless/01065_array_zip_mixed_const.reference b/dbms/tests/queries/0_stateless/01065_array_zip_mixed_const.reference new file mode 100644 index 00000000000..dc683e2bf4d --- /dev/null +++ b/dbms/tests/queries/0_stateless/01065_array_zip_mixed_const.reference @@ -0,0 +1,24 @@ +[(0,'hello'),(1,'world')] +[(0,'hello'),(1,'world')] +[(0,'hello'),(1,'world')] +[(0,'hello'),(1,'world')] +[(0,'0'),(0,'world')] +[(0,'1'),(1,'world')] +[(0,'2'),(2,'world')] +[(0,'3'),(3,'world')] +[(0,'4'),(4,'world')] +[(0,'5'),(5,'world')] +[(0,'6'),(6,'world')] +[(0,'7'),(7,'world')] +[(0,'8'),(8,'world')] +[(0,'9'),(9,'world')] +[(1,[]),(0,[]),(0,[])] +[(1,[]),(1,[]),(1,[])] +[(1,[]),(2,[]),(4,[])] +[(1,[]),(3,[]),(9,[])] +[(1,[]),(4,[]),(16,[])] +[(1,[]),(5,[]),(25,[])] +[(1,[]),(6,[]),(36,[])] +[(1,[]),(7,[]),(49,[])] +[(1,[]),(8,[]),(64,[])] +[(1,[]),(9,[]),(81,[])] diff --git a/dbms/tests/queries/0_stateless/01065_array_zip_mixed_const.sql b/dbms/tests/queries/0_stateless/01065_array_zip_mixed_const.sql new file mode 100644 index 00000000000..0cd369739f4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01065_array_zip_mixed_const.sql @@ -0,0 +1,7 @@ +SELECT arrayZip([0, 1], ['hello', 'world']); +SELECT arrayZip(materialize([0, 1]), ['hello', 'world']); +SELECT arrayZip([0, 1], materialize(['hello', 'world'])); +SELECT arrayZip(materialize([0, 1]), materialize(['hello', 'world'])); + +SELECT arrayZip([0, number], [toString(number), 'world']) FROM numbers(10); +SELECT arrayZip([1, number, number * number], [[], [], []]) FROM numbers(10); diff --git a/dbms/tests/queries/0_stateless/01065_if_not_finite.reference b/dbms/tests/queries/0_stateless/01065_if_not_finite.reference new file mode 100644 index 00000000000..cccd6c68047 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01065_if_not_finite.reference @@ -0,0 +1,17 @@ +111 +1 +0.5 +0.33 +0.25 +0.2 +0.17 +0.14 +0.12 +0.11 +1 +-1 +2 +2 +\N +-42 +1 diff --git a/dbms/tests/queries/0_stateless/01065_if_not_finite.sql b/dbms/tests/queries/0_stateless/01065_if_not_finite.sql new file mode 100644 index 00000000000..495932692ea --- /dev/null +++ b/dbms/tests/queries/0_stateless/01065_if_not_finite.sql @@ -0,0 +1,11 @@ +SELECT ifNotFinite(round(1 / number, 2), 111) FROM numbers(10); + +SELECT ifNotFinite(1, 2); +SELECT ifNotFinite(-1.0, 2); +SELECT ifNotFinite(nan, 2); +SELECT ifNotFinite(-1 / 0, 2); +SELECT ifNotFinite(log(0), NULL); +SELECT ifNotFinite(sqrt(-1), -42); +SELECT ifNotFinite(1234567890123456789, -1234567890123456789); -- { serverError 386 } + +SELECT ifNotFinite(NULL, 1); diff --git a/dbms/tests/queries/0_stateless/01066_bit_count.reference b/dbms/tests/queries/0_stateless/01066_bit_count.reference new file mode 100644 index 00000000000..4a3b084b4a2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01066_bit_count.reference @@ -0,0 +1,21 @@ +0 +1 +1 +2 +1 +2 +2 +3 +1 +2 +4 +0 +1 +8 +64 +32 +16 +8 +1 10 000000000000F03F +-1 11 000000000000F0BF +inf 11 000000000000F07F diff --git a/dbms/tests/queries/0_stateless/01066_bit_count.sql b/dbms/tests/queries/0_stateless/01066_bit_count.sql new file mode 100644 index 00000000000..d50b2657542 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01066_bit_count.sql @@ -0,0 +1,13 @@ +SELECT bitCount(number) FROM numbers(10); +SELECT avg(bitCount(number)) FROM numbers(256); + +SELECT bitCount(0); +SELECT bitCount(1); +SELECT bitCount(-1); + +SELECT bitCount(toInt64(-1)); +SELECT bitCount(toInt32(-1)); +SELECT bitCount(toInt16(-1)); +SELECT bitCount(toInt8(-1)); + +SELECT x, bitCount(x), hex(reinterpretAsString(x)) FROM VALUES ('x Float64', (1), (-1), (inf)); diff --git a/dbms/tests/queries/0_stateless/01067_join_null.reference b/dbms/tests/queries/0_stateless/01067_join_null.reference new file mode 100644 index 00000000000..e06b673c8c5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01067_join_null.reference @@ -0,0 +1,9 @@ +1 +\N +\N +\N +\N +--- +--- +\N +\N diff --git a/dbms/tests/queries/0_stateless/01067_join_null.sql b/dbms/tests/queries/0_stateless/01067_join_null.sql new file mode 100644 index 00000000000..60672312368 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01067_join_null.sql @@ -0,0 +1,42 @@ +SELECT id +FROM +( + SELECT 1 AS id + UNION ALL + SELECT NULL + UNION ALL + SELECT NULL +) +ALL FULL OUTER JOIN +( + SELECT 1 AS id + UNION ALL + SELECT NULL + UNION ALL + SELECT NULL +) USING (id) +ORDER BY id; + +SELECT '---'; + +SELECT * +FROM +( + SELECT NULL AS x +) +INNER JOIN +( + SELECT NULL AS x +) USING (x); + +SELECT '---'; + +SELECT * +FROM +( + SELECT NULL AS x +) +FULL OUTER JOIN +( + SELECT NULL AS x +) USING (x); diff --git a/dbms/tests/queries/0_stateless/01068_parens.reference b/dbms/tests/queries/0_stateless/01068_parens.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01068_parens.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/01068_parens.sql b/dbms/tests/queries/0_stateless/01068_parens.sql new file mode 100644 index 00000000000..7cb4f097b15 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01068_parens.sql @@ -0,0 +1 @@ +((((((((((((((SELECT((((((((((((((((((((((((((((((((1)))))))))))))))))))))))))))))))))))))))))))))) diff --git a/dbms/tests/queries/0_stateless/01069_database_memory.reference b/dbms/tests/queries/0_stateless/01069_database_memory.reference new file mode 100644 index 00000000000..b2fa2bd204c --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_database_memory.reference @@ -0,0 +1,7 @@ +CREATE DATABASE memory_01069 ENGINE = Memory() +1 +2 +3 +4 +3 +4 diff --git a/dbms/tests/queries/0_stateless/01069_database_memory.sql b/dbms/tests/queries/0_stateless/01069_database_memory.sql new file mode 100644 index 00000000000..645790e3f5e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_database_memory.sql @@ -0,0 +1,18 @@ +DROP DATABASE IF EXISTS memory_01069; +CREATE DATABASE memory_01069 ENGINE = Memory; +SHOW CREATE DATABASE memory_01069; + +CREATE TABLE memory_01069.mt (n UInt8) ENGINE = MergeTree() ORDER BY n; +CREATE TABLE memory_01069.file (n UInt8) ENGINE = File(CSV); + +INSERT INTO memory_01069.mt VALUES (1), (2); +INSERT INTO memory_01069.file VALUES (3), (4); + +SELECT * FROM memory_01069.mt ORDER BY n; +SELECT * FROM memory_01069.file ORDER BY n; + +DROP TABLE memory_01069.mt; +SELECT * FROM memory_01069.mt ORDER BY n; -- { serverError 60 } +SELECT * FROM memory_01069.file ORDER BY n; + +DROP DATABASE memory_01069; diff --git a/dbms/tests/queries/0_stateless/01069_insert_float_as_nullable_unit8.reference b/dbms/tests/queries/0_stateless/01069_insert_float_as_nullable_unit8.reference new file mode 100644 index 00000000000..51993f072d5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_insert_float_as_nullable_unit8.reference @@ -0,0 +1,2 @@ +2 +2 diff --git a/dbms/tests/queries/0_stateless/01069_insert_float_as_nullable_unit8.sql b/dbms/tests/queries/0_stateless/01069_insert_float_as_nullable_unit8.sql new file mode 100644 index 00000000000..f1780fa1e53 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_insert_float_as_nullable_unit8.sql @@ -0,0 +1,7 @@ +create temporary table t1 (a Nullable(UInt8)); +insert into t1 values (2.4); +select * from t1; + +create temporary table t2 (a UInt8); +insert into t2 values (2.4); +select * from t2; diff --git a/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.reference b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.reference new file mode 100644 index 00000000000..a07a1e62f3a --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.reference @@ -0,0 +1,8 @@ +1 +1 +2 +3 +1 0 +1 0 +2 0 +3 0 diff --git a/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.sql b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.sql new file mode 100644 index 00000000000..6be401597f6 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS mv; +DROP TABLE IF EXISTS mv_source; +DROP TABLE IF EXISTS mv_target; + +CREATE TABLE mv_source (`a` UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE mv_target (`a` UInt64) ENGINE = MergeTree ORDER BY tuple(); + +CREATE MATERIALIZED VIEW mv TO mv_target AS SELECT * FROM mv_source; + +INSERT INTO mv_source VALUES (1); + +ALTER TABLE mv_target ADD COLUMN b UInt8; +INSERT INTO mv_source VALUES (1),(2),(3); + +SELECT * FROM mv ORDER BY a; +SELECT * FROM mv_target ORDER BY a; diff --git a/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table_with_default_expression.reference b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table_with_default_expression.reference new file mode 100644 index 00000000000..97a02d3b487 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table_with_default_expression.reference @@ -0,0 +1,8 @@ +1 +1 +2 +3 +1 2 +1 2 +2 3 +3 4 diff --git a/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table_with_default_expression.sql b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table_with_default_expression.sql new file mode 100644 index 00000000000..dcfba76f9c2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table_with_default_expression.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS mv; +DROP TABLE IF EXISTS mv_source; +DROP TABLE IF EXISTS mv_target; + +CREATE TABLE mv_source (`a` UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE mv_target (`a` UInt64) ENGINE = MergeTree ORDER BY tuple(); + +CREATE MATERIALIZED VIEW mv TO mv_target AS SELECT * FROM mv_source; + +INSERT INTO mv_source VALUES (1); + +ALTER TABLE mv_target ADD COLUMN b UInt8 DEFAULT a + 1; +INSERT INTO mv_source VALUES (1),(2),(3); + +SELECT * FROM mv ORDER BY a; +SELECT * FROM mv_target ORDER BY a; diff --git a/dbms/tests/queries/0_stateless/01070_alter_with_ttl.reference b/dbms/tests/queries/0_stateless/01070_alter_with_ttl.reference new file mode 100644 index 00000000000..308d004ebf7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_alter_with_ttl.reference @@ -0,0 +1,2 @@ +CREATE TABLE default.alter_ttl (`i` Int, `s` String TTL toDate(\'2020-01-01\')) ENGINE = MergeTree ORDER BY i TTL toDate(\'2020-05-05\') SETTINGS index_granularity = 8192 +CREATE TABLE default.alter_ttl (`d` Date, `s` String TTL d + toIntervalDay(1)) ENGINE = MergeTree ORDER BY d TTL d + toIntervalMonth(1) SETTINGS index_granularity = 8192 diff --git a/dbms/tests/queries/0_stateless/01070_alter_with_ttl.sql b/dbms/tests/queries/0_stateless/01070_alter_with_ttl.sql new file mode 100644 index 00000000000..3adc3ccd6ae --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_alter_with_ttl.sql @@ -0,0 +1,12 @@ +drop table if exists alter_ttl; + +create table alter_ttl(i Int) engine = MergeTree order by i ttl toDate('2020-05-05'); +alter table alter_ttl add column s String; +alter table alter_ttl modify column s String ttl toDate('2020-01-01'); +show create table alter_ttl; +drop table alter_ttl; + +create table alter_ttl(d Date, s String) engine = MergeTree order by d ttl d + interval 1 month; +alter table alter_ttl modify column s String ttl d + interval 1 day; +show create table alter_ttl; +drop table alter_ttl; diff --git a/dbms/tests/queries/0_stateless/01070_exception_code_in_query_log_table.reference b/dbms/tests/queries/0_stateless/01070_exception_code_in_query_log_table.reference new file mode 100644 index 00000000000..1dee767cd4e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_exception_code_in_query_log_table.reference @@ -0,0 +1,3 @@ +0 +0 +60 diff --git a/dbms/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql b/dbms/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql new file mode 100644 index 00000000000..9cca089ce08 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table; +SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; -- { serverError 60 } +CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory(); +SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; +SYSTEM FLUSH LOGS; +SELECT exception_code FROM system.query_log WHERE query='SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' ORDER BY exception_code; +DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table; diff --git a/dbms/tests/queries/0_stateless/01070_template_empty_file.reference b/dbms/tests/queries/0_stateless/01070_template_empty_file.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01070_template_empty_file.sql b/dbms/tests/queries/0_stateless/01070_template_empty_file.sql new file mode 100644 index 00000000000..46a8f38f80b --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_template_empty_file.sql @@ -0,0 +1,2 @@ +select 1 format Template settings format_template_row='01070_nonexistent_file.txt'; -- { clientError 107 } +select 1 format Template settings format_template_row='/dev/null'; -- { clientError 474 } diff --git a/dbms/tests/queries/0_stateless/01070_to_decimal_or_null_exception.reference b/dbms/tests/queries/0_stateless/01070_to_decimal_or_null_exception.reference new file mode 100644 index 00000000000..b981dd4e1b6 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_to_decimal_or_null_exception.reference @@ -0,0 +1,3 @@ +\N 1 +\N 1 +\N 1 diff --git a/dbms/tests/queries/0_stateless/01070_to_decimal_or_null_exception.sql b/dbms/tests/queries/0_stateless/01070_to_decimal_or_null_exception.sql new file mode 100644 index 00000000000..9283cc76cd7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_to_decimal_or_null_exception.sql @@ -0,0 +1,7 @@ +SELECT toDecimal32('e', 1); -- { serverError 72 } +SELECT toDecimal64('e', 2); -- { serverError 72 } +SELECT toDecimal128('e', 3); -- { serverError 72 } + +SELECT toDecimal32OrNull('e', 1) x, isNull(x); +SELECT toDecimal64OrNull('e', 2) x, isNull(x); +SELECT toDecimal128OrNull('e', 3) x, isNull(x); diff --git a/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.reference b/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.sql b/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.sql new file mode 100644 index 00000000000..f12d5f8846d --- /dev/null +++ b/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.sql @@ -0,0 +1,26 @@ +set optimize_skip_unused_shards=1; + +drop table if exists data_01068; +drop table if exists dist_01068; + +create table data_01068 (key Int) Engine=Null(); + +create table dist_01068 as data_01068 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01068); +set force_optimize_skip_unused_shards=0; +select * from dist_01068; +set force_optimize_skip_unused_shards=1; +select * from dist_01068; +set force_optimize_skip_unused_shards=2; +select * from dist_01068; -- { serverError 507 } + +drop table if exists dist_01068; +create table dist_01068 as data_01068 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01068, key%2); +set force_optimize_skip_unused_shards=0; +select * from dist_01068; +set force_optimize_skip_unused_shards=1; +select * from dist_01068; -- { serverError 507 } +set force_optimize_skip_unused_shards=2; +select * from dist_01068; -- { serverError 507 } + +drop table if exists data_01068; +drop table if exists dist_01068; diff --git a/dbms/tests/queries/0_stateless/01071_http_header_exception_code.reference b/dbms/tests/queries/0_stateless/01071_http_header_exception_code.reference new file mode 100644 index 00000000000..0ca95142bb7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01071_http_header_exception_code.reference @@ -0,0 +1 @@ +True diff --git a/dbms/tests/queries/0_stateless/01071_http_header_exception_code.sh b/dbms/tests/queries/0_stateless/01071_http_header_exception_code.sh new file mode 100755 index 00000000000..09b6b63ee3e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01071_http_header_exception_code.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +if [[ `${CLICKHOUSE_CURL_COMMAND} -I "${CLICKHOUSE_URL}&query=BADREQUEST" 2>&1 | grep -c 'X-ClickHouse-Exception-Code: 62'` -eq 1 ]]; then + echo "True" +fi diff --git a/dbms/tests/queries/0_stateless/01071_live_view_detach_dependency.reference b/dbms/tests/queries/0_stateless/01071_live_view_detach_dependency.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01071_live_view_detach_dependency.sql b/dbms/tests/queries/0_stateless/01071_live_view_detach_dependency.sql new file mode 100644 index 00000000000..22d8adc503c --- /dev/null +++ b/dbms/tests/queries/0_stateless/01071_live_view_detach_dependency.sql @@ -0,0 +1,8 @@ +SET allow_experimental_live_view = 1; +DROP TABLE IF EXISTS test; +DROP TABLE IF EXISTS lv; +CREATE TABLE test (n Int8) ENGINE = Memory; +CREATE LIVE VIEW lv AS SELECT * FROM test; +DETACH TABLE lv; +INSERT INTO test VALUES (42); +DROP TABLE test; diff --git a/dbms/tests/queries/0_stateless/data_avro/complex.avro b/dbms/tests/queries/0_stateless/data_avro/complex.avro new file mode 100644 index 00000000000..0880f581882 Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_avro/complex.avro differ diff --git a/dbms/tests/queries/0_stateless/data_avro/complex.avsc b/dbms/tests/queries/0_stateless/data_avro/complex.avsc new file mode 100644 index 00000000000..325169aeb57 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/complex.avsc @@ -0,0 +1,20 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a_enum_to_string", "type": { "type": "enum", "name": "enum_1", "symbols" : ["A", "B", "C"]}}, + {"name": "b_enum_to_enum", "type": { "type": "enum", "name": "enum_2", "symbols" : ["t", "f"]}}, + {"name": "c_array_string", "type": { "type": "array", "items": "string"}}, + {"name": "d_array_array_string", "type": { "type": "array", "items": {"type": "array", "items": "string"}}}, + {"name": "e_union_null_string", "type": ["null", "string"]}, + {"name": "f_union_long_null", "type": ["long", "null"]}, + {"name": "g_fixed", "type": {"type":"fixed", "size": 32, "name": "fixed_1"}}, + {"name": "h_record_skip", "type": { + "type": "record", + "name": "subrecord", + "fields": [ + {"name": "a", "type": "string"} + ] + }} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/complex.json b/dbms/tests/queries/0_stateless/data_avro/complex.json new file mode 100644 index 00000000000..d05e09c72fc --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/complex.json @@ -0,0 +1,2 @@ +{"a_enum_to_string":"A","b_enum_to_enum":"t","c_array_string":["s1", "s2"],"d_array_array_string":[["a1"], ["a2"]],"e_union_null_string":{"string": "s1"},"f_union_long_null":null,"g_fixed":"79cd909892d7e7ade1987cc7422628ba","h_record_skip":{"a": "a"}} +{"a_enum_to_string":"C","b_enum_to_enum":"f","c_array_string":[],"d_array_array_string":[],"e_union_null_string":null,"f_union_long_null":{"long": 123},"g_fixed":"79cd909892d7e7ade1987cc7422628ba","h_record_skip":{"a": "a"}} \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/empty.avro b/dbms/tests/queries/0_stateless/data_avro/empty.avro new file mode 100644 index 00000000000..7cfae81758c Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_avro/empty.avro differ diff --git a/dbms/tests/queries/0_stateless/data_avro/empty.avsc b/dbms/tests/queries/0_stateless/data_avro/empty.avsc new file mode 100644 index 00000000000..923eda71054 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/empty.avsc @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a", "type": "long"} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/empty.json b/dbms/tests/queries/0_stateless/data_avro/empty.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/data_avro/generate_avro.sh b/dbms/tests/queries/0_stateless/data_avro/generate_avro.sh new file mode 100755 index 00000000000..3538c8693e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/generate_avro.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +#avro tools: https://www.apache.org/dyn/closer.cgi?path=avro/avro-1.9.1/java/avro-tools-1.9.1.jar + + +avro-tools fromjson --schema-file primitive.avsc primitive.json > primitive.avro +avro-tools fromjson --schema-file complex.avsc complex.json > complex.avro +avro-tools fromjson --schema-file logical_types.avsc logical_types.json > logical_types.avro +avro-tools fromjson --schema-file empty.avsc empty.json > empty.avro + +#compression +avro-tools fromjson --codec null --schema-file simple.avsc simple.json > simple.null.avro +avro-tools fromjson --codec deflate --schema-file simple.avsc simple.json > simple.deflate.avro +avro-tools fromjson --codec snappy --schema-file simple.avsc simple.json > simple.snappy.avro \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/logical_types.avro b/dbms/tests/queries/0_stateless/data_avro/logical_types.avro new file mode 100644 index 00000000000..7b8a3f60b7a Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_avro/logical_types.avro differ diff --git a/dbms/tests/queries/0_stateless/data_avro/logical_types.avsc b/dbms/tests/queries/0_stateless/data_avro/logical_types.avsc new file mode 100644 index 00000000000..5d9fd96821f --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/logical_types.avsc @@ -0,0 +1,9 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a_date", "type": { "type": "int", "logicalType": "date"}}, + {"name": "b_timestamp_millis", "type": { "type": "long", "logicalType": "timestamp-millis"}}, + {"name": "c_timestamp_micros", "type": { "type": "long", "logicalType": "timestamp-micros"}} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/logical_types.json b/dbms/tests/queries/0_stateless/data_avro/logical_types.json new file mode 100644 index 00000000000..652b85246e7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/logical_types.json @@ -0,0 +1 @@ +{"a_date":18250,"b_timestamp_millis":1578641516227,"c_timestamp_micros":1578641516227000} diff --git a/dbms/tests/queries/0_stateless/data_avro/primitive.avro b/dbms/tests/queries/0_stateless/data_avro/primitive.avro new file mode 100644 index 00000000000..ef5eb36639f Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_avro/primitive.avro differ diff --git a/dbms/tests/queries/0_stateless/data_avro/primitive.avsc b/dbms/tests/queries/0_stateless/data_avro/primitive.avsc new file mode 100644 index 00000000000..a4f06d02b01 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/primitive.avsc @@ -0,0 +1,14 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a_bool", "type": "boolean"}, + {"name": "b_int", "type": "int"}, + {"name": "c_long", "type": "long"}, + {"name": "d_float", "type": "float"}, + {"name": "e_double", "type": "double"}, + {"name": "f_bytes", "type": "bytes"}, + {"name": "g_string", "type": "string"}, + {"name": "h_null", "type": "null"} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/primitive.json b/dbms/tests/queries/0_stateless/data_avro/primitive.json new file mode 100644 index 00000000000..fc521c8829c --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/primitive.json @@ -0,0 +1,2 @@ +{"a_bool":true,"b_int":1,"c_long":2,"d_float":3.4,"e_double":5.6,"f_bytes":"b1","g_string":"s1","h_null": null} +{"a_bool":false,"b_int":-1,"c_long":9223372036854775807,"d_float":3.00004,"e_double":0.00001,"f_bytes":"","g_string":"","h_null": null} \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.avsc b/dbms/tests/queries/0_stateless/data_avro/simple.avsc new file mode 100644 index 00000000000..923eda71054 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/simple.avsc @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a", "type": "long"} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.deflate.avro b/dbms/tests/queries/0_stateless/data_avro/simple.deflate.avro new file mode 100644 index 00000000000..d4ba226b447 Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_avro/simple.deflate.avro differ diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.json b/dbms/tests/queries/0_stateless/data_avro/simple.json new file mode 100644 index 00000000000..c09fc0b732f --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/simple.json @@ -0,0 +1,1000 @@ +{"a":1} +{"a":2} +{"a":3} +{"a":4} +{"a":5} +{"a":6} +{"a":7} +{"a":8} +{"a":9} +{"a":10} +{"a":11} +{"a":12} +{"a":13} +{"a":14} +{"a":15} +{"a":16} +{"a":17} +{"a":18} +{"a":19} +{"a":20} +{"a":21} +{"a":22} +{"a":23} +{"a":24} +{"a":25} +{"a":26} +{"a":27} +{"a":28} +{"a":29} +{"a":30} +{"a":31} +{"a":32} +{"a":33} +{"a":34} +{"a":35} +{"a":36} +{"a":37} +{"a":38} +{"a":39} +{"a":40} +{"a":41} +{"a":42} +{"a":43} +{"a":44} +{"a":45} +{"a":46} +{"a":47} +{"a":48} +{"a":49} +{"a":50} +{"a":51} +{"a":52} +{"a":53} +{"a":54} +{"a":55} +{"a":56} +{"a":57} +{"a":58} +{"a":59} +{"a":60} +{"a":61} +{"a":62} +{"a":63} +{"a":64} +{"a":65} +{"a":66} +{"a":67} +{"a":68} +{"a":69} +{"a":70} +{"a":71} +{"a":72} +{"a":73} +{"a":74} +{"a":75} +{"a":76} +{"a":77} +{"a":78} +{"a":79} +{"a":80} +{"a":81} +{"a":82} +{"a":83} +{"a":84} +{"a":85} +{"a":86} +{"a":87} +{"a":88} +{"a":89} +{"a":90} +{"a":91} +{"a":92} +{"a":93} +{"a":94} +{"a":95} +{"a":96} +{"a":97} +{"a":98} +{"a":99} +{"a":100} +{"a":101} +{"a":102} +{"a":103} +{"a":104} +{"a":105} +{"a":106} +{"a":107} +{"a":108} +{"a":109} +{"a":110} +{"a":111} +{"a":112} +{"a":113} +{"a":114} +{"a":115} +{"a":116} +{"a":117} +{"a":118} +{"a":119} +{"a":120} +{"a":121} +{"a":122} +{"a":123} +{"a":124} +{"a":125} +{"a":126} +{"a":127} +{"a":128} +{"a":129} +{"a":130} +{"a":131} +{"a":132} +{"a":133} +{"a":134} +{"a":135} +{"a":136} +{"a":137} +{"a":138} +{"a":139} +{"a":140} +{"a":141} +{"a":142} +{"a":143} +{"a":144} +{"a":145} +{"a":146} +{"a":147} +{"a":148} +{"a":149} +{"a":150} +{"a":151} +{"a":152} +{"a":153} +{"a":154} +{"a":155} +{"a":156} +{"a":157} +{"a":158} +{"a":159} +{"a":160} +{"a":161} +{"a":162} +{"a":163} +{"a":164} +{"a":165} +{"a":166} +{"a":167} +{"a":168} +{"a":169} +{"a":170} +{"a":171} +{"a":172} +{"a":173} +{"a":174} +{"a":175} +{"a":176} +{"a":177} +{"a":178} +{"a":179} +{"a":180} +{"a":181} +{"a":182} +{"a":183} +{"a":184} +{"a":185} +{"a":186} +{"a":187} +{"a":188} +{"a":189} +{"a":190} +{"a":191} +{"a":192} +{"a":193} +{"a":194} +{"a":195} +{"a":196} +{"a":197} +{"a":198} +{"a":199} +{"a":200} +{"a":201} +{"a":202} +{"a":203} +{"a":204} +{"a":205} +{"a":206} +{"a":207} +{"a":208} +{"a":209} +{"a":210} +{"a":211} +{"a":212} +{"a":213} +{"a":214} +{"a":215} +{"a":216} +{"a":217} +{"a":218} +{"a":219} +{"a":220} +{"a":221} +{"a":222} +{"a":223} +{"a":224} +{"a":225} +{"a":226} +{"a":227} +{"a":228} +{"a":229} +{"a":230} +{"a":231} +{"a":232} +{"a":233} +{"a":234} +{"a":235} +{"a":236} +{"a":237} +{"a":238} +{"a":239} +{"a":240} +{"a":241} +{"a":242} +{"a":243} +{"a":244} +{"a":245} +{"a":246} +{"a":247} +{"a":248} +{"a":249} +{"a":250} +{"a":251} +{"a":252} +{"a":253} +{"a":254} +{"a":255} +{"a":256} +{"a":257} +{"a":258} +{"a":259} +{"a":260} +{"a":261} +{"a":262} +{"a":263} +{"a":264} +{"a":265} +{"a":266} +{"a":267} +{"a":268} +{"a":269} +{"a":270} +{"a":271} +{"a":272} +{"a":273} +{"a":274} +{"a":275} +{"a":276} +{"a":277} +{"a":278} +{"a":279} +{"a":280} +{"a":281} +{"a":282} +{"a":283} +{"a":284} +{"a":285} +{"a":286} +{"a":287} +{"a":288} +{"a":289} +{"a":290} +{"a":291} +{"a":292} +{"a":293} +{"a":294} +{"a":295} +{"a":296} +{"a":297} +{"a":298} +{"a":299} +{"a":300} +{"a":301} +{"a":302} +{"a":303} +{"a":304} +{"a":305} +{"a":306} +{"a":307} +{"a":308} +{"a":309} +{"a":310} +{"a":311} +{"a":312} +{"a":313} +{"a":314} +{"a":315} +{"a":316} +{"a":317} +{"a":318} +{"a":319} +{"a":320} +{"a":321} +{"a":322} +{"a":323} +{"a":324} +{"a":325} +{"a":326} +{"a":327} +{"a":328} +{"a":329} +{"a":330} +{"a":331} +{"a":332} +{"a":333} +{"a":334} +{"a":335} +{"a":336} +{"a":337} +{"a":338} +{"a":339} +{"a":340} +{"a":341} +{"a":342} +{"a":343} +{"a":344} +{"a":345} +{"a":346} +{"a":347} +{"a":348} +{"a":349} +{"a":350} +{"a":351} +{"a":352} +{"a":353} +{"a":354} +{"a":355} +{"a":356} +{"a":357} +{"a":358} +{"a":359} +{"a":360} +{"a":361} +{"a":362} +{"a":363} +{"a":364} +{"a":365} +{"a":366} +{"a":367} +{"a":368} +{"a":369} +{"a":370} +{"a":371} +{"a":372} +{"a":373} +{"a":374} +{"a":375} +{"a":376} +{"a":377} +{"a":378} +{"a":379} +{"a":380} +{"a":381} +{"a":382} +{"a":383} +{"a":384} +{"a":385} +{"a":386} +{"a":387} +{"a":388} +{"a":389} +{"a":390} +{"a":391} +{"a":392} +{"a":393} +{"a":394} +{"a":395} +{"a":396} +{"a":397} +{"a":398} +{"a":399} +{"a":400} +{"a":401} +{"a":402} +{"a":403} +{"a":404} +{"a":405} +{"a":406} +{"a":407} +{"a":408} +{"a":409} +{"a":410} +{"a":411} +{"a":412} +{"a":413} +{"a":414} +{"a":415} +{"a":416} +{"a":417} +{"a":418} +{"a":419} +{"a":420} +{"a":421} +{"a":422} +{"a":423} +{"a":424} +{"a":425} +{"a":426} +{"a":427} +{"a":428} +{"a":429} +{"a":430} +{"a":431} +{"a":432} +{"a":433} +{"a":434} +{"a":435} +{"a":436} +{"a":437} +{"a":438} +{"a":439} +{"a":440} +{"a":441} +{"a":442} +{"a":443} +{"a":444} +{"a":445} +{"a":446} +{"a":447} +{"a":448} +{"a":449} +{"a":450} +{"a":451} +{"a":452} +{"a":453} +{"a":454} +{"a":455} +{"a":456} +{"a":457} +{"a":458} +{"a":459} +{"a":460} +{"a":461} +{"a":462} +{"a":463} +{"a":464} +{"a":465} +{"a":466} +{"a":467} +{"a":468} +{"a":469} +{"a":470} +{"a":471} +{"a":472} +{"a":473} +{"a":474} +{"a":475} +{"a":476} +{"a":477} +{"a":478} +{"a":479} +{"a":480} +{"a":481} +{"a":482} +{"a":483} +{"a":484} +{"a":485} +{"a":486} +{"a":487} +{"a":488} +{"a":489} +{"a":490} +{"a":491} +{"a":492} +{"a":493} +{"a":494} +{"a":495} +{"a":496} +{"a":497} +{"a":498} +{"a":499} +{"a":500} +{"a":501} +{"a":502} +{"a":503} +{"a":504} +{"a":505} +{"a":506} +{"a":507} +{"a":508} +{"a":509} +{"a":510} +{"a":511} +{"a":512} +{"a":513} +{"a":514} +{"a":515} +{"a":516} +{"a":517} +{"a":518} +{"a":519} +{"a":520} +{"a":521} +{"a":522} +{"a":523} +{"a":524} +{"a":525} +{"a":526} +{"a":527} +{"a":528} +{"a":529} +{"a":530} +{"a":531} +{"a":532} +{"a":533} +{"a":534} +{"a":535} +{"a":536} +{"a":537} +{"a":538} +{"a":539} +{"a":540} +{"a":541} +{"a":542} +{"a":543} +{"a":544} +{"a":545} +{"a":546} +{"a":547} +{"a":548} +{"a":549} +{"a":550} +{"a":551} +{"a":552} +{"a":553} +{"a":554} +{"a":555} +{"a":556} +{"a":557} +{"a":558} +{"a":559} +{"a":560} +{"a":561} +{"a":562} +{"a":563} +{"a":564} +{"a":565} +{"a":566} +{"a":567} +{"a":568} +{"a":569} +{"a":570} +{"a":571} +{"a":572} +{"a":573} +{"a":574} +{"a":575} +{"a":576} +{"a":577} +{"a":578} +{"a":579} +{"a":580} +{"a":581} +{"a":582} +{"a":583} +{"a":584} +{"a":585} +{"a":586} +{"a":587} +{"a":588} +{"a":589} +{"a":590} +{"a":591} +{"a":592} +{"a":593} +{"a":594} +{"a":595} +{"a":596} +{"a":597} +{"a":598} +{"a":599} +{"a":600} +{"a":601} +{"a":602} +{"a":603} +{"a":604} +{"a":605} +{"a":606} +{"a":607} +{"a":608} +{"a":609} +{"a":610} +{"a":611} +{"a":612} +{"a":613} +{"a":614} +{"a":615} +{"a":616} +{"a":617} +{"a":618} +{"a":619} +{"a":620} +{"a":621} +{"a":622} +{"a":623} +{"a":624} +{"a":625} +{"a":626} +{"a":627} +{"a":628} +{"a":629} +{"a":630} +{"a":631} +{"a":632} +{"a":633} +{"a":634} +{"a":635} +{"a":636} +{"a":637} +{"a":638} +{"a":639} +{"a":640} +{"a":641} +{"a":642} +{"a":643} +{"a":644} +{"a":645} +{"a":646} +{"a":647} +{"a":648} +{"a":649} +{"a":650} +{"a":651} +{"a":652} +{"a":653} +{"a":654} +{"a":655} +{"a":656} +{"a":657} +{"a":658} +{"a":659} +{"a":660} +{"a":661} +{"a":662} +{"a":663} +{"a":664} +{"a":665} +{"a":666} +{"a":667} +{"a":668} +{"a":669} +{"a":670} +{"a":671} +{"a":672} +{"a":673} +{"a":674} +{"a":675} +{"a":676} +{"a":677} +{"a":678} +{"a":679} +{"a":680} +{"a":681} +{"a":682} +{"a":683} +{"a":684} +{"a":685} +{"a":686} +{"a":687} +{"a":688} +{"a":689} +{"a":690} +{"a":691} +{"a":692} +{"a":693} +{"a":694} +{"a":695} +{"a":696} +{"a":697} +{"a":698} +{"a":699} +{"a":700} +{"a":701} +{"a":702} +{"a":703} +{"a":704} +{"a":705} +{"a":706} +{"a":707} +{"a":708} +{"a":709} +{"a":710} +{"a":711} +{"a":712} +{"a":713} +{"a":714} +{"a":715} +{"a":716} +{"a":717} +{"a":718} +{"a":719} +{"a":720} +{"a":721} +{"a":722} +{"a":723} +{"a":724} +{"a":725} +{"a":726} +{"a":727} +{"a":728} +{"a":729} +{"a":730} +{"a":731} +{"a":732} +{"a":733} +{"a":734} +{"a":735} +{"a":736} +{"a":737} +{"a":738} +{"a":739} +{"a":740} +{"a":741} +{"a":742} +{"a":743} +{"a":744} +{"a":745} +{"a":746} +{"a":747} +{"a":748} +{"a":749} +{"a":750} +{"a":751} +{"a":752} +{"a":753} +{"a":754} +{"a":755} +{"a":756} +{"a":757} +{"a":758} +{"a":759} +{"a":760} +{"a":761} +{"a":762} +{"a":763} +{"a":764} +{"a":765} +{"a":766} +{"a":767} +{"a":768} +{"a":769} +{"a":770} +{"a":771} +{"a":772} +{"a":773} +{"a":774} +{"a":775} +{"a":776} +{"a":777} +{"a":778} +{"a":779} +{"a":780} +{"a":781} +{"a":782} +{"a":783} +{"a":784} +{"a":785} +{"a":786} +{"a":787} +{"a":788} +{"a":789} +{"a":790} +{"a":791} +{"a":792} +{"a":793} +{"a":794} +{"a":795} +{"a":796} +{"a":797} +{"a":798} +{"a":799} +{"a":800} +{"a":801} +{"a":802} +{"a":803} +{"a":804} +{"a":805} +{"a":806} +{"a":807} +{"a":808} +{"a":809} +{"a":810} +{"a":811} +{"a":812} +{"a":813} +{"a":814} +{"a":815} +{"a":816} +{"a":817} +{"a":818} +{"a":819} +{"a":820} +{"a":821} +{"a":822} +{"a":823} +{"a":824} +{"a":825} +{"a":826} +{"a":827} +{"a":828} +{"a":829} +{"a":830} +{"a":831} +{"a":832} +{"a":833} +{"a":834} +{"a":835} +{"a":836} +{"a":837} +{"a":838} +{"a":839} +{"a":840} +{"a":841} +{"a":842} +{"a":843} +{"a":844} +{"a":845} +{"a":846} +{"a":847} +{"a":848} +{"a":849} +{"a":850} +{"a":851} +{"a":852} +{"a":853} +{"a":854} +{"a":855} +{"a":856} +{"a":857} +{"a":858} +{"a":859} +{"a":860} +{"a":861} +{"a":862} +{"a":863} +{"a":864} +{"a":865} +{"a":866} +{"a":867} +{"a":868} +{"a":869} +{"a":870} +{"a":871} +{"a":872} +{"a":873} +{"a":874} +{"a":875} +{"a":876} +{"a":877} +{"a":878} +{"a":879} +{"a":880} +{"a":881} +{"a":882} +{"a":883} +{"a":884} +{"a":885} +{"a":886} +{"a":887} +{"a":888} +{"a":889} +{"a":890} +{"a":891} +{"a":892} +{"a":893} +{"a":894} +{"a":895} +{"a":896} +{"a":897} +{"a":898} +{"a":899} +{"a":900} +{"a":901} +{"a":902} +{"a":903} +{"a":904} +{"a":905} +{"a":906} +{"a":907} +{"a":908} +{"a":909} +{"a":910} +{"a":911} +{"a":912} +{"a":913} +{"a":914} +{"a":915} +{"a":916} +{"a":917} +{"a":918} +{"a":919} +{"a":920} +{"a":921} +{"a":922} +{"a":923} +{"a":924} +{"a":925} +{"a":926} +{"a":927} +{"a":928} +{"a":929} +{"a":930} +{"a":931} +{"a":932} +{"a":933} +{"a":934} +{"a":935} +{"a":936} +{"a":937} +{"a":938} +{"a":939} +{"a":940} +{"a":941} +{"a":942} +{"a":943} +{"a":944} +{"a":945} +{"a":946} +{"a":947} +{"a":948} +{"a":949} +{"a":950} +{"a":951} +{"a":952} +{"a":953} +{"a":954} +{"a":955} +{"a":956} +{"a":957} +{"a":958} +{"a":959} +{"a":960} +{"a":961} +{"a":962} +{"a":963} +{"a":964} +{"a":965} +{"a":966} +{"a":967} +{"a":968} +{"a":969} +{"a":970} +{"a":971} +{"a":972} +{"a":973} +{"a":974} +{"a":975} +{"a":976} +{"a":977} +{"a":978} +{"a":979} +{"a":980} +{"a":981} +{"a":982} +{"a":983} +{"a":984} +{"a":985} +{"a":986} +{"a":987} +{"a":988} +{"a":989} +{"a":990} +{"a":991} +{"a":992} +{"a":993} +{"a":994} +{"a":995} +{"a":996} +{"a":997} +{"a":998} +{"a":999} +{"a":1000} diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.null.avro b/dbms/tests/queries/0_stateless/data_avro/simple.null.avro new file mode 100644 index 00000000000..789ab45101f Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_avro/simple.null.avro differ diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.snappy.avro b/dbms/tests/queries/0_stateless/data_avro/simple.snappy.avro new file mode 100644 index 00000000000..b812ed6c7ea Binary files /dev/null and b/dbms/tests/queries/0_stateless/data_avro/simple.snappy.avro differ diff --git a/dbms/tests/queries/1_stateful/00154_avro.reference b/dbms/tests/queries/1_stateful/00154_avro.reference new file mode 100644 index 00000000000..7e243047e8b --- /dev/null +++ b/dbms/tests/queries/1_stateful/00154_avro.reference @@ -0,0 +1,2 @@ +17300372046749301651 +17300372046749301651 diff --git a/dbms/tests/queries/1_stateful/00154_avro.sql b/dbms/tests/queries/1_stateful/00154_avro.sql new file mode 100644 index 00000000000..3d43a23e516 --- /dev/null +++ b/dbms/tests/queries/1_stateful/00154_avro.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS test.avro; + +CREATE TABLE test.avro AS test.hits ENGINE = File(Avro); +INSERT INTO test.avro SELECT * FROM test.hits WHERE intHash64(WatchID) % 100 = 0; + +SELECT sum(cityHash64(*)) FROM test.hits WHERE intHash64(WatchID) % 100 = 0; +SELECT sum(cityHash64(*)) FROM test.avro; + +DROP TABLE test.avro; diff --git a/docker/packager/packager b/docker/packager/packager index 5e8ffbf1cb9..a31a387d502 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -176,7 +176,8 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") if unbundled: - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0') + # TODO: fix build with ENABLE_RDKAFKA + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') diff --git a/docker/server/README.md b/docker/server/README.md index b1225e8a27a..b36bbe40dc7 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -28,7 +28,7 @@ Container exposes 8123 port for [HTTP interface](https://clickhouse.yandex/docs/ ClickHouse configuration represented with a file "config.xml" ([documentation](https://clickhouse.yandex/docs/en/operations/configuration_files/)) -### start server instance with custom configuration +### Start server instance with custom configuration ```bash $ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml yandex/clickhouse-server ``` @@ -36,10 +36,15 @@ $ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v / ### Start server as custom user ``` # $(pwd)/data/clickhouse should exist and be owned by current user -$ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/data/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server +$ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server ``` When you use the image with mounting local directories inside you probably would like to not mess your directory tree with files owner and permissions. Then you could use `--user` argument. In this case, you should mount every necessary directory (`/var/lib/clickhouse` and `/var/log/clickhouse-server`) inside the container. Otherwise, image will complain and not start. +### Start server from root (useful in case of userns enabled) +``` +$ docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server +``` + ## How to extend this image If you would like to do additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb` it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 03bacb7ac88..fbab44de2b7 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -1,17 +1,28 @@ #!/bin/bash -# set some vars -CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" +DO_CHOWN=1 +if [ "$CLICKHOUSE_DO_NOT_CHOWN" = 1 ]; then + DO_CHOWN=0 +fi + +CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" +CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" + +# support --user if [ x"$UID" == x0 ]; then - USER="$(id -u clickhouse)" - GROUP="$(id -g clickhouse)" + USER=$CLICKHOUSE_UID + GROUP=$CLICKHOUSE_GID gosu="gosu $USER:$GROUP" else USER="$(id -u)" GROUP="$(id -g)" gosu="" + DO_CHOWN=0 fi +# set some vars +CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" + # port is needed to check if clickhouse-server is ready for connections HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" @@ -41,7 +52,7 @@ do exit 1 fi - if [ x"$UID" == x0 ] && [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then + if [ "$DO_CHOWN" = "1" ]; then # ensure proper directories permissions chown -R "$USER:$GROUP" "$dir" elif [ "$(stat -c %u "$dir")" != "$USER" ]; then diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile new file mode 100644 index 00000000000..f9d239ef8ef --- /dev/null +++ b/docker/test/codebrowser/Dockerfile @@ -0,0 +1,48 @@ +# docker build --network=host -t yandex/clickhouse-codebrowser . +# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output yandex/clickhouse-codebrowser +FROM ubuntu:18.04 + +RUN apt-get --allow-unauthenticated update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ + bash \ + sudo \ + wget \ + software-properties-common \ + ca-certificates \ + apt-transport-https \ + build-essential \ + gpg-agent \ + git + +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - +RUN sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' +RUN sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list + +RUN sudo apt-get --yes --allow-unauthenticated update +# To build woboq +RUN sudo apt-get --yes --allow-unauthenticated install cmake clang-8 libllvm8 libclang-8-dev + +# repo versions doesn't work correctly with C++17 +# also we push reports to s3, so we add index.html to subfolder urls +# https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b +RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser +RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release && make -j + +ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator +ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator +ENV STATIC_DATA=/woboq_codebrowser/data + +ENV SOURCE_DIRECTORY=/repo_folder +ENV BUILD_DIRECTORY=/build +ENV HTML_RESULT_DIRECTORY=$BUILD_DIRECTORY/html_report +ENV SHA=nosha +ENV DATA="data" + +CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \ + cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-8 -DCMAKE_C_COMPILER=/usr/bin/clang-8 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON && \ + mkdir -p $HTML_RESULT_DIRECTORY && \ + $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA && \ + cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\ + $CODEINDEX $HTML_RESULT_DIRECTORY -d $DATA && \ + mv $HTML_RESULT_DIRECTORY /test_output diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 6c67e724477..d76d333c973 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -1,9 +1,12 @@ # docker build -t yandex/clickhouse-performance-comparison . FROM ubuntu:18.04 +ENV TZ=Europe/Moscow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + RUN apt-get update \ - && apt-get install --yes --no-install-recommends \ - p7zip-full bash git ncdu wget psmisc python3 python3-pip python3-dev g++ \ + && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ + p7zip-full bash git moreutils ncdu wget psmisc python3 python3-pip tzdata tree python3-dev g++ \ && pip3 --no-cache-dir install clickhouse_driver \ && apt-get purge --yes python3-dev g++ \ && apt-get autoremove --yes \ diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 7ecf715403f..4d69bd47f26 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -23,22 +23,28 @@ function download la="$left_pr-$left_sha.tgz" ra="$right_pr-$right_sha.tgz" - wget -q -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/performance/performance.tgz" -O "$la" && tar -C left --strip-components=1 -zxvf "$la" & - wget -q -nd -c "https://clickhouse-builds.s3.yandex.net/$right_pr/$right_sha/performance/performance.tgz" -O "$ra" && tar -C right --strip-components=1 -zxvf "$ra" & - cd db0 && wget -q -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar" && tar -xvf hits_10m_single.tar & - cd db0 && wget -q -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_100m_single/partitions/hits_100m_single.tar" && tar -xvf hits_100m_single.tar & - cd db0 && wget -q -nd -c "https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar" && tar -xvf hits_v1.tar & + + # might have the same version on left and right + if ! [ "$la" = "$ra" ] + then + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/performance/performance.tgz" -O "$la" && tar -C left --strip-components=1 -zxvf "$la" & + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$right_pr/$right_sha/performance/performance.tgz" -O "$ra" && tar -C right --strip-components=1 -zxvf "$ra" & + else + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/performance/performance.tgz" -O "$la" && { tar -C left --strip-components=1 -zxvf "$la" & tar -C right --strip-components=1 -zxvf "$ra" & } & + fi + + cd db0 && wget -nv -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar" && tar -xvf hits_10m_single.tar & + cd db0 && wget -nv -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_100m_single/partitions/hits_100m_single.tar" && tar -xvf hits_100m_single.tar & + cd db0 && wget -nv -nd -c "https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar" && tar -xvf hits_v1.tar & wait - # Use hardlinks instead of copying - cp -al db0/ left/db/ - cp -al db0/ right/db/ } download function configure { - sed -i 's/9000/9001/g' right/config/config.xml + sed -i 's/9000/9001/g' left/config/config.xml + sed -i 's/9000/9002/g' right/config/config.xml cat > right/config/config.d/zz-perf-test-tweaks.xml < @@ -60,6 +66,22 @@ EOF rm left/config/config.d/text_log.xml ||: rm right/config/config.d/metric_log.xml ||: rm right/config/config.d/text_log.xml ||: + + # Start a temporary server to rename the tables + while killall clickhouse ; do echo . ; sleep 1 ; done + echo all killed + + set -m # Spawn temporary in its own process groups + left/clickhouse server --config-file=left/config/config.xml -- --path db0 &> setup-log.txt & + left_pid=$! + kill -0 $left_pid + disown $left_pid + set +m + while ! left/clickhouse client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done + echo server for setup started + + left/clickhouse client --port 9001 --query "create database test" ||: + left/clickhouse client --port 9001 --query "rename table datasets.hits_v1 to test.hits" ||: } configure @@ -68,8 +90,14 @@ function restart while killall clickhouse ; do echo . ; sleep 1 ; done echo all killed - # Spawn servers in their own process groups - set -m + # Make copies of the original db for both servers. Use hardlinks instead + # of copying. + rm -r left/db ||: + rm -r right/db ||: + cp -al db0/ left/db/ + cp -al db0/ right/db/ + + set -m # Spawn servers in their own process groups left/clickhouse server --config-file=left/config/config.xml -- --path left/db &> left/log.txt & left_pid=$! @@ -83,16 +111,13 @@ function restart set +m - while ! left/clickhouse client --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done + while ! left/clickhouse client --port 9001 --query "select 1" ; do kill -0 $left_pid ; echo . ; sleep 1 ; done echo left ok - - while ! right/clickhouse client --port 9001 --query "select 1" ; do kill -0 $right_pid ; echo . ; sleep 1 ; done + while ! right/clickhouse client --port 9002 --query "select 1" ; do kill -0 $right_pid ; echo . ; sleep 1 ; done echo right ok - right/clickhouse client --port 9001 --query "create database test" ||: - right/clickhouse client --port 9001 --query "rename table datasets.hits_v1 to test.hits" ||: - left/clickhouse client --port 9000 --query "create database test" ||: - left/clickhouse client --port 9000 --query "rename table datasets.hits_v1 to test.hits" ||: + left/clickhouse client --port 9001 --query "select * from system.tables where database != 'system'" + right/clickhouse client --port 9002 --query "select * from system.tables where database != 'system'" } restart @@ -101,18 +126,29 @@ function run_tests # Just check that the script runs at all "$script_dir/perf.py" --help > /dev/null + # FIXME remove some broken long tests + rm left/performance/{IPv4,IPv6,modulo,parse_engine_file,number_formatting_formats,select_format}.xml ||: + # Run the tests for test in left/performance/*.xml do test_name=$(basename $test ".xml") - "$script_dir/perf.py" "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" || continue - right/clickhouse local --file "$test_name-raw.tsv" --structure 'query text, run int, version UInt32, time float' --query "$(cat $script_dir/eqmed.sql)" > "$test_name-report.tsv" + echo test $test_name + TIMEFORMAT=$(printf "time\t$test_name\t%%3R\t%%3U\t%%3S\n") + #time "$script_dir/perf.py" "$test" > >(tee "$test_name-raw.tsv") 2> >(tee "$test_name-err.log") || continue + time "$script_dir/perf.py" "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" || continue + grep ^query "$test_name-raw.tsv" | cut -f2- > "$test_name-queries.tsv" + grep ^client-time "$test_name-raw.tsv" | cut -f2- > "$test_name-client-time.tsv" + right/clickhouse local --file "$test_name-queries.tsv" --structure 'query text, run int, version UInt32, time float' --query "$(cat $script_dir/eqmed.sql)" > "$test_name-report.tsv" done } run_tests # Analyze results result_structure="left float, right float, diff float, rd Array(float), query text" -right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where rd[3] > 0.05 order by rd[3] desc" > flap-prone.tsv -right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where diff > 0.05 and diff > rd[3] order by diff desc" > bad-perf.tsv +right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where abs(diff) < 0.05 and rd[3] > 0.05 order by rd[3] desc" > unstable.tsv +right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "select * from table where abs(diff) > 0.05 and abs(diff) > rd[3] order by diff desc" > changed-perf.tsv +right/clickhouse local --file '*-client-time.tsv' -S "query text, client float, server float" -q "select client, server, floor(client/server, 3) p, query from table where p > 1.01 order by p desc" > slow-on-client.tsv grep Exception:[^:] *-err.log > run-errors.log + +./report.py > report.html diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 823832f2881..3f86599b547 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -1,7 +1,11 @@ #!/bin/bash set -ex -cd /workspace +chown nobody workspace output +chgrp nogroup workspace output +chmod 777 workspace output + +cd workspace # We will compare to the most recent testing tag in master branch, let's find it. rm -rf ch ||: @@ -11,9 +15,26 @@ echo Reference tag is $ref_tag # We use annotated tags which have their own shas, so we have to further # dereference the tag to get the commit it points to, hence the '~0' thing. ref_sha=$(cd ch && git rev-parse $ref_tag~0) + +# Show what we're testing echo Reference SHA is $ref_sha +(cd ch && git log -1 --decorate $ref_sha) ||: +echo +echo SHA to test is $SHA_TO_TEST +(cd ch && git log -1 --decorate $SHA_TO_TEST) ||: +echo -../compare.sh 0 $ref_sha $PR_TO_TEST $SHA_TO_TEST > compare.log 2>&1 +# Set python output encoding so that we can print queries with Russian letters. +export PYTHONIOENCODING=utf-8 -7z a /output/output.7z *.log *.tsv +# Even if we have some errors, try our best to save the logs. +set +e +# compare.sh kills its process group, so put it into a separate one. +# It's probably at fault for using `kill 0` as an error handling mechanism, +# but I can't be bothered to change this now. +set -m +time ../compare.sh 0 $ref_sha $PR_TO_TEST $SHA_TO_TEST 2>&1 | ts | tee compare.log +set +m + +7z a /output/output.7z *.log *.tsv *.html cp compare.log /output diff --git a/docker/test/performance-comparison/eqmed.sql b/docker/test/performance-comparison/eqmed.sql index 5e8d842b7df..d7265533208 100644 --- a/docker/test/performance-comparison/eqmed.sql +++ b/docker/test/performance-comparison/eqmed.sql @@ -1,10 +1,10 @@ -- input is table(query text, run UInt32, version int, time float) select -- abs(diff_percent) > rd_quantiles_percent[3] fail, - floor(original_medians_array.time_by_version[1], 4) m1, - floor(original_medians_array.time_by_version[2], 4) m2, - floor((m1 - m2) / m1, 3) diff_percent, - arrayMap(x -> floor(x / m1, 3), rd.rd_quantiles) rd_quantiles_percent, + floor(original_medians_array.time_by_version[1], 4) l, + floor(original_medians_array.time_by_version[2], 4) r, + floor((r - l) / l, 3) diff_percent, + arrayMap(x -> floor(x / l, 3), rd.rd_quantiles) rd_quantiles_percent, query from ( diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 8d2fe1bc476..4eeb6e76828 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -1,14 +1,21 @@ #!/usr/bin/python3 +import os +import sys import itertools import clickhouse_driver import xml.etree.ElementTree as et import argparse import pprint +import time +import traceback parser = argparse.ArgumentParser(description='Run performance test.') # Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') +parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1']) +parser.add_argument('--port', nargs='*', default=[9001, 9002]) +parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7))) args = parser.parse_args() tree = et.parse(args.file[0]) @@ -25,14 +32,14 @@ if infinite_sign is not None: raise Exception('Looks like the test is infinite (sign 1)') # Open connections -servers = [{'host': 'localhost', 'port': 9000, 'client_name': 'left'}, {'host': 'localhost', 'port': 9001, 'client_name': 'right'}] +servers = [{'host': host, 'port': port} for (host, port) in zip(args.host, args.port)] connections = [clickhouse_driver.Client(**server) for server in servers] # Check tables that should exist tables = [e.text for e in root.findall('preconditions/table_exists')] for t in tables: for c in connections: - res = c.execute("select 1 from {}".format(t)) + res = c.execute("show create table {}".format(t)) # Apply settings settings = root.findall('settings/*') @@ -62,7 +69,8 @@ for c in connections: try: c.execute(q) except: - print("Error:", sys.exc_info()[0], file=sys.stderr) + traceback.print_exc() + pass # Run create queries create_query_templates = [q.text for q in root.findall('create_query')] @@ -86,10 +94,19 @@ test_query_templates = [q.text for q in root.findall('query')] test_queries = substitute_parameters(test_query_templates, parameter_combinations) for q in test_queries: - for run in range(0, 7): + # Track the time spent by the client to process this query, so that we can notice + # out the queries that take long to process on the client side, e.g. by sending + # excessive data. + start_seconds = time.perf_counter() + server_seconds = 0 + for run in range(0, args.runs): for conn_index, c in enumerate(connections): res = c.execute(q) - print(tsv_escape(q) + '\t' + str(run) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) + print('query\t' + tsv_escape(q) + '\t' + str(run) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) + server_seconds += c.last_query.elapsed + + client_seconds = time.perf_counter() - start_seconds + print('client-time\t{}\t{}\t{}'.format(tsv_escape(q), client_seconds, server_seconds)) # Run drop queries drop_query_templates = [q.text for q in root.findall('drop_query')] diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py new file mode 100755 index 00000000000..64461ba0587 --- /dev/null +++ b/docker/test/performance-comparison/report.py @@ -0,0 +1,105 @@ +#!/usr/bin/python3 + +import collections +import csv +import os +import sys + +doc_template = """ + + + + {header} + + +
+ +

{header}

+{test_part} +

+{raw_log_name} +{branch_name} +Commit +{additional_urls} +Test output +Task (private network) +

+ + +""" + +table_template = """ +

{caption}

+ +{header} +{rows} +
+""" + +def tr(x): + return '' + str(x) + '' + +def td(x): + return '' + str(x) + '' + +def th(x): + return '' + str(x) + '' + +def table_row(r): + return tr(''.join([td(f) for f in r])) + +def table_header(r): + return tr(''.join([th(f) for f in r])) + +def tsv_rows(n): + result = '' + with open(n) as fd: + for row in csv.reader(fd, delimiter="\t", quotechar='"'): + result += table_row(row) + return result + +params = collections.defaultdict(str) +params['header'] = "ClickHouse Performance Comparison" +params['test_part'] = (table_template.format_map( + collections.defaultdict(str, + caption = 'Changes in performance', + header = table_header(['Left', 'Right', 'Diff', 'RD', 'Query']), + rows = tsv_rows('changed-perf.tsv'))) + + table_template.format( + caption = 'Slow on client', + header = table_header(['Client', 'Server', 'Ratio', 'Query']), + rows = tsv_rows('slow-on-client.tsv')) + + table_template.format( + caption = 'Unstable', + header = table_header(['Left', 'Right', 'Diff', 'RD', 'Query']), + rows = tsv_rows('unstable.tsv')) + + table_template.format( + caption = 'Run errors', + header = table_header(['A', 'B']), + rows = tsv_rows('run-errors.log')) +) +print(doc_template.format_map(params)) diff --git a/docs/README.md b/docs/README.md index fc7fabc268b..91883444459 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,62 +1,209 @@ -# How to Contribute to ClickHouse Documentation +# Contributing to ClickHouse Documentation -ClickHouse uses the "documentation as code" approach, so you can edit Markdown files in this folder from the GitHub web interface. Alternatively, fork the ClickHouse repository, edit, commit, push, and open a pull request. +## Why You Need to Document ClickHouse -At the moment documentation is bilingual in English and Russian. Try to keep all languages in sync if you can, but this is not strictly required. There are people who are responsible for monitoring language versions and syncing them. If you add a new article, you should also add it to `toc_{en,ru,zh,ja,fa}.yaml` files with the pages index. +The main reason is that ClickHouse is an open source project, and if you don't write the docs, nobody does. "Incomplete or Confusing Documentation" is the top complaint about open source software by the results of a [Github Open Source Survey](http://opensourcesurvey.org/2017/) of 2017. Documentation is highly valued but often overlooked. One of the most important contributions someone can make to an open source repository is a documentation update. -The master branch is then asynchronously published to the ClickHouse official website: +Many developers can say that the code is the best docs by itself, and they are right. But, ClickHouse is not a project for C++ developers. Most of its users don't know C++, and they can't understand the code quickly. ClickHouse is large enough to absorb almost any change without a noticeable trace. Nobody will find your very useful function, or an important setting, or a very informative new column in a system table if it is not referenced in the documentation. -* In English: https://clickhouse.yandex/docs/en/ -* In Russian: https://clickhouse.yandex/docs/ru/ -* In Chinese: https://clickhouse.yandex/docs/zh/ -* In Japanese: https://clickhouse.yandex/docs/ja/ -* In Farsi: https://clickhouse.yandex/docs/fa/ +If you want to help ClickHouse with documentation you can face, for example, the following questions: -The infrastructure to build Markdown for publishing on the documentation website resides in the [tools](tools) folder. It has its own [README.md](tools/README.md) file with more details. +- "I don't know how to write." + + We have prepared some [recommendations](#what-to-write) for you. -# How to Write Content for ClickHouse Documentation +- "I know what I want to write, but I don't know how to contribute to docs." -## Target Audience + Here are some [tips](#how-to-contribute). -When you write pretty much any text, the first thing you should think about is who will read it and which terms you should use for communicating with them. +Writing the docs is extremely useful for project's users and developers, and grows your karma. -ClickHouse can be directly used by all sorts of analysts and engineers. For generic parts of documentation (like the query language, tutorials or overviews), assume that the reader only has a basic technical background. For more technical sections (like articles that describe ClickHouse internals, guides for operating ClickHouse clusters, or rules for contributing to C++ code), you can use technical language and concepts. +**Contents** -## Specific Recommendations +- [What is the ClickHouse Documentation](#clickhouse-docs) +- [How to Contribute to ClickHouse Documentation](#how-to-contribute) + - [Markdown Dialect Cheatsheet](#markdown-cheatsheet) + - [Adding a New File](#adding-a-new-file) + - [Adding a New Language](#adding-a-new-language) +- [How to Write Content for ClickHouse Documentation](#what-to-write) + - [Documentation for Different Audience](#target-audience) + - [Common Recommendations](#common-recommendations) + - [Description Templates](#templates) +- [How to Build Documentation](#how-to-build-docs) -* Documentation should make sense when you read it through from beginning to end. If you add new content, try to place it where the necessary concepts have already been explained. -* If a documentation section consists of many similar items, like functions or operators, try to order them from more generic (usable by a wide audience) to more specific (for specific use cases or application types). If several items are intended to be mostly used together, group them together in the documentation. -* Try to avoid slang. Use the most common and specific terms possible for everything. If some terms are used as synonyms, state this explicitly. -* All descriptions of functionality should be accompanied by examples. Basic examples are acceptable, but real world examples are welcome, too. -* Sensitive topics like politics, religion, race, and so on are strictly prohibited in documentation, examples, comments, and code. -* Proofread your text before publishing. Look for typos, missing punctuation, or repetitions that could be avoided. -* Try to avoid addressing the reader directly, although this is not strictly prohibited. -# How to Add a New Language + + +## What is the ClickHouse Documentation + +The documentation contains information about all the aspects of the ClickHouse lifecycle: developing, testing, installing, operating, and using. The base language of the documentation is English. The English version is the most actual. All other languages are supported as much as they can by contributors from different countries. + +At the moment, [documentation](https://clickhouse.yandex/docs) exists in English, Russian, Chinese, Japanese, and Farsi. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs). + +Each language lays in the corresponding folder. Files that are not translated from English are the symbolic links to the English ones. + + + +## How to Contribute to ClickHouse Documentation + +You can contribute to the documentation in many ways, for example: + +- Fork the ClickHouse repository, edit, commit, push, and open a pull request. + + Add the `documentation` label to this pull request for proper automatic checks applying. If you have no permissions for adding labels, the reviewer of your PR adds it. + +- Open a required file in the ClickHouse repository and edit it from the GitHub web interface. + + You can do it on GitHub, or on the [ClickHouse Documentation](https://clickhouse.yandex/docs/en/) site. Each page of ClickHouse Documentation site contains an "Edit this page" (🖋) element in the upper right corner. Clicking this symbol, you get to the ClickHouse docs file opened for editing. + + When you are saving a file, GitHub opens a pull-request for your contribution. Add the `documentation` label to this pull request for proper automatic checks applying. If you have no permissions for adding labels, the reviewer of your PR adds it. + +Contribute all new information in English language. Other languages are translations from English. + + + +### Markdown Dialect Cheatsheet + +- Headings: Place them on a separate line and start with `# `, `## ` or `### `. Use the [Title Case](https://titlecase.com/) for them. Example: + + ```text + # The First Obligatory Title on a Page. + ``` + +- Bold text: `**asterisks**` or `__underlines__`. +- Links: `[link text](uri)`. Examples: + + - External link: `[ClickHouse repo](https://github.com/ClickHouse/ClickHouse)` + - Cross link: `[How to build docs](tools/README.md)` + +- Images: `![Exclamation sign](uri)`. You can refer to local images as well as remote in internet. +- Lists: Lists can be of two types: + + - `- unordered`: Each item starts from the `-`. + - `1. ordered`: Each item starts from the number. + + A list must be separated from the text by an empty line. Nested lists must be indented with 4 spaces. + +- Inline code: `` `in backticks` ``. +- Multiline code blocks: + 
```lang_name
+    code
+    lines
+    ```
+- Note: + + ```text + !!! info "Header" + 4 spaces indented text. + ``` + +- Warning: + + ```text + !!! warning "Header" + 4 spaces indented text. + ``` + +- Text hidden behind a cut (single sting that opens on click): + + ```text +
Visible text + Hidden content. +
`. + ``` +- Colored text: `text`. +- Heading anchor to be linked to: `# Title {#anchor-name}`. +- Table: + ``` + | Header 1 | Header 2 | Header 3 | + | ----------- | ----------- | ----------- | + | Cell A1 | Cell A2 | Cell A3 | + | Cell B1 | Cell B2 | Cell B3 | + | Cell C1 | Cell C2 | Cell C3 | + ``` + + + +### Adding a New File + +When adding a new file: + +- Make symbolic links for all other languages. You can use the following commands: + + ```bash + $ cd /ClickHouse/clone/directory/docs + $ ln -sr en/new/file.md lang/new/file.md + ``` + +- Reference the file from `toc_{en,ru,zh,ja,fa}.yaml` files with the pages index. + + + + +### Adding a New Language 1. Create a new docs subfolder named using the [ISO-639-1 language code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes). 2. Add Markdown files with the translation, mirroring the folder structure of other languages. 3. Commit and open a pull request with the new content. -Some additional configuration has to be done to actually make a new language live on the official website, but it's not automated or documented yet, so we'll do it on our own after the pull request with the content is merged. +When everything is ready, we will add the new language to the website. -# Markdown Dialect Cheatsheet + -* Headings are on a separate line starting with `# `, `## ` or `### `. -* Bold is in `**asterisks**` or `__underlines__`. -* Links `[anchor](http://...)`, images `![with exclamation sign](http://...jpeg)`. -* Lists are on lines starting with `* unordered` or `1. ordered`, but there should be an empty line before the first list item. Sub-lists must be indented with 4 spaces. -* Inline code fragments are `in backticks`. -* Multiline code blocks are ```in triple backtick quotes ```. -* Brightly highlighted text starts with `!!! info "Header"`, followed by 4 spaces on the next line and content. For a warning, replace `info` with `warning`. -* Hidden block that opens on click: `
Header hidden content
`. -* Colored text: `text`. -* Heading anchor to be linked to: `Title {#anchor-name}`. -* Table: -``` -| Header 1 | Header 2 | Header 3 | -| ----------- | ----------- | ----------- | -| Cell A1 | Cell A2 | Cell A3 | -| Cell B1 | Cell B2 | Cell B3 | -| Cell C1 | Cell C2 | Cell C3 | -``` +## How to Write Content for ClickHouse Documentation + + + + +### Documentation for Different Audience + +When writing documentation, think about people who read it. Each audience has specific requirements for terms they use in communications. + +ClickHouse documentation can be divided by the audience for the following parts: + +- Conceptual topics in [Introduction](https://clickhouse.yandex/docs/en/), tutorials and overviews, changelog. + + These topics are for the most common auditory. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills. + +- Query language reference and related topics. + + These parts of the documentation are dedicated to those who use ClickHouse for data analysis. Carefully describe syntax, input, and output data for expressions. Don't forget the examples. + +- Description of table engines and operation details. + + Operation engineers who help data analysts to solve their tasks should know how to install/update a ClickHouse server, maintain the ClickHouse cluster, how to integrate it with other tools and systems, how to get the maximum performance of their entire environment. + +- Developer's guides. + + The documentation provides code writers with information about how to write code for ClickHouse and how to build it in different environments. + + + +### Common Recommendations + +- When searching for a position for your text, try to place it in the most anticipated place. +- Group entities. For example, if several functions solve similar tasks or belong to a specific group by use case or an application type, place them together. +- Try to avoid slang. Use the most common and specific terms possible. If some terms are used as synonyms, state this explicitly. +- Add examples for all the functionality. Add basic examples to show how the function works by itself. Add use case examples to show how the function participates in solving specific tasks. +- Any text concerning politics, religion, or other social related themes are strictly prohibited in all the ClickHouse texts. +- Proofread your text before publishing. Look for typos, missing punctuation, or repetitions that could be avoided. + + + +### Description Templates + +When writing docs, you can use prepared templates. Copy the code of a template and use it in your contribution. Sometimes you just need to change level of headers. + +Templates: + +- [Function](dscr-templates/template-function.md) +- [Setting](dscr-templates/template-setting.md) +- [Table engine](dscr-templates/template-table-engine.md) +- [System table](dscr-templates/template-system-table.md) + + + + +## How to Build Documentation + +You can build your documentation manually by following the instructions in [docs/tools/README.md](docs/tools/README.md). Also, our CI runs the documentation build after the `documentation` label is added to PR. You can see the results of a build in the GitHub interface. If you have no permissions to add labels, a reviewer of your PR will add it. diff --git a/docs/dscr-templates/template-function.md b/docs/dscr-templates/template-function.md new file mode 100644 index 00000000000..c4f4973e9ac --- /dev/null +++ b/docs/dscr-templates/template-function.md @@ -0,0 +1,48 @@ +## function-name {#function-name-in-lower-case} + +Short description. + +**Syntax** (without SELECT) + +```sql + +``` + +Alias: ``. (Optional) + +More text (Optional). + +**Parameters** (Optional) + +- `x` — Description. [Type name](relative/path/to/type/dscr.md#type). +- `y` — Description. [Type name](relative/path/to/type/dscr.md#type). + +**Returned value(s)** + +- Returned values list. + +Type: [Type](relative/path/to/type/dscr.md#type). + +**Example** + +The example must show usage and/or a use cases. The following text contains recommended parts of an example. + +Input table (Optional): + +```text +``` + +Query: + +```sql +``` + +Result: + +```text +``` + +**See Also** (Optional) + +- [link](#) + diff --git a/docs/dscr-templates/template-setting.md b/docs/dscr-templates/template-setting.md new file mode 100644 index 00000000000..c0c1e05d3f1 --- /dev/null +++ b/docs/dscr-templates/template-setting.md @@ -0,0 +1,27 @@ +## setting-name {#setting-name-in-lower-case} + +Description. + +For switcher setting, use the typical phrase: "Enables or disables something ...". + +Possible values: + +*For switcher setting:* + +- 0 — Disabled. +- 1 — Enabled. + +*For another setting (typical phrases):* + +- Positive integer. +- 0 — Disabled or unlimited or smth. else. + +Default value: `value`. + +**Additional Info** (Optional) + +The name of additional section can be any, for example **Usage**. + +**See Also** (Optional) + +- [link](#) diff --git a/docs/dscr-templates/template-system-table.md b/docs/dscr-templates/template-system-table.md new file mode 100644 index 00000000000..8bcd1f27c1e --- /dev/null +++ b/docs/dscr-templates/template-system-table.md @@ -0,0 +1,21 @@ +## system.table_name {#system_tables-table_name} + +Description. + +Columns: + +- `column_name` ([data_type_name](data_type_uri)) — Description. + +**Example** + +```sql +SELECT * FROM system.table_name +``` + +```text +Some output. It shouldn't be long. +``` + +**See Also** + +- [Article name](uri) — Some words about referenced information. diff --git a/docs/dscr-templates/template-table-engine.md b/docs/dscr-templates/template-table-engine.md new file mode 100644 index 00000000000..822944e9ecf --- /dev/null +++ b/docs/dscr-templates/template-table-engine.md @@ -0,0 +1,51 @@ +# EngineName + +- What the engine does. +- Relations with other engines if they exist. + +## Creating a Table + +``` +Syntax codeblock +``` + +**Engine Parameters** + +**Query Clauses** + + +## Virtual columns + +If they exist. + +## Specifics and recommendations +Algorithms +Specifics of read/write processes +Examples of tasks +Recommendations for usage +Specifics of data storage + +## Usage Example + +The example must show usage and/or a use cases. The following text contains recommended parts of an example. + +Input table: + +```text +``` + +Query: + +```sql +``` + +Result: + +```text +``` + +Any text clarifying the example. + +## See Also + +- [link](#) diff --git a/docs/en/data_types/datetime.md b/docs/en/data_types/datetime.md index 967e184ba99..7e30f3ae426 100644 --- a/docs/en/data_types/datetime.md +++ b/docs/en/data_types/datetime.md @@ -1,6 +1,6 @@ # DateTime {#data_type-datetime} -Data structure storing Unix timestamp. Also, it can store a time zone. +Allows to store an instant in time, that can be expressed as a calendar date and a time of a day. `DateTime` allows to take into account time zones for stored values. Syntax: @@ -8,38 +8,68 @@ Syntax: DateTime([timezone]) ``` -Range of values in the Unix timestamp: [1970-01-01 00:00:00, 2105-12-31 23:59:59]. +Supported range of values: [1970-01-01 00:00:00, 2105-12-31 23:59:59]. Resolution: 1 second. -## Usage remarks +SELECT toDateTime(0) +FORMAT TSV -ClickHouse stores date and time values in the Unix timestamp format that is independent of the time zones and daylight saving rules. The time zone value affects displaying `DateTime` values in text formats and parsing the input strings for storage. You can find the list of supported time zones in [IANA Time Zone Database](https://www.iana.org/time-zones). +0000-00-00 00:00:00 -You can explicitly set a time zone for `DateTime`-type column when creating a table. If time zone isn't set, ClickHouse uses the value of the [timezone](../operations/server_settings/settings.md#server_settings-timezone) server configuration parameter or the operating system settings at the moment of the ClickHouse server start. -The [clickhouse-client](../interfaces/cli.md) applies the server time zone by default if a time zone isn't explicitly defined when initializing the data type. To use the client time zone, run it with the `--use_client_time_zone` parameter. +## Usage Remarks -ClickHouse outputs values in the `YYYY-MM-DD hh:mm:ss` text format by default. You can change the format with the [formatDateTime](../query_language/functions/date_time_functions.md#formatdatetime) function. +A moment of time is stored as [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time), independently of time zones and daylight savings. Additionally `DateTime` can store time zone, that affects how `DateTime` values are displayed in text format and how input strings are parsed for storage. The `tzdata` package, containing [IANA Time Zone Database](https://www.iana.org/time-zones), should be installed in the system. Use the `timedatectl list-timezones` command to list timezones known by a local system. -When inserting data into ClickHouse, you can use different formats of date and time strings, depending on the [date_time_input_format](../operations/settings/settings.md#settings-date_time_input_format) setting value. +You can explicitly set a time zone for `DateTime`-type columns when creating a table. If the time zone isn't set, ClickHouse uses the value of the [timezone](../operations/server_settings/settings.md#server_settings-timezone) parameter in the server settings or the operating system settings at the moment of the ClickHouse server start. + +The [clickhouse-client](../interfaces/cli.md) applies the server time zone by default if a time zone isn't explicitly set when initializing the data type. To use the client time zone, run `clickhouse-client` with the `--use_client_time_zone` parameter. + +ClickHouse outputs values in `YYYY-MM-DD hh:mm:ss` text format by default. You can change the format with the [formatDateTime](../query_language/functions/date_time_functions.md#formatdatetime) function. + +When inserting data into ClickHouse, you can use different formats of date and time strings, depending on the value of the [date_time_input_format](../operations/settings/settings.md#settings-date_time_input_format) setting. ## Examples -Creating a table with a `DateTime`-type column: +**1.** Creating a table with a `DateTime`-type column and inserting data into it: ```sql -CREATE TABLE dt( - timestamp DateTime('Europe/Moscow') +CREATE TABLE dt +( + `timestamp` DateTime('Europe/Moscow'), + `event_id` UInt8 ) +ENGINE = TinyLog +``` +```sql +INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2) +``` +```sql +SELECT * FROM dt +``` +```text +┌───────────timestamp─┬─event_id─┐ +│ 2019-01-01 03:00:00 │ 1 │ +│ 2019-01-01 00:00:00 │ 2 │ +└─────────────────────┴──────────┘ ``` -Getting a time zone for a `DateTime`-type value: +Unix timestamp `1546300800` represents the `'2019-01-01 00:00:00'` date and time in `Europe/London` (UTC+0) time zone, but the `timestamp` column stores values in the `Europe/Moscow` (UTC+3) timezone, so the value inserted as Unix timestamp is formatted as `2019-01-01 03:00:00`. ```sql -SELECT - toDateTime(now(), 'Europe/Moscow') AS column, - toTypeName(column) AS x +SELECT * FROM dt WHERE timestamp = toDateTime('2019-01-01 00:00:00', 'Europe/Moscow') +``` +```text +┌───────────timestamp─┬─event_id─┐ +│ 2019-01-01 00:00:00 │ 2 │ +└─────────────────────┴──────────┘ +``` + +**2.** Getting a time zone for a `DateTime`-type value: + +```sql +SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x ``` ```text ┌──────────────column─┬─x─────────────────────────┐ @@ -49,12 +79,12 @@ SELECT ## See Also -- [Type Conversion Functions](../query_language/functions/type_conversion_functions.md) -- [Functions for Working with Dates and Times](../query_language/functions/date_time_functions.md) -- [Functions for Working with Arrays](../query_language/functions/array_functions.md) +- [Type conversion functions](../query_language/functions/type_conversion_functions.md) +- [Functions for working with dates and times](../query_language/functions/date_time_functions.md) +- [Functions for working with arrays](../query_language/functions/array_functions.md) - [The `date_time_input_format` setting](../operations/settings/settings.md#settings-date_time_input_format) - [The `timezone` server configuration parameter](../operations/server_settings/settings.md#server_settings-timezone) -- [Operator for Working with Dates and Times](../query_language/operators.md#operators-datetime) +- [Operators for working with dates and times](../query_language/operators.md#operators-datetime) - [The `Date` data type](date.md) [Original article](https://clickhouse.yandex/docs/en/data_types/datetime/) diff --git a/docs/en/development/browse_code.md b/docs/en/development/browse_code.md new file mode 100644 index 00000000000..0a99f5128cb --- /dev/null +++ b/docs/en/development/browse_code.md @@ -0,0 +1,7 @@ +# Browse ClickHouse Source Code + +You can use **Woboq** online code browser available [here](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). It provides code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. + +Also you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. + +If you're interested what IDE to use, we recommend CLion, QT Creator, VS Code and KDevelop (with caveats). You can use any favourite IDE. Vim and Emacs also count. diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 2df8e7eeb25..1686f013559 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -58,12 +58,6 @@ $ export CC=gcc-9 $ export CXX=g++-9 ``` -## Install Required Libraries from Packages - -```bash -$ sudo apt-get install libreadline-dev -``` - ## Checkout ClickHouse Sources ```bash diff --git a/docs/en/development/build_osx.md b/docs/en/development/build_osx.md index 23fe52ddb64..ae08d3abb3f 100644 --- a/docs/en/development/build_osx.md +++ b/docs/en/development/build_osx.md @@ -11,7 +11,7 @@ $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/inst ## Install Required Compilers, Tools, and Libraries ```bash -$ brew install cmake ninja libtool gettext readline +$ brew install cmake ninja libtool gettext ``` ## Checkout ClickHouse Sources diff --git a/docs/en/development/developer_instruction.md b/docs/en/development/developer_instruction.md index 40eb60fc5da..591fc7c486c 100644 --- a/docs/en/development/developer_instruction.md +++ b/docs/en/development/developer_instruction.md @@ -4,6 +4,10 @@ Building of ClickHouse is supported on Linux, FreeBSD and Mac OS X. If you use Windows, you need to create a virtual machine with Ubuntu. To start working with a virtual machine please install VirtualBox. You can download Ubuntu from the website: https://www.ubuntu.com/#download. Please create a virtual machine from the downloaded image (you should reserve at least 4GB of RAM for it). To run a command line terminal in Ubuntu, please locate a program containing the word "terminal" in its name (gnome-terminal, konsole etc.) or just press Ctrl+Alt+T. +# If you use 32-bit system + +ClickHouse cannot work or build on 32-bit system. You should acquire access to 64-bit system and you can continue reading. + # Creating a repository on GitHub @@ -96,15 +100,7 @@ Next, check the version of CMake: `cmake --version`. If it is below 3.3, you sho # Optional External Libraries -ClickHouse uses several external libraries for building. Most of them do not need to be installed separately as they are built together with ClickHouse from the sources located in the submodules. You can check the list in `contrib`. - -There is one library that is not built from sources but is supplied by the system: Readline, and thus is recommended to be installed. - -Ubuntu: `sudo apt install libreadline-dev` - -Mac OS X: `brew install readline` - -However, these libraries are optional and ClickHouse can well be built without them. ICU is used for support of `COLLATE` in `ORDER BY` (i.e. for sorting in turkish alphabet). Readline is used for more convenient command input in clickhouse-client. +ClickHouse uses several external libraries for building. All of them do not need to be installed separately as they are built together with ClickHouse from the sources located in the submodules. You can check the list in `contrib`. # C++ Compiler diff --git a/docs/en/faq/general.md b/docs/en/faq/general.md index 3e6daf6ed9a..8afe995f724 100644 --- a/docs/en/faq/general.md +++ b/docs/en/faq/general.md @@ -21,11 +21,11 @@ If you use Oracle through the ODBC driver as a source of external dictionaries, NLS_LANG=RUSSIAN_RUSSIA.UTF8 ``` -## How to export data from ClickHouse to the file? +## How Do I Export Data from ClickHouse to a File? {#how-to-export-to-file} ### Using INTO OUTFILE Clause -Add [INTO OUTFILE](../query_language/select/#into-outfile-clause) clause to your query. +Add an [INTO OUTFILE](../query_language/select/#into-outfile-clause) clause to your query. For example: @@ -41,11 +41,11 @@ For example: SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV ``` -### Using File-engine Table +### Using a File-Engine Table See [File](../operations/table_engines/file.md). -### Using Command-line Redirection +### Using Command-Line Redirection ```sql $ clickhouse-client --query "SELECT * from table" > result.txt diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index 86c7a104670..0fcc5e8899f 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -53,7 +53,7 @@ Only a single query is run, so everything after the semicolon is ignored. You can specify `\G` instead of or after the semicolon. This indicates Vertical format. In this format, each value is printed on a separate line, which is convenient for wide tables. This unusual feature was added for compatibility with the MySQL CLI. -The command line is based on 'readline' (and 'history' or 'libedit', or without a library, depending on the build). In other words, it uses the familiar keyboard shortcuts and keeps a history. +The command line is based on 'replxx' (similar to 'readline'). In other words, it uses the familiar keyboard shortcuts and keeps a history. The history is written to `~/.clickhouse-client-history`. By default, the format used is PrettyCompact. You can change the format in the FORMAT clause of the query, or by specifying `\G` at the end of the query, using the `--format` or `--vertical` argument in the command line, or using the client configuration file. diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index b37c9cdddb2..9992467047a 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -968,7 +968,7 @@ To exchange data with Hadoop, you can use [HDFS table engine](../operations/tabl ## ORC {#data-format-orc} -[Apache ORC](https://orc.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. ClickHouse supports only read operations for this format. +[Apache ORC](https://orc.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. You can only insert data in this format to ClickHouse. ### Data Types Matching @@ -991,18 +991,18 @@ The table below shows supported data types and how they match ClickHouse [data t | `STRING`, `BINARY` | [String](../data_types/string.md) | | `DECIMAL` | [Decimal](../data_types/decimal.md) | -ClickHouse supports configurable precision of `Decimal` type. The `INSERT` query treats the ORC `DECIMAL` type as the ClickHouse `Decimal128` type. +ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` query treats the ORC `DECIMAL` type as the ClickHouse `Decimal128` type. Unsupported ORC data types: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Data types of a ClickHouse table columns can differ from the corresponding fields of the ORC data inserted. When inserting data, ClickHouse interprets data types according to the table above and then [cast](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) the data to that data type which is set for the ClickHouse table column. +The data types of ClickHouse table columns don't have to match the corresponding ORC data fields. When inserting data, ClickHouse interprets data types according to the table above and then [casts](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) the data to the data type set for the ClickHouse table column. ### Inserting Data -You can insert Parquet data from a file into ClickHouse table by the following command: +You can insert ORC data from a file into ClickHouse table by the following command: ```bash -$ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT ORC" +$ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" ``` To exchange data with Hadoop, you can use [HDFS table engine](../operations/table_engines/hdfs.md). diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 4383318f98f..f8f4b159a4e 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -172,7 +172,7 @@ $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?data By default, the database that is registered in the server settings is used as the default database. By default, this is the database called 'default'. Alternatively, you can always specify the database using a dot before the table name. -The username and password can be indicated in one of two ways: +The username and password can be indicated in one of three ways: 1. Using HTTP Basic Authentication. Example: @@ -186,6 +186,12 @@ $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- $ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- ``` +3. Using ‘X-ClickHouse-User’ and ‘X-ClickHouse-Key’ headers. Example: + +```bash +$ echo 'SELECT 1' | curl -H 'X-ClickHouse-User: user' -H 'X-ClickHouse-Key: password' 'http://localhost:8123/' -d @- +``` + If the user name is not specified, the `default` name is used. If the password is not specified, the empty password is used. You can also use the URL parameters to specify any settings for processing a single query, or entire profiles of settings. Example:http://localhost:8123/?profile=web&max_rows_to_read=1000000000&query=SELECT+1 diff --git a/docs/en/interfaces/mysql.md b/docs/en/interfaces/mysql.md new file mode 100644 index 00000000000..454cdb9160d --- /dev/null +++ b/docs/en/interfaces/mysql.md @@ -0,0 +1,37 @@ +# MySQL interface {#mysql_interface} + +ClickHouse supports MySQL wire protocol. It can be enabled by [mysql_port](../operations/server_settings/settings.md#server_settings-mysql_port) setting in configuration file: +```xml +9004 +``` + +Example of connecting using command-line tool mysql: +```bash +$ mysql --protocol tcp -u default -P 9004 +``` + +Output if connection succeeded: +```text +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 4 +Server version: 20.2.1.1-ClickHouse + +Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> +``` + +For compatibility with all MySQL clients, it is recommended to specify user password with [double SHA1](../operations/settings/settings_users.md#password_double_sha1_hex) in configuration file. +If user password is specified using [SHA256](../operations/settings/settings_users.md#password_sha256_hex), some clients won't be able to authenticate (mysqljs and old versions of command-line tool mysql). + +Restrictions: + +- prepared queries are not supported + +- some data types are sent as strings diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index f6842a63adb..447173b3103 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -88,18 +88,6 @@ Features: ## Commercial -### Holistics Software - -[Holistics](https://www.holistics.io/) was listed by Gartner's Frontrunners in 2019 as one of the top 2 highest ranked business intelligence tools globally for usability. Holistics is a full-stack data platform and business intelligence tool for setting up your analytics processes, built on SQL. - -Features: - -- Automated email, Slack and Google Sheet schedules of reports. -- Powerful SQL editor with visualizations, version control, auto-completion, reusable query components and dynamic filters. -- Embedded analytics of reports and dashboards via iframe. -- Data preparation and ETL capabilities. -- SQL data modeling support for relational mapping of data. - ### DataGrip [DataGrip](https://www.jetbrains.com/datagrip/) is a database IDE from JetBrains with dedicated support for ClickHouse. It is also embedded into other IntelliJ-based tools: PyCharm, IntelliJ IDEA, GoLand, PhpStorm and others. @@ -113,6 +101,18 @@ Features: - Refactorings. - Search and Navigation. +### Holistics Software + +[Holistics](https://www.holistics.io/) is a full-stack data platform and business intelligence tool. + +Features: + +- Automated email, Slack and Google Sheet schedules of reports. +- SQL editor with visualizations, version control, auto-completion, reusable query components and dynamic filters. +- Embedded analytics of reports and dashboards via iframe. +- Data preparation and ETL capabilities. +- SQL data modeling support for relational mapping of data. + ### Looker [Looker](https://looker.com) is a data platform and business intelligence tool with support for 50+ database dialects including ClickHouse. Looker is available as a SaaS platform and self-hosted. Users can use Looker via the browser to explore data, build visualizations and dashboards, schedule reports, and share their diff --git a/docs/en/operations/performance_test.md b/docs/en/operations/performance_test.md index a56490ac8ba..f5f249a75e8 100644 --- a/docs/en/operations/performance_test.md +++ b/docs/en/operations/performance_test.md @@ -1,16 +1,14 @@ # How To Test Your Hardware With ClickHouse -Draft. - With this instruction you can run basic ClickHouse performance test on any server without installation of ClickHouse packages. -1. Go to "commits" page: https://github.com/ClickHouse/ClickHouse/commits/master +\1. Go to "commits" page: [https://github.com/ClickHouse/ClickHouse/commits/master](https://github.com/ClickHouse/ClickHouse/commits/master) -2. Click on the first green check mark or red cross with green "ClickHouse Build Check" and click on the "Details" link near "ClickHouse Build Check". +\2. Click on the first green check mark or red cross with green "ClickHouse Build Check" and click on the "Details" link near "ClickHouse Build Check". -3. Copy the link to "clickhouse" binary for amd64 or aarch64. +\3. Copy the link to "clickhouse" binary for amd64 or aarch64. -4. ssh to the server and download it with wget: +\4. ssh to the server and download it with wget: ``` # For amd64: wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578163263_binary/clickhouse @@ -20,7 +18,7 @@ wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f7907 chmod a+x clickhouse ``` -5. Download configs: +\5. Download configs: ``` wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/dbms/programs/server/config.xml wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/dbms/programs/server/users.xml @@ -29,16 +27,19 @@ wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/dbms/program wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/dbms/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml ``` -6. Download benchmark files: +\6. Download benchmark files: ``` wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/dbms/benchmark/clickhouse/benchmark-new.sh chmod a+x benchmark-new.sh wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/dbms/benchmark/clickhouse/queries.sql ``` -7. Download test data: +\7. Download test data: + According to the instruction: -https://clickhouse.yandex/docs/en/getting_started/example_datasets/metrica/ + +[https://clickhouse.yandex/docs/en/getting_started/example_datasets/metrica/](https://clickhouse.yandex/docs/en/getting_started/example_datasets/metrica/) + ("hits" table containing 100 million rows) ``` @@ -47,26 +48,29 @@ tar xvf hits_100m_obfuscated_v1.tar.xz -C . mv hits_100m_obfuscated_v1/* . ``` -8. Run the server: +\8. Run the server: ``` ./clickhouse server ``` -9. Check the data: +\9. Check the data: + ssh to the server in another terminal ``` ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" 100000000 ``` -10. Edit the benchmark-new.sh, change "clickhouse-client" to "./clickhouse client" and add "--max_memory_usage 100000000000" parameter. +\10. Edit the benchmark-new.sh, change "clickhouse-client" to "./clickhouse client" and add "--max_memory_usage 100000000000" parameter. ``` mcedit benchmark-new.sh ``` -11. Run the benchmark: +\11. Run the benchmark: ``` ./benchmark-new.sh hits_100m_obfuscated ``` -12. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com +\12. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com + +All the results are published here: [https://clickhouse.yandex/benchmark_hardware.html](https://clickhouse.yandex/benchmark_hardware.html) diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index 89bb7ef33ae..550a84350d9 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -709,7 +709,21 @@ Positive integer. 9440 ``` -## tmp_path +## mysql_port {#server_settings-mysql_port} + +Port for communicating with clients over MySQL protocol. + +**Possible values** + +Positive integer. + +Example + +```xml +9004 +``` + +## tmp_path {#server-settings-tmp_path} Path to temporary data for processing large queries. @@ -723,6 +737,17 @@ Path to temporary data for processing large queries. ``` +## tmp_policy {#server-settings-tmp_policy} + +Policy from [`storage_configuration`](mergetree.md#table_engine-mergetree-multiple-volumes) to store temporary files. +If not set [`tmp_path`](#server-settings-tmp_path) is used, otherwise it is ignored. + +!!! note + - `move_factor` is ignored + - `keep_free_space_bytes` is ignored + - `max_data_part_size_bytes` is ignored + - you must have exactly one volume in that policy + ## uncompressed_cache_size {#server-settings-uncompressed_cache_size} Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../../operations/table_engines/mergetree.md). diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c49a5f25cf8..f380cd2d4d7 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -968,6 +968,24 @@ Possible values: Default value: 0. +## optimize_skip_unused_shards {#settings-optimize_skip_unused_shards} + +Enables or disables skipping of unused shards for SELECT queries that has sharding key condition in PREWHERE/WHERE (assumes that the data is distributed by sharding key, otherwise do nothing). + +Default value: 0 + +## force_optimize_skip_unused_shards {#settings-force_optimize_skip_unused_shards} + +Enables or disables query execution if [`optimize_skip_unused_shards`](#settings-optimize_skip_unused_shards) enabled and skipping of unused shards is not possible. If the skipping is not possible and the setting is enabled exception will be thrown. + +Possible values: + +- 0 - Disabled (do not throws) +- 1 - Disable query execution only if the table has sharding key +- 2 - Disable query execution regardless sharding key is defined for the table + +Default value: 0 + ## optimize_throw_if_noop {#setting-optimize_throw_if_noop} Enables or disables throwing an exception if an [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) query didn't perform a merge. diff --git a/docs/en/operations/settings/settings_users.md b/docs/en/operations/settings/settings_users.md index 8937c59b667..04f7900382c 100644 --- a/docs/en/operations/settings/settings_users.md +++ b/docs/en/operations/settings/settings_users.md @@ -33,12 +33,14 @@ Structure of the `users` section: ### user_name/password -Password could be specified in plaintext or in SHA256 (hex format). +Password can be specified in plaintext or in SHA256 (hex format). - To assign a password in plaintext (**not recommended**), place it in a `password` element. For example, `qwerty`. The password can be left blank. + + - To assign a password using its SHA256 hash, place it in a `password_sha256_hex` element. For example, `65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5`. @@ -51,6 +53,19 @@ Password could be specified in plaintext or in SHA256 (hex format). The first line of the result is the password. The second line is the corresponding SHA256 hash. + + +- For compatibility with MySQL clients, password can be specified in double SHA1 hash. Place it in `password_double_sha1_hex` element. + + For example, `08b4a0f1de6ad37da17359e592c8d74788a83eb0`. + + Example of how to generate a password from shell: + + ``` + PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' + ``` + + The first line of the result is the password. The second line is the corresponding double SHA1 hash. ### user_name/networks diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 77964c7377f..563bd3864bd 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -395,6 +395,35 @@ Columns: - `query` (String) – The query text. For `INSERT`, it doesn't include the data to insert. - `query_id` (String) – Query ID, if defined. +## system.text_log {#system_tables-text_log} + +Contains logging entries. Logging level which goes to this table can be limited with `text_log.level` server setting. + +Columns: + +- `event_date` (`Date`) - Date of the entry. +- `event_time` (`DateTime`) - Time of the entry. +- `microseconds` (`UInt32`) - Microseconds of the entry. +- `thread_name` (String) — Name of the thread from which the logging was done. +- `thread_number` (UInt32) — Internal thread ID. +- `os_thread_id` (Int32) — OS thread ID. +- `level` (`Enum8`) - Entry level. + - `'Fatal' = 1` + - `'Critical' = 2` + - `'Error' = 3` + - `'Warning' = 4` + - `'Notice' = 5` + - `'Information' = 6` + - `'Debug' = 7` + - `'Trace' = 8` +- `query_id` (`String`) - ID of the query. +- `logger_name` (`LowCardinality(String)`) - Name of the logger (i.e. `DDLWorker`) +- `message` (`String`) - The message itself. +- `revision` (`UInt32`) - ClickHouse revision. +- `source_file` (`LowCardinality(String)`) - Source file from which the logging was done. +- `source_line` (`UInt64`) - Source line from which the logging was done. + + ## system.query_log {#system_tables-query_log} Contains information about execution of queries. For each query, you can see processing start time, duration of processing, error messages and other information. @@ -744,6 +773,43 @@ WHERE changed └────────────────────────┴─────────────┴─────────┘ ``` +## system.table_engines + +Contains description of table engines supported by server and their feature support information. + +This table contains the following columns (the column type is shown in brackets): + +- `name` (String) — The name of table engine. +- `supports_settings` (UInt8) — Flag that indicates if table engine supports `SETTINGS` clause. +- `supports_skipping_indices` (UInt8) — Flag that indicates if table engine supports [skipping indices](table_engines/mergetree/#table_engine-mergetree-data_skipping-indexes). +- `supports_ttl` (UInt8) — Flag that indicates if table engine supports [TTL](table_engines/mergetree/#table_engine-mergetree-ttl). +- `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` and `SAMPLE_BY`. +- `supports_replication` (UInt8) — Flag that indicates if table engine supports [data replication](table_engines/replication/). +- `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. + +Example: + +```sql +SELECT * +FROM system.table_engines +WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') +``` + +```text +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ +``` + +**See also** + +- MergeTree family [query clauses](table_engines/mergetree/#sektsii-zaprosa) +- Kafka [settings](table_engines/kafka.md#table_engine-kafka-creating-a-table) +- Join [settings](table_engines/join/#limitations-and-settings) + + ## system.tables Contains metadata of each table that the server knows about. Detached tables are not shown in `system.tables`. diff --git a/docs/en/operations/table_engines/distributed.md b/docs/en/operations/table_engines/distributed.md index 24a61998b39..1e0145dda15 100644 --- a/docs/en/operations/table_engines/distributed.md +++ b/docs/en/operations/table_engines/distributed.md @@ -3,11 +3,23 @@ **The Distributed engine does not store data itself**, but allows distributed query processing on multiple servers. Reading is automatically parallelized. During a read, the table indexes on remote servers are used, if there are any. -The Distributed engine accepts parameters: the cluster name in the server's config file, the name of a remote database, the name of a remote table, and (optionally) a sharding key. + +The Distributed engine accepts parameters: + +- the cluster name in the server's config file +- the name of a remote database +- the name of a remote table +- (optionally) sharding key +- (optionally) policy name, it will be used to store temporary files for async send + + See also: + - `insert_distributed_sync` setting + - [MergeTree](../mergetree.md#table_engine-mergetree-multiple-volumes) for the examples + Example: ```sql -Distributed(logs, default, hits[, sharding_key]) +Distributed(logs, default, hits[, sharding_key[, policy_name]]) ``` Data will be read from all servers in the 'logs' cluster, from the default.hits table located on every server in the cluster. diff --git a/docs/en/operations/table_engines/hdfs.md b/docs/en/operations/table_engines/hdfs.md index 22760c02d83..2f98e4dc452 100644 --- a/docs/en/operations/table_engines/hdfs.md +++ b/docs/en/operations/table_engines/hdfs.md @@ -100,4 +100,13 @@ Create table with files named `file000`, `file001`, ... , `file999`: CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV') ``` +## Virtual Columns + +- `_path` — Path to the file. +- `_file` — Name of the file. + +**See Also** + +- [Virtual columns](https://clickhouse.yandex/docs/en/operations/table_engines/#table_engines-virtual_columns) + [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/hdfs/) diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index e486e269865..e9eab89bf9c 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -189,8 +189,10 @@ The following operations with [partitions](../operations/table_engines/custom_pa - [DETACH PARTITION](#alter_detach-partition) – Moves a partition to the `detached` directory and forget it. - [DROP PARTITION](#alter_drop-partition) – Deletes a partition. - [ATTACH PART|PARTITION](#alter_attach-partition) – Adds a part or partition from the `detached` directory to the table. +- [REPLACE PARTITION](#alter_replace-partition) - Copies the data partition from one table to another. - [ATTACH PARTITION FROM](#alter_attach-partition-from) – Copies the data partition from one table to another and adds. - [REPLACE PARTITION](#alter_replace-partition) - Copies the data partition from one table to another and replaces. +- [MOVE PARTITION](#alter_move-partition) - Move the data partition from one table to another. - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) - Resets the value of a specified column in a partition. - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) - Resets the specified secondary index in a partition. - [FREEZE PARTITION](#alter_freeze-partition) – Creates a backup of a partition. @@ -283,6 +285,23 @@ For the query to run successfully, the following conditions must be met: - Both tables must have the same structure. - Both tables must have the same partition key. +#### MOVE PARTITION {#alter_move-partition} + +``` sql +ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest +``` + +This query move the data partition from the `table_source` to `table_dest` with deleting the data from `table_source`. + +For the query to run successfully, the following conditions must be met: + +- Both tables must have the same structure. +- Both tables must have the same partition key. +- Both tables must be the same engine family. (replicated or non-replicated) +- Both tables must have the same storage policy. + + + #### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} ```sql diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index 3ca6249acfa..2705fd123c1 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -259,7 +259,9 @@ SELECT a, b, c FROM (SELECT ...) Materialized views store data transformed by the corresponding SELECT query. -When creating a materialized view, you must specify ENGINE – the table engine for storing data. +When creating a materialized view without `TO [db].[table]`, you must specify ENGINE – the table engine for storing data. + +When creating a materialized view with `TO [db].[table]`, you must not use `POPULATE`. A materialized view is arranged as follows: when inserting data to the table specified in SELECT, part of the inserted data is converted by this SELECT query, and the result is inserted in the view. diff --git a/docs/en/query_language/functions/rounding_functions.md b/docs/en/query_language/functions/rounding_functions.md index 3fe58a05c46..ec365235381 100644 --- a/docs/en/query_language/functions/rounding_functions.md +++ b/docs/en/query_language/functions/rounding_functions.md @@ -78,6 +78,89 @@ round(3.55, 1) = 3.6 round(3.65, 1) = 3.6 ``` +**See Also** + +- [roundBankers](#roundbankers) + +## roundBankers {#roundbankers} + +Rounds a number to a specified decimal position. + +- If rounding number is a half between two numbers, the function uses banker's rounding. + + Banker's rounding is a method of rounding fractional numbers. When the rounding number is a half between two numbers, it is rounded to the nearest even number. E.g. 3.5 rounds up to 4, 2.5 rounds down to 2. + +- In other cases function rounds numbers to the nearest integer. + +Using banker's rounding, you can reduce the effect of rounding numbers on the result of summing or subtracting these numbers. + +For example, sum numbers 1.5, 2.5, 3.5, 4.5 with different rounding: + +- No rounding: 1.5 + 2.5 + 3.5 + 4.5 = 12. +- Banker's rounding: 2 + 2 + 4 + 4 = 12. +- Rounding to the nearest integer: 2 + 3 + 4 + 5 = 14. + +**Syntax** + +```sql +roundBankers(expression [, decimal_places]) +``` + +**Parameters** + +- `expression` — A number to be rounded. Can be any [expression](../syntax.md#syntax-expressions) returning the numeric [data type](../../data_types/index.md#data_types). +- `decimal-places` — Decimal places. An integer number. + - `decimal-places > 0` — The function rounds the number to the given position right of the decimal point. E.g. `roundBankers(3.55, 1) = 3.6`. + - `decimal-places < 0` — The function rounds the number to the given position left of the decimal point. E.g. `roundBankers(24.55, -1) = 20`. + - `decimal-places = 0` — The function rounds the number to integer. In this case the argument can be omitted. E.g. `roundBankers(2.5) = 2`. + +**Returned value** + +A value rounded by banker's rounding method. + +### Examples + +**Example of use** + +Query: + +```sql + SELECT number / 2 AS x, roundBankers(x, 0) AS b fROM system.numbers limit 10 +``` + +Result: + +```text +┌───x─┬─b─┐ +│ 0 │ 0 │ +│ 0.5 │ 0 │ +│ 1 │ 1 │ +│ 1.5 │ 2 │ +│ 2 │ 2 │ +│ 2.5 │ 2 │ +│ 3 │ 3 │ +│ 3.5 │ 4 │ +│ 4 │ 4 │ +│ 4.5 │ 4 │ +└─────┴───┘ +``` + +**Examples of Banker's rounding** + +```text +roundBankers(0.4) = 0 +roundBankers(-3.5) = -4 +roundBankers(4.5) = 4 +roundBankers(3.55, 1) = 3.6 +roundBankers(3.65, 1) = 3.6 +roundBankers(10.35, 1) = 10.4 +roundBankers(10.755, 2) = 11,76 +``` + +**See Also** + +- [round](#rounding_functions-round) + ## roundToExp2(num) Accepts a number. If the number is less than one, it returns 0. Otherwise, it rounds the number down to the nearest (whole non-negative) degree of two. @@ -95,9 +178,3 @@ Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rou Accept a number, round it down to an element in the specified array. If the value is less than the lowest bound, the lowest bound is returned. [Original article](https://clickhouse.yandex/docs/en/query_language/functions/rounding_functions/) - -## roundBankers(x\[, N\]) - -Rounds a value to a specified number of decimal places. - -The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function always return the number having the nearest even digit (banker's rounding). diff --git a/docs/en/query_language/table_functions/file.md b/docs/en/query_language/table_functions/file.md index b4f74bd2d8c..6ad324bb0d9 100644 --- a/docs/en/query_language/table_functions/file.md +++ b/docs/en/query_language/table_functions/file.md @@ -98,4 +98,13 @@ SELECT count(*) FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') ``` +## Virtual Columns + +- `_path` — Path to the file. +- `_file` — Name of the file. + +**See Also** + +- [Virtual columns](https://clickhouse.yandex/docs/en/operations/table_engines/#table_engines-virtual_columns) + [Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/file/) diff --git a/docs/en/query_language/table_functions/hdfs.md b/docs/en/query_language/table_functions/hdfs.md index b069ce5a9c0..2c13ef0c95b 100644 --- a/docs/en/query_language/table_functions/hdfs.md +++ b/docs/en/query_language/table_functions/hdfs.md @@ -83,4 +83,13 @@ SELECT count(*) FROM hdfs('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') ``` +## Virtual Columns + +- `_path` — Path to the file. +- `_file` — Name of the file. + +**See Also** + +- [Virtual columns](https://clickhouse.yandex/docs/en/operations/table_engines/#table_engines-virtual_columns) + [Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/hdfs/) diff --git a/docs/fa/development/browse_code.md b/docs/fa/development/browse_code.md new file mode 120000 index 00000000000..8c08c622129 --- /dev/null +++ b/docs/fa/development/browse_code.md @@ -0,0 +1 @@ +../../en/development/browse_code.md \ No newline at end of file diff --git a/docs/fa/interfaces/cli.md b/docs/fa/interfaces/cli.md index e5f869e1c0d..4c366ac0104 100644 --- a/docs/fa/interfaces/cli.md +++ b/docs/fa/interfaces/cli.md @@ -53,7 +53,7 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA شما میتوانید از `\G` به جای سیمی کالن یا بعد از سیمی کالن استفاده کنید. این علامت، فرمت Vertical را نشان می دهد. در این فرمت، هر مقدار در یک خط جدا چاپ می شود که برای جداول عریض مناسب است. این ویژگی غیرمعمول برای سازگاری با MySQL CLI اضافه شد. -command line برا پایه 'readline' (و 'history' یا 'libedit'، یه بدون کتابخانه بسته به build) می باشد. به عبارت دیگر، این محیط از shortcut های آشنا استفاده می کند و history دستورات را نگه می دار. history ها در فایل ~/.clickhouse-client-history نوشته می شوند. +command line برا پایه 'replxx' می باشد. به عبارت دیگر، این محیط از shortcut های آشنا استفاده می کند و history دستورات را نگه می دار. history ها در فایل ~/.clickhouse-client-history نوشته می شوند. به صورت پیش فرض فرمت خروجی PrettyCompact می باشد. شما میتوانید از طریق دستور FORMAT در یک query، یا با مشخص کردن `\G` در انتهای query، استفاده از آرگومان های `--format` یا `--vertical` یا از کانفیگ فایل کلاینت، فرمت خروجی را مشخص کنید. diff --git a/docs/ja/development/browse_code.md b/docs/ja/development/browse_code.md new file mode 120000 index 00000000000..8c08c622129 --- /dev/null +++ b/docs/ja/development/browse_code.md @@ -0,0 +1 @@ +../../en/development/browse_code.md \ No newline at end of file diff --git a/docs/ru/data_types/datetime.md b/docs/ru/data_types/datetime.md index 19d701ddad7..e2bc3e1482b 100644 --- a/docs/ru/data_types/datetime.md +++ b/docs/ru/data_types/datetime.md @@ -1,16 +1,83 @@ # DateTime {#data_type-datetime} -Дата-с-временем. Хранится в 4 байтах, в виде (беззнакового) unix timestamp. Позволяет хранить значения в том же интервале, что и для типа Date. Минимальное значение выводится как 0000-00-00 00:00:00. -Время хранится с точностью до одной секунды (без учёта секунд координации). +Позволяет хранить момент времени, который может быть представлен как календарная дата и время. `DateTime` позволяет учесть часовой пояс для хранимых значений. -## Часовые пояса +Синтаксис: -Дата-с-временем преобразуется из текстового (разбитого на составляющие) в бинарный вид и обратно, с использованием системного часового пояса на момент старта клиента или сервера. В текстовом виде, теряется информация о том, был ли произведён перевод стрелок. +```sql +DateTime([timezone]) +``` -По умолчанию клиент переключается на часовой пояс сервера при подключении. Это поведение можно изменить, включив у клиента параметр командной строки `--use_client_time_zone`. +Диапазон значений: [1970-01-01 00:00:00, 2105-12-31 23:59:59]. -Поддерживаются только часовые пояса, для которых для всего диапазона времён, с которым вы будете работать, не существовало моментов времени, в которые время отличалось от UTC на нецелое число часов (без учёта секунд координации). +Точность: 1 секунда. -То есть, при работе с датой в виде текста (например, при сохранении текстовых дампов), следует иметь ввиду о проблемах с неоднозначностью во время перевода стрелок назад, и о проблемах с соответствием данных, при смене часового пояса. +## Использование + +Момент времени сохраняется как Unix timestamp, независимо от часового пояса и переходов на летнее/зимнее время. Дополнительно, `DateTime` позволяет хранить часовой пояс, который влияет на то, как буду отображаться значения типа `DateTime` в текстовом виде и как будут парситься входные строки. Список поддержанных временных зон можно найти в [IANA Time Zone Database](https://www.iana.org/time-zones). + +Часовой пояс для столбца типа `DateTime` можно в явном виде установить при создании таблицы. Если часовой пояс не установлен, то ClickHouse использует значение параметра [timezone](../operations/server_settings/settings.md#server_settings-timezone), установленное в конфигурации сервера или в настройках операционной системы на момент запуска сервера. + +Консольный клиент ClickHouse по умолчанию использует часовой пояс сервера, если для значения `DateTime` часовой пояс не был задан в явном виде при инициализации типа данных. Чтобы использовать часовой пояс клиента, запустите [clickhouse-client](../interfaces/cli.md) с параметром `--use_client_time_zone`. + +ClickHouse по умолчанию выводит значение в формате `YYYY-MM-DD hh:mm:ss`. Формат можно поменять с помощь функции [formatDateTime](../query_language/functions/date_time_functions.md#formatdatetime). + +При вставке данных в ClickHouse, можно использовать различные форматы даты и времени в зависимости от значения настройки [date_time_input_format](../operations/settings/settings.md#settings-date_time_input_format). + +## Примеры + +**1.** Создание таблицы с столбцом типа `DateTime` и вставка данных в неё: + +CREATE TABLE dt +( + `timestamp` DateTime('Europe/Moscow'), + `event_id` UInt8 +) +ENGINE = TinyLog +``` +```sql +INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2) +``` +```sql +SELECT * FROM dt +``` +```text +┌───────────timestamp─┬─event_id─┐ +│ 2019-01-01 03:00:00 │ 1 │ +│ 2019-01-01 00:00:00 │ 2 │ +└─────────────────────┴──────────┘ +``` + +Unix timestamp `1546300800` в часовом поясе `Europe/London (UTC+0)` представляет время `'2019-01-01 00:00:00'`, однако столбец `timestamp` хранит время в часовом поясе `Europe/Moscow (UTC+3)`, таким образом значение, вставленное в виде Unix timestamp, представляет время `2019-01-01 03:00:00`. + +```sql +SELECT * FROM dt WHERE timestamp = toDateTime('2019-01-01 00:00:00', 'Europe/Moscow') +``` +```text +┌───────────timestamp─┬─event_id─┐ +│ 2019-01-01 00:00:00 │ 2 │ +└─────────────────────┴──────────┘ +``` + +**2.** Получение часового пояса для значения типа `DateTime`: + +```sql +SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x +``` +```text +┌──────────────column─┬─x─────────────────────────┐ +│ 2019-10-16 04:12:04 │ DateTime('Europe/Moscow') │ +└─────────────────────┴───────────────────────────┘ +``` + +## See Also + +- [Функции преобразования типов](../query_language/functions/type_conversion_functions.md) +- [Функции для работы с датой и временем](../query_language/functions/date_time_functions.md) +- [Функции для работы с массивами](../query_language/functions/array_functions.md) +- [Настройка `date_time_input_format`](../operations/settings/settings.md#settings-date_time_input_format) +- [Конфигурационный параметр сервера `timezone`](../operations/server_settings/settings.md#server_settings-timezone) +- [Операторы для работы с датой и временем](../query_language/operators.md#operators-datetime) +- [Тип данных `Date`](date.md) [Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/datetime/) diff --git a/docs/ru/development/browse_code.md b/docs/ru/development/browse_code.md new file mode 100644 index 00000000000..e18ba21b889 --- /dev/null +++ b/docs/ru/development/browse_code.md @@ -0,0 +1,7 @@ +# Навигация по коду ClickHouse + +Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. + +Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). + +Если вы интересуетесь, какую среду разработки выбрать для работы с ClickHouse, мы рекомендуем CLion, QT Creator, VSCode или KDevelop (с некоторыми предостережениями). Вы можете использовать свою любимую среду разработки, Vim и Emacs тоже считаются. diff --git a/docs/ru/development/developer_instruction.md b/docs/ru/development/developer_instruction.md index 61be36a7089..4686847041d 100644 --- a/docs/ru/development/developer_instruction.md +++ b/docs/ru/development/developer_instruction.md @@ -5,6 +5,10 @@ Если вы используете Windows, вам потребуется создать виртуальную машину с Ubuntu. Для работы с виртуальной машиной, установите VirtualBox. Скачать Ubuntu можно на сайте: https://www.ubuntu.com/#download Создайте виртуальную машину из полученного образа. Выделите для неё не менее 4 GB оперативной памяти. Для запуска терминала в Ubuntu, найдите в меню программу со словом terminal (gnome-terminal, konsole или что-то в этом роде) или нажмите Ctrl+Alt+T. +# Если вы используете 32-битную систему + +ClickHouse не работает и не собирается на 32-битных системах. Получите доступ к 64-битной системе и продолжайте. + # Создание репозитория на GitHub @@ -96,15 +100,7 @@ brew install cmake ninja # Необязательные внешние библиотеки -ClickHouse использует для сборки некоторое количество внешних библиотек. Большинство из них не требуется отдельно устанавливать, так как они собираются вместе с ClickHouse, из исходников, которые расположены в submodules. Посмотреть набор этих библиотек можно в директории contrib. - -Одна библиотека не собирается из исходников, а используется из системы: Readline, и её рекомендуется установить. - -Ubuntu: `sudo apt install libreadline-dev` - -Mac OS X: `brew install readline` - -Впрочем, эти библиотеки не обязательны для работы и ClickHouse может быть собран без них. ICU используется для поддержки `COLLATE` в `ORDER BY` (например, для сортировки с учётом турецкого алфавита). Readline используется для более удобного набора команд в интерактивном режиме в clickhouse-client. +ClickHouse использует для сборки некоторое количество внешних библиотек. Но ни одну из них не требуется отдельно устанавливать, так как они собираются вместе с ClickHouse, из исходников, которые расположены в submodules. Посмотреть набор этих библиотек можно в директории contrib. # Компилятор C++ diff --git a/docs/ru/extended_roadmap.md b/docs/ru/extended_roadmap.md index d5cbcf5f8c8..145779cfccb 100644 --- a/docs/ru/extended_roadmap.md +++ b/docs/ru/extended_roadmap.md @@ -16,9 +16,11 @@ Изначально делал [Андрей Чулков](https://github.com/achulkov2), ВШЭ, теперь доделывает [Ольга Хвостикова](https://github.com/stavrolia), но сроки немного сдвинуты из-за задачи 25.9. Будем надеятся на реализацию к концу ноября. Впрочем, [Андрей Чулков](https://github.com/achulkov2) скоро сможет помочь её доделать. +Upd. Доделывать будет другой человек. Хотелось бы в Q1, но приоритет не высокий. + ### 1.2. Wait-free каталог баз данных. -Делает [Александр Токмаков](https://github.com/tavplubix), первый рабочий вариант в декабре 2019. Нужно для DataLens и Яндекс.Метрики. +Q1. Делает [Александр Токмаков](https://github.com/tavplubix), первый рабочий вариант в декабре 2019. Нужно для DataLens и Яндекс.Метрики. Манипуляции с каталогом баз данных: запросы CREATE TABLE, DROP TABLE, RENAME TABLE и DATABASE, требуют синхронизации с помощью блокировок. Эта синхронизация становится весьма сложной, так как на неё полагается много внутренних структур данных. @@ -26,7 +28,7 @@ ### 1.3. Неблокирующие ALTER. -И полностью immutable куски. Делает [Александр Сапин](https://github.com/alesapin). Готов приступить к задаче в конце ноября 2019. Нужно для Яндекс.Метрики. +Q1. И полностью immutable куски. Делает [Александр Сапин](https://github.com/alesapin). Готов приступить к задаче в конце ноября 2019. Нужно для Яндекс.Метрики. ### 1.4. Нетранзитивные ALTER столбцов. @@ -38,6 +40,8 @@ ### 1.6. Полиморфные куски данных. +Компактные куски - Q1, куски в оперативке Q1/Q2. + Делает [Антон Попов](https://github.com/CurtizJ), первый рабочий вариант в декабре. Пререквизит чтобы снизить сложность мелких INSERT, что в свою очередь нужно для 1.12, иначе задача 1.12 не сможет нормально работать. Особенно нужно для Яндекс.Облака. Данные в таблицах типа MergeTree в ClickHouse хранятся в виде набора независимых "кусков". Внутри куска, каждый столбец, а также индекс, хранится в отдельных файлах. Это сделано для возможности быстрых манипуляций со столбцами (пример - запрос ALTER DROP COLUMN). При вставке данных (INSERT), создаётся новый кусок. Для таблиц с большим количеством столбцов, запросы INSERT с маленьким количеством строк являются неэффективными, так как требуют создания большого количества файлов в файловой системе. Это является врождённой особенностью ClickHouse - одной из первой проблем, с которыми сталкиваются пользователи. Пользователям приходится буферизовывать данные и собирать их в более крупные пачки перед вставкой в ClickHouse. @@ -54,6 +58,8 @@ Делает [Владимир Чеботарёв](https://github.com/excitoon), Altinity. Декабрь 2019. +Q1. Закоммичено, но есть технический долг, который исправляется сейчас. + ### 1.9. Использование TTL для прореживания данных. Будет делать Сорокин Николай, ВШЭ и Яндекс. @@ -86,6 +92,8 @@ ### 1.11. Виртуальная файловая система. +Q2. + Нужно для Яндекс.Облака. Делает Александр, Яндекс.Облако, а также Олег Ершов, ВШЭ и Яндекс. ClickHouse использует для хранения данных локальную файловую систему. Существует сценарий работы, в котором размещение старых (архивных) данных было бы выгодно на удалённой файловой системе. Если файловая система POSIX совместимая, то это не составляет проблем: ClickHouse успешно работает с Ceph, GlusterFS, MooseFS. Также востребованным является сценарий использования S3 (из-за доступности в облаке) или HDFS (для интеграции с Hadoop). Но эти файловые системы не являются POSIX совместимыми. Хотя для них существуют FUSE драйверы, но скорость работы сильно страдает и поддержка неполная. @@ -94,6 +102,8 @@ ClickHouse использует небольшое подмножество фу ### 1.12. Экспериментальная реализация VFS поверх S3 и HDFS. +Q2. + Нужно для Яндекс.Облака. Требует 1.11. Желательно 1.6 и 1.18. Делает Александр, Яндекс.Облако (сначала часть для S3), а также Олег Ершов, ВШЭ и Яндекс. @@ -103,8 +113,10 @@ ClickHouse использует небольшое подмножество фу ### 1.14. Не писать столбцы, полностью состоящие из нулей. +Антон Попов. Q1/Q2. В очереди. Простая задача, является небольшим пререквизитом для потенциальной поддержки полуструктурированных данных. + ### 1.15. Возможность иметь разный первичный ключ в разных кусках. Сложная задача, только после 1.3. @@ -130,6 +142,11 @@ ClickHouse использует небольшое подмножество фу Делает [Николай Кочетов](https://github.com/KochetovNicolai). Финальная стадия разработки. Включение по-умолчанию в конце декабря 2019. Удаление старого кода в начале 2020. +Upd. На данный момент исправляются проблемы с регрессиями производительности в отдельных случаях. Кажется, что все проблемы исправлены. +Включение по-умолчанию в Q1, но остаётся вторая часть задачи по корректному выделению async части. + +Upd. Включили по-умолчанию. Удаление старого кода не раньше, чем после первого релиза, в котором это включено по-умолчанию и всё ещё можно выключить обратно. + ### 2.2. Инфраструктура событий/метрик/ограничений/квот/трассировки. В очереди. https://gist.github.com/alexey-milovidov/d62d73222d83b9319dc519cbb13aeff6 @@ -152,11 +169,11 @@ ClickHouse использует небольшое подмножество фу ### 2.7. Нормализация Context. -В очереди. +В очереди. Нужно для YQL. ### 2.8. Декларативный парсер запросов. -Низкий приоритет. Задачу хочет сделать [Иван Лежанкин](https://github.com/abyss7) в свободное время, но пока ничего нет. +Средний приоритет. Нужно для YQL. ### 2.9. Логгировние в format-стиле. @@ -193,15 +210,17 @@ ClickHouse использует небольшое подмножество фу ### 3.1. Перенос документации по функциям в код. -Требует 2.12 и 2.13. +Требует 2.12 и 2.13. Хотим в Q1/Q2, средний приоритет. ### 3.2. Перенос однородных частей документации в код. Требует 3.1. -### 3.3. Исправить катастрофически отвратительно неприемлемый поиск по документации. +### + 3.3. Исправить катастрофически отвратительно неприемлемый поиск по документации. -[Иван Блинков](https://github.com/blinkov/) - очень хороший человек. Сам сайт документации основан на технологиях, не удовлетворяющих требованиям задачи, и эти технологии трудно исправить. +[Иван Блинков](https://github.com/blinkov/) - очень хороший человек. Сам сайт документации основан на технологиях, не удовлетворяющих требованиям задачи, и эти технологии трудно исправить. Задачу будет делать первый встретившийся нам frontend разработчик, которого мы сможем заставить это сделать. + +Upd. Иван Блинков сделал эту задачу путём замены треш-технологий на нормальные. ### 3.4. + Добавить японский язык в документацию. @@ -212,7 +231,7 @@ ClickHouse использует небольшое подмножество фу ### 4.1. Уменьшение числа потоков при распределённых запросах. -[Никита Лапков](https://github.com/laplab), весна 2020. +[Никита Лапков](https://github.com/laplab), весна 2020. Upd. Есть прототип. ### 4.2. Спекулятивное выполнение запросов на нескольких репликах. @@ -231,7 +250,7 @@ ClickHouse использует небольшое подмножество фу ### 4.4. Ограничение сетевой полосы при репликации. -Дмитрий Григорьев, ВШЭ. +Дмитрий Григорьев, ВШЭ. Нужно для Метрики. ### 4.5. Возможность продолжить передачу куска данных при репликации после сбоя. @@ -251,19 +270,20 @@ ClickHouse использует небольшое подмножество фу ### 5.1. Разделение задач на более мелкие куски в clickhouse-copier. -Нужно для Метрики, в очереди, но исполнитель не назначен, есть шанс успеть в 2019. +Q1. Нужно для Метрики, в очереди. Никита Михайлов. ### 5.2. Автонастройка лимита на оперативку и размера кэшей. -### 5.3. Встроенная ручка для Prometheus и, возможно, Solomon. +### 5.3. + Встроенная ручка для Prometheus. -Простая задача. https://github.com/Vdimir +Сделано. https://github.com/Vdimir ### 5.4. Opt-in сообщать в клиенте, если вышла новая версия. -### 5.5. LTS релизы. +### 5.5. + LTS релизы. Требует 7.5. Задачу хочет Метрика, Облако, БК, Маркет и Altinity. Первой LTS версией уже стала версия 19.14. +Метрика, БК, Маркет уже используют более свежие версии чем LTS. ## 6. Инструментирование. @@ -271,15 +291,15 @@ ClickHouse использует небольшое подмножество фу ### 6.1. Исправления сэмплирующего профайлера запросов. Михаил Филимонов, Altinity. Ноябрь 2019. Сделано. -Осталось ещё проверить работоспособность профайлера в первом потоке (что важно для INSERT). +Осталось ещё проверить работоспособность профайлера в первом потоке (что важно для INSERT). Иван Лежанкин. Q1. ### 6.2. Добавление memory profiler. -Сравнительно простая задача, но только для опытных разработчиков. Нужна всем. +Сравнительно простая задача, но только для опытных разработчиков. Нужна всем. Иван Лежанкин. Q1. ### 6.3. Учёт оперативки total расширить не только на запросы. -Исправление долгоживущей проблемы с дрифтом учёта оперативки. Нужна для Метрики и БК. +Исправление долгоживущей проблемы с дрифтом учёта оперативки. Нужна для Метрики и БК. Иван Лежанкин. Q1. ### 6.4. Поддержка perf events как метрик запроса. @@ -291,11 +311,11 @@ ClickHouse использует небольшое подмножество фу Требует 2.2. -### 6.6. Стек трейс для любых исключений. +### 6.6. + Стек трейс для любых исключений. Сейчас есть стек трейс для почти всех, но не всех исключений. Требует 7.4. -### 6.7. Таблица system.stack_trace. +### 6.7. + Таблица system.stack_trace. Сравнительно простая задача, но только для опытных разработчиков. @@ -327,7 +347,7 @@ ClickHouse использует небольшое подмножество фу Сейчас включено только при сборке с clang, но продакшен сборка использует gcc. Требует 7.2 и, возможно, 7.1 (только в случае новой версии ICU). -### 7.5. Начать публиковать LTS релизы. +### 7.5. + Начать публиковать LTS релизы. [Александр Сапин](https://github.com/alesapin). @@ -364,11 +384,17 @@ UBSan включен в функциональных тестах, но не в При сборке с clang, -Weverything уже включено. Но в gcc есть уникальные warning-и, отсутствующие в clang. Сделал Wolf Kreuzerkrieg. -### 7.14. Альтернатива для readline и libedit. +### 7.14. + Альтернатива для readline и libedit. -Тагир Кускаров, ВШЭ. Посмотрим на https://github.com/AmokHuginnsson/replxx +Подключение replxx вместо readline сделал Иван Лежанкин. -Для ввода запросов в интерактивном режиме в клиенте командной строки clickhouse-client используется библиотека readline или libedit. +### 7.14.1. Улучшение возможностей интерактивного режима clickhouse-client. + +Тагир Кускаров, ВШЭ. + +Upd. В рамках данной задачи добавляем подстветку синтаксиса и исправление проблем со вставкой больших запросов. + +Для ввода запросов в интерактивном режиме в клиенте командной строки clickhouse-client использовалась библиотека readline или libedit. Библиотеки readline и libedit обладает следующими недостатками: - (исправлено в новых версиях readline) Очень низкая производительность вставки больших кусков текста. Вставка каждого следующего символа имеет сложность O(n = количество предыдущих символов) и при вставке 1 МБ текста, скорость падает до десятков байт в секунду. @@ -407,11 +433,13 @@ UBSan включен в функциональных тестах, но не в Уже давно собираются универсальные tgz пакеты, но по нелепой случайности из них исчез install скрипт. [Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. +Upd. Сделано всё кроме инструкции на сайте. Для этого требуется создать директории testing/stable/prestable на repo.yandex.ru. Внезапно оказалось, что человек, отвечающий за это, в отпуске, и он не отвечает на вопрос, кто его заместитель. Q1. ### 7.18.1. Доделать бинарники под Mac. Уже есть автосборка бинарников под Mac на каждый коммит и PR, но с недостатками. [Иван Лежанкин](https://github.com/abyss7). Требует 7.1, 7.2. Рекомендуется 7.14. Сейчас не хватает по крайней мере SSL и ICU. Нужно для Яндекс.Облака. +Upd. Сделано SSL. Ориентируемся в Q1, но приоритет средний и может потеряться. ### 7.18. Поместить ссылку на собранные бинарники под Mac на сайт. @@ -480,6 +508,8 @@ Fuzzing тестирование - это тестирование случай Также можно сделать функции с детерминированным генератором случайных чисел (аргументом передаётся seed) для воспроизводимости тестовых кейсов. +Upd. Сергей Штыков сделал функцию `randomPrintableASCII`. + ### 7.24. Fuzzing лексера и парсера запросов; кодеков и форматов. Андрей Некрашевич, ВШЭ. @@ -496,6 +526,9 @@ Fuzzing тестирование - это тестирование случай Затем, возможно, [Иван Лежанкин](https://github.com/abyss7). Но сейчас приостановлено, так как Максим из YT должен исправить регрессию производительности в анализе индекса. Максим из YT сказал, что сделает это после нового года. +Максим из YT сказал, что "мы планируем в январе добиться". + +Нужно для CHYT и YQL. ### 7.26. Побайтовая идентичность репозитория с Аркадией. @@ -529,11 +562,11 @@ Fuzzing тестирование - это тестирование случай ### 7.33. Выкладывать патч релизы в репозиторий автоматически. -[Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. +В очереди. Иван Лежанкин. ### 7.34. Бэкпортировать bugfix автоматически. -[Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. +В очереди. Иван Лежанкин. ### 7.35. Начальные правила для авто-merge. @@ -564,6 +597,8 @@ Altinity. [Ольга Хвостикова](https://github.com/stavrolia). +Уменьшение числа stream-ов сделано, а вот правильная поддержка диапазонов - нет. Будем надеяться на Q1/Q2. + ### 8.4. Унификация File, HDFS, S3 под URL. ### 8.5. + Аутентификация в S3. @@ -579,7 +614,7 @@ Altinity. В ядрах 2.6 отсутствует один системный вызов, который библиотека hdfs3 использует без необходимости. Сделал Amos Bird. -### 8.8. Поддержка виртуальных столбцов с именем файла и путём. +### 8.8. + Поддержка виртуальных столбцов с именем файла и путём. [Ольга Хвостикова](https://github.com/stavrolia). @@ -611,10 +646,26 @@ Altinity. ### 8.16. Поддержка формата Avro. -Павел Круглов, ВШЭ и Яндекс. +Andrew Onyshchuk. Есть pull request. Q1. Формат Apache Avro является компактным структурированным построчным бинарным форматом данных с внешней схемой. Этот формат часто используется совместно с Kafka и поддержка его в качестве одного из форматов ввода-вывода в ClickHouse является востребованной пользователями. +### 8.16.1. Поддержка формата JSONEachRow, засунутого в массив. + +Павел Круглов, ВШЭ и Яндекс. + +### 8.16.2. Поддержка формата Thrift. + +Павел Круглов, ВШЭ и Яндекс. + +### 8.16.3. Поддержка формата MsgPack. + +Павел Круглов, ВШЭ и Яндекс. + +### 8.16.4. Формат Regexp. + +Павел Круглов, ВШЭ и Яндекс. + ### 8.17. ClickHouse как MySQL реплика. Ильяс Адюгамов, ВШЭ. @@ -642,6 +693,7 @@ Maxim Fedotov, Wargaming + Yuri Baranov, Яндекс. ### 8.21. Поддержка произвольного количества языков для имён регионов. Нужно для БК. Декабрь 2019. +В декабре для БК сделан минимальный вариант этой задачи. ### 8.22. Поддержка синтаксиса для переменных в стиле MySQL. @@ -689,7 +741,7 @@ ClickHouse предоставляет возможность обратитьс ### 10.4. Словарь из YDB (KikiMR). -Нужно для Метрики, а делать будет таинственный незнакомец из команды KikiMR (под вопросом). +Нужно для Метрики, а делать будет таинственный незнакомец из команды KikiMR (под вопросом). Таинственный незнакомец не подтверждает, что он будет делать эту задачу. ### 10.5. Закрытие соединений и уменьшение числа соединений для MySQL и ODBC. @@ -707,15 +759,15 @@ ClickHouse предоставляет возможность обратитьс ### 10.9. Уменьшение блокировок для cache словарей за счёт одновременных запросов одного и того же. -Нужно для БК, но мотивация задачи находится под вопросом, так как есть рабочее предположение о том, что данная задача не устраняет причину проблемы. +Никита Михайлов. Q1. Нужно для БК, но мотивация задачи находится под вопросом, так как есть рабочее предположение о том, что данная задача не устраняет причину проблемы. ### 10.10. Возможность использования старых значений из cache словаря пока они перезапрашиваются. -Нужно для БК и Метрики. +Никита Михайлов. Q1. Нужно для БК и Метрики. ### 10.11. Возможность исключительно асинхронных запросов в cache словарях. -Нужно для БК и Метрики. Требует 10.10. +Никита Михайлов. Q1. Нужно для БК и Метрики. Требует 10.10. ### 10.12. Layout direct для словарей. @@ -731,7 +783,7 @@ ClickHouse предоставляет возможность обратитьс ### 10.16. Словари на локальном SSD. -Никита Васильев, ВШЭ и Яндекс. +Никита Васильев, ВШЭ и Яндекс. Есть pull request. Реализовать в ClickHouse специализированный движок таблиц, подходящий для быстрых key-value запросов и оптимизированный для расположения данных на SSD. Это может быть: реализация на основе RocksDB; сериализованные RowBinary данные с индексом в оперативке; секретная очень эффективная структура данных, о которой я расскажу. @@ -778,7 +830,7 @@ ClickHouse предоставляет возможность обратитьс ### 11.10. Преднастроенные HTTP handlers для запросов. -zhang2014 +zhang2014, есть pull request. Возможность описать в конфигурационном файле handler (путь в URL) для HTTP запросов к серверу, которому соответствует некоторый параметризованный запрос. Пользователь может вызвать этот обработчик и не должен передавать SQL запрос. @@ -787,15 +839,18 @@ zhang2014 ### 12.1. Role Based Access Control. -[Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в декабре 2019. +[Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в начале февраля 2019. +Q1. Сейчас сделаны все интерфейсы в коде и запросы, но не сделаны варианты хранения прав кроме прототипа. ### 12.2. Управление пользователями и правами доступа с помощью SQL запросов. [Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в декабре 2019. +Q1. ### 12.3. Подключение справочника пользователей и прав доступа из LDAP. [Виталий Баранов](https://github.com/vitlibar). Требует 12.1. +Q1/Q2. ### 12.4. Подключение IDM системы Яндекса как справочника пользователей и прав доступа. @@ -823,6 +878,7 @@ zhang2014 ### 13.3. Пулы ресурсов. Требует 13.2 или сможем сделать более неудобную реализацию раньше. +Обсуждается вариант неудобной реализации. Пока средний приоритет, целимся на Q1/Q2. ## 14. Диалект SQL. @@ -830,9 +886,12 @@ zhang2014 ### 14.1. Исправление семантики CAST для Nullable. Нужно для DataLens. А также для внедрения в BI инструмент Looker. +Павел Потёмкин, ВШЭ. ### 14.2. Поддержка WITH для подзапросов. +Павел Потёмкин, ВШЭ. + ### 14.3. Поддержка подстановок для множеств в правой части IN. ### 14.4. Поддержка подстановок для идентификаторов (имён) в SQL запросе. @@ -845,8 +904,12 @@ zhang2014 ### 14.6. Глобальный scope для WITH. +Павел Потёмкин, ВШЭ. + ### 14.7. Nullable для WITH ROLLUP, WITH CUBE, WITH TOTALS. +Павел Потёмкин, ВШЭ. + Простая задача. ### 14.8. Модификаторы DISTINCT, ORDER BY для агрегатных функций. @@ -893,18 +956,23 @@ zhang2014. ### 14.18. UNION DISTINCT и возможность включить его по-умолчанию. +Павел Потёмкин, ВШЭ. Для BI систем. ### 14.19. Совместимость парсера типов данных с SQL. +Павел Потёмкин, ВШЭ. Для BI систем. ### 14.20. Позиционные аргументы для GROUP BY и ORDER BY. +Павел Потёмкин, ВШЭ. Тривиально и используется многими системами, но не входит в стандарт SQL. ### 14.21. Приведение типов для IN (подзапрос) и для JOIN. +Павел Потёмкин, ВШЭ. + ## 15. Улучшение поддержки JOIN. @@ -912,6 +980,15 @@ zhang2014. Артём Зуйков. Сейчас merge JOIN включается вручную опцией и всегда замедляет запросы. Хотим, чтобы он замедлял запросы только когда это неизбежно. Кстати, смысл merge JOIN появляется только совместно с 15.2 и 15.3. +Q1. + +### 15.1.1. Алгоритм two-level merge JOIN. + +Александр Кузьменков. В очереди. + +### 15.1.2. Тестирование реализации JOIN в Greenplum. + +В очереди. ### 15.2. Прокидывание условий в OUTER JOIN. @@ -934,7 +1011,7 @@ zhang2014. ## 16. Типы данных и функции. -### 16.1. DateTime64. +### 16.1. + DateTime64. Василий Немков, Altinity, декабрь 2019. @@ -964,6 +1041,8 @@ ClickHouse не является geospatial СУБД. Тем не менее, в Похожая, но более сложная задача, которую ClickHouse пока не умеет решать - определение полигона среди множества полигонов, в которые попадают точки. Для примера: определение района города по географическим координатам. Для решения этой задачи нужно будет реализовать поддержку словарей с полигонами, в которых данные проиндексированы для быстрого поиска. +Upd. Андрей сделал прототип интерфейса и реализацию-заглушку внутри него. + ### 17.2. GIS типы данных и операции. Алексей Коряков, Алексей Илюхов, ВШЭ, Яндекс.Карты. @@ -1070,13 +1149,15 @@ Hold. Полезно для заказчиков внутри Яндекса, н Начинал Олег Ершов, доделывает Никита Михайлов, помогает [Александр Кузьменков](https://github.com/akuzm). Готово. +### 21.1.1. Избавление от лишнего копирование при параллельном парсинге форматов, если возможен mmap файла целиком. + ### 21.2. Параллельное форматирование форматов. После 21.1, предположительно Никита Михайлов. Задача сильно проще чем 21.1. ### 21.3. Исправление низкой производительности анализа индекса в случае большого множества в секции IN. -Нужно всем (Zen, БК, DataLens...) Пока ещё не выбран исполнитель. +Нужно всем (Zen, БК, DataLens, TestEnv...). Антон Попов, Q1/Q2. ### 21.4. Использование ORDER BY ключа для оптимизации GROUP BY и DISTINCT. @@ -1091,12 +1172,14 @@ Hold. Полезно для заказчиков внутри Яндекса, н ### 21.5. Распараллеливание INSERT при INSERT SELECT, если это необходимо. [Vxider](https://github.com/Vxider), ICT +Есть pull request. ### 21.6. Уменьшение числа потоков для SELECT в случае тривиального INSERT SELECT. ### 21.7. Кэш результатов запросов. [Achimbab](https://github.com/achimbab). +Есть pull request. ### 21.8. Взаимная интеграция аллокатора и кэша. @@ -1131,6 +1214,8 @@ Amos Bird. - Вынесение любых функций наружу any, anyLast. - При GROUP BY по transform или if по строкам, замена строк на Enum. +Сделана замена цепочек if на multiIf, но внезапно оказалось, что это является не оптимизацией, а наоборот. + ### 21.12. Алгебраические оптимизации запросов. Руслан Камалов, Михаил Малафеев, Виктор Гришанин, ВШЭ @@ -1200,9 +1285,9 @@ Constraints позволяют задать выражение, истиннос В ByteDance есть готовая реализация, но они её боятся из-за, возможно, низкого качества кода. -### 21.21. Чтение больших файлов с помощью mmap. +### 21.21. + Чтение больших файлов с помощью mmap. -Тривиально, почти всё готово. +Сделан вариант, но достаточно топорный. Без тестирования в продакшене включать по-умолчанию нельзя. ### 21.22. Userspace page cache. @@ -1211,6 +1296,7 @@ Constraints позволяют задать выражение, истиннос ### 21.23. Ускорение работы с вторичными индексами. zhang2014. +Есть pull request. ## 22. Долги и недоделанные возможности. @@ -1219,9 +1305,9 @@ zhang2014. Нужно для Яндекс.Облака. Сделал Алексей Миловидов. -### 22.2. Убрать возможность изменить настройки в native протоколе в случае readonly. +### 22.2. + Убрать возможность изменить настройки в native протоколе в случае readonly. -Алексей Миловидов или [Виталий Баранов](https://github.com/vitlibar). +N.Vartolomei. ### 22.3. Защита от абсурдно заданных пользователем кодеков. @@ -1229,25 +1315,27 @@ zhang2014. ### 22.4. Исправление оставшихся deadlocks в табличных RWLock-ах. -Александр Казаков. Нужно для Яндекс.Метрики и Datalens. +Александр Казаков. Нужно для Яндекс.Метрики и Datalens. Задача постепенно тащится и исправлениями в соседних местах стала менее актуальна. +В Q1 будет сделана или отменена с учётом 1.2. и 1.3. -### 22.5. Исправление редких срабатываний TSan в stress тестах в CI. +### 22.5. + Исправление редких срабатываний TSan в stress тестах в CI. Александр Казаков. ### 22.6. Изменение только DEFAULT в ALTER TABLE может поменять тип столбца. -### 22.7. Row-Level Security не работает в случае наличия в запросе IN подзапросов. +### 22.7. + Row-Level Security не работает в случае наличия в запросе IN подзапросов. -[Виталий Баранов](https://github.com/vitlibar). Нужно для Метрики. +Нужно для Метрики. Иван Лежанкин. -### 22.8. Исправить десериализацию параметров для параметризованных запросов. +### 22.8. + Исправить десериализацию параметров для параметризованных запросов. Хотел исправить Василий Немков, Altinity, но есть маленькие затруднения, наверное переделает Алексей Миловидов. ### 22.9. Разобраться с десериализацией массивов со значениями по-умолчанию в Protobuf формате в случае protobuf 3. [Виталий Баранов](https://github.com/vitlibar). Возможно, это - фундаментальная проблема и следует её только документировать. +Кажется, отменяем, но пока ещё не всё ясно. ### 22.10. Исправление дрифта при отслеживании потребления памяти запросами. @@ -1278,11 +1366,10 @@ zhang2014. Altinity. -### 22.16. Исправление низкой производительности кодека DoubleDelta. +### 22.16. + Исправление низкой производительности кодека DoubleDelta. Василий Немков, Altinity - в процессе. - -Мы считаем важным, что код в ClickHouse содержит разумные оптимизации, основанные на анализе производительности. Но иногда бывают досадные исключения. +Можно считать, что сделано, хотя отсутствие SIMD оптимизаций для variable length кодеков - это ужасно. ### 22.17. Консистентно работающий POPULATE для MaterializedView. @@ -1331,14 +1418,14 @@ https://github.com/ClickHouse/ClickHouse/issues/2655 Altinity. -### 22.29. Уязвимость DDL для словарей executable. +### 22.29. + Уязвимость DDL для словарей executable. [Александр Сапин](https://github.com/alesapin) ## 23. Default Festival. -### 23.1. Включение minimalistic_part_header в ZooKeeper. +### 23.1. + Включение minimalistic_part_header в ZooKeeper. Сильно уменьшает объём данных в ZooKeeper. Уже год в продакшене в Яндекс.Метрике. Алексей Миловидов, ноябрь 2019. @@ -1367,13 +1454,13 @@ Altinity. Просто аккуратно включить. -### 23.8. Включение оптимизации VALUES. +### 23.8. + Включение оптимизации VALUES. Просто аккуратно включить. ### 23.9. Включение Processors. -[Николай Кочетов](https://github.com/KochetovNicolai). +Q1. [Николай Кочетов](https://github.com/KochetovNicolai). ### 23.10. Включение mlock бинарника. @@ -1578,12 +1665,16 @@ ucasFL, ICT. Алгоритмы min-hash и sim-hash позволяют вычислить для текста несколько хэш-значений таких, что при небольшом изменении текста, по крайней мере один из хэшей не меняется. Вычисления можно реализовать на n-грамах и словарных шинглах. Предлагается добавить поддержку этих алгоритмов в виде функций в ClickHouse и изучить их применимость для задачи нечёткого поиска полудубликатов. +Есть pull request, есть что доделывать. + ### 24.28. Другой sketch для квантилей. -Похоже на quantileTiming, но с логарифмическими корзинами. +Похоже на quantileTiming, но с логарифмическими корзинами. См. DDSketch. ### 24.29. Поддержка Arrow Flight. +Жанна Зосимова, ВШЭ. + ### 24.30. ClickHouse как графовая СУБД. Amos Bird, но его решение слишком громоздкое и пока не open-source. @@ -1613,7 +1704,7 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.2. Вычитка и выкладка статьи про обфускацию данных на английском. -Эми, Александр Казаков, Алексей Миловидов, ноябрь 2019. +Эми, Александр Казаков, Алексей Миловидов, Q1. Готово к выкладке. ### 25.3. Подготовка статьи "Секреты оптимизации производительности ClickHouse". @@ -1640,9 +1731,9 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.9. Подготовка докладчиков: khvostikao, ilezhankin, nikitamikhailov, akuzm и другие. -[Ольга Хвостикова](https://github.com/stavrolia), [Иван Лежанкин](https://github.com/abyss7), Никита Михайлов, [Александр Кузьменков](https://github.com/akuzm). +[Ольга Хвостикова](https://github.com/stavrolia), [Иван Лежанкин](https://github.com/abyss7), Никита Михайлов, [Александр Кузьменков](https://github.com/akuzm), Артём Зуйков. Уже готовые докладчики: Алексей Миловидов, [Николай Кочетов](https://github.com/KochetovNicolai), [Александр Сапин](https://github.com/alesapin). -Получаем минимум 7 докладчиков в 2020 году. +Получаем минимум 8 докладчиков в 2020 году. ### 25.10. Митапы в России и Беларуси: Москва x2 + митап для разработчиков или хакатон, Санкт-Петербург, Минск, Нижний Новгород, Екатеринбург, Новосибирск и/или Академгородок, Иннополис или Казань. @@ -1650,7 +1741,7 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.11. Митапы зарубежные: восток США (Нью Йорк, возможно Raleigh), возможно северо-запад (Сиэтл), Китай (Пекин снова, возможно митап для разработчиков или хакатон), Лондон. -[Иван Блинков](https://github.com/blinkov/) - организация +[Иван Блинков](https://github.com/blinkov/) - организация. Две штуки в США запланированы. ### 25.12. Статья "научная" - про устройство хранения данных и индексов или whitepaper по архитектуре. Есть вариант подать на VLDB. diff --git a/docs/ru/faq/general.md b/docs/ru/faq/general.md index 010926d2cf9..970069bc641 100644 --- a/docs/ru/faq/general.md +++ b/docs/ru/faq/general.md @@ -21,4 +21,37 @@ NLS_LANG=RUSSIAN_RUSSIA.UTF8 ``` +## Как экспортировать данные из ClickHouse в файл? {#how-to-export-to-file} + +### Секция INTO OUTFILE + +Добавьте секцию [INTO OUTFILE](../query_language/select/#into-outfile-clause) к своему запросу. + +Например: + +```sql +SELECT * FROM table INTO OUTFILE 'file' +``` + +По умолчанию, для выдачи данных ClickHouse использует формат [TabSeparated](../interfaces/formats.md#tabseparated). Чтобы выбрать [формат данных](../interfaces/formats.md), используйте [секцию FORMAT](../query_language/select/#format-clause). + +Например: + +```sql +SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV +``` + +### Таблица с движком File + +Смотрите [File](../operations/table_engines/file.md). + +### Перенаправление в командой строке + +```sql +$ clickhouse-client --query "SELECT * from table" > result.txt +``` + +Смотрите [clickhouse-client](../interfaces/cli.md). + + [Оригинальная статья ](https://clickhouse.yandex/docs/en/faq/general/) diff --git a/docs/ru/index.md b/docs/ru/index.md index fbb80bfb8a9..15ad4c5853c 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -125,7 +125,7 @@ SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIM 2. Кодогенерация. Для запроса генерируется код, в котором подставлены все косвенные вызовы. -В "обычных" БД этого не делается, так как не имеет смысла при выполнении простых запросов. Хотя есть исключения. Например, в MemSQL кодогенерация используется для уменьшения latency при выполнении SQL запросов. (Для сравнения - в аналитических СУБД, требуется оптимизация throughput, а не latency). +В "обычных" БД этого не делается, так как не имеет смысла при выполнении простых запросов. Хотя есть исключения. Например, в MemSQL кодогенерация используется для уменьшения latency при выполнении SQL запросов. Для сравнения, в аналитических СУБД требуется оптимизация throughput, а не latency. Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K). То есть, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации. diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 36f7ae462c4..ad941b23345 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -28,6 +28,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT | [PrettySpace](#prettyspace) | ✗ | ✔ | | [Protobuf](#protobuf) | ✔ | ✔ | | [Parquet](#data-format-parquet) | ✔ | ✔ | +| [ORC](#data-format-orc) | ✔ | ✗ | | [RowBinary](#rowbinary) | ✔ | ✔ | | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [Native](#native) | ✔ | ✔ | @@ -132,28 +133,28 @@ world `delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`, - где `delimiter_i` - разделители между значениями (символ `$` в разделителе экранируется как `$$`), - `column_i` - имена или номера столбцов, значения которых должны быть выведены или считаны (если имя не указано - столбец пропускается), + где `delimiter_i` - разделители между значениями (символ `$` в разделителе экранируется как `$$`), + `column_i` - имена или номера столбцов, значения которых должны быть выведены или считаны (если имя не указано - столбец пропускается), `serializeAs_i` - тип экранирования для значений соответствующего столбца. Поддерживаются следующие типы экранирования: - + - `CSV`, `JSON`, `XML` (как в одноимённых форматах) - `Escaped` (как в `TSV`) - `Quoted` (как в `Values`) - `Raw` (без экранирования, как в `TSVRaw`) - `None` (тип экранирования отсутствует, см. далее) - - Если для столбца не указан тип экранирования, используется `None`. `XML` и `Raw` поддерживаются только для вывода. - - Так, в форматной строке - + + Если для столбца не указан тип экранирования, используется `None`. `XML` и `Raw` поддерживаются только для вывода. + + Так, в форматной строке + `Search phrase: ${SearchPhrase:Quoted}, count: ${c:Escaped}, ad price: $$${price:JSON};` - + между разделителями `Search phrase: `, `, count: `, `, ad price: $` и `;` при выводе будут подставлены (при вводе - будут ожидаться) значения столбцов `SearchPhrase`, `c` и `price`, сериализованные как `Quoted`, `Escaped` и `JSON` соответственно, например: `Search phrase: 'bathroom interior design', count: 2166, ad price: $3;` - + Настройка `format_template_rows_between_delimiter` задаёт разделитель между строками, который выводится (или ожмдается при вводе) после каждой строки, кроме последней. По умолчанию `\n`. - + Настройка `format_template_resultset` задаёт путь к файлу, содержащему форматную строку для результата. Форматная строка для результата имеет синтаксис аналогичный форматной строке для строк таблицы и позволяет указать префикс, суффикс и способ вывода дополнительной информации. Вместо имён столбцов в ней указываются следующие имена подстановок: - `data` - строки с данными в формате `format_template_row`, разделённые `format_template_rows_between_delimiter`. Эта подстановка должна быть первой подстановкой в форматной строке. @@ -165,15 +166,15 @@ world - `time` - время выполнения запроса в секундах - `rows_read` - сколько строк было прочитано при выполнении запроса - `bytes_read` - сколько байт (несжатых) было прочитано при выполнении запроса - + У подстановок `data`, `totals`, `min` и `max` не должны быть указаны типы экранирования (или должен быть указан `None`). Остальные подстановки - это отдельные значения, для них может быть указан любой тип экранирования. Если строка `format_template_resultset` пустая, то по-умолчанию используется `${data}`. Из всех перечисленных подстановок форматная строка `format_template_resultset` для ввода может содержать только `data`. Также при вводе формат поддерживает пропуск значений столбцов и пропуск значений в префиксе и суффиксе (см. пример). - + Пример вывода: ```sql -SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 FORMAT Template SETTINGS +SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 FORMAT Template SETTINGS format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = '\n ' ``` `/some/path/resultset.format`: @@ -225,7 +226,7 @@ Page views: 6, User id: 4324182021466249494, Useless field: world, Duration: 185 Total rows: 2 ``` ```sql -INSERT INTO UserActivity FORMAT Template SETTINGS +INSERT INTO UserActivity FORMAT Template SETTINGS format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format' ``` `/some/path/resultset.format`: @@ -238,7 +239,7 @@ Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Du ``` `PageViews`, `UserID`, `Duration` и `Sign` внутри подстановок - имена столбцов в таблице, в которую вставляются данные. Значения после `Useless field` в строках и значение после `\nTotal rows: ` в суффиксе будут проигнорированы. Все разделители во входных данных должны строго соответствовать разделителям в форматных строках. - + ## TemplateIgnoreSpaces {#templateignorespaces} Подходит только для ввода. Отличается от формата `Template` тем, что пропускает пробельные символы между разделителями и значениями во входном потоке. Также в этом формате можно указать пустые подстановки с типом экранирования `None` (`${}` или `${:None}`), чтобы разбить разделители на несколько частей, пробелы между которыми должны игнорироваться. Такие подстановки используются только для пропуска пробелов. С помощью этого формата можно считывать `JSON`, если значения столбцов в нём всегда идут в одном порядке в каждой строке. Например, для вставки данных из примера вывода формата [JSON](#json) в таблицу со столбцами `phrase` и `cnt` можно использовать следующий запрос: @@ -941,7 +942,7 @@ ClickHouse поддерживает настраиваемую точность Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. -### Inserting and Selecting Data +### Вставка и выборка данных Чтобы вставить в ClickHouse данные из файла в формате Parquet, выполните команду следующего вида: @@ -955,7 +956,49 @@ $ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Pa $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} ``` -Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../operations/table_engines/hdfs.md) и `URL`. +Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../operations/table_engines/hdfs.md). + + +## ORC {#data-format-orc} + +[Apache ORC](https://orc.apache.org/) - это column-oriented формат данных, распространённый в экосистеме Hadoop. Вы можете только вставлять данные этого формата в ClickHouse. + +### Соответствие типов данных + +Таблица показывает поддержанные типы данных и их соответствие [типам данных](../data_types/index.md) ClickHouse для запросов `INSERT`. + +| Тип данных ORC (`INSERT`) | Тип данных ClickHouse | +| -------------------- | ------------------ | +| `UINT8`, `BOOL` | [UInt8](../data_types/int_uint.md) | +| `INT8` | [Int8](../data_types/int_uint.md) | +| `UINT16` | [UInt16](../data_types/int_uint.md) | +| `INT16` | [Int16](../data_types/int_uint.md) | +| `UINT32` | [UInt32](../data_types/int_uint.md) | +| `INT32` | [Int32](../data_types/int_uint.md) | +| `UINT64` | [UInt64](../data_types/int_uint.md) | +| `INT64` | [Int64](../data_types/int_uint.md) | +| `FLOAT`, `HALF_FLOAT` | [Float32](../data_types/float.md) | +| `DOUBLE` | [Float64](../data_types/float.md) | +| `DATE32` | [Date](../data_types/date.md) | +| `DATE64`, `TIMESTAMP` | [DateTime](../data_types/datetime.md) | +| `STRING`, `BINARY` | [String](../data_types/string.md) | +| `DECIMAL` | [Decimal](../data_types/decimal.md) | + +ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. + +Неподдержанные типы данных ORC: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. + +Типы данных столбцов в таблицах ClickHouse могут отличаться от типов данных для соответствующих полей ORC. При вставке данных, ClickHouse интерпретирует типы данных ORC согласно таблице соответствия, а затем [приводит](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) данные к типу, установленному для столбца таблицы ClickHouse. + +### Вставка данных + +Данные ORC можно вставить в таблицу ClickHouse командой: + +```bash +$ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" +``` + +Для обмена данных с Hadoop можно использовать [движок таблиц HDFS](../operations/table_engines/hdfs.md). ## Схема формата {#formatschema} diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 4d3b831cd5c..4779388cde7 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -3,7 +3,7 @@ HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах, HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. По умолчанию, clickhouse-server слушает HTTP на порту 8123 (это можно изменить в конфиге). -Если запросить GET / без параметров, то вернётся строка "Ok." (с переводом строки на конце). Это может быть использовано в скриптах проверки живости. +Если запросить GET / без параметров, то вернётся строка "Ok." (с переводом строки на конце). Это может быть использовано в скриптах проверки доступности. ```bash $ curl 'http://localhost:8123/' @@ -173,7 +173,7 @@ $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?data По умолчанию используется БД, которая прописана в настройках сервера, как БД по умолчанию. По умолчанию, это - БД default. Также вы всегда можете указать БД через точку перед именем таблицы. -Имя пользователя и пароль могут быть указаны в одном из двух вариантов: +Имя пользователя и пароль могут быть указаны в одном из трёх вариантов: 1. С использованием HTTP Basic Authentication. Пример: @@ -187,6 +187,12 @@ $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- $ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- ``` +3. С использованием заголовков ‘X-ClickHouse-User’ и ‘X-ClickHouse-Key’. Пример: + +```bash +$ echo 'SELECT 1' | curl -H 'X-ClickHouse-User: user' -H 'X-ClickHouse-Key: password' 'http://localhost:8123/' -d @- +``` + Если пользователь не задан,то используется `default`. Если пароль не задан, то используется пустой пароль. Также в параметрах URL вы можете указать любые настройки, которые будут использованы для обработки одного запроса, или целые профили настроек. Пример:http://localhost:8123/?profile=web&max_rows_to_read=1000000000&query=SELECT+1 diff --git a/docs/ru/interfaces/mysql.md b/docs/ru/interfaces/mysql.md new file mode 100644 index 00000000000..d0a36174d2b --- /dev/null +++ b/docs/ru/interfaces/mysql.md @@ -0,0 +1,37 @@ +# MySQL-интерфейс {#mysql_interface} + +ClickHouse поддерживает взаимодействие по протоколу MySQL. Данная функция включается настройкой [mysql_port](../operations/server_settings/settings.md#server_settings-mysql_port) в конфигурационном файле: +```xml +9004 +``` + +Пример подключения с помощью стандартного клиента mysql: +```bash +$ mysql --protocol tcp -u default -P 9004 +``` + +Вывод в случае успешного подключения: +```text +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 4 +Server version: 20.2.1.1-ClickHouse + +Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> +``` + +Для совместимости со всеми клиентами рекомендуется задавать пароль пользователя в конфигурационном файле с помощью двойного хэша [SHA1](../operations/settings/settings_users.md#password_double_sha1_hex). +В случае указания пароля с помощью [SHA256](../operations/settings/settings_users.md#password_sha256_hex) некоторые клиенты не смогут пройти аутентификацию (mysqljs и старые версии стандартного клиента mysql). + +Ограничения: + +- не поддерживаются подготовленные запросы + +- некоторые типы данных отправляются как строки diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index 50a82c7e169..c618fd3a6e1 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -91,18 +91,6 @@ ## Коммерческие -### Holistics Software - -[Holistics](https://www.holistics.io/) вошёл в топ-2 наиболее удобных инструментов для бизнес-аналитики по рейтингу Gartner's Frontrunners в 2019 году. Holistics — full-stack платформа для обработки данных и инструмент бизнес-аналитики, позволяющий вам построить свои процессы с помощью SQL. - -Основные возможности: - -- Автоматизированные отчёты на почту, Slack, и Google Sheet. -- Мощный редактор SQL c визуализацией, контролем версий, автодополнением, повторным использованием частей запроса и динамическими фильтрами. -- Встроенные инструменты анализа отчётов и всплывающие (iframe) дашборды. -- Подготовка данных и возможности ETL. -- Моделирование данных с помощью SQL для их реляционного отображения. - ### DataGrip [DataGrip](https://www.jetbrains.com/datagrip/) — это IDE для баз данных о JetBrains с выделенной поддержкой ClickHouse. Он также встроен в другие инструменты на основе IntelliJ: PyCharm, IntelliJ IDEA, GoLand, PhpStorm и другие. @@ -116,4 +104,16 @@ - Рефакторинги. - Поиск и навигация. +### Holistics Software + +[Holistics](https://www.holistics.io/) — full-stack платформа для обработки данных и бизнес-аналитики. + +Основные возможности: + +- Автоматизированные отчёты на почту, Slack, и Google Sheet. +- Редактор SQL c визуализацией, контролем версий, автодополнением, повторным использованием частей запроса и динамическими фильтрами. +- Встроенные инструменты анализа отчётов и всплывающие (iframe) дашборды. +- Подготовка данных и возможности ETL. +- Моделирование данных с помощью SQL для их реляционного отображения. + [Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/third-party/gui/) diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md index 9a0fa781f63..de0cf7d2a41 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_settings/settings.md @@ -622,7 +622,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat - [skip_unavailable_shards](../settings/settings.md#settings-skip_unavailable_shards) -## timezone +## timezone {#server_settings-timezone} Временная зона сервера. @@ -661,6 +661,16 @@ TCP порт для защищённого обмена данными с кли 9440 ``` +## mysql_port {#server_settings-mysql_port} + +Порт для взаимодействия с клиентами по протоколу MySQL. + +Пример + +```xml +9004 +``` + ## tmp_path Путь ко временным данным для обработки больших запросов. diff --git a/docs/ru/operations/settings/settings_users.md b/docs/ru/operations/settings/settings_users.md index dfa3e40c660..2f39e8e86de 100644 --- a/docs/ru/operations/settings/settings_users.md +++ b/docs/ru/operations/settings/settings_users.md @@ -39,6 +39,8 @@ Например, `qwerty`. Пароль можно оставить пустым. + + - Чтобы назначить пароль в виде SHA256, поместите хэш в элемент `password_sha256_hex`. Например, `65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5`. @@ -51,6 +53,20 @@ Первая строка результата — пароль. Вторая строка — соответствующий ему хэш SHA256. + + +- Для совместимости с клиентами MySQL, пароль можно задать с помощью двойного хэша SHA1, поместив его в элемент `password_double_sha1_hex`. + + Например, `08b4a0f1de6ad37da17359e592c8d74788a83eb0`. + + Пример создания пароля в командной строке: + + ``` + PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' + ``` + + Первая строка результата — пароль. Вторая строка — соответствующий ему двойной хэш SHA1. + ### user_name/networks Список сетей, из которых пользователь может подключиться к серверу ClickHouse. diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index 7aaf7f9ee4f..086e079358b 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -695,6 +695,43 @@ WHERE changed └────────────────────────┴─────────────┴─────────┘ ``` +## system.table_engines + +Содержит информацию про движки таблиц, поддерживаемые сервером, а также об их возможностях. + +Эта таблица содержит следующие столбцы (тип столбца показан в скобках): + +- `name` (String) — имя движка. +- `supports_settings` (UInt8) — флаг, показывающий поддержку секции `SETTINGS`. +- `supports_skipping_indices` (UInt8) — флаг, показывающий поддержку [индексов пропуска данных](table_engines/mergetree/#table_engine-mergetree-data_skipping-indexes). +- `supports_ttl` (UInt8) — флаг, показывающий поддержку [TTL](table_engines/mergetree/#table_engine-mergetree-ttl). +- `supports_sort_order` (UInt8) — флаг, показывающий поддержку секций `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` и `SAMPLE_BY`. +- `supports_replication` (UInt8) — флаг, показвыающий поддержку [репликации](table_engines/replication/). +- `supports_duduplication` (UInt8) — флаг, показывающий наличие в движке дедупликации данных. + +Пример: + +```sql +SELECT * +FROM system.table_engines +WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') +``` + +```text +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ +``` + +**Смотрите также** + +- [Секции движка](table_engines/mergetree/#sektsii-zaprosa) семейства MergeTree +- [Настройки](table_engines/kafka.md#table_engine-kafka-creating-a-table) Kafka +- [Настройки](table_engines/join/#limitations-and-settings) Join + + ## system.tables Содержит метаданные каждой таблицы, о которой знает сервер. Отсоединённые таблицы не отображаются в `system.tables`. diff --git a/docs/ru/operations/table_engines/hdfs.md b/docs/ru/operations/table_engines/hdfs.md index b384eb3bf60..ca889a5d4ee 100644 --- a/docs/ru/operations/table_engines/hdfs.md +++ b/docs/ru/operations/table_engines/hdfs.md @@ -97,4 +97,13 @@ CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV') ``` +## Виртуальные столбцы + +- `_path` — Путь к файлу. +- `_file` — Имя файла. + +**Смотрите также** + +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) + [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/hdfs/) diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md index cf25e3a2d39..fbdf5ce5265 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/query_language/create.md @@ -258,7 +258,9 @@ SELECT a, b, c FROM (SELECT ...) Материализованные (MATERIALIZED) представления хранят данные, преобразованные соответствующим запросом SELECT. -При создании материализованного представления, нужно обязательно указать ENGINE - движок таблицы для хранения данных. +При создании материализованного представления без использования `TO [db].[table]`, нужно обязательно указать ENGINE - движок таблицы для хранения данных. + +При создании материализованного представления с испольованием `TO [db].[table]`, нельзя указывать `POPULATE` Материализованное представление устроено следующим образом: при вставке данных в таблицу, указанную в SELECT-е, кусок вставляемых данных преобразуется этим запросом SELECT, и полученный результат вставляется в представление. diff --git a/docs/ru/query_language/dicts/external_dicts.md b/docs/ru/query_language/dicts/external_dicts.md index d03b33942ef..6df7bb501b2 100644 --- a/docs/ru/query_language/dicts/external_dicts.md +++ b/docs/ru/query_language/dicts/external_dicts.md @@ -2,10 +2,7 @@ Существует возможность подключать собственные словари из различных источников данных. Источником данных для словаря может быть локальный текстовый/исполняемый файл, HTTP(s) ресурс или другая СУБД. Подробнее смотрите в разделе "[Источники внешних словарей](external_dicts_dict_sources.md)". -ClickHouse: - -- Полностью или частично хранит словари в оперативной памяти. -- Периодически обновляет их и динамически подгружает отсутствующие значения. Т.е. словари можно подгружать динамически. +ClickHouse полностью или частично хранит словари в оперативной памяти. Словари можно подгружать динамически, ClickHouse периодически обновляет их и динамически подгружает отсутствующие значения. Конфигурация внешних словарей находится в одном или нескольких файлах. Путь к конфигурации указывается в параметре [dictionaries_config](../../operations/server_settings/settings.md). diff --git a/docs/ru/query_language/functions/date_time_functions.md b/docs/ru/query_language/functions/date_time_functions.md index 11156285434..3f087505464 100644 --- a/docs/ru/query_language/functions/date_time_functions.md +++ b/docs/ru/query_language/functions/date_time_functions.md @@ -135,7 +135,7 @@ SELECT Например, `timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. Это нужно для поиска хитов, входящих в соответствующий визит. -## formatDateTime(Time, Format\[, Timezone\]) +## formatDateTime(Time, Format\[, Timezone\]) {#formatdatetime} Функция преобразования даты-с-временем в String согласно заданному шаблону. Важно - шаблон является константным выражением, т.е. невозможно использование разных шаблонов в одной колонке. Поддерживаемые модификаторы в шаблоне Format: diff --git a/docs/ru/query_language/functions/rounding_functions.md b/docs/ru/query_language/functions/rounding_functions.md index 61bf6d94419..8f5eeca5dc8 100644 --- a/docs/ru/query_language/functions/rounding_functions.md +++ b/docs/ru/query_language/functions/rounding_functions.md @@ -75,6 +75,91 @@ round(3.55, 1) = 3.6 round(3.65, 1) = 3.6 ``` +**См. также** + +- [roundBankers](#roundbankers) + +## roundBankers {#roundbankers} + +Округляет число до указанного десятичного разряда. + +- Если округляемое число равноудалено от соседних чисел, то используется банковское округление. + + Банковское округление (англ. banker's rounding) — метод округления дробных чисел. Если округляемое число равноудалёно от соседних чисел, то оно округляется до ближайшего чётного числа. К примеру, 3,5 округляется до 4, а 2,5 до 2. + +- В других случаях функция округляет к ближайшему целому. + +Банковское округление позволяет уменьшить влияние округления чисел на результат суммирования или вычитания этих чисел. + +Пример суммирования чисел 1.5, 2.5, 3.5 и 4.5 с различным округлением: + +- Без округления: 1.5 + 2.5 + 3.5 + 4.5 = 12. +- Банковское округление: 2 + 2 + 4 + 4 = 12. +- Округление до ближайшего целого: 2 + 3 + 4 + 5 = 14. + + +**Синтаксис** + +```sql +roundBankers(expression [, decimal_places]) +``` + +**Параметры** + +- `expression` — Число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../data_types/index.md#data_types). +- `decimal-places` — Десятичный разряд. Целое число. + - `decimal-places > 0` — Функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции справа от запятой. Например, `roundBankers(3.55, 1) = 3.6`. + - `decimal-places < 0` — Функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции слева от запятой. Например, `roundBankers(24.55, -1) = 20`. + - `decimal-places = 0` — Функция округляет значение до целого. В этом случае аргумент можно не передавать. Например, `roundBankers(2.5) = 2`. + + +**Возвращаемое значение** + +Округлённое значение по методу банковского округления. + +**Пример использования** + +Запрос: + +```sql + SELECT number / 2 AS x, roundBankers(x, 0) AS b fROM system.numbers limit 10 +``` + +Результат: + +```text +┌───x─┬─b─┐ +│ 0 │ 0 │ +│ 0.5 │ 0 │ +│ 1 │ 1 │ +│ 1.5 │ 2 │ +│ 2 │ 2 │ +│ 2.5 │ 2 │ +│ 3 │ 3 │ +│ 3.5 │ 4 │ +│ 4 │ 4 │ +│ 4.5 │ 4 │ +└─────┴───┘ +``` + + +**Примеры банковского округления** + +```text +roundBankers(0.4) = 0 +roundBankers(-3.5) = -4 +roundBankers(4.5) = 4 +roundBankers(3.55, 1) = 3.6 +roundBankers(3.65, 1) = 3.6 +roundBankers(10.35, 1) = 10.4 +roundBankers(10.755, 2) = 11,76 +``` + +**См. также** + +- [round](#rounding_functions-round) + + ## roundToExp2(num) Принимает число. Если число меньше единицы - возвращает 0. Иначе округляет число вниз до ближайшей (целой неотрицательной) степени двух. diff --git a/docs/ru/query_language/select.md b/docs/ru/query_language/select.md index fb659e4355f..9bc5c7a4c04 100644 --- a/docs/ru/query_language/select.md +++ b/docs/ru/query_language/select.md @@ -1147,7 +1147,7 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Запросы - части `UNION ALL` нельзя заключить в скобки. `ORDER BY` и `LIMIT` применяются к отдельным запросам, а не к общему результату. Если вам нужно применить какое-либо преобразование к общему результату, то вы можете разместить все запросы с `UNION ALL` в подзапросе в секции `FROM`. -### Секция INTO OUTFILE +### Секция INTO OUTFILE {#into-outfile-clause} При указании `INTO OUTFILE filename` (где filename - строковый литерал), результат запроса будет сохранён в файл filename. В отличие от MySQL, файл создаётся на стороне клиента. Если файл с таким именем уже существует, это приведёт к ошибке. @@ -1155,7 +1155,7 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Формат вывода по умолчанию - TabSeparated, как и в не интерактивном режиме клиента командной строки. -### Секция FORMAT +### Секция FORMAT {#format-clause} При указании FORMAT format вы можете получить данные в любом указанном формате. Это может использоваться для удобства или для создания дампов. diff --git a/docs/ru/query_language/table_functions/file.md b/docs/ru/query_language/table_functions/file.md index 9938077de0a..d43f2773a42 100644 --- a/docs/ru/query_language/table_functions/file.md +++ b/docs/ru/query_language/table_functions/file.md @@ -91,4 +91,13 @@ SELECT count(*) FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32') ``` +## Виртуальные столбцы + +- `_path` — Путь к файлу. +- `_file` — Имя файла. + +**Смотрите также** + +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/file/) diff --git a/docs/ru/query_language/table_functions/hdfs.md b/docs/ru/query_language/table_functions/hdfs.md index 8787c23e826..768cf4e3b1d 100644 --- a/docs/ru/query_language/table_functions/hdfs.md +++ b/docs/ru/query_language/table_functions/hdfs.md @@ -47,4 +47,13 @@ LIMIT 2 Шаблоны могут содержаться в разных частях пути. Обрабатываться будут ровно те файлы, которые и удовлетворяют всему шаблону пути, и существуют в файловой системе. +## Виртуальные столбцы + +- `_path` — Путь к файлу. +- `_file` — Имя файла. + +**Смотрите также** + +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/hdfs/) diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 604aca5d18e..042b0332bbc 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -25,6 +25,7 @@ nav: - 'Command-Line Client': 'interfaces/cli.md' - 'Native Interface (TCP)': 'interfaces/tcp.md' - 'HTTP Interface': 'interfaces/http.md' + - 'MySQL Interface': 'interfaces/mysql.md' - 'Input and Output Formats': 'interfaces/formats.md' - 'JDBC Driver': 'interfaces/jdbc.md' - 'ODBC Driver': 'interfaces/odbc.md' @@ -217,14 +218,15 @@ nav: - 'Development': - 'hidden': 'development/index.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Overview of ClickHouse Architecture': 'development/architecture.md' + - 'Browse ClickHouse Source Code': 'development/browse_code.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' - 'How to Write C++ Code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index b13c8092889..9da4346dbbc 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -59,7 +59,7 @@ nav: - 'Expression': 'data_types/special_data_types/expression.md' - 'Set': 'data_types/special_data_types/set.md' - 'Nothing': 'data_types/special_data_types/nothing.md' - - 'Interval': 'data_types/special_data_types/interval.md' + - 'Interval': 'data_types/special_data_types/interval.md' - 'Domains': - 'Overview': 'data_types/domains/overview.md' - 'IPv4': 'data_types/domains/ipv4.md' @@ -167,7 +167,7 @@ nav: - 'jdbc': 'query_language/table_functions/jdbc.md' - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - - 'input': 'query_language/table_functions/input.md' + - 'input': 'query_language/table_functions/input.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': @@ -210,18 +210,19 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' - 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md' - + - 'F.A.Q.': - 'General Questions': 'faq/general.md' - 'Development': - 'hidden': 'development/index.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Overview of ClickHouse Architecture': 'development/architecture.md' + - 'Browse ClickHouse Source Code': 'development/browse_code.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Write C++ code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index 31b384f97b5..786559124f5 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -214,17 +214,18 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' - 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md' - + - 'Development': - 'hidden': 'development/index.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Overview of ClickHouse Architecture': 'development/architecture.md' + - 'Browse ClickHouse Source Code': 'development/browse_code.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' - 'How to Write C++ Code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index dc6a0d9227c..c2b7ab6961a 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -26,6 +26,7 @@ nav: - 'Клиент командной строки': 'interfaces/cli.md' - 'Нативный интерфейс (TCP)': 'interfaces/tcp.md' - 'HTTP-интерфейс': 'interfaces/http.md' + - 'MySQL-интерфейс': 'interfaces/mysql.md' - 'Форматы входных и выходных данных': 'interfaces/formats.md' - 'JDBC-драйвер': 'interfaces/jdbc.md' - 'ODBC-драйвер': 'interfaces/odbc.md' @@ -137,7 +138,7 @@ nav: - 'mysql': 'query_language/table_functions/mysql.md' - 'jdbc': 'query_language/table_functions/jdbc.md' - 'odbc': 'query_language/table_functions/odbc.md' - - 'hdfs': 'query_language/table_functions/hdfs.md' + - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - 'Словари': - 'Введение': 'query_language/dicts/index.md' @@ -215,13 +216,14 @@ nav: - 'Разработка': - 'hidden': 'development/index.md' + - 'Инструкция для начинающего разработчика ClickHouse': 'development/developer_instruction.md' - 'Обзор архитектуры ClickHouse': 'development/architecture.md' + - 'Навигация по коду ClickHouse': 'development/browse_code.md' - 'Как собрать ClickHouse на Linux': 'development/build.md' - 'Как собрать ClickHouse на Mac OS X': 'development/build_osx.md' - 'Как собрать ClickHouse на Linux для Mac OS X': 'development/build_cross_osx.md' - 'Как писать код на C++': 'development/style.md' - 'Как запустить тесты': 'development/tests.md' - - 'Инструкция для начинающего разработчика ClickHouse': 'development/developer_instruction.md' - 'Сторонние библиотеки': 'development/contrib.md' - 'Что нового': diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index c07c3bb2711..8f33e5579d2 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -59,7 +59,7 @@ nav: - 'Expression': 'data_types/special_data_types/expression.md' - 'Set': 'data_types/special_data_types/set.md' - 'Nothing': 'data_types/special_data_types/nothing.md' - - 'Interval': 'data_types/special_data_types/interval.md' + - 'Interval': 'data_types/special_data_types/interval.md' - 'Domain类型': - '介绍': 'data_types/domains/overview.md' - 'IPv4': 'data_types/domains/ipv4.md' @@ -209,19 +209,20 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' - 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md' - + - '常见问题': - '一般的问题': 'faq/general.md' - '开发者指南': - 'hidden': 'development/index.md' + - '开发者指南': 'development/developer_instruction.md' - 'ClickHouse架构概述': 'development/architecture.md' + - 'ClickHouse Code Browser': 'development/browse_code.md' - '如何在Linux中编译ClickHouse': 'development/build.md' - '如何在Mac OS X中编译ClickHouse': 'development/build_osx.md' - '如何在Linux中编译Mac OS X ClickHouse': 'development/build_cross_osx.md' - '如何编写C++代码': 'development/style.md' - '如何运行ClickHouse测试': 'development/tests.md' - - '开发者指南': 'development/developer_instruction.md' - '使用的第三方库': 'development/contrib.md' - '新功能特性': diff --git a/docs/tools/build.py b/docs/tools/build.py index 0e855ce9f1e..d6762a708d2 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -66,7 +66,7 @@ def build_for_lang(lang, args): 'logo': 'images/logo.svg', 'favicon': 'assets/images/favicon.ico', 'include_search_page': False, - 'search_index_only': True, + 'search_index_only': False, 'static_templates': ['404.html'], 'extra': { 'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching @@ -113,15 +113,8 @@ def build_for_lang(lang, args): } } ], - plugins=[{ - 'search': { - 'lang': ['en', 'ru'] if lang == 'ru' else ['en'] - } - }], + plugins=[], extra={ - 'search': { - 'language': 'en,ru' if lang == 'ru' else 'en' - }, 'stable_releases': args.stable_releases, 'version_prefix': args.version_prefix } diff --git a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css index e452e88ad66..21d6ab80b38 100644 --- a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css +++ b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css @@ -209,3 +209,23 @@ h1, h2, h3, .md-logo { content: "" !important; } } + +.algolia-autocomplete .algolia-docsearch-suggestion--subcategory-column { + display: none !important; +} + +.algolia-autocomplete .algolia-docsearch-suggestion--content { + float: none !important; + width: 100% !important; +} + +.algolia-autocomplete .algolia-docsearch-suggestion--content:before { + content: none !important; +} + +.algolia-autocomplete .ds-dropdown-menu { + width: 100% !important; + max-height: 512px; + overflow-x: hidden; + overflow-y: auto; +} diff --git a/docs/tools/mkdocs-material-theme/base.html b/docs/tools/mkdocs-material-theme/base.html index 8b04617c721..850d3647dda 100644 --- a/docs/tools/mkdocs-material-theme/base.html +++ b/docs/tools/mkdocs-material-theme/base.html @@ -3,7 +3,7 @@ {% set palette = config.theme.palette %} {% set font = config.theme.font %} - + {% block site_meta %} @@ -49,6 +49,7 @@ {% endif %} {% endblock %} {% block styles %} + {% if palette.primary or palette.accent %} @@ -269,6 +270,19 @@ ) }); + + +