mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 17:12:03 +00:00
Merge remote-tracking branch 'origin/master' into HEAD
This commit is contained in:
Alexander Kuzmenkov
commit
432cadae15
45
.github/workflows/anchore-analysis.yml
vendored
Normal file
45
.github/workflows/anchore-analysis.yml
vendored
Normal file
@ -0,0 +1,45 @@
|
||||
# This workflow checks out code, performs an Anchore container image
|
||||
# vulnerability and compliance scan, and integrates the results with
|
||||
# GitHub Advanced Security code scanning feature. For more information on
|
||||
# the Anchore scan action usage and parameters, see
|
||||
# https://github.com/anchore/scan-action. For more information on
|
||||
# Anchore container image scanning in general, see
|
||||
# https://docs.anchore.com.
|
||||
|
||||
name: Docker Container Scan (clickhouse-server)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- docker/server/Dockerfile
|
||||
- .github/workflows/anchore-analysis.yml
|
||||
schedule:
|
||||
- cron: '0 21 * * *'
|
||||
|
||||
jobs:
|
||||
Anchore-Build-Scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout the code
|
||||
uses: actions/checkout@v2
|
||||
- name: Build the Docker image
|
||||
run: |
|
||||
cd docker/server
|
||||
perl -pi -e 's|=\$version||g' Dockerfile
|
||||
docker build . --file Dockerfile --tag localbuild/testimage:latest
|
||||
- name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled
|
||||
uses: anchore/scan-action@master
|
||||
with:
|
||||
image-reference: "localbuild/testimage:latest"
|
||||
dockerfile-path: "docker/server/Dockerfile"
|
||||
acs-report-enable: true
|
||||
fail-build: true
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v1.0.0
|
||||
with:
|
||||
name: AnchoreReports
|
||||
path: ./anchore-reports/
|
||||
- name: Upload Anchore Scan Report
|
||||
uses: github/codeql-action/upload-sarif@v1
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
33
.github/workflows/codeql-analysis.yml
vendored
Normal file
33
.github/workflows/codeql-analysis.yml
vendored
Normal file
@ -0,0 +1,33 @@
|
||||
name: "CodeQL Scanning"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 19 * * *'
|
||||
jobs:
|
||||
CodeQL-Build:
|
||||
|
||||
runs-on: self-hosted
|
||||
timeout-minutes: 1440
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 2
|
||||
submodules: 'recursive'
|
||||
|
||||
- run: git checkout HEAD^2
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
|
||||
with:
|
||||
languages: cpp
|
||||
|
||||
- run: sudo apt-get update && sudo apt-get install -y git cmake python ninja-build gcc-9 g++-9 && mkdir build
|
||||
- run: cd build && CC=gcc-9 CXX=g++-9 cmake ..
|
||||
- run: cd build && ninja
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -157,6 +157,9 @@
|
||||
[submodule "contrib/openldap"]
|
||||
path = contrib/openldap
|
||||
url = https://github.com/openldap/openldap.git
|
||||
[submodule "contrib/AMQP-CPP"]
|
||||
path = contrib/AMQP-CPP
|
||||
url = https://github.com/CopernicaMarketingSoftware/AMQP-CPP.git
|
||||
[submodule "contrib/cassandra"]
|
||||
path = contrib/cassandra
|
||||
url = https://github.com/ClickHouse-Extras/cpp-driver.git
|
||||
|
||||
506
CHANGELOG.md
506
CHANGELOG.md
@ -1,5 +1,435 @@
|
||||
## ClickHouse release 20.5
|
||||
|
||||
### ClickHouse release v20.5.2.7-stable 2020-07-02
|
||||
|
||||
#### Backward Incompatible Change
|
||||
|
||||
* Return non-Nullable result from COUNT(DISTINCT), and `uniq` aggregate functions family. If all passed values are NULL, return zero instead. This improves SQL compatibility. [#11661](https://github.com/ClickHouse/ClickHouse/pull/11661) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added a check for the case when user-level setting is specified in a wrong place. User-level settings should be specified in `users.xml` inside `<profile>` section for specific user profile (or in `<default>` for default settings). The server won't start with exception message in log. This fixes [#9051](https://github.com/ClickHouse/ClickHouse/issues/9051). If you want to skip the check, you can either move settings to the appropriate place or add `<skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings>` to config.xml. [#11449](https://github.com/ClickHouse/ClickHouse/pull/11449) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* The setting `input_format_with_names_use_header` is enabled by default. It will affect parsing of input formats `-WithNames` and `-WithNamesAndTypes`. [#10937](https://github.com/ClickHouse/ClickHouse/pull/10937) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove `experimental_use_processors` setting. It is enabled by default. [#10924](https://github.com/ClickHouse/ClickHouse/pull/10924) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Update `zstd` to 1.4.4. It has some minor improvements in performance and compression ratio. If you run replicas with different versions of ClickHouse you may see reasonable error messages `Data after merge is not byte-identical to data on another replicas.` with explanation. These messages are Ok and you should not worry. This change is backward compatible but we list it here in changelog in case you will wonder about these messages. [#10663](https://github.com/ClickHouse/ClickHouse/pull/10663) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added a check for meaningless codecs and a setting `allow_suspicious_codecs` to control this check. This closes [#4966](https://github.com/ClickHouse/ClickHouse/issues/4966). [#10645](https://github.com/ClickHouse/ClickHouse/pull/10645) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Several Kafka setting changes their defaults. See [#11388](https://github.com/ClickHouse/ClickHouse/pull/11388).
|
||||
|
||||
#### New Feature
|
||||
|
||||
* `TTL DELETE WHERE` and `TTL GROUP BY` for automatic data coarsening and rollup in tables. [#10537](https://github.com/ClickHouse/ClickHouse/pull/10537) ([expl0si0nn](https://github.com/expl0si0nn)).
|
||||
* Implementation of PostgreSQL wire protocol. [#10242](https://github.com/ClickHouse/ClickHouse/pull/10242) ([Movses](https://github.com/MovElb)).
|
||||
* Added system tables for users, roles, grants, settings profiles, quotas, row policies; added commands SHOW USER, SHOW [CURRENT|ENABLED] ROLES, SHOW SETTINGS PROFILES. [#10387](https://github.com/ClickHouse/ClickHouse/pull/10387) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Support writes in ODBC Table function [#10554](https://github.com/ClickHouse/ClickHouse/pull/10554) ([ageraab](https://github.com/ageraab)). [#10901](https://github.com/ClickHouse/ClickHouse/pull/10901) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Add query performance metrics based on Linux `perf_events` (these metrics are calculated with hardware CPU counters and OS counters). It is optional and requires `CAP_SYS_ADMIN` to be set on clickhouse binary. [#9545](https://github.com/ClickHouse/ClickHouse/pull/9545) [Andrey Skobtsov](https://github.com/And42). [#11226](https://github.com/ClickHouse/ClickHouse/pull/11226) ([Alexander Kuzmenkov](https://github.com/akuzm)).
|
||||
* Now support `NULL` and `NOT NULL` modifiers for data types in `CREATE` query. [#11057](https://github.com/ClickHouse/ClickHouse/pull/11057) ([Павел Потемкин](https://github.com/Potya)).
|
||||
* Added a new layout `direct` which loads all the data directly from the source for each query, without storing or caching data. [#10622](https://github.com/ClickHouse/ClickHouse/pull/10622) ([Artem Streltsov](https://github.com/kekekekule)).
|
||||
* Added new `complex_key_direct` layout to dictionaries, that does not store anything locally during query execution. [#10850](https://github.com/ClickHouse/ClickHouse/pull/10850) ([Artem Streltsov](https://github.com/kekekekule)).
|
||||
* Added support for MySQL style global variables syntax (stub). This is needed for compatibility of MySQL protocol. [#11832](https://github.com/ClickHouse/ClickHouse/pull/11832) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* `minMap` and `maxMap` functions were added. [#11603](https://github.com/ClickHouse/ClickHouse/pull/11603) ([Ildus Kurbangaliev](https://github.com/ildus)).
|
||||
* Add the `system.asynchronous_metric_log` table that logs historical metrics from `system.asynchronous_metrics`. [#11588](https://github.com/ClickHouse/ClickHouse/pull/11588) ([Alexander Kuzmenkov](https://github.com/akuzm)).
|
||||
* Add functions `extractAllGroupsHorizontal(haystack, re)` and `extractAllGroupsVertical(haystack, re)`. [#11554](https://github.com/ClickHouse/ClickHouse/pull/11554) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Add SHOW CLUSTER(S) queries. [#11467](https://github.com/ClickHouse/ClickHouse/pull/11467) ([hexiaoting](https://github.com/hexiaoting)).
|
||||
* Add `netloc` function for extracting network location, similar to `urlparse(url)`, `netloc` in python. [#11356](https://github.com/ClickHouse/ClickHouse/pull/11356) ([Guillaume Tassery](https://github.com/YiuRULE)).
|
||||
* Add 2 more virtual columns for engine=Kafka to access message headers. [#11283](https://github.com/ClickHouse/ClickHouse/pull/11283) ([filimonov](https://github.com/filimonov)).
|
||||
* Add `_timestamp_ms` virtual column for Kafka engine (type is `Nullable(DateTime64(3))`). [#11260](https://github.com/ClickHouse/ClickHouse/pull/11260) ([filimonov](https://github.com/filimonov)).
|
||||
* Add function `fuzzBits` that randomly flips bits in a string with given probability. [#11237](https://github.com/ClickHouse/ClickHouse/pull/11237) ([Andrei Nekrashevich](https://github.com/xolm)).
|
||||
* Allow comparison of numbers with constant string in comparison operators, IN and VALUES sections. [#11647](https://github.com/ClickHouse/ClickHouse/pull/11647) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add `round_robin` load_balancing mode. [#11645](https://github.com/ClickHouse/ClickHouse/pull/11645) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Add `cast_keep_nullable` setting. If set `CAST(something_nullable AS Type)` return `Nullable(Type)`. [#11733](https://github.com/ClickHouse/ClickHouse/pull/11733) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Added column `position` to `system.columns` table and `column_position` to `system.parts_columns` table. It contains ordinal position of a column in a table starting with 1. This closes [#7744](https://github.com/ClickHouse/ClickHouse/issues/7744). [#11655](https://github.com/ClickHouse/ClickHouse/pull/11655) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* ON CLUSTER support for SYSTEM {FLUSH DISTRIBUTED,STOP/START DISTRIBUTED SEND}. [#11415](https://github.com/ClickHouse/ClickHouse/pull/11415) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Add system.distribution_queue table. [#11394](https://github.com/ClickHouse/ClickHouse/pull/11394) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Support for all format settings in Kafka, expose some setting on table level, adjust the defaults for better performance. [#11388](https://github.com/ClickHouse/ClickHouse/pull/11388) ([filimonov](https://github.com/filimonov)).
|
||||
* Add `port` function (to extract port from URL). [#11120](https://github.com/ClickHouse/ClickHouse/pull/11120) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Now `dictGet*` functions accept table names. [#11050](https://github.com/ClickHouse/ClickHouse/pull/11050) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* The `clickhouse-format` tool is now able to format multiple queries when the `-n` argument is used. [#10852](https://github.com/ClickHouse/ClickHouse/pull/10852) ([Darío](https://github.com/dgrr)).
|
||||
* Possibility to configure proxy-resolver for DiskS3. [#10744](https://github.com/ClickHouse/ClickHouse/pull/10744) ([Pavel Kovalenko](https://github.com/Jokser)).
|
||||
* Make `pointInPolygon` work with non-constant polygon. PointInPolygon now can take Array(Array(Tuple(..., ...))) as second argument, array of polygon and holes. [#10623](https://github.com/ClickHouse/ClickHouse/pull/10623) ([Alexey Ilyukhov](https://github.com/livace)) [#11421](https://github.com/ClickHouse/ClickHouse/pull/11421) ([Alexey Ilyukhov](https://github.com/livace)).
|
||||
* Added `move_ttl_info` to `system.parts` in order to provide introspection of move TTL functionality. [#10591](https://github.com/ClickHouse/ClickHouse/pull/10591) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
* Possibility to work with S3 through proxies. [#10576](https://github.com/ClickHouse/ClickHouse/pull/10576) ([Pavel Kovalenko](https://github.com/Jokser)).
|
||||
* Add `NCHAR` and `NVARCHAR` synonims for data types. [#11025](https://github.com/ClickHouse/ClickHouse/pull/11025) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Resolved [#7224](https://github.com/ClickHouse/ClickHouse/issues/7224): added `FailedQuery`, `FailedSelectQuery` and `FailedInsertQuery` metrics to `system.events` table. [#11151](https://github.com/ClickHouse/ClickHouse/pull/11151) ([Nikita Orlov](https://github.com/naorlov)).
|
||||
* Add more `jemalloc` statistics to `system.asynchronous_metrics`, and ensure that we see up-to-date values for them. [#11748](https://github.com/ClickHouse/ClickHouse/pull/11748) ([Alexander Kuzmenkov](https://github.com/akuzm)).
|
||||
* Allow to specify default S3 credentials and custom auth headers. [#11134](https://github.com/ClickHouse/ClickHouse/pull/11134) ([Grigory Pervakov](https://github.com/GrigoryPervakov)).
|
||||
* Added new functions to import/export DateTime64 as Int64 with various precision: `to-/fromUnixTimestamp64Milli/-Micro/-Nano`. [#10923](https://github.com/ClickHouse/ClickHouse/pull/10923) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Allow specifying `mongodb://` URI for MongoDB dictionaries. [#10915](https://github.com/ClickHouse/ClickHouse/pull/10915) ([Alexander Kuzmenkov](https://github.com/akuzm)).
|
||||
* OFFSET keyword can now be used without an affiliated LIMIT clause. [#10802](https://github.com/ClickHouse/ClickHouse/pull/10802) ([Guillaume Tassery](https://github.com/YiuRULE)).
|
||||
* Added `system.licenses` table. This table contains licenses of third-party libraries that are located in `contrib` directory. This closes [#2890](https://github.com/ClickHouse/ClickHouse/issues/2890). [#10795](https://github.com/ClickHouse/ClickHouse/pull/10795) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* New function function toStartOfSecond(DateTime64) -> DateTime64 that nullifies sub-second part of DateTime64 value. [#10722](https://github.com/ClickHouse/ClickHouse/pull/10722) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Add new input format `JSONAsString` that accepts a sequence of JSON objects separated by newlines, spaces and/or commas. [#10607](https://github.com/ClickHouse/ClickHouse/pull/10607) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Allowed to profile memory with finer granularity steps than 4 MiB. Added sampling memory profiler to capture random allocations/deallocations. [#10598](https://github.com/ClickHouse/ClickHouse/pull/10598) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* `SimpleAggregateFunction` now also supports `sumMap`. [#10000](https://github.com/ClickHouse/ClickHouse/pull/10000) ([Ildus Kurbangaliev](https://github.com/ildus)).
|
||||
* Support `ALTER RENAME COLUMN` for the distributed table engine. Continuation of [#10727](https://github.com/ClickHouse/ClickHouse/issues/10727). Fixes [#10747](https://github.com/ClickHouse/ClickHouse/issues/10747). [#10887](https://github.com/ClickHouse/ClickHouse/pull/10887) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix UBSan report in Decimal parse. This fixes [#7540](https://github.com/ClickHouse/ClickHouse/issues/7540). [#10512](https://github.com/ClickHouse/ClickHouse/pull/10512) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potential floating point exception when parsing DateTime64. This fixes [#11374](https://github.com/ClickHouse/ClickHouse/issues/11374). [#11875](https://github.com/ClickHouse/ClickHouse/pull/11875) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix rare crash caused by using `Nullable` column in prewhere condition. [#11895](https://github.com/ClickHouse/ClickHouse/pull/11895) [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608) [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Don't allow arrayJoin inside higher order functions. It was leading to broken protocol synchronization. This closes [#3933](https://github.com/ClickHouse/ClickHouse/issues/3933). [#11846](https://github.com/ClickHouse/ClickHouse/pull/11846) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix wrong result of comparison of FixedString with constant String. This fixes [#11393](https://github.com/ClickHouse/ClickHouse/issues/11393). This bug appeared in version 20.4. [#11828](https://github.com/ClickHouse/ClickHouse/pull/11828) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix wrong result for `if` with NULLs in condition. [#11807](https://github.com/ClickHouse/ClickHouse/pull/11807) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fix using too many threads for queries. [#11788](https://github.com/ClickHouse/ClickHouse/pull/11788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed `Scalar doesn't exist` exception when using `WITH <scalar subquery> ...` in `SELECT ... FROM merge_tree_table ...` https://github.com/ClickHouse/ClickHouse/issues/11621. [#11767](https://github.com/ClickHouse/ClickHouse/pull/11767) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)).
|
||||
* Now replicated fetches will be cancelled during metadata alter. [#11744](https://github.com/ClickHouse/ClickHouse/pull/11744) ([alesapin](https://github.com/alesapin)).
|
||||
* Parse metadata stored in zookeeper before checking for equality. [#11739](https://github.com/ClickHouse/ClickHouse/pull/11739) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fixed LOGICAL_ERROR caused by wrong type deduction of complex literals in Values input format. [#11732](https://github.com/ClickHouse/ClickHouse/pull/11732) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fix `ORDER BY ... WITH FILL` over const columns. [#11697](https://github.com/ClickHouse/ClickHouse/pull/11697) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix very rare race condition in SYSTEM SYNC REPLICA. If the replicated table is created and at the same time from the separate connection another client is issuing `SYSTEM SYNC REPLICA` command on that table (this is unlikely, because another client should be aware that the table is created), it's possible to get nullptr dereference. [#11691](https://github.com/ClickHouse/ClickHouse/pull/11691) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Pass proper timeouts when communicating with XDBC bridge. Recently timeouts were not respected when checking bridge liveness and receiving meta info. [#11690](https://github.com/ClickHouse/ClickHouse/pull/11690) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix `LIMIT n WITH TIES` usage together with `ORDER BY` statement, which contains aliases. [#11689](https://github.com/ClickHouse/ClickHouse/pull/11689) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix possible `Pipeline stuck` for selects with parallel `FINAL`. Fixes [#11636](https://github.com/ClickHouse/ClickHouse/issues/11636). [#11682](https://github.com/ClickHouse/ClickHouse/pull/11682) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix error which leads to an incorrect state of `system.mutations`. It may show that whole mutation is already done but the server still has `MUTATE_PART` tasks in the replication queue and tries to execute them. This fixes [#11611](https://github.com/ClickHouse/ClickHouse/issues/11611). [#11681](https://github.com/ClickHouse/ClickHouse/pull/11681) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix syntax hilite in CREATE USER query. [#11664](https://github.com/ClickHouse/ClickHouse/pull/11664) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add support for regular expressions with case-insensitive flags. This fixes [#11101](https://github.com/ClickHouse/ClickHouse/issues/11101) and fixes [#11506](https://github.com/ClickHouse/ClickHouse/issues/11506). [#11649](https://github.com/ClickHouse/ClickHouse/pull/11649) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove trivial count query optimization if row-level security is set. In previous versions the user get total count of records in a table instead filtered. This fixes [#11352](https://github.com/ClickHouse/ClickHouse/issues/11352). [#11644](https://github.com/ClickHouse/ClickHouse/pull/11644) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bloom filters for String (data skipping indices). [#11638](https://github.com/ClickHouse/ClickHouse/pull/11638) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Without `-q` option the database does not get created at startup. [#11604](https://github.com/ClickHouse/ClickHouse/pull/11604) ([giordyb](https://github.com/giordyb)).
|
||||
* Fix error `Block structure mismatch` for queries with sampling reading from `Buffer` table. [#11602](https://github.com/ClickHouse/ClickHouse/pull/11602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix wrong exit code of the clickhouse-client, when `exception.code() % 256 == 0`. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)).
|
||||
* Fix race conditions in CREATE/DROP of different replicas of ReplicatedMergeTree. Continue to work if the table was not removed completely from ZooKeeper or not created successfully. This fixes [#11432](https://github.com/ClickHouse/ClickHouse/issues/11432). [#11592](https://github.com/ClickHouse/ClickHouse/pull/11592) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed rare segfault in `SHOW CREATE TABLE` Fixes [#11490](https://github.com/ClickHouse/ClickHouse/issues/11490). [#11579](https://github.com/ClickHouse/ClickHouse/pull/11579) ([tavplubix](https://github.com/tavplubix)).
|
||||
* All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)).
|
||||
* Fix the error `Data compressed with different methods` that can happen if `min_bytes_to_use_direct_io` is enabled and PREWHERE is active and using SAMPLE or high number of threads. This fixes [#11539](https://github.com/ClickHouse/ClickHouse/issues/11539). [#11540](https://github.com/ClickHouse/ClickHouse/pull/11540) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix shard_num/replica_num for `<node>` (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix async INSERT into Distributed for prefer_localhost_replica=0 and w/o internal_replication. [#11527](https://github.com/ClickHouse/ClickHouse/pull/11527) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix memory leak when exception is thrown in the middle of aggregation with `-State` functions. This fixes [#8995](https://github.com/ClickHouse/ClickHouse/issues/8995). [#11496](https://github.com/ClickHouse/ClickHouse/pull/11496) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix `Pipeline stuck` exception for `INSERT SELECT FINAL` where `SELECT` (`max_threads`>1) has multiple streams but `INSERT` has only one (`max_insert_threads`==0). [#11455](https://github.com/ClickHouse/ClickHouse/pull/11455) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix wrong result in queries like `select count() from t, u`. [#11454](https://github.com/ClickHouse/ClickHouse/pull/11454) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fix return compressed size for codecs. [#11448](https://github.com/ClickHouse/ClickHouse/pull/11448) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix server crash when a column has compression codec with non-literal arguments. Fixes [#11365](https://github.com/ClickHouse/ClickHouse/issues/11365). [#11431](https://github.com/ClickHouse/ClickHouse/pull/11431) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix potential uninitialized memory read in MergeTree shutdown if table was not created successfully. [#11420](https://github.com/ClickHouse/ClickHouse/pull/11420) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix crash in JOIN over `LowCarinality(T)` and `Nullable(T)`. [#11380](https://github.com/ClickHouse/ClickHouse/issues/11380). [#11414](https://github.com/ClickHouse/ClickHouse/pull/11414) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fix error code for wrong `USING` key. [#11373](https://github.com/ClickHouse/ClickHouse/issues/11373). [#11404](https://github.com/ClickHouse/ClickHouse/pull/11404) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fixed `geohashesInBox` with arguments outside of latitude/longitude range. [#11403](https://github.com/ClickHouse/ClickHouse/pull/11403) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Better errors for `joinGet()` functions. [#11389](https://github.com/ClickHouse/ClickHouse/pull/11389) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fix possible `Pipeline stuck` error for queries with external sort and limit. Fixes [#11359](https://github.com/ClickHouse/ClickHouse/issues/11359). [#11366](https://github.com/ClickHouse/ClickHouse/pull/11366) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Remove redundant lock during parts send in ReplicatedMergeTree. [#11354](https://github.com/ClickHouse/ClickHouse/pull/11354) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix support for `\G` (vertical output) in clickhouse-client in multiline mode. This closes [#9933](https://github.com/ClickHouse/ClickHouse/issues/9933). [#11350](https://github.com/ClickHouse/ClickHouse/pull/11350) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potential segfault when using `Lazy` database. [#11348](https://github.com/ClickHouse/ClickHouse/pull/11348) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix crash in direct selects from `Join` table engine (without JOIN) and wrong nullability. [#11340](https://github.com/ClickHouse/ClickHouse/pull/11340) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fix crash in `quantilesExactWeightedArray`. [#11337](https://github.com/ClickHouse/ClickHouse/pull/11337) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Now merges stopped before change metadata in `ALTER` queries. [#11335](https://github.com/ClickHouse/ClickHouse/pull/11335) ([alesapin](https://github.com/alesapin)).
|
||||
* Make writing to `MATERIALIZED VIEW` with setting `parallel_view_processing = 1` parallel again. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#11330](https://github.com/ClickHouse/ClickHouse/pull/11330) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix `visitParamExtractRaw` when extracted JSON has strings with unbalanced { or [. [#11318](https://github.com/ClickHouse/ClickHouse/pull/11318) ([Ewout](https://github.com/devwout)).
|
||||
* Fix very rare race condition in ThreadPool. [#11314](https://github.com/ClickHouse/ClickHouse/pull/11314) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix insignificant data race in `clickhouse-copier`. Found by integration tests. [#11313](https://github.com/ClickHouse/ClickHouse/pull/11313) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potential uninitialized memory in conversion. Example: `SELECT toIntervalSecond(now64())`. [#11311](https://github.com/ClickHouse/ClickHouse/pull/11311) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix the issue when index analysis cannot work if a table has Array column in primary key and if a query is filtering by this column with `empty` or `notEmpty` functions. This fixes [#11286](https://github.com/ClickHouse/ClickHouse/issues/11286). [#11303](https://github.com/ClickHouse/ClickHouse/pull/11303) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bug when query speed estimation can be incorrect and the limit of `min_execution_speed` may not work or work incorrectly if the query is throttled by `max_network_bandwidth`, `max_execution_speed` or `priority` settings. Change the default value of `timeout_before_checking_execution_speed` to non-zero, because otherwise the settings `min_execution_speed` and `max_execution_speed` have no effect. This fixes [#11297](https://github.com/ClickHouse/ClickHouse/issues/11297). This fixes [#5732](https://github.com/ClickHouse/ClickHouse/issues/5732). This fixes [#6228](https://github.com/ClickHouse/ClickHouse/issues/6228). Usability improvement: avoid concatenation of exception message with progress bar in `clickhouse-client`. [#11296](https://github.com/ClickHouse/ClickHouse/pull/11296) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix crash when `SET DEFAULT ROLE` is called with wrong arguments. This fixes https://github.com/ClickHouse/ClickHouse/issues/10586. [#11278](https://github.com/ClickHouse/ClickHouse/pull/11278) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix crash while reading malformed data in `Protobuf` format. This fixes https://github.com/ClickHouse/ClickHouse/issues/5957, fixes https://github.com/ClickHouse/ClickHouse/issues/11203. [#11258](https://github.com/ClickHouse/ClickHouse/pull/11258) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fixed a bug when `cache` dictionary could return default value instead of normal (when there are only expired keys). This affects only string fields. [#11233](https://github.com/ClickHouse/ClickHouse/pull/11233) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix error `Block structure mismatch in QueryPipeline` while reading from `VIEW` with constants in inner query. Fixes [#11181](https://github.com/ClickHouse/ClickHouse/issues/11181). [#11205](https://github.com/ClickHouse/ClickHouse/pull/11205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix possible exception `Invalid status for associated output`. [#11200](https://github.com/ClickHouse/ClickHouse/pull/11200) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Now `primary.idx` will be checked if it's defined in `CREATE` query. [#11199](https://github.com/ClickHouse/ClickHouse/pull/11199) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix possible error `Cannot capture column` for higher-order functions with `Array(Array(LowCardinality))` captured argument. [#11185](https://github.com/ClickHouse/ClickHouse/pull/11185) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed `S3` globbing which could fail in case of more than 1000 keys and some backends. [#11179](https://github.com/ClickHouse/ClickHouse/pull/11179) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
* If data skipping index is dependent on columns that are going to be modified during background merge (for SummingMergeTree, AggregatingMergeTree as well as for TTL GROUP BY), it was calculated incorrectly. This issue is fixed by moving index calculation after merge so the index is calculated on merged data. [#11162](https://github.com/ClickHouse/ClickHouse/pull/11162) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix for the hang which was happening sometimes during DROP of table engine=Kafka (or during server restarts). [#11145](https://github.com/ClickHouse/ClickHouse/pull/11145) ([filimonov](https://github.com/filimonov)).
|
||||
* Fix excessive reserving of threads for simple queries (optimization for reducing the number of threads, which was partly broken after changes in pipeline). [#11114](https://github.com/ClickHouse/ClickHouse/pull/11114) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Remove logging from mutation finalization task if nothing was finalized. [#11109](https://github.com/ClickHouse/ClickHouse/pull/11109) ([alesapin](https://github.com/alesapin)).
|
||||
* Fixed deadlock during server startup after update with changes in structure of system log tables. [#11106](https://github.com/ClickHouse/ClickHouse/pull/11106) ([alesapin](https://github.com/alesapin)).
|
||||
* Fixed memory leak in registerDiskS3. [#11074](https://github.com/ClickHouse/ClickHouse/pull/11074) ([Pavel Kovalenko](https://github.com/Jokser)).
|
||||
* Fix error `No such name in Block::erase()` when JOIN appears with PREWHERE or `optimize_move_to_prewhere` makes PREWHERE from WHERE. [#11051](https://github.com/ClickHouse/ClickHouse/pull/11051) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fixes the potential missed data during termination of Kafka engine table. [#11048](https://github.com/ClickHouse/ClickHouse/pull/11048) ([filimonov](https://github.com/filimonov)).
|
||||
* Fixed parseDateTime64BestEffort argument resolution bugs. [#10925](https://github.com/ClickHouse/ClickHouse/issues/10925). [#11038](https://github.com/ClickHouse/ClickHouse/pull/11038) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Now it's possible to `ADD/DROP` and `RENAME` the same one column in a single `ALTER` query. Exception message for simultaneous `MODIFY` and `RENAME` became more clear. Partially fixes [#10669](https://github.com/ClickHouse/ClickHouse/issues/10669). [#11037](https://github.com/ClickHouse/ClickHouse/pull/11037) ([alesapin](https://github.com/alesapin)).
|
||||
* Fixed parsing of S3 URLs. [#11036](https://github.com/ClickHouse/ClickHouse/pull/11036) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
* Fix very rare potential use-after-free error in MergeTree if table was not created successfully. [#10986](https://github.com/ClickHouse/ClickHouse/pull/10986) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix metadata (relative path for rename) and data (relative path for symlink) handling for Atomic database. [#10980](https://github.com/ClickHouse/ClickHouse/pull/10980) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix server crash on concurrent `ALTER` and `DROP DATABASE` queries with `Atomic` database engine. [#10968](https://github.com/ClickHouse/ClickHouse/pull/10968) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fix incorrect raw data size in method getRawData(). [#10964](https://github.com/ClickHouse/ClickHouse/pull/10964) ([Igr](https://github.com/ObjatieGroba)).
|
||||
* Fix incompatibility of two-level aggregation between versions 20.1 and earlier. This incompatibility happens when different versions of ClickHouse are used on initiator node and remote nodes and the size of GROUP BY result is large and aggregation is performed by a single String field. It leads to several unmerged rows for a single key in result. [#10952](https://github.com/ClickHouse/ClickHouse/pull/10952) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Avoid sending partially written files by the DistributedBlockOutputStream. [#10940](https://github.com/ClickHouse/ClickHouse/pull/10940) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix crash in `SELECT count(notNullIn(NULL, []))`. [#10920](https://github.com/ClickHouse/ClickHouse/pull/10920) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix for the hang which was happening sometimes during DROP of table engine=Kafka (or during server restarts). [#10910](https://github.com/ClickHouse/ClickHouse/pull/10910) ([filimonov](https://github.com/filimonov)).
|
||||
* Now it's possible to execute multiple `ALTER RENAME` like `a TO b, c TO a`. [#10895](https://github.com/ClickHouse/ClickHouse/pull/10895) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix possible race which could happen when you get result from aggregate function state from multiple thread for the same column. The only way (which I found) it can happen is when you use `finalizeAggregation` function while reading from table with `Memory` engine which stores `AggregateFunction` state for `quanite*` function. [#10890](https://github.com/ClickHouse/ClickHouse/pull/10890) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix backward compatibility with tuples in Distributed tables. [#10889](https://github.com/ClickHouse/ClickHouse/pull/10889) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix SIGSEGV in StringHashTable (if such key does not exist). [#10870](https://github.com/ClickHouse/ClickHouse/pull/10870) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fixed `WATCH` hangs after `LiveView` table was dropped from database with `Atomic` engine. [#10859](https://github.com/ClickHouse/ClickHouse/pull/10859) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fixed bug in `ReplicatedMergeTree` which might cause some `ALTER` on `OPTIMIZE` query to hang waiting for some replica after it become inactive. [#10849](https://github.com/ClickHouse/ClickHouse/pull/10849) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Now constraints are updated if the column participating in `CONSTRAINT` expression was renamed. Fixes [#10844](https://github.com/ClickHouse/ClickHouse/issues/10844). [#10847](https://github.com/ClickHouse/ClickHouse/pull/10847) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix potential read of uninitialized memory in cache dictionary. [#10834](https://github.com/ClickHouse/ClickHouse/pull/10834) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix columns order after Block::sortColumns() (also add a test that shows that it affects some real use case - Buffer engine). [#10826](https://github.com/ClickHouse/ClickHouse/pull/10826) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix the issue with ODBC bridge when no quoting of identifiers is requested. This fixes [#7984](https://github.com/ClickHouse/ClickHouse/issues/7984). [#10821](https://github.com/ClickHouse/ClickHouse/pull/10821) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix UBSan and MSan report in DateLUT. [#10798](https://github.com/ClickHouse/ClickHouse/pull/10798) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Make use of `src_type` for correct type conversion in key conditions. Fixes [#6287](https://github.com/ClickHouse/ClickHouse/issues/6287). [#10791](https://github.com/ClickHouse/ClickHouse/pull/10791) ([Andrew Onyshchuk](https://github.com/oandrew)).
|
||||
* Get rid of old libunwind patches. https://github.com/ClickHouse-Extras/libunwind/commit/500aa227911bd185a94bfc071d68f4d3b03cb3b1#r39048012 This allows to disable `-fno-omit-frame-pointer` in `clang` builds that improves performance at least by 1% in average. [#10761](https://github.com/ClickHouse/ClickHouse/pull/10761) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix avgWeighted when using floating-point weight over multiple shards. [#10758](https://github.com/ClickHouse/ClickHouse/pull/10758) ([Baudouin Giard](https://github.com/bgiard)).
|
||||
* Fix `parallel_view_processing` behavior. Now all insertions into `MATERIALIZED VIEW` without exception should be finished if exception happened. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#10757](https://github.com/ClickHouse/ClickHouse/pull/10757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix combinator -OrNull and -OrDefault when combined with -State. [#10741](https://github.com/ClickHouse/ClickHouse/pull/10741) ([hcz](https://github.com/hczhcz)).
|
||||
* Fix crash in `generateRandom` with nested types. Fixes [#10583](https://github.com/ClickHouse/ClickHouse/issues/10583). [#10734](https://github.com/ClickHouse/ClickHouse/pull/10734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix data corruption for `LowCardinality(FixedString)` key column in `SummingMergeTree` which could have happened after merge. Fixes [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489). [#10721](https://github.com/ClickHouse/ClickHouse/pull/10721) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix usage of primary key wrapped into a function with 'FINAL' modifier and 'ORDER BY' optimization. [#10715](https://github.com/ClickHouse/ClickHouse/pull/10715) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix possible buffer overflow in function `h3EdgeAngle`. [#10711](https://github.com/ClickHouse/ClickHouse/pull/10711) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix disappearing totals. Totals could have being filtered if query had had join or subquery with external where condition. Fixes [#10674](https://github.com/ClickHouse/ClickHouse/issues/10674). [#10698](https://github.com/ClickHouse/ClickHouse/pull/10698) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix atomicity of HTTP insert. This fixes [#9666](https://github.com/ClickHouse/ClickHouse/issues/9666). [#10687](https://github.com/ClickHouse/ClickHouse/pull/10687) ([Andrew Onyshchuk](https://github.com/oandrew)).
|
||||
* Fix multiple usages of `IN` operator with the identical set in one query. [#10686](https://github.com/ClickHouse/ClickHouse/pull/10686) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fixed bug, which causes http requests stuck on client close when `readonly=2` and `cancel_http_readonly_queries_on_client_close=1`. Fixes [#7939](https://github.com/ClickHouse/ClickHouse/issues/7939), [#7019](https://github.com/ClickHouse/ClickHouse/issues/7019), [#7736](https://github.com/ClickHouse/ClickHouse/issues/7736), [#7091](https://github.com/ClickHouse/ClickHouse/issues/7091). [#10684](https://github.com/ClickHouse/ClickHouse/pull/10684) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fix order of parameters in AggregateTransform constructor. [#10667](https://github.com/ClickHouse/ClickHouse/pull/10667) ([palasonic1](https://github.com/palasonic1)).
|
||||
* Fix the lack of parallel execution of remote queries with `distributed_aggregation_memory_efficient` enabled. Fixes [#10655](https://github.com/ClickHouse/ClickHouse/issues/10655). [#10664](https://github.com/ClickHouse/ClickHouse/pull/10664) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix possible incorrect number of rows for queries with `LIMIT`. Fixes [#10566](https://github.com/ClickHouse/ClickHouse/issues/10566), [#10709](https://github.com/ClickHouse/ClickHouse/issues/10709). [#10660](https://github.com/ClickHouse/ClickHouse/pull/10660) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix bug which locks concurrent alters when table has a lot of parts. [#10659](https://github.com/ClickHouse/ClickHouse/pull/10659) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix nullptr dereference in StorageBuffer if server was shutdown before table startup. [#10641](https://github.com/ClickHouse/ClickHouse/pull/10641) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix predicates optimization for distributed queries (`enable_optimize_predicate_expression=1`) for queries with `HAVING` section (i.e. when filtering on the server initiator is required), by preserving the order of expressions (and this is enough to fix), and also force aggregator use column names over indexes. Fixes: [#10613](https://github.com/ClickHouse/ClickHouse/issues/10613), [#11413](https://github.com/ClickHouse/ClickHouse/issues/11413). [#10621](https://github.com/ClickHouse/ClickHouse/pull/10621) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix optimize_skip_unused_shards with LowCardinality. [#10611](https://github.com/ClickHouse/ClickHouse/pull/10611) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix segfault in StorageBuffer when exception on server startup. Fixes [#10550](https://github.com/ClickHouse/ClickHouse/issues/10550). [#10609](https://github.com/ClickHouse/ClickHouse/pull/10609) ([tavplubix](https://github.com/tavplubix)).
|
||||
* On `SYSTEM DROP DNS CACHE` query also drop caches, which are used to check if user is allowed to connect from some IP addresses. [#10608](https://github.com/ClickHouse/ClickHouse/pull/10608) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fixed incorrect scalar results inside inner query of `MATERIALIZED VIEW` in case if this query contained dependent table. [#10603](https://github.com/ClickHouse/ClickHouse/pull/10603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed handling condition variable for synchronous mutations. In some cases signals to that condition variable could be lost. [#10588](https://github.com/ClickHouse/ClickHouse/pull/10588) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
* Fixes possible crash `createDictionary()` is called before `loadStoredObject()` has finished. [#10587](https://github.com/ClickHouse/ClickHouse/pull/10587) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix error `the BloomFilter false positive must be a double number between 0 and 1` [#10551](https://github.com/ClickHouse/ClickHouse/issues/10551). [#10569](https://github.com/ClickHouse/ClickHouse/pull/10569) ([Winter Zhang](https://github.com/zhang2014)).
|
||||
* Fix SELECT of column ALIAS which default expression type different from column type. [#10563](https://github.com/ClickHouse/ClickHouse/pull/10563) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Implemented comparison between DateTime64 and String values (just like for DateTime). [#10560](https://github.com/ClickHouse/ClickHouse/pull/10560) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Fix index corruption, which may accur in some cases after merge compact parts into another compact part. [#10531](https://github.com/ClickHouse/ClickHouse/pull/10531) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Disable GROUP BY sharding_key optimization by default (`optimize_distributed_group_by_sharding_key` had been introduced and turned of by default, due to trickery of sharding_key analyzing, simple example is `if` in sharding key) and fix it for WITH ROLLUP/CUBE/TOTALS. [#10516](https://github.com/ClickHouse/ClickHouse/pull/10516) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fixes: [#10263](https://github.com/ClickHouse/ClickHouse/issues/10263) (after that PR dist send via INSERT had been postponing on each INSERT) Fixes: [#8756](https://github.com/ClickHouse/ClickHouse/issues/8756) (that PR breaks distributed sends with all of the following conditions met (unlikely setup for now I guess): `internal_replication == false`, multiple local shards (activates the hardlinking code) and `distributed_storage_policy` (makes `link(2)` fails on `EXDEV`)). [#10486](https://github.com/ClickHouse/ClickHouse/pull/10486) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fixed error with "max_rows_to_sort" limit. [#10268](https://github.com/ClickHouse/ClickHouse/pull/10268) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Get dictionary and check access rights only once per each call of any function reading external dictionaries. [#10928](https://github.com/ClickHouse/ClickHouse/pull/10928) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Improvement
|
||||
|
||||
* Apply `TTL` for old data, after `ALTER MODIFY TTL` query. This behaviour is controlled by setting `materialize_ttl_after_modify`, which is enabled by default. [#11042](https://github.com/ClickHouse/ClickHouse/pull/11042) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* When parsing C-style backslash escapes in string literals, VALUES and various text formats (this is an extension to SQL standard that is endemic for ClickHouse and MySQL), keep backslash if unknown escape sequence is found (e.g. `\%` or `\w`) that will make usage of `LIKE` and `match` regular expressions more convenient (it's enough to write `name LIKE 'used\_cars'` instead of `name LIKE 'used\\_cars'`) and more compatible at the same time. This fixes [#10922](https://github.com/ClickHouse/ClickHouse/issues/10922). [#11208](https://github.com/ClickHouse/ClickHouse/pull/11208) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* When reading Decimal value, cut extra digits after point. This behaviour is more compatible with MySQL and PostgreSQL. This fixes [#10202](https://github.com/ClickHouse/ClickHouse/issues/10202). [#11831](https://github.com/ClickHouse/ClickHouse/pull/11831) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Allow to DROP replicated table if the metadata in ZooKeeper was already removed and does not exist (this is also the case when using TestKeeper for testing and the server was restarted). Allow to RENAME replicated table even if there is an error communicating with ZooKeeper. This fixes [#10720](https://github.com/ClickHouse/ClickHouse/issues/10720). [#11652](https://github.com/ClickHouse/ClickHouse/pull/11652) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Slightly improve diagnostic of reading decimal from string. This closes [#10202](https://github.com/ClickHouse/ClickHouse/issues/10202). [#11829](https://github.com/ClickHouse/ClickHouse/pull/11829) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix sleep invocation in signal handler. It was sleeping for less amount of time than expected. [#11825](https://github.com/ClickHouse/ClickHouse/pull/11825) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* (Only Linux) OS related performance metrics (for CPU and I/O) will work even without `CAP_NET_ADMIN` capability. [#10544](https://github.com/ClickHouse/ClickHouse/pull/10544) ([Alexander Kazakov](https://github.com/Akazz)).
|
||||
* Added `hostname` as an alias to function `hostName`. This feature was suggested by Victor Tarnavskiy from Yandex.Metrica. [#11821](https://github.com/ClickHouse/ClickHouse/pull/11821) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added support for distributed `DDL` (update/delete/drop partition) on cross replication clusters. [#11703](https://github.com/ClickHouse/ClickHouse/pull/11703) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Emit warning instead of error in server log at startup if we cannot listen one of the listen addresses (e.g. IPv6 is unavailable inside Docker). Note that if server fails to listen all listed addresses, it will refuse to startup as before. This fixes [#4406](https://github.com/ClickHouse/ClickHouse/issues/4406). [#11687](https://github.com/ClickHouse/ClickHouse/pull/11687) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Default user and database creation on docker image starting. [#10637](https://github.com/ClickHouse/ClickHouse/pull/10637) ([Paramtamtam](https://github.com/tarampampam)).
|
||||
* When multiline query is printed to server log, the lines are joined. Make it to work correct in case of multiline string literals, identifiers and single-line comments. This fixes [#3853](https://github.com/ClickHouse/ClickHouse/issues/3853). [#11686](https://github.com/ClickHouse/ClickHouse/pull/11686) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Multiple names are now allowed in commands: CREATE USER, CREATE ROLE, ALTER USER, SHOW CREATE USER, SHOW GRANTS and so on. [#11670](https://github.com/ClickHouse/ClickHouse/pull/11670) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Clear password from command line in `clickhouse-client` and `clickhouse-benchmark` if the user has specified it with explicit value. This prevents password exposure by `ps` and similar tools. [#11665](https://github.com/ClickHouse/ClickHouse/pull/11665) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Don't use debug info from ELF file if it doesn't correspond to the running binary. It is needed to avoid printing wrong function names and source locations in stack traces. This fixes [#7514](https://github.com/ClickHouse/ClickHouse/issues/7514). [#11657](https://github.com/ClickHouse/ClickHouse/pull/11657) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Return NULL/zero when value is not parsed completely in parseDateTimeBestEffortOrNull/Zero functions. This fixes [#7876](https://github.com/ClickHouse/ClickHouse/issues/7876). [#11653](https://github.com/ClickHouse/ClickHouse/pull/11653) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Skip empty parameters in requested URL. They may appear when you write `http://localhost:8123/?&a=b` or `http://localhost:8123/?a=b&&c=d`. This closes [#10749](https://github.com/ClickHouse/ClickHouse/issues/10749). [#11651](https://github.com/ClickHouse/ClickHouse/pull/11651) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Allow using `groupArrayArray` and `groupUniqArrayArray` as `SimpleAggregateFunction`. [#11650](https://github.com/ClickHouse/ClickHouse/pull/11650) ([Volodymyr Kuznetsov](https://github.com/ksvladimir)).
|
||||
* Allow comparison with constant strings by implicit conversions when analysing index conditions on other types. This may close [#11630](https://github.com/ClickHouse/ClickHouse/issues/11630). [#11648](https://github.com/ClickHouse/ClickHouse/pull/11648) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* https://github.com/ClickHouse/ClickHouse/pull/7572#issuecomment-642815377 Support config default HTTPHandlers. [#11628](https://github.com/ClickHouse/ClickHouse/pull/11628) ([Winter Zhang](https://github.com/zhang2014)).
|
||||
* Make more input formats to work with Kafka engine. Fix the issue with premature flushes. Fix the performance issue when `kafka_num_consumers` is greater than number of partitions in topic. [#11599](https://github.com/ClickHouse/ClickHouse/pull/11599) ([filimonov](https://github.com/filimonov)).
|
||||
* Improve `multiple_joins_rewriter_version=2` logic. Fix unknown columns error for lambda aliases. [#11587](https://github.com/ClickHouse/ClickHouse/pull/11587) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Better exception message when cannot parse columns declaration list. This closes [#10403](https://github.com/ClickHouse/ClickHouse/issues/10403). [#11537](https://github.com/ClickHouse/ClickHouse/pull/11537) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Improve `enable_optimize_predicate_expression=1` logic for VIEW. [#11513](https://github.com/ClickHouse/ClickHouse/pull/11513) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Adding support for PREWHERE in live view tables. [#11495](https://github.com/ClickHouse/ClickHouse/pull/11495) ([vzakaznikov](https://github.com/vzakaznikov)).
|
||||
* Automatically update DNS cache, which is used to check if user is allowed to connect from an address. [#11487](https://github.com/ClickHouse/ClickHouse/pull/11487) ([tavplubix](https://github.com/tavplubix)).
|
||||
* OPTIMIZE FINAL will force merge even if concurrent merges are performed. This closes [#11309](https://github.com/ClickHouse/ClickHouse/issues/11309) and closes [#11322](https://github.com/ClickHouse/ClickHouse/issues/11322). [#11346](https://github.com/ClickHouse/ClickHouse/pull/11346) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Suppress output of cancelled queries in clickhouse-client. In previous versions result may continue to print in terminal even after you press Ctrl+C to cancel query. This closes [#9473](https://github.com/ClickHouse/ClickHouse/issues/9473). [#11342](https://github.com/ClickHouse/ClickHouse/pull/11342) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Better log messages in while reloading configuration. [#11341](https://github.com/ClickHouse/ClickHouse/pull/11341) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove trailing whitespaces from formatted queries in `clickhouse-client` or `clickhouse-format` in some cases. [#11325](https://github.com/ClickHouse/ClickHouse/pull/11325) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add setting "output_format_pretty_max_value_width". If value is longer, it will be cut to avoid output of too large values in terminal. This closes [#11140](https://github.com/ClickHouse/ClickHouse/issues/11140). [#11324](https://github.com/ClickHouse/ClickHouse/pull/11324) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Better exception message in case when there is shortage of memory mappings. This closes [#11027](https://github.com/ClickHouse/ClickHouse/issues/11027). [#11316](https://github.com/ClickHouse/ClickHouse/pull/11316) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Support (U)Int8, (U)Int16, Date in ASOF JOIN. [#11301](https://github.com/ClickHouse/ClickHouse/pull/11301) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Support kafka_client_id parameter for Kafka tables. It also changes the default `client.id` used by ClickHouse when communicating with Kafka to be more verbose and usable. [#11252](https://github.com/ClickHouse/ClickHouse/pull/11252) ([filimonov](https://github.com/filimonov)).
|
||||
* Keep the value of `DistributedFilesToInsert` metric on exceptions. In previous versions, the value was set when we are going to send some files, but it is zero, if there was an exception and some files are still pending. Now it corresponds to the number of pending files in filesystem. [#11220](https://github.com/ClickHouse/ClickHouse/pull/11220) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add support for multi-word data type names (such as `DOUBLE PRECISION` and `CHAR VARYING`) for better SQL compatibility. [#11214](https://github.com/ClickHouse/ClickHouse/pull/11214) ([Павел Потемкин](https://github.com/Potya)).
|
||||
* The query log is now enabled by default. [#11184](https://github.com/ClickHouse/ClickHouse/pull/11184) ([Ivan Blinkov](https://github.com/blinkov)).
|
||||
* Show authentication type in table system.users and while executing SHOW CREATE USER query. [#11080](https://github.com/ClickHouse/ClickHouse/pull/11080) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Remove data on explicit `DROP DATABASE` for `Memory` database engine. Fixes [#10557](https://github.com/ClickHouse/ClickHouse/issues/10557). [#11021](https://github.com/ClickHouse/ClickHouse/pull/11021) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Set thread names for internal threads of rdkafka library. Make logs from rdkafka available in server logs. [#10983](https://github.com/ClickHouse/ClickHouse/pull/10983) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Support for unicode whitespaces in queries. This helps when queries are copy-pasted from Word or from web page. This fixes [#10896](https://github.com/ClickHouse/ClickHouse/issues/10896). [#10903](https://github.com/ClickHouse/ClickHouse/pull/10903) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Allow large UInt types as the index in function `tupleElement`. [#10874](https://github.com/ClickHouse/ClickHouse/pull/10874) ([hcz](https://github.com/hczhcz)).
|
||||
* Respect prefer_localhost_replica/load_balancing on INSERT into Distributed. [#10867](https://github.com/ClickHouse/ClickHouse/pull/10867) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Introduce `min_insert_block_size_rows_for_materialized_views`, `min_insert_block_size_bytes_for_materialized_views` settings. This settings are similar to `min_insert_block_size_rows` and `min_insert_block_size_bytes`, but applied only for blocks inserted into `MATERIALIZED VIEW`. It helps to control blocks squashing while pushing to MVs and avoid excessive memory usage. [#10858](https://github.com/ClickHouse/ClickHouse/pull/10858) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Get rid of exception from replicated queue during server shutdown. Fixes [#10819](https://github.com/ClickHouse/ClickHouse/issues/10819). [#10841](https://github.com/ClickHouse/ClickHouse/pull/10841) ([alesapin](https://github.com/alesapin)).
|
||||
* Ensure that `varSamp`, `varPop` cannot return negative results due to numerical errors and that `stddevSamp`, `stddevPop` cannot be calculated from negative variance. This fixes [#10532](https://github.com/ClickHouse/ClickHouse/issues/10532). [#10829](https://github.com/ClickHouse/ClickHouse/pull/10829) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Better DNS exception message. This fixes [#10813](https://github.com/ClickHouse/ClickHouse/issues/10813). [#10828](https://github.com/ClickHouse/ClickHouse/pull/10828) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Change HTTP response code in case of some parse errors to 400 Bad Request. This fix [#10636](https://github.com/ClickHouse/ClickHouse/issues/10636). [#10640](https://github.com/ClickHouse/ClickHouse/pull/10640) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Print a message if clickhouse-client is newer than clickhouse-server. [#10627](https://github.com/ClickHouse/ClickHouse/pull/10627) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Adding support for `INSERT INTO [db.]table WATCH` query. [#10498](https://github.com/ClickHouse/ClickHouse/pull/10498) ([vzakaznikov](https://github.com/vzakaznikov)).
|
||||
* Allow to pass quota_key in clickhouse-client. This closes [#10227](https://github.com/ClickHouse/ClickHouse/issues/10227). [#10270](https://github.com/ClickHouse/ClickHouse/pull/10270) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### Performance Improvement
|
||||
|
||||
* Allow multiple replicas to assign merges, mutations, partition drop, move and replace concurrently. This closes [#10367](https://github.com/ClickHouse/ClickHouse/issues/10367). [#11639](https://github.com/ClickHouse/ClickHouse/pull/11639) ([alexey-milovidov](https://github.com/alexey-milovidov)) [#11795](https://github.com/ClickHouse/ClickHouse/pull/11795) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Optimization of GROUP BY with respect to table sorting key, enabled with `optimize_aggregation_in_order` setting. [#9113](https://github.com/ClickHouse/ClickHouse/pull/9113) ([dimarub2000](https://github.com/dimarub2000)).
|
||||
* Selects with final are executed in parallel. Added setting `max_final_threads` to limit the number of threads used. [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Improve performance for INSERT queries via `INSERT SELECT` or INSERT with clickhouse-client when small blocks are generated (typical case with parallel parsing). This fixes [#11275](https://github.com/ClickHouse/ClickHouse/issues/11275). Fix the issue that CONSTRAINTs were not working for DEFAULT fields. This fixes [#11273](https://github.com/ClickHouse/ClickHouse/issues/11273). Fix the issue that CONSTRAINTS were ignored for TEMPORARY tables. This fixes [#11274](https://github.com/ClickHouse/ClickHouse/issues/11274). [#11276](https://github.com/ClickHouse/ClickHouse/pull/11276) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Optimization that eliminates min/max/any aggregators of GROUP BY keys in SELECT section, enabled with `optimize_aggregators_of_group_by_keys` setting. [#11667](https://github.com/ClickHouse/ClickHouse/pull/11667) ([xPoSx](https://github.com/xPoSx)). [#11806](https://github.com/ClickHouse/ClickHouse/pull/11806) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* New optimization that takes all operations out of `any` function, enabled with `optimize_move_functions_out_of_any` [#11529](https://github.com/ClickHouse/ClickHouse/pull/11529) ([Ruslan](https://github.com/kamalov-ruslan)).
|
||||
* Improve performance of `clickhouse-client` in interactive mode when Pretty formats are used. In previous versions, significant amount of time can be spent calculating visible width of UTF-8 string. This closes [#11323](https://github.com/ClickHouse/ClickHouse/issues/11323). [#11323](https://github.com/ClickHouse/ClickHouse/pull/11323) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Improved performance for queries with `ORDER BY` and small `LIMIT` (less, then `max_block_size`). [#11171](https://github.com/ClickHouse/ClickHouse/pull/11171) ([Albert Kidrachev](https://github.com/Provet)).
|
||||
* Enable `mlock` of clickhouse binary by default. It will prevent clickhouse executable from being paged out under high IO load. [#11139](https://github.com/ClickHouse/ClickHouse/pull/11139) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Make queries with `sum` aggregate function and without GROUP BY keys to run multiple times faster. [#10992](https://github.com/ClickHouse/ClickHouse/pull/10992) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Improving radix sort (used in `ORDER BY` with simple keys) by removing some redundant data moves. [#10981](https://github.com/ClickHouse/ClickHouse/pull/10981) ([Arslan Gumerov](https://github.com/g-arslan)).
|
||||
* Sort bigger parts of the left table in MergeJoin. Buffer left blocks in memory. Add `partial_merge_join_left_table_buffer_bytes` setting to manage the left blocks buffers sizes. [#10601](https://github.com/ClickHouse/ClickHouse/pull/10601) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Remove duplicate ORDER BY and DISTINCT from subqueries, this optimization is enabled with `optimize_duplicate_order_by_and_distinct` [#10067](https://github.com/ClickHouse/ClickHouse/pull/10067) ([Mikhail Malafeev](https://github.com/demo-99)).
|
||||
* This feature eliminates functions of other keys in GROUP BY section, enabled with `optimize_group_by_function_keys` [#10051](https://github.com/ClickHouse/ClickHouse/pull/10051) ([xPoSx](https://github.com/xPoSx)).
|
||||
* New optimization that takes arithmetic operations out of aggregate functions, enabled with `optimize_arithmetic_operations_in_aggregate_functions` [#10047](https://github.com/ClickHouse/ClickHouse/pull/10047) ([Ruslan](https://github.com/kamalov-ruslan)).
|
||||
* Use HTTP client for S3 based on Poco instead of curl. This will improve performance and lower memory usage of s3 storage and table functions. [#11230](https://github.com/ClickHouse/ClickHouse/pull/11230) ([Pavel Kovalenko](https://github.com/Jokser)).
|
||||
* Fix Kafka performance issue related to reschedules based on limits, which were always applied. [#11149](https://github.com/ClickHouse/ClickHouse/pull/11149) ([filimonov](https://github.com/filimonov)).
|
||||
* Enable percpu_arena:percpu for jemalloc (This will reduce memory fragmentation due to thread pool). [#11084](https://github.com/ClickHouse/ClickHouse/pull/11084) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Optimize memory usage when reading a response from an S3 HTTP client. [#11561](https://github.com/ClickHouse/ClickHouse/pull/11561) ([Pavel Kovalenko](https://github.com/Jokser)).
|
||||
* Adjust the default Kafka settings for better performance. [#11388](https://github.com/ClickHouse/ClickHouse/pull/11388) ([filimonov](https://github.com/filimonov)).
|
||||
|
||||
#### Experimental Feature
|
||||
|
||||
* Add data type `Point` (Tuple(Float64, Float64)) and `Polygon` (Array(Array(Tuple(Float64, Float64))). [#10678](https://github.com/ClickHouse/ClickHouse/pull/10678) ([Alexey Ilyukhov](https://github.com/livace)).
|
||||
* Add's a `hasSubstr` function that allows for look for subsequences in arrays. Note: this function is likely to be renamed without further notice. [#11071](https://github.com/ClickHouse/ClickHouse/pull/11071) ([Ryad Zenine](https://github.com/r-zenine)).
|
||||
* Added OpenCL support and bitonic sort algorithm, which can be used for sorting integer types of data in single column. Needs to be build with flag `-DENABLE_OPENCL=1`. For using bitonic sort algorithm instead of others you need to set `bitonic_sort` for Setting's option `special_sort` and make sure that OpenCL is available. This feature does not improve performance or anything else, it is only provided as an example and for demonstration purposes. It is likely to be removed in near future if there will be no further development in this direction. [#10232](https://github.com/ClickHouse/ClickHouse/pull/10232) ([Ri](https://github.com/margaritiko)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
|
||||
* Enable clang-tidy for programs and utils. [#10991](https://github.com/ClickHouse/ClickHouse/pull/10991) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove dependency on `tzdata`: do not fail if `/usr/share/zoneinfo` directory does not exist. Note that all timezones work in ClickHouse even without tzdata installed in system. [#11827](https://github.com/ClickHouse/ClickHouse/pull/11827) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added MSan and UBSan stress tests. Note that we already have MSan, UBSan for functional tests and "stress" test is another kind of tests. [#10871](https://github.com/ClickHouse/ClickHouse/pull/10871) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Print compiler build id in crash messages. It will make us slightly more certain about what binary has crashed. Added new function `buildId`. [#11824](https://github.com/ClickHouse/ClickHouse/pull/11824) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added a test to ensure that mutations continue to work after FREEZE query. [#11820](https://github.com/ClickHouse/ClickHouse/pull/11820) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Don't allow tests with "fail" substring in their names because it makes looking at the tests results in browser less convenient when you type Ctrl+F and search for "fail". [#11817](https://github.com/ClickHouse/ClickHouse/pull/11817) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Removes unused imports from HTTPHandlerFactory. [#11660](https://github.com/ClickHouse/ClickHouse/pull/11660) ([Bharat Nallan](https://github.com/bharatnc)).
|
||||
* Added a random sampling of instances where copier is executed. It is needed to avoid `Too many simultaneous queries` error. Also increased timeout and decreased fault probability. [#11573](https://github.com/ClickHouse/ClickHouse/pull/11573) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix missed include. [#11525](https://github.com/ClickHouse/ClickHouse/pull/11525) ([Matwey V. Kornilov](https://github.com/matwey)).
|
||||
* Speed up build by removing old example programs. Also found some orphan functional test. [#11486](https://github.com/ClickHouse/ClickHouse/pull/11486) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Increase ccache size for builds in CI. [#11450](https://github.com/ClickHouse/ClickHouse/pull/11450) ([alesapin](https://github.com/alesapin)).
|
||||
* Leave only unit_tests_dbms in deb build. [#11429](https://github.com/ClickHouse/ClickHouse/pull/11429) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Update librdkafka to version [1.4.2](https://github.com/edenhill/librdkafka/releases/tag/v1.4.2). [#11256](https://github.com/ClickHouse/ClickHouse/pull/11256) ([filimonov](https://github.com/filimonov)).
|
||||
* Refactor CMake build files. [#11390](https://github.com/ClickHouse/ClickHouse/pull/11390) ([Ivan](https://github.com/abyss7)).
|
||||
* Fix several flaky integration tests. [#11355](https://github.com/ClickHouse/ClickHouse/pull/11355) ([alesapin](https://github.com/alesapin)).
|
||||
* Add support for unit tests run with UBSan. [#11345](https://github.com/ClickHouse/ClickHouse/pull/11345) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove redundant timeout from integration test `test_insertion_sync_fails_with_timeout`. [#11343](https://github.com/ClickHouse/ClickHouse/pull/11343) ([alesapin](https://github.com/alesapin)).
|
||||
* Better check for hung queries in clickhouse-test. [#11321](https://github.com/ClickHouse/ClickHouse/pull/11321) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Emit a warning if server was build in debug or with sanitizers. [#11304](https://github.com/ClickHouse/ClickHouse/pull/11304) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Now clickhouse-test check the server aliveness before tests run. [#11285](https://github.com/ClickHouse/ClickHouse/pull/11285) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix potentially flacky test `00731_long_merge_tree_select_opened_files.sh`. It does not fail frequently but we have discovered potential race condition in this test while experimenting with ThreadFuzzer: [#9814](https://github.com/ClickHouse/ClickHouse/issues/9814) See [link](https://clickhouse-test-reports.s3.yandex.net/9814/40e3023e215df22985d275bf85f4d2290897b76b/functional_stateless_tests_(unbundled).html#fail1) for the example. [#11270](https://github.com/ClickHouse/ClickHouse/pull/11270) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Repeat test in CI if `curl` invocation was timed out. It is possible due to system hangups for 10+ seconds that are typical in our CI infrastructure. This fixes [#11267](https://github.com/ClickHouse/ClickHouse/issues/11267). [#11268](https://github.com/ClickHouse/ClickHouse/pull/11268) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add a test for Join table engine from @donmikel. This closes [#9158](https://github.com/ClickHouse/ClickHouse/issues/9158). [#11265](https://github.com/ClickHouse/ClickHouse/pull/11265) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix several non significant errors in unit tests. [#11262](https://github.com/ClickHouse/ClickHouse/pull/11262) ([alesapin](https://github.com/alesapin)).
|
||||
* Now parts of linker command for `cctz` library will not be shuffled with other libraries. [#11213](https://github.com/ClickHouse/ClickHouse/pull/11213) ([alesapin](https://github.com/alesapin)).
|
||||
* Split /programs/server into actual program and library. [#11186](https://github.com/ClickHouse/ClickHouse/pull/11186) ([Ivan](https://github.com/abyss7)).
|
||||
* Improve build scripts for protobuf & gRPC. [#11172](https://github.com/ClickHouse/ClickHouse/pull/11172) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Enable performance test that was not working. [#11158](https://github.com/ClickHouse/ClickHouse/pull/11158) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Create root S3 bucket for tests before any CH instance is started. [#11142](https://github.com/ClickHouse/ClickHouse/pull/11142) ([Pavel Kovalenko](https://github.com/Jokser)).
|
||||
* Add performance test for non-constant polygons. [#11141](https://github.com/ClickHouse/ClickHouse/pull/11141) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixing `00979_live_view_watch_continuous_aggregates` test. [#11024](https://github.com/ClickHouse/ClickHouse/pull/11024) ([vzakaznikov](https://github.com/vzakaznikov)).
|
||||
* Add ability to run zookeeper in integration tests over tmpfs. [#11002](https://github.com/ClickHouse/ClickHouse/pull/11002) ([alesapin](https://github.com/alesapin)).
|
||||
* Wait for odbc-bridge with exponential backoff. Previous wait time of 200 ms was not enough in our CI environment. [#10990](https://github.com/ClickHouse/ClickHouse/pull/10990) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix non-deterministic test. [#10989](https://github.com/ClickHouse/ClickHouse/pull/10989) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added a test for empty external data. [#10926](https://github.com/ClickHouse/ClickHouse/pull/10926) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Database is recreated for every test. This improves separation of tests. [#10902](https://github.com/ClickHouse/ClickHouse/pull/10902) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added more asserts in columns code. [#10833](https://github.com/ClickHouse/ClickHouse/pull/10833) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Better cooperation with sanitizers. Print information about query_id in the message of sanitizer failure. [#10832](https://github.com/ClickHouse/ClickHouse/pull/10832) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix obvious race condition in "Split build smoke test" check. [#10820](https://github.com/ClickHouse/ClickHouse/pull/10820) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix (false) MSan report in MergeTreeIndexFullText. The issue first appeared in [#9968](https://github.com/ClickHouse/ClickHouse/issues/9968). [#10801](https://github.com/ClickHouse/ClickHouse/pull/10801) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add MSan suppression for MariaDB Client library. [#10800](https://github.com/ClickHouse/ClickHouse/pull/10800) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* GRPC make couldn't find protobuf files, changed make file by adding the right link. [#10794](https://github.com/ClickHouse/ClickHouse/pull/10794) ([mnkonkova](https://github.com/mnkonkova)).
|
||||
* Enable extra warnings (`-Weverything`) for base, utils, programs. Note that we already have it for the most of the code. [#10779](https://github.com/ClickHouse/ClickHouse/pull/10779) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Suppressions of warnings from libraries was mistakenly declared as public in [#10396](https://github.com/ClickHouse/ClickHouse/issues/10396). [#10776](https://github.com/ClickHouse/ClickHouse/pull/10776) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Restore a patch that was accidentially deleted in [#10396](https://github.com/ClickHouse/ClickHouse/issues/10396). [#10774](https://github.com/ClickHouse/ClickHouse/pull/10774) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix performance tests errors, part 2. [#10773](https://github.com/ClickHouse/ClickHouse/pull/10773) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix performance test errors. [#10766](https://github.com/ClickHouse/ClickHouse/pull/10766) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Update cross-builds to use clang-10 compiler. [#10724](https://github.com/ClickHouse/ClickHouse/pull/10724) ([Ivan](https://github.com/abyss7)).
|
||||
* Update instruction to install RPM packages. This was suggested by Denis (TG login @ldviolet) and implemented by Arkady Shejn. [#10707](https://github.com/ClickHouse/ClickHouse/pull/10707) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Trying to fix `tests/queries/0_stateless/01246_insert_into_watch_live_view.py` test. [#10670](https://github.com/ClickHouse/ClickHouse/pull/10670) ([vzakaznikov](https://github.com/vzakaznikov)).
|
||||
* Fixing and re-enabling 00979_live_view_watch_continuous_aggregates.py test. [#10658](https://github.com/ClickHouse/ClickHouse/pull/10658) ([vzakaznikov](https://github.com/vzakaznikov)).
|
||||
* Fix OOM in ASan stress test. [#10646](https://github.com/ClickHouse/ClickHouse/pull/10646) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix UBSan report (adding zero to nullptr) in HashTable that appeared after migration to clang-10. [#10638](https://github.com/ClickHouse/ClickHouse/pull/10638) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove external call to `ld` (bfd) linker during tzdata processing in compile time. [#10634](https://github.com/ClickHouse/ClickHouse/pull/10634) ([alesapin](https://github.com/alesapin)).
|
||||
* Allow to use `lld` to link blobs (resources). [#10632](https://github.com/ClickHouse/ClickHouse/pull/10632) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix UBSan report in `LZ4` library. [#10631](https://github.com/ClickHouse/ClickHouse/pull/10631) ([alexey-milovidov](https://github.com/alexey-milovidov)). See also [https://github.com/lz4/lz4/issues/857](https://github.com/lz4/lz4/issues/857)
|
||||
* Update LZ4 to the latest dev branch. [#10630](https://github.com/ClickHouse/ClickHouse/pull/10630) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added auto-generated machine-readable file with the list of stable versions. [#10628](https://github.com/ClickHouse/ClickHouse/pull/10628) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix `capnproto` version check for `capnp::UnalignedFlatArrayMessageReader`. [#10618](https://github.com/ClickHouse/ClickHouse/pull/10618) ([Matwey V. Kornilov](https://github.com/matwey)).
|
||||
* Lower memory usage in tests. [#10617](https://github.com/ClickHouse/ClickHouse/pull/10617) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixing hard coded timeouts in new live view tests. [#10604](https://github.com/ClickHouse/ClickHouse/pull/10604) ([vzakaznikov](https://github.com/vzakaznikov)).
|
||||
* Increasing timeout when opening a client in tests/queries/0_stateless/helpers/client.py. [#10599](https://github.com/ClickHouse/ClickHouse/pull/10599) ([vzakaznikov](https://github.com/vzakaznikov)).
|
||||
* Enable ThinLTO for clang builds, continuation of https://github.com/ClickHouse/ClickHouse/pull/10435. [#10585](https://github.com/ClickHouse/ClickHouse/pull/10585) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Adding fuzzers and preparing for oss-fuzz integration. [#10546](https://github.com/ClickHouse/ClickHouse/pull/10546) ([kyprizel](https://github.com/kyprizel)).
|
||||
* Fix FreeBSD build. [#10150](https://github.com/ClickHouse/ClickHouse/pull/10150) ([Ivan](https://github.com/abyss7)).
|
||||
* Add new build for query tests using pytest framework. [#10039](https://github.com/ClickHouse/ClickHouse/pull/10039) ([Ivan](https://github.com/abyss7)).
|
||||
|
||||
|
||||
## ClickHouse release v20.4
|
||||
|
||||
### ClickHouse release v20.4.6.53-stable 2020-06-25
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix rare crash caused by using `Nullable` column in prewhere condition. Continuation of [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608). [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Don't allow arrayJoin inside higher order functions. It was leading to broken protocol synchronization. This closes [#3933](https://github.com/ClickHouse/ClickHouse/issues/3933). [#11846](https://github.com/ClickHouse/ClickHouse/pull/11846) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix wrong result of comparison of FixedString with constant String. This fixes [#11393](https://github.com/ClickHouse/ClickHouse/issues/11393). This bug appeared in version 20.4. [#11828](https://github.com/ClickHouse/ClickHouse/pull/11828) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix wrong result for `if()` with NULLs in condition. [#11807](https://github.com/ClickHouse/ClickHouse/pull/11807) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Fix using too many threads for queries. [#11788](https://github.com/ClickHouse/ClickHouse/pull/11788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)).
|
||||
* Now replicated fetches will be cancelled during metadata alter. [#11744](https://github.com/ClickHouse/ClickHouse/pull/11744) ([alesapin](https://github.com/alesapin)).
|
||||
* Fixed LOGICAL_ERROR caused by wrong type deduction of complex literals in Values input format. [#11732](https://github.com/ClickHouse/ClickHouse/pull/11732) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fix `ORDER BY ... WITH FILL` over const columns. [#11697](https://github.com/ClickHouse/ClickHouse/pull/11697) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Pass proper timeouts when communicating with XDBC bridge. Recently timeouts were not respected when checking bridge liveness and receiving meta info. [#11690](https://github.com/ClickHouse/ClickHouse/pull/11690) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix `LIMIT n WITH TIES` usage together with `ORDER BY` statement, which contains aliases. [#11689](https://github.com/ClickHouse/ClickHouse/pull/11689) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix error which leads to an incorrect state of `system.mutations`. It may show that whole mutation is already done but the server still has `MUTATE_PART` tasks in the replication queue and tries to execute them. This fixes [#11611](https://github.com/ClickHouse/ClickHouse/issues/11611). [#11681](https://github.com/ClickHouse/ClickHouse/pull/11681) ([alesapin](https://github.com/alesapin)).
|
||||
* Add support for regular expressions with case-insensitive flags. This fixes [#11101](https://github.com/ClickHouse/ClickHouse/issues/11101) and fixes [#11506](https://github.com/ClickHouse/ClickHouse/issues/11506). [#11649](https://github.com/ClickHouse/ClickHouse/pull/11649) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove trivial count query optimization if row-level security is set. In previous versions the user get total count of records in a table instead filtered. This fixes [#11352](https://github.com/ClickHouse/ClickHouse/issues/11352). [#11644](https://github.com/ClickHouse/ClickHouse/pull/11644) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bloom filters for String (data skipping indices). [#11638](https://github.com/ClickHouse/ClickHouse/pull/11638) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix rare crash caused by using `Nullable` column in prewhere condition. (Probably it is connected with [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572) somehow). [#11608](https://github.com/ClickHouse/ClickHouse/pull/11608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix error `Block structure mismatch` for queries with sampling reading from `Buffer` table. [#11602](https://github.com/ClickHouse/ClickHouse/pull/11602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix wrong exit code of the clickhouse-client, when exception.code() % 256 = 0. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)).
|
||||
* Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed rare segfault in `SHOW CREATE TABLE` Fixes [#11490](https://github.com/ClickHouse/ClickHouse/issues/11490). [#11579](https://github.com/ClickHouse/ClickHouse/pull/11579) ([tavplubix](https://github.com/tavplubix)).
|
||||
* All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)).
|
||||
* Fix shard_num/replica_num for `<node>` (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix race condition which may lead to an exception during table drop. It's a bit tricky and not dangerous at all. If you want an explanation, just notice me in telegram. [#11523](https://github.com/ClickHouse/ClickHouse/pull/11523) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix memory leak when exception is thrown in the middle of aggregation with -State functions. This fixes [#8995](https://github.com/ClickHouse/ClickHouse/issues/8995). [#11496](https://github.com/ClickHouse/ClickHouse/pull/11496) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* If data skipping index is dependent on columns that are going to be modified during background merge (for SummingMergeTree, AggregatingMergeTree as well as for TTL GROUP BY), it was calculated incorrectly. This issue is fixed by moving index calculation after merge so the index is calculated on merged data. [#11162](https://github.com/ClickHouse/ClickHouse/pull/11162) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Get rid of old libunwind patches. https://github.com/ClickHouse-Extras/libunwind/commit/500aa227911bd185a94bfc071d68f4d3b03cb3b1#r39048012 This allows to disable `-fno-omit-frame-pointer` in `clang` builds that improves performance at least by 1% in average. [#10761](https://github.com/ClickHouse/ClickHouse/pull/10761) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix usage of primary key wrapped into a function with 'FINAL' modifier and 'ORDER BY' optimization. [#10715](https://github.com/ClickHouse/ClickHouse/pull/10715) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
|
||||
* Fix several non significant errors in unit tests. [#11262](https://github.com/ClickHouse/ClickHouse/pull/11262) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix (false) MSan report in MergeTreeIndexFullText. The issue first appeared in [#9968](https://github.com/ClickHouse/ClickHouse/issues/9968). [#10801](https://github.com/ClickHouse/ClickHouse/pull/10801) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
|
||||
### ClickHouse release v20.4.5.36-stable 2020-06-10
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix the error `Data compressed with different methods` that can happen if `min_bytes_to_use_direct_io` is enabled and PREWHERE is active and using SAMPLE or high number of threads. This fixes [#11539](https://github.com/ClickHouse/ClickHouse/issues/11539). [#11540](https://github.com/ClickHouse/ClickHouse/pull/11540) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix return compressed size for codecs. [#11448](https://github.com/ClickHouse/ClickHouse/pull/11448) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix server crash when a column has compression codec with non-literal arguments. Fixes [#11365](https://github.com/ClickHouse/ClickHouse/issues/11365). [#11431](https://github.com/ClickHouse/ClickHouse/pull/11431) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix pointInPolygon with nan as point. Fixes https://github.com/ClickHouse/ClickHouse/issues/11375. [#11421](https://github.com/ClickHouse/ClickHouse/pull/11421) ([Alexey Ilyukhov](https://github.com/livace)).
|
||||
* Fix potential uninitialized memory read in MergeTree shutdown if table was not created successfully. [#11420](https://github.com/ClickHouse/ClickHouse/pull/11420) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixed geohashesInBox with arguments outside of latitude/longitude range. [#11403](https://github.com/ClickHouse/ClickHouse/pull/11403) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Fix possible `Pipeline stuck` error for queries with external sort and limit. Fixes [#11359](https://github.com/ClickHouse/ClickHouse/issues/11359). [#11366](https://github.com/ClickHouse/ClickHouse/pull/11366) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Remove redundant lock during parts send in ReplicatedMergeTree. [#11354](https://github.com/ClickHouse/ClickHouse/pull/11354) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix support for `\G` (vertical output) in clickhouse-client in multiline mode. This closes [#9933](https://github.com/ClickHouse/ClickHouse/issues/9933). [#11350](https://github.com/ClickHouse/ClickHouse/pull/11350) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potential segfault when using `Lazy` database. [#11348](https://github.com/ClickHouse/ClickHouse/pull/11348) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix crash in `quantilesExactWeightedArray`. [#11337](https://github.com/ClickHouse/ClickHouse/pull/11337) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Now merges stopped before change metadata in `ALTER` queries. [#11335](https://github.com/ClickHouse/ClickHouse/pull/11335) ([alesapin](https://github.com/alesapin)).
|
||||
* Make writing to `MATERIALIZED VIEW` with setting `parallel_view_processing = 1` parallel again. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#11330](https://github.com/ClickHouse/ClickHouse/pull/11330) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix visitParamExtractRaw when extracted JSON has strings with unbalanced { or [. [#11318](https://github.com/ClickHouse/ClickHouse/pull/11318) ([Ewout](https://github.com/devwout)).
|
||||
* Fix very rare race condition in ThreadPool. [#11314](https://github.com/ClickHouse/ClickHouse/pull/11314) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix insignificant data race in clickhouse-copier. Found by integration tests. [#11313](https://github.com/ClickHouse/ClickHouse/pull/11313) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potential uninitialized memory in conversion. Example: `SELECT toIntervalSecond(now64())`. [#11311](https://github.com/ClickHouse/ClickHouse/pull/11311) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix the issue when index analysis cannot work if a table has Array column in primary key and if a query is filtering by this column with `empty` or `notEmpty` functions. This fixes [#11286](https://github.com/ClickHouse/ClickHouse/issues/11286). [#11303](https://github.com/ClickHouse/ClickHouse/pull/11303) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bug when query speed estimation can be incorrect and the limit of `min_execution_speed` may not work or work incorrectly if the query is throttled by `max_network_bandwidth`, `max_execution_speed` or `priority` settings. Change the default value of `timeout_before_checking_execution_speed` to non-zero, because otherwise the settings `min_execution_speed` and `max_execution_speed` have no effect. This fixes [#11297](https://github.com/ClickHouse/ClickHouse/issues/11297). This fixes [#5732](https://github.com/ClickHouse/ClickHouse/issues/5732). This fixes [#6228](https://github.com/ClickHouse/ClickHouse/issues/6228). Usability improvement: avoid concatenation of exception message with progress bar in `clickhouse-client`. [#11296](https://github.com/ClickHouse/ClickHouse/pull/11296) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix crash when SET DEFAULT ROLE is called with wrong arguments. This fixes https://github.com/ClickHouse/ClickHouse/issues/10586. [#11278](https://github.com/ClickHouse/ClickHouse/pull/11278) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix crash while reading malformed data in Protobuf format. This fixes https://github.com/ClickHouse/ClickHouse/issues/5957, fixes https://github.com/ClickHouse/ClickHouse/issues/11203. [#11258](https://github.com/ClickHouse/ClickHouse/pull/11258) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fixed a bug when cache-dictionary could return default value instead of normal (when there are only expired keys). This affects only string fields. [#11233](https://github.com/ClickHouse/ClickHouse/pull/11233) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix error `Block structure mismatch in QueryPipeline` while reading from `VIEW` with constants in inner query. Fixes [#11181](https://github.com/ClickHouse/ClickHouse/issues/11181). [#11205](https://github.com/ClickHouse/ClickHouse/pull/11205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix possible exception `Invalid status for associated output`. [#11200](https://github.com/ClickHouse/ClickHouse/pull/11200) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix possible error `Cannot capture column` for higher-order functions with `Array(Array(LowCardinality))` captured argument. [#11185](https://github.com/ClickHouse/ClickHouse/pull/11185) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed S3 globbing which could fail in case of more than 1000 keys and some backends. [#11179](https://github.com/ClickHouse/ClickHouse/pull/11179) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
* If data skipping index is dependent on columns that are going to be modified during background merge (for SummingMergeTree, AggregatingMergeTree as well as for TTL GROUP BY), it was calculated incorrectly. This issue is fixed by moving index calculation after merge so the index is calculated on merged data. [#11162](https://github.com/ClickHouse/ClickHouse/pull/11162) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix Kafka performance issue related to reschedules based on limits, which were always applied. [#11149](https://github.com/ClickHouse/ClickHouse/pull/11149) ([filimonov](https://github.com/filimonov)).
|
||||
* Fix for the hang which was happening sometimes during DROP of table engine=Kafka (or during server restarts). [#11145](https://github.com/ClickHouse/ClickHouse/pull/11145) ([filimonov](https://github.com/filimonov)).
|
||||
* Fix excessive reserving of threads for simple queries (optimization for reducing the number of threads, which was partly broken after changes in pipeline). [#11114](https://github.com/ClickHouse/ClickHouse/pull/11114) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix predicates optimization for distributed queries (`enable_optimize_predicate_expression=1`) for queries with `HAVING` section (i.e. when filtering on the server initiator is required), by preserving the order of expressions (and this is enough to fix), and also force aggregator use column names over indexes. Fixes: [#10613](https://github.com/ClickHouse/ClickHouse/issues/10613), [#11413](https://github.com/ClickHouse/ClickHouse/issues/11413). [#10621](https://github.com/ClickHouse/ClickHouse/pull/10621) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
|
||||
* Fix several flaky integration tests. [#11355](https://github.com/ClickHouse/ClickHouse/pull/11355) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
### ClickHouse release v20.4.4.18-stable 2020-05-26
|
||||
|
||||
No changes compared to v20.4.3.16-stable.
|
||||
|
||||
### ClickHouse release v20.4.3.16-stable 2020-05-23
|
||||
|
||||
#### Bug Fix
|
||||
@ -799,6 +1229,82 @@
|
||||
|
||||
## ClickHouse release v20.1
|
||||
|
||||
### ClickHouse release v20.1.16.120-stable 2020-60-26
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix rare crash caused by using `Nullable` column in prewhere condition. Continuation of [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608). [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Don't allow arrayJoin inside higher order functions. It was leading to broken protocol synchronization. This closes [#3933](https://github.com/ClickHouse/ClickHouse/issues/3933). [#11846](https://github.com/ClickHouse/ClickHouse/pull/11846) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)).
|
||||
* Fixed LOGICAL_ERROR caused by wrong type deduction of complex literals in Values input format. [#11732](https://github.com/ClickHouse/ClickHouse/pull/11732) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fix `ORDER BY ... WITH FILL` over const columns. [#11697](https://github.com/ClickHouse/ClickHouse/pull/11697) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Pass proper timeouts when communicating with XDBC bridge. Recently timeouts were not respected when checking bridge liveness and receiving meta info. [#11690](https://github.com/ClickHouse/ClickHouse/pull/11690) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add support for regular expressions with case-insensitive flags. This fixes [#11101](https://github.com/ClickHouse/ClickHouse/issues/11101) and fixes [#11506](https://github.com/ClickHouse/ClickHouse/issues/11506). [#11649](https://github.com/ClickHouse/ClickHouse/pull/11649) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bloom filters for String (data skipping indices). [#11638](https://github.com/ClickHouse/ClickHouse/pull/11638) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix rare crash caused by using `Nullable` column in prewhere condition. (Probably it is connected with [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572) somehow). [#11608](https://github.com/ClickHouse/ClickHouse/pull/11608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix wrong exit code of the clickhouse-client, when exception.code() % 256 = 0. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)).
|
||||
* Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)).
|
||||
* Fix memory leak when exception is thrown in the middle of aggregation with -State functions. This fixes [#8995](https://github.com/ClickHouse/ClickHouse/issues/8995). [#11496](https://github.com/ClickHouse/ClickHouse/pull/11496) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix usage of primary key wrapped into a function with 'FINAL' modifier and 'ORDER BY' optimization. [#10715](https://github.com/ClickHouse/ClickHouse/pull/10715) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
|
||||
### ClickHouse release v20.1.15.109-stable 2020-06-19
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix excess lock for structure during alter. [#11790](https://github.com/ClickHouse/ClickHouse/pull/11790) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
|
||||
### ClickHouse release v20.1.14.107-stable 2020-06-11
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
|
||||
### ClickHouse release v20.1.13.105-stable 2020-06-10
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix the error `Data compressed with different methods` that can happen if `min_bytes_to_use_direct_io` is enabled and PREWHERE is active and using SAMPLE or high number of threads. This fixes [#11539](https://github.com/ClickHouse/ClickHouse/issues/11539). [#11540](https://github.com/ClickHouse/ClickHouse/pull/11540) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix return compressed size for codecs. [#11448](https://github.com/ClickHouse/ClickHouse/pull/11448) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix server crash when a column has compression codec with non-literal arguments. Fixes [#11365](https://github.com/ClickHouse/ClickHouse/issues/11365). [#11431](https://github.com/ClickHouse/ClickHouse/pull/11431) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix pointInPolygon with nan as point. Fixes https://github.com/ClickHouse/ClickHouse/issues/11375. [#11421](https://github.com/ClickHouse/ClickHouse/pull/11421) ([Alexey Ilyukhov](https://github.com/livace)).
|
||||
* Fixed geohashesInBox with arguments outside of latitude/longitude range. [#11403](https://github.com/ClickHouse/ClickHouse/pull/11403) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Fix possible `Pipeline stuck` error for queries with external sort and limit. Fixes [#11359](https://github.com/ClickHouse/ClickHouse/issues/11359). [#11366](https://github.com/ClickHouse/ClickHouse/pull/11366) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix crash in `quantilesExactWeightedArray`. [#11337](https://github.com/ClickHouse/ClickHouse/pull/11337) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Make writing to `MATERIALIZED VIEW` with setting `parallel_view_processing = 1` parallel again. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#11330](https://github.com/ClickHouse/ClickHouse/pull/11330) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix visitParamExtractRaw when extracted JSON has strings with unbalanced { or [. [#11318](https://github.com/ClickHouse/ClickHouse/pull/11318) ([Ewout](https://github.com/devwout)).
|
||||
* Fix very rare race condition in ThreadPool. [#11314](https://github.com/ClickHouse/ClickHouse/pull/11314) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potential uninitialized memory in conversion. Example: `SELECT toIntervalSecond(now64())`. [#11311](https://github.com/ClickHouse/ClickHouse/pull/11311) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix the issue when index analysis cannot work if a table has Array column in primary key and if a query is filtering by this column with `empty` or `notEmpty` functions. This fixes [#11286](https://github.com/ClickHouse/ClickHouse/issues/11286). [#11303](https://github.com/ClickHouse/ClickHouse/pull/11303) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bug when query speed estimation can be incorrect and the limit of `min_execution_speed` may not work or work incorrectly if the query is throttled by `max_network_bandwidth`, `max_execution_speed` or `priority` settings. Change the default value of `timeout_before_checking_execution_speed` to non-zero, because otherwise the settings `min_execution_speed` and `max_execution_speed` have no effect. This fixes [#11297](https://github.com/ClickHouse/ClickHouse/issues/11297). This fixes [#5732](https://github.com/ClickHouse/ClickHouse/issues/5732). This fixes [#6228](https://github.com/ClickHouse/ClickHouse/issues/6228). Usability improvement: avoid concatenation of exception message with progress bar in `clickhouse-client`. [#11296](https://github.com/ClickHouse/ClickHouse/pull/11296) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix crash while reading malformed data in Protobuf format. This fixes https://github.com/ClickHouse/ClickHouse/issues/5957, fixes https://github.com/ClickHouse/ClickHouse/issues/11203. [#11258](https://github.com/ClickHouse/ClickHouse/pull/11258) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix possible error `Cannot capture column` for higher-order functions with `Array(Array(LowCardinality))` captured argument. [#11185](https://github.com/ClickHouse/ClickHouse/pull/11185) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* If data skipping index is dependent on columns that are going to be modified during background merge (for SummingMergeTree, AggregatingMergeTree as well as for TTL GROUP BY), it was calculated incorrectly. This issue is fixed by moving index calculation after merge so the index is calculated on merged data. [#11162](https://github.com/ClickHouse/ClickHouse/pull/11162) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Remove logging from mutation finalization task if nothing was finalized. [#11109](https://github.com/ClickHouse/ClickHouse/pull/11109) ([alesapin](https://github.com/alesapin)).
|
||||
* Fixed parseDateTime64BestEffort argument resolution bugs. [#10925](https://github.com/ClickHouse/ClickHouse/issues/10925). [#11038](https://github.com/ClickHouse/ClickHouse/pull/11038) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Fix incorrect raw data size in method getRawData(). [#10964](https://github.com/ClickHouse/ClickHouse/pull/10964) ([Igr](https://github.com/ObjatieGroba)).
|
||||
* Fix backward compatibility with tuples in Distributed tables. [#10889](https://github.com/ClickHouse/ClickHouse/pull/10889) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix SIGSEGV in StringHashTable (if such key does not exist). [#10870](https://github.com/ClickHouse/ClickHouse/pull/10870) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fixed bug in `ReplicatedMergeTree` which might cause some `ALTER` on `OPTIMIZE` query to hang waiting for some replica after it become inactive. [#10849](https://github.com/ClickHouse/ClickHouse/pull/10849) ([tavplubix](https://github.com/tavplubix)).
|
||||
* Fix columns order after Block::sortColumns() (also add a test that shows that it affects some real use case - Buffer engine). [#10826](https://github.com/ClickHouse/ClickHouse/pull/10826) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix the issue with ODBC bridge when no quoting of identifiers is requested. This fixes [#7984](https://github.com/ClickHouse/ClickHouse/issues/7984). [#10821](https://github.com/ClickHouse/ClickHouse/pull/10821) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix UBSan and MSan report in DateLUT. [#10798](https://github.com/ClickHouse/ClickHouse/pull/10798) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* - Make use of `src_type` for correct type conversion in key conditions. Fixes [#6287](https://github.com/ClickHouse/ClickHouse/issues/6287). [#10791](https://github.com/ClickHouse/ClickHouse/pull/10791) ([Andrew Onyshchuk](https://github.com/oandrew)).
|
||||
* Fix `parallel_view_processing` behavior. Now all insertions into `MATERIALIZED VIEW` without exception should be finished if exception happened. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#10757](https://github.com/ClickHouse/ClickHouse/pull/10757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix combinator -OrNull and -OrDefault when combined with -State. [#10741](https://github.com/ClickHouse/ClickHouse/pull/10741) ([hcz](https://github.com/hczhcz)).
|
||||
* Fix disappearing totals. Totals could have being filtered if query had had join or subquery with external where condition. Fixes [#10674](https://github.com/ClickHouse/ClickHouse/issues/10674). [#10698](https://github.com/ClickHouse/ClickHouse/pull/10698) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix multiple usages of `IN` operator with the identical set in one query. [#10686](https://github.com/ClickHouse/ClickHouse/pull/10686) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix order of parameters in AggregateTransform constructor. [#10667](https://github.com/ClickHouse/ClickHouse/pull/10667) ([palasonic1](https://github.com/palasonic1)).
|
||||
* Fix the lack of parallel execution of remote queries with `distributed_aggregation_memory_efficient` enabled. Fixes [#10655](https://github.com/ClickHouse/ClickHouse/issues/10655). [#10664](https://github.com/ClickHouse/ClickHouse/pull/10664) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix predicates optimization for distributed queries (`enable_optimize_predicate_expression=1`) for queries with `HAVING` section (i.e. when filtering on the server initiator is required), by preserving the order of expressions (and this is enough to fix), and also force aggregator use column names over indexes. Fixes: [#10613](https://github.com/ClickHouse/ClickHouse/issues/10613), [#11413](https://github.com/ClickHouse/ClickHouse/issues/11413). [#10621](https://github.com/ClickHouse/ClickHouse/pull/10621) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix error `the BloomFilter false positive must be a double number between 0 and 1` [#10551](https://github.com/ClickHouse/ClickHouse/issues/10551). [#10569](https://github.com/ClickHouse/ClickHouse/pull/10569) ([Winter Zhang](https://github.com/zhang2014)).
|
||||
* Fix SELECT of column ALIAS which default expression type different from column type. [#10563](https://github.com/ClickHouse/ClickHouse/pull/10563) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* * Implemented comparison between DateTime64 and String values (just like for DateTime). [#10560](https://github.com/ClickHouse/ClickHouse/pull/10560) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
|
||||
|
||||
### ClickHouse release v20.1.12.86, 2020-05-26
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
@ -342,6 +342,7 @@ include (cmake/find/sparsehash.cmake)
|
||||
include (cmake/find/re2.cmake)
|
||||
include (cmake/find/libgsasl.cmake)
|
||||
include (cmake/find/rdkafka.cmake)
|
||||
include (cmake/find/amqpcpp.cmake)
|
||||
include (cmake/find/capnp.cmake)
|
||||
include (cmake/find/llvm.cmake)
|
||||
include (cmake/find/opencl.cmake)
|
||||
|
||||
@ -13,3 +13,7 @@ ClickHouse is an open-source column-oriented database management system that all
|
||||
* [Yandex.Messenger channel](https://yandex.ru/chat/#/join/20e380d9-c7be-4123-ab06-e95fb946975e) shares announcements and useful links in Russian.
|
||||
* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
|
||||
* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
* [ClickHouse at Yandex Cloud Webinar (in Russian)](https://cloud.yandex.ru/events/144) on July 7, 2020.
|
||||
|
||||
@ -77,10 +77,8 @@ target_link_libraries (common
|
||||
Poco::Util
|
||||
Poco::Foundation
|
||||
replxx
|
||||
fmt
|
||||
|
||||
PRIVATE
|
||||
cctz
|
||||
fmt
|
||||
)
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
|
||||
@ -48,7 +48,7 @@ protected:
|
||||
};
|
||||
|
||||
const String history_file_path;
|
||||
static constexpr char word_break_characters[] = " \t\v\f\a\b\r\n`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?_";
|
||||
static constexpr char word_break_characters[] = " \t\v\f\a\b\r\n`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?";
|
||||
|
||||
String input;
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@ LIBRARY()
|
||||
|
||||
ADDINCL(
|
||||
GLOBAL clickhouse/base
|
||||
contrib/libs/cctz/include
|
||||
GLOBAL contrib/libs/cctz/include
|
||||
)
|
||||
|
||||
CFLAGS (GLOBAL -DARCADIA_BUILD)
|
||||
|
||||
@ -628,7 +628,7 @@ void BaseDaemon::initialize(Application & self)
|
||||
|
||||
/// Create pid file.
|
||||
if (config().has("pid"))
|
||||
pid.emplace(config().getString("pid"));
|
||||
pid.emplace(config().getString("pid"), DB::StatusFile::write_pid);
|
||||
|
||||
/// Change path for logging.
|
||||
if (!log_path.empty())
|
||||
@ -812,63 +812,6 @@ void BaseDaemon::defineOptions(Poco::Util::OptionSet & new_options)
|
||||
Poco::Util::ServerApplication::defineOptions(new_options);
|
||||
}
|
||||
|
||||
bool isPidRunning(pid_t pid)
|
||||
{
|
||||
return getpgid(pid) >= 0;
|
||||
}
|
||||
|
||||
BaseDaemon::PID::PID(const std::string & file_)
|
||||
{
|
||||
file = Poco::Path(file_).absolute().toString();
|
||||
Poco::File poco_file(file);
|
||||
|
||||
if (poco_file.exists())
|
||||
{
|
||||
pid_t pid_read = 0;
|
||||
{
|
||||
std::ifstream in(file);
|
||||
if (in.good())
|
||||
{
|
||||
in >> pid_read;
|
||||
if (pid_read && isPidRunning(pid_read))
|
||||
throw Poco::Exception("Pid file exists and program running with pid = " + std::to_string(pid_read) + ", should not start daemon.");
|
||||
}
|
||||
}
|
||||
std::cerr << "Old pid file exists (with pid = " << pid_read << "), removing." << std::endl;
|
||||
poco_file.remove();
|
||||
}
|
||||
|
||||
int fd = open(file.c_str(),
|
||||
O_CREAT | O_EXCL | O_WRONLY,
|
||||
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
|
||||
|
||||
if (-1 == fd)
|
||||
{
|
||||
if (EEXIST == errno)
|
||||
throw Poco::Exception("Pid file exists, should not start daemon.");
|
||||
throw Poco::CreateFileException("Cannot create pid file.");
|
||||
}
|
||||
|
||||
SCOPE_EXIT({ close(fd); });
|
||||
|
||||
std::stringstream s;
|
||||
s << getpid();
|
||||
if (static_cast<ssize_t>(s.str().size()) != write(fd, s.str().c_str(), s.str().size()))
|
||||
throw Poco::Exception("Cannot write to pid file.");
|
||||
}
|
||||
|
||||
BaseDaemon::PID::~PID()
|
||||
{
|
||||
try
|
||||
{
|
||||
Poco::File(file).remove();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
void BaseDaemon::handleSignal(int signal_id)
|
||||
{
|
||||
if (signal_id == SIGINT ||
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <common/getThreadId.h>
|
||||
#include <daemon/GraphiteWriter.h>
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Common/StatusFile.h>
|
||||
#include <loggers/Loggers.h>
|
||||
|
||||
|
||||
@ -163,16 +164,7 @@ protected:
|
||||
|
||||
std::unique_ptr<Poco::TaskManager> task_manager;
|
||||
|
||||
/// RAII wrapper for pid file.
|
||||
struct PID
|
||||
{
|
||||
std::string file;
|
||||
|
||||
PID(const std::string & file_);
|
||||
~PID();
|
||||
};
|
||||
|
||||
std::optional<PID> pid;
|
||||
std::optional<DB::StatusFile> pid;
|
||||
|
||||
std::atomic_bool is_cancelled{false};
|
||||
|
||||
|
||||
@ -8,6 +8,5 @@ target_include_directories (daemon PUBLIC ..)
|
||||
target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES})
|
||||
|
||||
if (USE_SENTRY)
|
||||
target_link_libraries (daemon PRIVATE curl)
|
||||
target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY})
|
||||
endif ()
|
||||
|
||||
@ -1,19 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <string>
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
#include <cctz/time_zone.h>
|
||||
|
||||
|
||||
namespace ext
|
||||
{
|
||||
inline std::string to_string(const std::time_t & time)
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << std::put_time(std::localtime(&time), "%Y-%m-%d %X");
|
||||
return ss.str();
|
||||
return cctz::format("%Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(time), cctz::local_time_zone());
|
||||
}
|
||||
|
||||
template <typename Clock, typename Duration = typename Clock::duration>
|
||||
|
||||
20
cmake/find/amqpcpp.cmake
Normal file
20
cmake/find/amqpcpp.cmake
Normal file
@ -0,0 +1,20 @@
|
||||
SET(ENABLE_AMQPCPP 1)
|
||||
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/CMakeLists.txt")
|
||||
message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init --recursive")
|
||||
set (ENABLE_AMQPCPP 0)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_AMQPCPP)
|
||||
|
||||
set (USE_AMQPCPP 1)
|
||||
set (AMQPCPP_LIBRARY AMQP-CPP)
|
||||
|
||||
set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include")
|
||||
|
||||
list (APPEND AMQPCPP_INCLUDE_DIR
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include"
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP")
|
||||
|
||||
endif()
|
||||
|
||||
message (STATUS "Using AMQP-CPP=${USE_AMQPCPP}: ${AMQPCPP_INCLUDE_DIR} : ${AMQPCPP_LIBRARY}")
|
||||
@ -6,9 +6,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h")
|
||||
endif ()
|
||||
|
||||
if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG))
|
||||
option (USE_SENTRY "Use Sentry" ON)
|
||||
set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib)
|
||||
set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include)
|
||||
option (USE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES})
|
||||
set (SENTRY_TRANSPORT "curl" CACHE STRING "")
|
||||
set (SENTRY_BACKEND "none" CACHE STRING "")
|
||||
set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "")
|
||||
|
||||
1
contrib/AMQP-CPP
vendored
Submodule
1
contrib/AMQP-CPP
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 1c08399ab0ab9e4042ef8e2bbe9e208e5dcbc13b
|
||||
27
contrib/CMakeLists.txt
vendored
27
contrib/CMakeLists.txt
vendored
@ -106,6 +106,12 @@ if (ENABLE_LDAP AND USE_INTERNAL_LDAP_LIBRARY)
|
||||
add_subdirectory (openldap-cmake)
|
||||
endif ()
|
||||
|
||||
# Should go before:
|
||||
# - mariadb-connector-c
|
||||
# - aws-s3-cmake
|
||||
# - sentry-native
|
||||
add_subdirectory (curl-cmake)
|
||||
|
||||
function(mysql_support)
|
||||
set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC)
|
||||
set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC)
|
||||
@ -263,23 +269,6 @@ if (USE_INTERNAL_GRPC_LIBRARY)
|
||||
add_subdirectory(grpc-cmake)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_AWS_S3_LIBRARY OR USE_SENTRY)
|
||||
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
|
||||
set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
|
||||
set (save_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
|
||||
set (save_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
|
||||
set (save_CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH})
|
||||
add_subdirectory(curl-cmake)
|
||||
set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS})
|
||||
set (CMAKE_REQUIRED_LIBRARIES ${save_CMAKE_REQUIRED_LIBRARIES})
|
||||
set (CMAKE_CMAKE_REQUIRED_INCLUDES ${save_CMAKE_REQUIRED_INCLUDES})
|
||||
set (CMAKE_REQUIRED_FLAGS ${save_CMAKE_REQUIRED_FLAGS})
|
||||
set (CMAKE_CMAKE_MODULE_PATH ${save_CMAKE_MODULE_PATH})
|
||||
|
||||
# The library is large - avoid bloat.
|
||||
target_compile_options (curl PRIVATE -g0)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_AWS_S3_LIBRARY)
|
||||
add_subdirectory(aws-s3-cmake)
|
||||
|
||||
@ -301,6 +290,10 @@ if (USE_FASTOPS)
|
||||
add_subdirectory (fastops-cmake)
|
||||
endif()
|
||||
|
||||
if (USE_AMQPCPP)
|
||||
add_subdirectory (amqpcpp-cmake)
|
||||
endif()
|
||||
|
||||
if (USE_CASSANDRA)
|
||||
add_subdirectory (libuv)
|
||||
add_subdirectory (cassandra)
|
||||
|
||||
44
contrib/amqpcpp-cmake/CMakeLists.txt
Normal file
44
contrib/amqpcpp-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,44 @@
|
||||
set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP)
|
||||
|
||||
set (SRCS
|
||||
${LIBRARY_DIR}/src/array.cpp
|
||||
${LIBRARY_DIR}/src/channel.cpp
|
||||
${LIBRARY_DIR}/src/channelimpl.cpp
|
||||
${LIBRARY_DIR}/src/connectionimpl.cpp
|
||||
${LIBRARY_DIR}/src/deferredcancel.cpp
|
||||
${LIBRARY_DIR}/src/deferredconfirm.cpp
|
||||
${LIBRARY_DIR}/src/deferredconsumer.cpp
|
||||
${LIBRARY_DIR}/src/deferredextreceiver.cpp
|
||||
${LIBRARY_DIR}/src/deferredget.cpp
|
||||
${LIBRARY_DIR}/src/deferredpublisher.cpp
|
||||
${LIBRARY_DIR}/src/deferredreceiver.cpp
|
||||
${LIBRARY_DIR}/src/field.cpp
|
||||
${LIBRARY_DIR}/src/flags.cpp
|
||||
${LIBRARY_DIR}/src/linux_tcp/openssl.cpp
|
||||
${LIBRARY_DIR}/src/linux_tcp/tcpconnection.cpp
|
||||
${LIBRARY_DIR}/src/receivedframe.cpp
|
||||
${LIBRARY_DIR}/src/table.cpp
|
||||
${LIBRARY_DIR}/src/watchable.cpp
|
||||
)
|
||||
|
||||
add_library(amqp-cpp ${SRCS})
|
||||
|
||||
target_compile_options (amqp-cpp
|
||||
PUBLIC
|
||||
-Wno-old-style-cast
|
||||
-Wno-inconsistent-missing-destructor-override
|
||||
-Wno-deprecated
|
||||
-Wno-unused-parameter
|
||||
-Wno-shadow
|
||||
-Wno-tautological-type-limit-compare
|
||||
-Wno-extra-semi
|
||||
# NOTE: disable all warnings at last because the warning:
|
||||
# "conversion function converting 'XXX' to itself will never be used"
|
||||
# doesn't have it's own diagnostic flag yet.
|
||||
-w
|
||||
)
|
||||
|
||||
target_include_directories (amqp-cpp PUBLIC ${LIBRARY_DIR}/include)
|
||||
|
||||
target_link_libraries (amqp-cpp PUBLIC ssl)
|
||||
|
||||
@ -1,152 +1,187 @@
|
||||
set (CURL_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
|
||||
set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib)
|
||||
set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include)
|
||||
option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES})
|
||||
|
||||
set (SRCS
|
||||
${CURL_DIR}/lib/file.c
|
||||
${CURL_DIR}/lib/timeval.c
|
||||
${CURL_DIR}/lib/base64.c
|
||||
${CURL_DIR}/lib/hostip.c
|
||||
${CURL_DIR}/lib/progress.c
|
||||
${CURL_DIR}/lib/formdata.c
|
||||
${CURL_DIR}/lib/cookie.c
|
||||
${CURL_DIR}/lib/http.c
|
||||
${CURL_DIR}/lib/sendf.c
|
||||
${CURL_DIR}/lib/url.c
|
||||
${CURL_DIR}/lib/dict.c
|
||||
${CURL_DIR}/lib/if2ip.c
|
||||
${CURL_DIR}/lib/speedcheck.c
|
||||
${CURL_DIR}/lib/ldap.c
|
||||
${CURL_DIR}/lib/version.c
|
||||
${CURL_DIR}/lib/getenv.c
|
||||
${CURL_DIR}/lib/escape.c
|
||||
${CURL_DIR}/lib/mprintf.c
|
||||
${CURL_DIR}/lib/telnet.c
|
||||
${CURL_DIR}/lib/netrc.c
|
||||
${CURL_DIR}/lib/getinfo.c
|
||||
${CURL_DIR}/lib/transfer.c
|
||||
${CURL_DIR}/lib/strcase.c
|
||||
${CURL_DIR}/lib/easy.c
|
||||
${CURL_DIR}/lib/security.c
|
||||
${CURL_DIR}/lib/curl_fnmatch.c
|
||||
${CURL_DIR}/lib/fileinfo.c
|
||||
${CURL_DIR}/lib/wildcard.c
|
||||
${CURL_DIR}/lib/krb5.c
|
||||
${CURL_DIR}/lib/memdebug.c
|
||||
${CURL_DIR}/lib/http_chunks.c
|
||||
${CURL_DIR}/lib/strtok.c
|
||||
${CURL_DIR}/lib/connect.c
|
||||
${CURL_DIR}/lib/llist.c
|
||||
${CURL_DIR}/lib/hash.c
|
||||
${CURL_DIR}/lib/multi.c
|
||||
${CURL_DIR}/lib/content_encoding.c
|
||||
${CURL_DIR}/lib/share.c
|
||||
${CURL_DIR}/lib/http_digest.c
|
||||
${CURL_DIR}/lib/md4.c
|
||||
${CURL_DIR}/lib/md5.c
|
||||
${CURL_DIR}/lib/http_negotiate.c
|
||||
${CURL_DIR}/lib/inet_pton.c
|
||||
${CURL_DIR}/lib/strtoofft.c
|
||||
${CURL_DIR}/lib/strerror.c
|
||||
${CURL_DIR}/lib/amigaos.c
|
||||
${CURL_DIR}/lib/hostasyn.c
|
||||
${CURL_DIR}/lib/hostip4.c
|
||||
${CURL_DIR}/lib/hostip6.c
|
||||
${CURL_DIR}/lib/hostsyn.c
|
||||
${CURL_DIR}/lib/inet_ntop.c
|
||||
${CURL_DIR}/lib/parsedate.c
|
||||
${CURL_DIR}/lib/select.c
|
||||
${CURL_DIR}/lib/splay.c
|
||||
${CURL_DIR}/lib/strdup.c
|
||||
${CURL_DIR}/lib/socks.c
|
||||
${CURL_DIR}/lib/curl_addrinfo.c
|
||||
${CURL_DIR}/lib/socks_gssapi.c
|
||||
${CURL_DIR}/lib/socks_sspi.c
|
||||
${CURL_DIR}/lib/curl_sspi.c
|
||||
${CURL_DIR}/lib/slist.c
|
||||
${CURL_DIR}/lib/nonblock.c
|
||||
${CURL_DIR}/lib/curl_memrchr.c
|
||||
${CURL_DIR}/lib/imap.c
|
||||
${CURL_DIR}/lib/pop3.c
|
||||
${CURL_DIR}/lib/smtp.c
|
||||
${CURL_DIR}/lib/pingpong.c
|
||||
${CURL_DIR}/lib/rtsp.c
|
||||
${CURL_DIR}/lib/curl_threads.c
|
||||
${CURL_DIR}/lib/warnless.c
|
||||
${CURL_DIR}/lib/hmac.c
|
||||
${CURL_DIR}/lib/curl_rtmp.c
|
||||
${CURL_DIR}/lib/openldap.c
|
||||
${CURL_DIR}/lib/curl_gethostname.c
|
||||
${CURL_DIR}/lib/gopher.c
|
||||
${CURL_DIR}/lib/idn_win32.c
|
||||
${CURL_DIR}/lib/http_proxy.c
|
||||
${CURL_DIR}/lib/non-ascii.c
|
||||
${CURL_DIR}/lib/asyn-thread.c
|
||||
${CURL_DIR}/lib/curl_gssapi.c
|
||||
${CURL_DIR}/lib/http_ntlm.c
|
||||
${CURL_DIR}/lib/curl_ntlm_wb.c
|
||||
${CURL_DIR}/lib/curl_ntlm_core.c
|
||||
${CURL_DIR}/lib/curl_sasl.c
|
||||
${CURL_DIR}/lib/rand.c
|
||||
${CURL_DIR}/lib/curl_multibyte.c
|
||||
${CURL_DIR}/lib/hostcheck.c
|
||||
${CURL_DIR}/lib/conncache.c
|
||||
${CURL_DIR}/lib/dotdot.c
|
||||
${CURL_DIR}/lib/x509asn1.c
|
||||
${CURL_DIR}/lib/http2.c
|
||||
${CURL_DIR}/lib/smb.c
|
||||
${CURL_DIR}/lib/curl_endian.c
|
||||
${CURL_DIR}/lib/curl_des.c
|
||||
${CURL_DIR}/lib/system_win32.c
|
||||
${CURL_DIR}/lib/mime.c
|
||||
${CURL_DIR}/lib/sha256.c
|
||||
${CURL_DIR}/lib/setopt.c
|
||||
${CURL_DIR}/lib/curl_path.c
|
||||
${CURL_DIR}/lib/curl_ctype.c
|
||||
${CURL_DIR}/lib/curl_range.c
|
||||
${CURL_DIR}/lib/psl.c
|
||||
${CURL_DIR}/lib/doh.c
|
||||
${CURL_DIR}/lib/urlapi.c
|
||||
${CURL_DIR}/lib/curl_get_line.c
|
||||
${CURL_DIR}/lib/altsvc.c
|
||||
${CURL_DIR}/lib/socketpair.c
|
||||
${CURL_DIR}/lib/vauth/vauth.c
|
||||
${CURL_DIR}/lib/vauth/cleartext.c
|
||||
${CURL_DIR}/lib/vauth/cram.c
|
||||
${CURL_DIR}/lib/vauth/digest.c
|
||||
${CURL_DIR}/lib/vauth/digest_sspi.c
|
||||
${CURL_DIR}/lib/vauth/krb5_gssapi.c
|
||||
${CURL_DIR}/lib/vauth/krb5_sspi.c
|
||||
${CURL_DIR}/lib/vauth/ntlm.c
|
||||
${CURL_DIR}/lib/vauth/ntlm_sspi.c
|
||||
${CURL_DIR}/lib/vauth/oauth2.c
|
||||
${CURL_DIR}/lib/vauth/spnego_gssapi.c
|
||||
${CURL_DIR}/lib/vauth/spnego_sspi.c
|
||||
${CURL_DIR}/lib/vtls/openssl.c
|
||||
${CURL_DIR}/lib/vtls/gtls.c
|
||||
${CURL_DIR}/lib/vtls/vtls.c
|
||||
${CURL_DIR}/lib/vtls/nss.c
|
||||
${CURL_DIR}/lib/vtls/polarssl.c
|
||||
${CURL_DIR}/lib/vtls/polarssl_threadlock.c
|
||||
${CURL_DIR}/lib/vtls/wolfssl.c
|
||||
${CURL_DIR}/lib/vtls/schannel.c
|
||||
${CURL_DIR}/lib/vtls/schannel_verify.c
|
||||
${CURL_DIR}/lib/vtls/sectransp.c
|
||||
${CURL_DIR}/lib/vtls/gskit.c
|
||||
${CURL_DIR}/lib/vtls/mbedtls.c
|
||||
${CURL_DIR}/lib/vtls/mesalink.c
|
||||
${CURL_DIR}/lib/vtls/bearssl.c
|
||||
${CURL_DIR}/lib/vquic/ngtcp2.c
|
||||
${CURL_DIR}/lib/vquic/quiche.c
|
||||
${CURL_DIR}/lib/vssh/libssh2.c
|
||||
${CURL_DIR}/lib/vssh/libssh.c
|
||||
)
|
||||
if (ENABLE_CURL)
|
||||
option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED})
|
||||
|
||||
add_library (curl ${SRCS})
|
||||
if (USE_INTERNAL_CURL)
|
||||
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl")
|
||||
|
||||
target_compile_definitions(curl PRIVATE HAVE_CONFIG_H BUILDING_LIBCURL CURL_HIDDEN_SYMBOLS libcurl_EXPORTS)
|
||||
target_include_directories(curl PUBLIC ${CURL_DIR}/include ${CURL_DIR}/lib .)
|
||||
set (SRCS
|
||||
${LIBRARY_DIR}/lib/file.c
|
||||
${LIBRARY_DIR}/lib/timeval.c
|
||||
${LIBRARY_DIR}/lib/base64.c
|
||||
${LIBRARY_DIR}/lib/hostip.c
|
||||
${LIBRARY_DIR}/lib/progress.c
|
||||
${LIBRARY_DIR}/lib/formdata.c
|
||||
${LIBRARY_DIR}/lib/cookie.c
|
||||
${LIBRARY_DIR}/lib/http.c
|
||||
${LIBRARY_DIR}/lib/sendf.c
|
||||
${LIBRARY_DIR}/lib/url.c
|
||||
${LIBRARY_DIR}/lib/dict.c
|
||||
${LIBRARY_DIR}/lib/if2ip.c
|
||||
${LIBRARY_DIR}/lib/speedcheck.c
|
||||
${LIBRARY_DIR}/lib/ldap.c
|
||||
${LIBRARY_DIR}/lib/version.c
|
||||
${LIBRARY_DIR}/lib/getenv.c
|
||||
${LIBRARY_DIR}/lib/escape.c
|
||||
${LIBRARY_DIR}/lib/mprintf.c
|
||||
${LIBRARY_DIR}/lib/telnet.c
|
||||
${LIBRARY_DIR}/lib/netrc.c
|
||||
${LIBRARY_DIR}/lib/getinfo.c
|
||||
${LIBRARY_DIR}/lib/transfer.c
|
||||
${LIBRARY_DIR}/lib/strcase.c
|
||||
${LIBRARY_DIR}/lib/easy.c
|
||||
${LIBRARY_DIR}/lib/security.c
|
||||
${LIBRARY_DIR}/lib/curl_fnmatch.c
|
||||
${LIBRARY_DIR}/lib/fileinfo.c
|
||||
${LIBRARY_DIR}/lib/wildcard.c
|
||||
${LIBRARY_DIR}/lib/krb5.c
|
||||
${LIBRARY_DIR}/lib/memdebug.c
|
||||
${LIBRARY_DIR}/lib/http_chunks.c
|
||||
${LIBRARY_DIR}/lib/strtok.c
|
||||
${LIBRARY_DIR}/lib/connect.c
|
||||
${LIBRARY_DIR}/lib/llist.c
|
||||
${LIBRARY_DIR}/lib/hash.c
|
||||
${LIBRARY_DIR}/lib/multi.c
|
||||
${LIBRARY_DIR}/lib/content_encoding.c
|
||||
${LIBRARY_DIR}/lib/share.c
|
||||
${LIBRARY_DIR}/lib/http_digest.c
|
||||
${LIBRARY_DIR}/lib/md4.c
|
||||
${LIBRARY_DIR}/lib/md5.c
|
||||
${LIBRARY_DIR}/lib/http_negotiate.c
|
||||
${LIBRARY_DIR}/lib/inet_pton.c
|
||||
${LIBRARY_DIR}/lib/strtoofft.c
|
||||
${LIBRARY_DIR}/lib/strerror.c
|
||||
${LIBRARY_DIR}/lib/amigaos.c
|
||||
${LIBRARY_DIR}/lib/hostasyn.c
|
||||
${LIBRARY_DIR}/lib/hostip4.c
|
||||
${LIBRARY_DIR}/lib/hostip6.c
|
||||
${LIBRARY_DIR}/lib/hostsyn.c
|
||||
${LIBRARY_DIR}/lib/inet_ntop.c
|
||||
${LIBRARY_DIR}/lib/parsedate.c
|
||||
${LIBRARY_DIR}/lib/select.c
|
||||
${LIBRARY_DIR}/lib/splay.c
|
||||
${LIBRARY_DIR}/lib/strdup.c
|
||||
${LIBRARY_DIR}/lib/socks.c
|
||||
${LIBRARY_DIR}/lib/curl_addrinfo.c
|
||||
${LIBRARY_DIR}/lib/socks_gssapi.c
|
||||
${LIBRARY_DIR}/lib/socks_sspi.c
|
||||
${LIBRARY_DIR}/lib/curl_sspi.c
|
||||
${LIBRARY_DIR}/lib/slist.c
|
||||
${LIBRARY_DIR}/lib/nonblock.c
|
||||
${LIBRARY_DIR}/lib/curl_memrchr.c
|
||||
${LIBRARY_DIR}/lib/imap.c
|
||||
${LIBRARY_DIR}/lib/pop3.c
|
||||
${LIBRARY_DIR}/lib/smtp.c
|
||||
${LIBRARY_DIR}/lib/pingpong.c
|
||||
${LIBRARY_DIR}/lib/rtsp.c
|
||||
${LIBRARY_DIR}/lib/curl_threads.c
|
||||
${LIBRARY_DIR}/lib/warnless.c
|
||||
${LIBRARY_DIR}/lib/hmac.c
|
||||
${LIBRARY_DIR}/lib/curl_rtmp.c
|
||||
${LIBRARY_DIR}/lib/openldap.c
|
||||
${LIBRARY_DIR}/lib/curl_gethostname.c
|
||||
${LIBRARY_DIR}/lib/gopher.c
|
||||
${LIBRARY_DIR}/lib/idn_win32.c
|
||||
${LIBRARY_DIR}/lib/http_proxy.c
|
||||
${LIBRARY_DIR}/lib/non-ascii.c
|
||||
${LIBRARY_DIR}/lib/asyn-thread.c
|
||||
${LIBRARY_DIR}/lib/curl_gssapi.c
|
||||
${LIBRARY_DIR}/lib/http_ntlm.c
|
||||
${LIBRARY_DIR}/lib/curl_ntlm_wb.c
|
||||
${LIBRARY_DIR}/lib/curl_ntlm_core.c
|
||||
${LIBRARY_DIR}/lib/curl_sasl.c
|
||||
${LIBRARY_DIR}/lib/rand.c
|
||||
${LIBRARY_DIR}/lib/curl_multibyte.c
|
||||
${LIBRARY_DIR}/lib/hostcheck.c
|
||||
${LIBRARY_DIR}/lib/conncache.c
|
||||
${LIBRARY_DIR}/lib/dotdot.c
|
||||
${LIBRARY_DIR}/lib/x509asn1.c
|
||||
${LIBRARY_DIR}/lib/http2.c
|
||||
${LIBRARY_DIR}/lib/smb.c
|
||||
${LIBRARY_DIR}/lib/curl_endian.c
|
||||
${LIBRARY_DIR}/lib/curl_des.c
|
||||
${LIBRARY_DIR}/lib/system_win32.c
|
||||
${LIBRARY_DIR}/lib/mime.c
|
||||
${LIBRARY_DIR}/lib/sha256.c
|
||||
${LIBRARY_DIR}/lib/setopt.c
|
||||
${LIBRARY_DIR}/lib/curl_path.c
|
||||
${LIBRARY_DIR}/lib/curl_ctype.c
|
||||
${LIBRARY_DIR}/lib/curl_range.c
|
||||
${LIBRARY_DIR}/lib/psl.c
|
||||
${LIBRARY_DIR}/lib/doh.c
|
||||
${LIBRARY_DIR}/lib/urlapi.c
|
||||
${LIBRARY_DIR}/lib/curl_get_line.c
|
||||
${LIBRARY_DIR}/lib/altsvc.c
|
||||
${LIBRARY_DIR}/lib/socketpair.c
|
||||
${LIBRARY_DIR}/lib/vauth/vauth.c
|
||||
${LIBRARY_DIR}/lib/vauth/cleartext.c
|
||||
${LIBRARY_DIR}/lib/vauth/cram.c
|
||||
${LIBRARY_DIR}/lib/vauth/digest.c
|
||||
${LIBRARY_DIR}/lib/vauth/digest_sspi.c
|
||||
${LIBRARY_DIR}/lib/vauth/krb5_gssapi.c
|
||||
${LIBRARY_DIR}/lib/vauth/krb5_sspi.c
|
||||
${LIBRARY_DIR}/lib/vauth/ntlm.c
|
||||
${LIBRARY_DIR}/lib/vauth/ntlm_sspi.c
|
||||
${LIBRARY_DIR}/lib/vauth/oauth2.c
|
||||
${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c
|
||||
${LIBRARY_DIR}/lib/vauth/spnego_sspi.c
|
||||
${LIBRARY_DIR}/lib/vtls/openssl.c
|
||||
${LIBRARY_DIR}/lib/vtls/gtls.c
|
||||
${LIBRARY_DIR}/lib/vtls/vtls.c
|
||||
${LIBRARY_DIR}/lib/vtls/nss.c
|
||||
${LIBRARY_DIR}/lib/vtls/polarssl.c
|
||||
${LIBRARY_DIR}/lib/vtls/polarssl_threadlock.c
|
||||
${LIBRARY_DIR}/lib/vtls/wolfssl.c
|
||||
${LIBRARY_DIR}/lib/vtls/schannel.c
|
||||
${LIBRARY_DIR}/lib/vtls/schannel_verify.c
|
||||
${LIBRARY_DIR}/lib/vtls/sectransp.c
|
||||
${LIBRARY_DIR}/lib/vtls/gskit.c
|
||||
${LIBRARY_DIR}/lib/vtls/mbedtls.c
|
||||
${LIBRARY_DIR}/lib/vtls/mesalink.c
|
||||
${LIBRARY_DIR}/lib/vtls/bearssl.c
|
||||
${LIBRARY_DIR}/lib/vquic/ngtcp2.c
|
||||
${LIBRARY_DIR}/lib/vquic/quiche.c
|
||||
${LIBRARY_DIR}/lib/vssh/libssh2.c
|
||||
${LIBRARY_DIR}/lib/vssh/libssh.c
|
||||
)
|
||||
|
||||
target_compile_definitions(curl PRIVATE OS="${CMAKE_SYSTEM_NAME}")
|
||||
add_library (curl ${SRCS})
|
||||
|
||||
target_link_libraries(curl PRIVATE ssl)
|
||||
target_compile_definitions (curl PRIVATE
|
||||
HAVE_CONFIG_H
|
||||
BUILDING_LIBCURL
|
||||
CURL_HIDDEN_SYMBOLS
|
||||
libcurl_EXPORTS
|
||||
OS="${CMAKE_SYSTEM_NAME}"
|
||||
)
|
||||
target_include_directories (curl PUBLIC
|
||||
${LIBRARY_DIR}/include
|
||||
${LIBRARY_DIR}/lib
|
||||
. # curl_config.h
|
||||
)
|
||||
|
||||
target_link_libraries (curl PRIVATE ssl)
|
||||
|
||||
# The library is large - avoid bloat (XXX: is it?)
|
||||
target_compile_options (curl PRIVATE -g0)
|
||||
|
||||
# find_package(CURL) compatibility for the following packages that uses
|
||||
# find_package(CURL)/include(FindCURL):
|
||||
# - mariadb-connector-c
|
||||
# - aws-s3-cmake
|
||||
# - sentry-native
|
||||
set (CURL_FOUND ON CACHE BOOL "")
|
||||
set (CURL_ROOT_DIR ${LIBRARY_DIR} CACHE PATH "")
|
||||
set (CURL_INCLUDE_DIR ${LIBRARY_DIR}/include CACHE PATH "")
|
||||
set (CURL_INCLUDE_DIRS ${LIBRARY_DIR}/include CACHE PATH "")
|
||||
set (CURL_LIBRARY curl CACHE STRING "")
|
||||
set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "")
|
||||
set (CURL_VERSION_STRING 7.67.0 CACHE STRING "")
|
||||
add_library (CURL::libcurl ALIAS ${CURL_LIBRARY})
|
||||
else ()
|
||||
find_package (CURL REQUIRED)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
message (STATUS "Using curl: ${CURL_INCLUDE_DIRS} : ${CURL_LIBRARIES}")
|
||||
|
||||
@ -76,7 +76,7 @@
|
||||
do { \
|
||||
fprintf(stderr, "libdivide.h:%d: %s(): Error: %s\n", \
|
||||
__LINE__, LIBDIVIDE_FUNCTION, msg); \
|
||||
exit(-1); \
|
||||
abort(); \
|
||||
} while (0)
|
||||
|
||||
#if defined(LIBDIVIDE_ASSERTIONS_ON)
|
||||
@ -85,7 +85,7 @@
|
||||
if (!(x)) { \
|
||||
fprintf(stderr, "libdivide.h:%d: %s(): Assertion failed: %s\n", \
|
||||
__LINE__, LIBDIVIDE_FUNCTION, #x); \
|
||||
exit(-1); \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
#else
|
||||
@ -290,10 +290,17 @@ static inline int32_t libdivide_count_leading_zeros32(uint32_t val) {
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
int32_t result = 0;
|
||||
uint32_t hi = 1U << 31;
|
||||
for (; ~val & hi; hi >>= 1) {
|
||||
result++;
|
||||
if (val == 0)
|
||||
return 32;
|
||||
int32_t result = 8;
|
||||
uint32_t hi = 0xFFU << 24;
|
||||
while ((val & hi) == 0) {
|
||||
hi >>= 8;
|
||||
result += 8;
|
||||
}
|
||||
while (val & hi) {
|
||||
result -= 1;
|
||||
hi <<= 1;
|
||||
}
|
||||
return result;
|
||||
#endif
|
||||
|
||||
@ -24,7 +24,7 @@ if (ENABLE_ODBC)
|
||||
target_include_directories (_poco_data_odbc SYSTEM PUBLIC ${LIBRARY_DIR}/Data/ODBC/include)
|
||||
target_link_libraries (_poco_data_odbc PUBLIC Poco::Data unixodbc)
|
||||
else ()
|
||||
add_library (Poco::Data::ODBC UNKNOWN IMPORTED)
|
||||
add_library (Poco::Data::ODBC UNKNOWN IMPORTED GLOBAL)
|
||||
|
||||
find_library(LIBRARY_POCO_DATA_ODBC PocoDataODBC)
|
||||
find_path(INCLUDE_POCO_DATA_ODBC Poco/Data/ODBC/ODBC.h)
|
||||
|
||||
@ -307,7 +307,7 @@ if (ENABLE_ODBC)
|
||||
set_target_properties (unixodbc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_ODBC})
|
||||
endif ()
|
||||
|
||||
target_compile_definitions (unixodbc PUBLIC USE_ODBC=1)
|
||||
target_compile_definitions (unixodbc INTERFACE USE_ODBC=1)
|
||||
|
||||
message (STATUS "Using unixodbc")
|
||||
else ()
|
||||
|
||||
2
docker/bare/Dockerfile
Normal file
2
docker/bare/Dockerfile
Normal file
@ -0,0 +1,2 @@
|
||||
FROM scratch
|
||||
ADD root /
|
||||
37
docker/bare/README.md
Normal file
37
docker/bare/README.md
Normal file
@ -0,0 +1,37 @@
|
||||
## The bare minimum ClickHouse Docker image.
|
||||
|
||||
It is intented as a showcase to check the amount of implicit dependencies of ClickHouse from the OS in addition to the OS kernel.
|
||||
|
||||
Example usage:
|
||||
|
||||
```
|
||||
./prepare
|
||||
docker build --tag clickhouse-bare .
|
||||
```
|
||||
|
||||
Run clickhouse-local:
|
||||
```
|
||||
docker run -it --rm --network host clickhouse-bare /clickhouse local --query "SELECT 1"
|
||||
```
|
||||
|
||||
Run clickhouse-client in interactive mode:
|
||||
```
|
||||
docker run -it --rm --network host clickhouse-bare /clickhouse client
|
||||
```
|
||||
|
||||
Run clickhouse-server:
|
||||
```
|
||||
docker run -it --rm --network host clickhouse-bare /clickhouse server
|
||||
```
|
||||
|
||||
It can be also run in chroot instead of Docker (first edit the `prepare` script to enable `proc`):
|
||||
|
||||
```
|
||||
sudo chroot . /clickhouse server
|
||||
```
|
||||
|
||||
## What does it miss?
|
||||
|
||||
- creation of `clickhouse` user to run the server;
|
||||
- VOLUME for server;
|
||||
- most of the details, see other docker images for comparison.
|
||||
24
docker/bare/prepare
Executable file
24
docker/bare/prepare
Executable file
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
SRC_DIR=../..
|
||||
BUILD_DIR=${SRC_DIR}/build
|
||||
|
||||
# BTW, .so files are acceptable from any Linux distribution for the last 12 years (at least).
|
||||
# See https://presentations.clickhouse.tech/cpp_russia_2020/ for the details.
|
||||
|
||||
mkdir root
|
||||
pushd root
|
||||
mkdir lib lib64 etc tmp root
|
||||
cp ${BUILD_DIR}/programs/clickhouse .
|
||||
cp ${SRC_DIR}/programs/server/{config,users}.xml .
|
||||
cp /lib/x86_64-linux-gnu/{libc.so.6,libdl.so.2,libm.so.6,libpthread.so.0,librt.so.1,libnss_dns.so.2,libresolv.so.2} lib
|
||||
cp /lib64/ld-linux-x86-64.so.2 lib64
|
||||
cp /etc/resolv.conf ./etc
|
||||
strip clickhouse
|
||||
|
||||
# This is needed for chroot but not needed for Docker:
|
||||
|
||||
# mkdir proc
|
||||
# sudo mount --bind /proc proc
|
||||
@ -68,6 +68,7 @@ RUN apt-get --allow-unauthenticated update -y \
|
||||
libre2-dev \
|
||||
libjemalloc-dev \
|
||||
libmsgpack-dev \
|
||||
libcurl4-openssl-dev \
|
||||
opencl-headers \
|
||||
ocl-icd-libopencl1 \
|
||||
intel-opencl-icd \
|
||||
|
||||
12
docker/test/integration/compose/docker_compose_rabbitmq.yml
Normal file
12
docker/test/integration/compose/docker_compose_rabbitmq.yml
Normal file
@ -0,0 +1,12 @@
|
||||
version: '2.3'
|
||||
|
||||
services:
|
||||
rabbitmq1:
|
||||
image: rabbitmq:3-management
|
||||
hostname: rabbitmq1
|
||||
ports:
|
||||
- "5672:5672"
|
||||
- "15672:15672"
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: "root"
|
||||
RABBITMQ_DEFAULT_PASS: "clickhouse"
|
||||
@ -26,6 +26,7 @@ RUN apt-get update \
|
||||
liblua5.1-dev \
|
||||
luajit \
|
||||
libssl-dev \
|
||||
libcurl4-openssl-dev \
|
||||
gdb \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
|
||||
@ -53,4 +53,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
|
||||
&& clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" \
|
||||
&& clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" \
|
||||
&& clickhouse-client --query "SHOW TABLES FROM test" \
|
||||
&& clickhouse-test --testname --shard --zookeeper --no-stateless $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
&& clickhouse-test --testname --shard --zookeeper --no-stateless --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
@ -105,7 +105,7 @@ LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABL
|
||||
LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
|
||||
LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
||||
LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
|
||||
LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --testname --shard --zookeeper --no-stateless $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --testname --shard --zookeeper --no-stateless --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
kill_clickhouse
|
||||
|
||||
|
||||
@ -83,4 +83,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
|
||||
if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/; fi; \
|
||||
ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml; \
|
||||
service zookeeper start; sleep 5; \
|
||||
service clickhouse-server start && sleep 5 && clickhouse-test --testname --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
service clickhouse-server start && sleep 5 && clickhouse-test --testname --shard --zookeeper --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
@ -76,7 +76,7 @@ start_clickhouse
|
||||
|
||||
sleep 10
|
||||
|
||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
kill_clickhouse
|
||||
|
||||
|
||||
@ -17,13 +17,13 @@ def run_perf_test(cmd, xmls_path, output_folder):
|
||||
def run_func_test(cmd, output_prefix, num_processes, skip_tests_option):
|
||||
output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)]
|
||||
f = open(output_paths[0], 'w')
|
||||
main_command = "{} {}".format(cmd, skip_tests_option)
|
||||
main_command = "{} --use-skip-list {}".format(cmd, skip_tests_option)
|
||||
logging.info("Run func tests main cmd '%s'", main_command)
|
||||
pipes = [Popen(main_command, shell=True, stdout=f, stderr=f)]
|
||||
for output_path in output_paths[1:]:
|
||||
time.sleep(0.5)
|
||||
f = open(output_path, 'w')
|
||||
full_command = "{} --order=random {}".format(cmd, skip_tests_option)
|
||||
full_command = "{} --use-skip-list --order=random {}".format(cmd, skip_tests_option)
|
||||
logging.info("Run func tests '%s'", full_command)
|
||||
p = Popen(full_command, shell=True, stdout=f, stderr=f)
|
||||
pipes.append(p)
|
||||
|
||||
@ -50,7 +50,6 @@ See also the description of [max\_memory\_usage](#settings_max_memory_usage).
|
||||
## max\_rows\_to\_read {#max-rows-to-read}
|
||||
|
||||
The following restrictions can be checked on each block (instead of on each row). That is, the restrictions can be broken a little.
|
||||
When running a query in multiple threads, the following restrictions apply to each thread separately.
|
||||
|
||||
A maximum number of rows that can be read from a table when running a query.
|
||||
|
||||
|
||||
@ -521,6 +521,80 @@ Result:
|
||||
- [toDate](#todate)
|
||||
- [toDateTime](#todatetime)
|
||||
|
||||
## parseDateTimeBestEffortUS {#parsedatetimebesteffortUS}
|
||||
|
||||
This function is similar to ['parseDateTimeBestEffort'](#parsedatetimebesteffort), the only difference is that this function prefers US style (`MM/DD/YYYY` etc) in case of ambiguouty.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
parseDateTimeBestEffortUS(time_string [, time_zone]);
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `time_string` — String containing a date and time to convert. [String](../../sql-reference/data-types/string.md).
|
||||
- `time_zone` — Time zone. The function parses `time_string` according to the time zone. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Supported non-standard formats**
|
||||
|
||||
- A string containing 9..10 digit [unix timestamp](https://en.wikipedia.org/wiki/Unix_time).
|
||||
- A string with a date and a time component: `YYYYMMDDhhmmss`, `MM/DD/YYYY hh:mm:ss`, `MM-DD-YY hh:mm`, `YYYY-MM-DD hh:mm:ss`, etc.
|
||||
- A string with a date, but no time component: `YYYY`, `YYYYMM`, `YYYY*MM`, `MM/DD/YYYY`, `MM-DD-YY` etc.
|
||||
- A string with a day and time: `DD`, `DD hh`, `DD hh:mm`. In this case `YYYY-MM` are substituted as `2000-01`.
|
||||
- A string that includes the date and time along with time zone offset information: `YYYY-MM-DD hh:mm:ss ±h:mm`, etc. For example, `2020-12-12 17:36:00 -5:00`.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- `time_string` converted to the `DateTime` data type.
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT parseDateTimeBestEffortUS('09/12/2020 12:12:57')
|
||||
AS parseDateTimeBestEffortUS;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─parseDateTimeBestEffortUS─┐
|
||||
│ 2020-09-12 12:12:57 │
|
||||
└─────────────────────────——┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT parseDateTimeBestEffortUS('09-12-2020 12:12:57')
|
||||
AS parseDateTimeBestEffortUS;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─parseDateTimeBestEffortUS─┐
|
||||
│ 2020-09-12 12:12:57 │
|
||||
└─────────────────────────——┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT parseDateTimeBestEffortUS('09.12.2020 12:12:57')
|
||||
AS parseDateTimeBestEffortUS;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─parseDateTimeBestEffortUS─┐
|
||||
│ 2020-09-12 12:12:57 │
|
||||
└─────────────────────────——┘
|
||||
```
|
||||
|
||||
## parseDateTimeBestEffortOrNull {#parsedatetimebesteffortornull}
|
||||
|
||||
Same as for [parseDateTimeBestEffort](#parsedatetimebesteffort) except that it returns null when it encounters a date format that cannot be processed.
|
||||
|
||||
@ -59,7 +59,6 @@ Ver también la descripción de [Método de codificación de datos:](#settings_m
|
||||
## ¿Qué puedes encontrar en Neodigit {#max-rows-to-read}
|
||||
|
||||
Las siguientes restricciones se pueden verificar en cada bloque (en lugar de en cada fila). Es decir, las restricciones se pueden romper un poco.
|
||||
Al ejecutar una consulta en varios subprocesos, las siguientes restricciones se aplican a cada subproceso por separado.
|
||||
|
||||
Un número máximo de filas que se pueden leer de una tabla al ejecutar una consulta.
|
||||
|
||||
|
||||
@ -45,7 +45,6 @@
|
||||
## max\_rows\_to\_read {#max-rows-to-read}
|
||||
|
||||
Следующие ограничения могут проверяться на каждый блок (а не на каждую строку). То есть, ограничения могут быть немного нарушены.
|
||||
При выполнении запроса в несколько потоков, следующие ограничения действуют в каждом потоке по отдельности.
|
||||
|
||||
Максимальное количество строчек, которое можно прочитать из таблицы при выполнении запроса.
|
||||
|
||||
|
||||
@ -60,7 +60,6 @@ Restrictions on the «maximum amount of something» can take the value 0, which
|
||||
## max\_rows\_to\_read {#max-rows-to-read}
|
||||
|
||||
可以在每个块（而不是每行）上检查以下限制。 也就是说，限制可以打破一点。
|
||||
在多个线程中运行查询时，以下限制单独应用于每个线程。
|
||||
|
||||
运行查询时可从表中读取的最大行数。
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# 数组函数 {#shu-zu-han-shu}
|
||||
|
||||
## 空 {#empty}
|
||||
## empty {#empty}
|
||||
|
||||
对于空数组返回1，对于非空数组返回0。
|
||||
结果类型是UInt8。
|
||||
@ -12,7 +12,7 @@
|
||||
结果类型是UInt8。
|
||||
该函数也适用于字符串。
|
||||
|
||||
## 长度 {#array_functions-length}
|
||||
## length {#array_functions-length}
|
||||
|
||||
返回数组中的元素个数。
|
||||
结果类型是UInt64。
|
||||
@ -24,9 +24,9 @@
|
||||
|
||||
## emptyArrayFloat32,emptyArrayFloat64 {#emptyarrayfloat32-emptyarrayfloat64}
|
||||
|
||||
## 空空漫步，空空漫步时间 {#emptyarraydate-emptyarraydatetime}
|
||||
## emptyArrayDate，emptyArrayDateTime {#emptyarraydate-emptyarraydatetime}
|
||||
|
||||
## 空字符串 {#emptyarraystring}
|
||||
## emptyArrayString {#emptyarraystring}
|
||||
|
||||
不接受任何参数并返回适当类型的空数组。
|
||||
|
||||
@ -34,7 +34,7 @@
|
||||
|
||||
接受一个空数组并返回一个仅包含一个默认值元素的数组。
|
||||
|
||||
## 范围(N) {#rangen}
|
||||
## range(N) {#rangen}
|
||||
|
||||
返回从0到N-1的数字数组。
|
||||
以防万一，如果在数据块中创建总长度超过100,000,000个元素的数组，则抛出异常。
|
||||
@ -74,7 +74,7 @@ SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res
|
||||
|
||||
如果索引超出数组的边界，则返回默认值（数字为0，字符串为空字符串等）。
|
||||
|
||||
## 有(arr,elem) {#hasarr-elem}
|
||||
## has(arr,elem) {#hasarr-elem}
|
||||
|
||||
检查’arr’数组是否具有’elem’元素。
|
||||
如果元素不在数组中，则返回0;如果在，则返回1。
|
||||
@ -186,7 +186,7 @@ SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res
|
||||
│ 2 │
|
||||
└──────────────────────────────────────┘
|
||||
|
||||
## ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人) {#array_functions-arrayenumerate}
|
||||
## arrayEnumerate(arr) {#array_functions-arrayenumerate}
|
||||
|
||||
返回 Array \[1, 2, 3, …, length (arr) \]
|
||||
|
||||
@ -658,7 +658,7 @@ SELECT
|
||||
|
||||
arrayReduce（‘agg\_func’，arr1，…） - 将聚合函数`agg_func`应用于数组`arr1 ...`。如果传递了多个数组，则相应位置上的元素将作为多个参数传递给聚合函数。例如：SELECT arrayReduce（‘max’，\[1,2,3\]）= 3
|
||||
|
||||
## ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人) {#arrayreversearr}
|
||||
## arrayReverse(arr) {#arrayreversearr}
|
||||
|
||||
返回与源数组大小相同的数组，包含反转源数组的所有元素的结果。
|
||||
|
||||
|
||||
@ -79,7 +79,7 @@ Suggest::Suggest()
|
||||
"IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE",
|
||||
"PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE",
|
||||
"IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "FOR", "RANDOMIZED",
|
||||
"INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP"};
|
||||
"INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE"};
|
||||
}
|
||||
|
||||
void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit)
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
#include "ClusterCopierApp.h"
|
||||
#include <Common/StatusFile.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -91,7 +93,7 @@ void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options)
|
||||
|
||||
void ClusterCopierApp::mainImpl()
|
||||
{
|
||||
StatusFile status_file(process_path + "/status");
|
||||
StatusFile status_file(process_path + "/status", StatusFile::write_full_info);
|
||||
ThreadStatus thread_status;
|
||||
|
||||
auto * log = &logger();
|
||||
|
||||
@ -66,7 +66,6 @@
|
||||
#include <Dictionaries/registerDictionaries.h>
|
||||
#include <Disks/registerDisks.h>
|
||||
#include <Databases/DatabaseMemory.h>
|
||||
#include <Common/StatusFile.h>
|
||||
|
||||
#include "Aliases.h"
|
||||
|
||||
|
||||
@ -248,7 +248,7 @@ try
|
||||
if (!context->getPath().empty())
|
||||
{
|
||||
/// Lock path directory before read
|
||||
status.emplace(context->getPath() + "status");
|
||||
status.emplace(context->getPath() + "status", StatusFile::write_full_info);
|
||||
|
||||
LOG_DEBUG(log, "Loading metadata from {}", context->getPath());
|
||||
loadMetadataSystem(*context);
|
||||
|
||||
@ -378,7 +378,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
global_context->setPath(path);
|
||||
|
||||
StatusFile status{path + "status"};
|
||||
StatusFile status{path + "status", StatusFile::write_full_info};
|
||||
|
||||
SCOPE_EXIT({
|
||||
/** Ask to cancel background jobs all table engines,
|
||||
|
||||
@ -66,7 +66,6 @@
|
||||
<https_port>8443</https_port>
|
||||
<tcp_port_secure>9440</tcp_port_secure>
|
||||
-->
|
||||
|
||||
<!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
|
||||
<openSSL>
|
||||
<server> <!-- Used for https server AND secure tcp port -->
|
||||
|
||||
@ -426,12 +426,17 @@ public:
|
||||
|
||||
friend bool operator!=(const Node & left, const Node & right) { return !(left == right); }
|
||||
|
||||
void merge(const Node & other, const Helper & helper)
|
||||
void makeUnion(const Node & other, const Helper & helper)
|
||||
{
|
||||
mergeAccessRec(other);
|
||||
makeUnionRec(other);
|
||||
calculateFinalAccessRec(helper);
|
||||
}
|
||||
|
||||
void makeIntersection(const Node & other, const Helper & helper)
|
||||
{
|
||||
makeIntersectionRec(other);
|
||||
calculateFinalAccessRec(helper);
|
||||
}
|
||||
|
||||
ProtoElements getElements() const
|
||||
{
|
||||
@ -723,12 +728,12 @@ private:
|
||||
max_access = final_access | max_access_among_children;
|
||||
}
|
||||
|
||||
void mergeAccessRec(const Node & rhs)
|
||||
void makeUnionRec(const Node & rhs)
|
||||
{
|
||||
if (rhs.children)
|
||||
{
|
||||
for (const auto & [rhs_childname, rhs_child] : *rhs.children)
|
||||
getChild(rhs_childname).mergeAccessRec(rhs_child);
|
||||
getChild(rhs_childname).makeUnionRec(rhs_child);
|
||||
}
|
||||
access |= rhs.access;
|
||||
if (children)
|
||||
@ -740,6 +745,24 @@ private:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void makeIntersectionRec(const Node & rhs)
|
||||
{
|
||||
if (rhs.children)
|
||||
{
|
||||
for (const auto & [rhs_childname, rhs_child] : *rhs.children)
|
||||
getChild(rhs_childname).makeIntersectionRec(rhs_child);
|
||||
}
|
||||
access &= rhs.access;
|
||||
if (children)
|
||||
{
|
||||
for (auto & [lhs_childname, lhs_child] : *children)
|
||||
{
|
||||
if (!rhs.tryGetChild(lhs_childname))
|
||||
lhs_child.access &= rhs.access;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -989,7 +1012,7 @@ bool operator ==(const AccessRights & left, const AccessRights & right)
|
||||
}
|
||||
|
||||
|
||||
void AccessRights::merge(const AccessRights & other)
|
||||
void AccessRights::makeUnion(const AccessRights & other)
|
||||
{
|
||||
auto helper = [](std::unique_ptr<Node> & root_node, const std::unique_ptr<Node> & other_root_node)
|
||||
{
|
||||
@ -1001,7 +1024,29 @@ void AccessRights::merge(const AccessRights & other)
|
||||
}
|
||||
if (other_root_node)
|
||||
{
|
||||
root_node->merge(*other_root_node, Helper::instance());
|
||||
root_node->makeUnion(*other_root_node, Helper::instance());
|
||||
if (!root_node->access && !root_node->children)
|
||||
root_node = nullptr;
|
||||
}
|
||||
};
|
||||
helper(root, other.root);
|
||||
helper(root_with_grant_option, other.root_with_grant_option);
|
||||
}
|
||||
|
||||
|
||||
void AccessRights::makeIntersection(const AccessRights & other)
|
||||
{
|
||||
auto helper = [](std::unique_ptr<Node> & root_node, const std::unique_ptr<Node> & other_root_node)
|
||||
{
|
||||
if (!root_node)
|
||||
{
|
||||
if (other_root_node)
|
||||
root_node = std::make_unique<Node>(*other_root_node);
|
||||
return;
|
||||
}
|
||||
if (other_root_node)
|
||||
{
|
||||
root_node->makeIntersection(*other_root_node, Helper::instance());
|
||||
if (!root_node->access && !root_node->children)
|
||||
root_node = nullptr;
|
||||
}
|
||||
|
||||
@ -93,7 +93,9 @@ public:
|
||||
|
||||
/// Merges two sets of access rights together.
|
||||
/// It's used to combine access rights from multiple roles.
|
||||
void merge(const AccessRights & other);
|
||||
void makeUnion(const AccessRights & other);
|
||||
|
||||
void makeIntersection(const AccessRights & other);
|
||||
|
||||
friend bool operator ==(const AccessRights & left, const AccessRights & right);
|
||||
friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }
|
||||
|
||||
@ -30,6 +30,73 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_USER;
|
||||
}
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
std::shared_ptr<AccessRights> mixAccessRightsFromUserAndRoles(const User & user, const EnabledRolesInfo & roles_info)
|
||||
{
|
||||
auto res = std::make_shared<AccessRights>(user.access);
|
||||
res->makeUnion(roles_info.access);
|
||||
return res;
|
||||
}
|
||||
|
||||
std::shared_ptr<AccessRights> applyParamsToAccessRights(const AccessRights & access, const ContextAccessParams & params)
|
||||
{
|
||||
auto res = std::make_shared<AccessRights>(access);
|
||||
|
||||
static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW
|
||||
| AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW
|
||||
| AccessType::TRUNCATE;
|
||||
|
||||
static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
|
||||
static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
|
||||
static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
|
||||
static const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
|
||||
|
||||
if (params.readonly)
|
||||
res->revoke(write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY);
|
||||
|
||||
if (params.readonly == 1)
|
||||
{
|
||||
/// Table functions are forbidden in readonly mode.
|
||||
/// For example, for readonly = 2 - allowed.
|
||||
res->revoke(AccessType::CREATE_TEMPORARY_TABLE);
|
||||
}
|
||||
|
||||
if (!params.allow_ddl)
|
||||
res->revoke(table_and_dictionary_ddl);
|
||||
|
||||
if (!params.allow_introspection)
|
||||
res->revoke(AccessType::INTROSPECTION);
|
||||
|
||||
/// Anyone has access to the "system" database.
|
||||
res->grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
|
||||
|
||||
if (params.readonly != 1)
|
||||
{
|
||||
/// User has access to temporary or external table if such table was resolved in session or query context
|
||||
res->grant(AccessFlags::allTableFlags() | AccessFlags::allColumnFlags(), DatabaseCatalog::TEMPORARY_DATABASE);
|
||||
}
|
||||
|
||||
if (params.readonly)
|
||||
{
|
||||
/// No grant option in readonly mode.
|
||||
res->revokeGrantOption(AccessType::ALL);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
std::array<UUID, 1> to_array(const UUID & id)
|
||||
{
|
||||
std::array<UUID, 1> ids;
|
||||
ids[0] = id;
|
||||
return ids;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ContextAccess::ContextAccess(const AccessControlManager & manager_, const Params & params_)
|
||||
: manager(&manager_)
|
||||
, params(params_)
|
||||
@ -54,6 +121,10 @@ void ContextAccess::setUser(const UserPtr & user_) const
|
||||
/// User has been dropped.
|
||||
auto nothing_granted = std::make_shared<AccessRights>();
|
||||
access = nothing_granted;
|
||||
access_without_readonly = nothing_granted;
|
||||
access_with_allow_ddl = nothing_granted;
|
||||
access_with_allow_introspection = nothing_granted;
|
||||
access_from_user_and_roles = nothing_granted;
|
||||
subscription_for_user_change = {};
|
||||
subscription_for_roles_changes = {};
|
||||
enabled_roles = nullptr;
|
||||
@ -108,56 +179,18 @@ void ContextAccess::setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> &
|
||||
enabled_row_policies = manager->getEnabledRowPolicies(*params.user_id, roles_info->enabled_roles);
|
||||
enabled_quota = manager->getEnabledQuota(*params.user_id, user_name, roles_info->enabled_roles, params.address, params.quota_key);
|
||||
enabled_settings = manager->getEnabledSettings(*params.user_id, user->settings, roles_info->enabled_roles, roles_info->settings_from_enabled_roles);
|
||||
setFinalAccess();
|
||||
calculateAccessRights();
|
||||
}
|
||||
|
||||
|
||||
void ContextAccess::setFinalAccess() const
|
||||
void ContextAccess::calculateAccessRights() const
|
||||
{
|
||||
auto final_access = std::make_shared<AccessRights>();
|
||||
*final_access = user->access;
|
||||
if (roles_info)
|
||||
final_access->merge(roles_info->access);
|
||||
access_from_user_and_roles = mixAccessRightsFromUserAndRoles(*user, *roles_info);
|
||||
access = applyParamsToAccessRights(*access_from_user_and_roles, params);
|
||||
|
||||
static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW
|
||||
| AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW
|
||||
| AccessType::TRUNCATE;
|
||||
|
||||
static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
|
||||
static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
|
||||
static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
|
||||
static const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
|
||||
|
||||
if (params.readonly)
|
||||
final_access->revoke(write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY);
|
||||
|
||||
if (params.readonly == 1)
|
||||
{
|
||||
/// Table functions are forbidden in readonly mode.
|
||||
/// For example, for readonly = 2 - allowed.
|
||||
final_access->revoke(AccessType::CREATE_TEMPORARY_TABLE);
|
||||
}
|
||||
|
||||
if (!params.allow_ddl)
|
||||
final_access->revoke(table_and_dictionary_ddl);
|
||||
|
||||
if (!params.allow_introspection)
|
||||
final_access->revoke(AccessType::INTROSPECTION);
|
||||
|
||||
/// Anyone has access to the "system" database.
|
||||
final_access->grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
|
||||
|
||||
if (params.readonly != 1)
|
||||
{
|
||||
/// User has access to temporary or external table if such table was resolved in session or query context
|
||||
final_access->grant(AccessFlags::allTableFlags() | AccessFlags::allColumnFlags(), DatabaseCatalog::TEMPORARY_DATABASE);
|
||||
}
|
||||
|
||||
if (params.readonly)
|
||||
{
|
||||
/// No grant option in readonly mode.
|
||||
final_access->revokeGrantOption(AccessType::ALL);
|
||||
}
|
||||
access_without_readonly = nullptr;
|
||||
access_with_allow_ddl = nullptr;
|
||||
access_with_allow_introspection = nullptr;
|
||||
|
||||
if (trace_log)
|
||||
{
|
||||
@ -168,10 +201,8 @@ void ContextAccess::setFinalAccess() const
|
||||
boost::algorithm::join(roles_info->getEnabledRolesNames(), ", "));
|
||||
}
|
||||
LOG_TRACE(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", params.readonly, params.allow_ddl, params.allow_introspection);
|
||||
LOG_TRACE(trace_log, "List of all grants: {}", final_access->toString());
|
||||
LOG_TRACE(trace_log, "List of all grants: {}", access->toString());
|
||||
}
|
||||
|
||||
access = final_access;
|
||||
}
|
||||
|
||||
|
||||
@ -361,9 +392,7 @@ void ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &...
|
||||
std::lock_guard lock{mutex};
|
||||
|
||||
if (!user)
|
||||
{
|
||||
show_error("User has been dropped", ErrorCodes::UNKNOWN_USER);
|
||||
}
|
||||
|
||||
if (grant_option && access->isGranted(flags, args...))
|
||||
{
|
||||
@ -381,7 +410,7 @@ void ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &...
|
||||
{
|
||||
Params changed_params = params;
|
||||
changed_params.readonly = 0;
|
||||
access_without_readonly = manager->getContextAccess(changed_params);
|
||||
access_without_readonly = applyParamsToAccessRights(*access_from_user_and_roles, changed_params);
|
||||
}
|
||||
|
||||
if (access_without_readonly->isGranted(flags, args...))
|
||||
@ -402,7 +431,7 @@ void ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &...
|
||||
{
|
||||
Params changed_params = params;
|
||||
changed_params.allow_ddl = true;
|
||||
access_with_allow_ddl = manager->getContextAccess(changed_params);
|
||||
access_with_allow_ddl = applyParamsToAccessRights(*access_from_user_and_roles, changed_params);
|
||||
}
|
||||
|
||||
if (access_with_allow_ddl->isGranted(flags, args...))
|
||||
@ -417,7 +446,7 @@ void ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &...
|
||||
{
|
||||
Params changed_params = params;
|
||||
changed_params.allow_introspection = true;
|
||||
access_with_allow_introspection = manager->getContextAccess(changed_params);
|
||||
access_with_allow_introspection = applyParamsToAccessRights(*access_from_user_and_roles, changed_params);
|
||||
}
|
||||
|
||||
if (access_with_allow_introspection->isGranted(flags, args...))
|
||||
@ -483,25 +512,65 @@ void ContextAccess::checkGrantOption(const AccessRightsElement & element) const
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true>(elements); }
|
||||
|
||||
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id) const
|
||||
template <typename Container, typename GetNameFunction>
|
||||
void ContextAccess::checkAdminOptionImpl(const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
{
|
||||
if (isGranted(AccessType::ROLE_ADMIN))
|
||||
return;
|
||||
|
||||
auto info = getRolesInfo();
|
||||
if (info && info->enabled_roles_with_admin_option.count(role_id))
|
||||
if (!info)
|
||||
{
|
||||
if (!user)
|
||||
throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!user)
|
||||
throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER);
|
||||
size_t i = 0;
|
||||
for (auto it = std::begin(role_ids); it != std::end(role_ids); ++it, ++i)
|
||||
{
|
||||
const UUID & role_id = *it;
|
||||
if (info->enabled_roles_with_admin_option.count(role_id))
|
||||
continue;
|
||||
|
||||
std::optional<String> role_name = manager->readName(role_id);
|
||||
if (!role_name)
|
||||
role_name = "ID {" + toString(role_id) + "}";
|
||||
throw Exception(
|
||||
getUserName() + ": Not enough privileges. To execute this query it's necessary to have the grant " + backQuoteIfNeed(*role_name)
|
||||
+ " WITH ADMIN OPTION ",
|
||||
ErrorCodes::ACCESS_DENIED);
|
||||
auto role_name = get_name_function(role_id, i);
|
||||
if (!role_name)
|
||||
role_name = "ID {" + toString(role_id) + "}";
|
||||
String msg = "To execute this query it's necessary to have the role " + backQuoteIfNeed(*role_name) + " granted with ADMIN option";
|
||||
if (info->enabled_roles.count(role_id))
|
||||
msg = "Role " + backQuote(*role_name) + " is granted, but without ADMIN option. " + msg;
|
||||
throw Exception(getUserName() + ": Not enough privileges. " + msg, ErrorCodes::ACCESS_DENIED);
|
||||
}
|
||||
}
|
||||
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id) const
|
||||
{
|
||||
checkAdminOptionImpl(to_array(role_id), [this](const UUID & id, size_t) { return manager->tryReadName(id); });
|
||||
}
|
||||
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const String & role_name) const
|
||||
{
|
||||
checkAdminOptionImpl(to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
|
||||
}
|
||||
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
checkAdminOptionImpl(to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids) const
|
||||
{
|
||||
checkAdminOptionImpl(role_ids, [this](const UUID & id, size_t) { return manager->tryReadName(id); });
|
||||
}
|
||||
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
|
||||
{
|
||||
checkAdminOptionImpl(role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
|
||||
}
|
||||
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
checkAdminOptionImpl(role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -135,6 +135,11 @@ public:
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and throws an exception if not.
|
||||
void checkAdminOption(const UUID & role_id) const;
|
||||
void checkAdminOption(const UUID & role_id, const String & role_name) const;
|
||||
void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
/// Makes an instance of ContextAccess which provides full access to everything
|
||||
/// without any limitations. This is used for the global context.
|
||||
@ -148,7 +153,7 @@ private:
|
||||
void setUser(const UserPtr & user_) const;
|
||||
void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const;
|
||||
void setSettingsAndConstraints() const;
|
||||
void setFinalAccess() const;
|
||||
void calculateAccessRights() const;
|
||||
|
||||
template <bool grant_option>
|
||||
bool isGrantedImpl(const AccessFlags & flags) const;
|
||||
@ -180,6 +185,9 @@ private:
|
||||
template <bool grant_option, typename... Args>
|
||||
void checkAccessImpl2(const AccessFlags & flags, const Args &... args) const;
|
||||
|
||||
template <typename Container, typename GetNameFunction>
|
||||
void checkAdminOptionImpl(const Container & role_ids, const GetNameFunction & get_name_function) const;
|
||||
|
||||
const AccessControlManager * manager = nullptr;
|
||||
const Params params;
|
||||
mutable Poco::Logger * trace_log = nullptr;
|
||||
@ -193,9 +201,10 @@ private:
|
||||
mutable std::shared_ptr<const EnabledRowPolicies> enabled_row_policies;
|
||||
mutable std::shared_ptr<const EnabledQuota> enabled_quota;
|
||||
mutable std::shared_ptr<const EnabledSettings> enabled_settings;
|
||||
mutable std::shared_ptr<const ContextAccess> access_without_readonly;
|
||||
mutable std::shared_ptr<const ContextAccess> access_with_allow_ddl;
|
||||
mutable std::shared_ptr<const ContextAccess> access_with_allow_introspection;
|
||||
mutable std::shared_ptr<const AccessRights> access_without_readonly;
|
||||
mutable std::shared_ptr<const AccessRights> access_with_allow_ddl;
|
||||
mutable std::shared_ptr<const AccessRights> access_with_allow_introspection;
|
||||
mutable std::shared_ptr<const AccessRights> access_from_user_and_roles;
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
|
||||
@ -43,7 +43,7 @@ namespace
|
||||
roles_info.enabled_roles_with_admin_option.emplace(role_id);
|
||||
|
||||
roles_info.names_of_roles[role_id] = role->getName();
|
||||
roles_info.access.merge(role->access);
|
||||
roles_info.access.makeUnion(role->access);
|
||||
roles_info.settings_from_enabled_roles.merge(role->settings);
|
||||
|
||||
for (const auto & granted_role : role->granted_roles.roles)
|
||||
|
||||
@ -19,6 +19,8 @@ namespace ErrorCodes
|
||||
template <typename T, typename Denominator>
|
||||
struct AggregateFunctionAvgData
|
||||
{
|
||||
using NumeratorType = T;
|
||||
|
||||
T numerator = 0;
|
||||
Denominator denominator = 0;
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@ public:
|
||||
const auto & values = static_cast<const ColVecType &>(*columns[0]);
|
||||
const auto & weights = static_cast<const ColVecType &>(*columns[1]);
|
||||
|
||||
this->data(place).numerator += values.getData()[row_num] * weights.getData()[row_num];
|
||||
this->data(place).numerator += static_cast<typename Data::NumeratorType>(values.getData()[row_num]) * weights.getData()[row_num];
|
||||
this->data(place).denominator += weights.getData()[row_num];
|
||||
}
|
||||
|
||||
|
||||
@ -28,7 +28,8 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons
|
||||
|
||||
void registerAggregateFunctionCount(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("count", {createAggregateFunctionCount, {true}}, AggregateFunctionFactory::CaseInsensitive);
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false };
|
||||
factory.registerFunction("count", {createAggregateFunctionCount, properties}, AggregateFunctionFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -162,6 +162,52 @@ AggregateFunctionPtr AggregateFunctionFactory::tryGet(
|
||||
}
|
||||
|
||||
|
||||
std::optional<AggregateFunctionProperties> AggregateFunctionFactory::tryGetPropertiesImpl(const String & name_param, int recursion_level) const
|
||||
{
|
||||
String name = getAliasToOrName(name_param);
|
||||
Value found;
|
||||
|
||||
/// Find by exact match.
|
||||
if (auto it = aggregate_functions.find(name); it != aggregate_functions.end())
|
||||
{
|
||||
found = it->second;
|
||||
}
|
||||
/// Find by case-insensitive name.
|
||||
/// Combinators cannot apply for case insensitive (SQL-style) aggregate function names. Only for native names.
|
||||
else if (recursion_level == 0)
|
||||
{
|
||||
if (auto jt = case_insensitive_aggregate_functions.find(Poco::toLower(name)); jt != case_insensitive_aggregate_functions.end())
|
||||
found = jt->second;
|
||||
}
|
||||
|
||||
if (found.creator)
|
||||
return found.properties;
|
||||
|
||||
/// Combinators of aggregate functions.
|
||||
/// For every aggregate function 'agg' and combiner '-Comb' there is combined aggregate function with name 'aggComb',
|
||||
/// that can have different number and/or types of arguments, different result type and different behaviour.
|
||||
|
||||
if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name))
|
||||
{
|
||||
if (combinator->isForInternalUsageOnly())
|
||||
return {};
|
||||
|
||||
String nested_name = name.substr(0, name.size() - combinator->getName().size());
|
||||
|
||||
/// NOTE: It's reasonable to also allow to transform properties by combinator.
|
||||
return tryGetPropertiesImpl(nested_name, recursion_level + 1);
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
std::optional<AggregateFunctionProperties> AggregateFunctionFactory::tryGetProperties(const String & name) const
|
||||
{
|
||||
return tryGetPropertiesImpl(name, 0);
|
||||
}
|
||||
|
||||
|
||||
bool AggregateFunctionFactory::isAggregateFunctionName(const String & name, int recursion_level) const
|
||||
{
|
||||
if (aggregate_functions.count(name) || isAlias(name))
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include <optional>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -72,6 +73,9 @@ public:
|
||||
const Array & parameters,
|
||||
AggregateFunctionProperties & out_properties) const;
|
||||
|
||||
/// Get properties if the aggregate function exists.
|
||||
std::optional<AggregateFunctionProperties> tryGetProperties(const String & name) const;
|
||||
|
||||
bool isAggregateFunctionName(const String & name, int recursion_level = 0) const;
|
||||
|
||||
private:
|
||||
@ -83,6 +87,8 @@ private:
|
||||
bool has_null_arguments,
|
||||
int recursion_level) const;
|
||||
|
||||
std::optional<AggregateFunctionProperties> tryGetPropertiesImpl(const String & name, int recursion_level) const;
|
||||
|
||||
private:
|
||||
using AggregateFunctions = std::unordered_map<String, Value>;
|
||||
|
||||
|
||||
@ -120,8 +120,10 @@ AggregateFunctionPtr createAggregateFunctionGroupArraySample(const std::string &
|
||||
|
||||
void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("groupArray", createAggregateFunctionGroupArray);
|
||||
factory.registerFunction("groupArraySample", createAggregateFunctionGroupArraySample);
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("groupArray", { createAggregateFunctionGroupArray, properties });
|
||||
factory.registerFunction("groupArraySample", { createAggregateFunctionGroupArraySample, properties });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -95,8 +95,10 @@ AggregateFunctionPtr createAggregateFunctionMoving(const std::string & name, con
|
||||
|
||||
void registerAggregateFunctionMoving(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("groupArrayMovingSum", createAggregateFunctionMoving<MovingSumTemplate>);
|
||||
factory.registerFunction("groupArrayMovingAvg", createAggregateFunctionMoving<MovingAvgTemplate>);
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("groupArrayMovingSum", { createAggregateFunctionMoving<MovingSumTemplate>, properties });
|
||||
factory.registerFunction("groupArrayMovingAvg", { createAggregateFunctionMoving<MovingAvgTemplate>, properties });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -71,7 +71,6 @@ struct MovingAvgData
|
||||
void add(T val, Arena * arena)
|
||||
{
|
||||
sum += val;
|
||||
|
||||
value.push_back(sum, arena);
|
||||
}
|
||||
|
||||
@ -96,7 +95,8 @@ class MovingImpl final
|
||||
|
||||
public:
|
||||
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
||||
using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>; // probably for overflow function in the future
|
||||
// probably for overflow function in the future
|
||||
using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
||||
|
||||
explicit MovingImpl(const DataTypePtr & data_type_, UInt64 win_size_ = std::numeric_limits<UInt64>::max())
|
||||
: IAggregateFunctionDataHelper<Data, MovingImpl<T, Tlimit_num_elems, Data>>({data_type_}, {})
|
||||
|
||||
@ -110,7 +110,9 @@ AggregateFunctionPtr createAggregateFunctionGroupUniqArray(const std::string & n
|
||||
|
||||
void registerAggregateFunctionGroupUniqArray(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("groupUniqArray", createAggregateFunctionGroupUniqArray);
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("groupUniqArray", { createAggregateFunctionGroupUniqArray, properties });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -49,13 +49,18 @@ AggregateFunctionPtr createAggregateFunctionArgMax(const std::string & name, con
|
||||
|
||||
void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("any", createAggregateFunctionAny);
|
||||
factory.registerFunction("anyLast", createAggregateFunctionAnyLast);
|
||||
factory.registerFunction("anyHeavy", createAggregateFunctionAnyHeavy);
|
||||
factory.registerFunction("min", createAggregateFunctionMin, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("max", createAggregateFunctionMax, AggregateFunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction("argMin", createAggregateFunctionArgMin);
|
||||
factory.registerFunction("argMax", createAggregateFunctionArgMax);
|
||||
|
||||
/// The functions below depend on the order of data.
|
||||
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("any", { createAggregateFunctionAny, properties });
|
||||
factory.registerFunction("anyLast", { createAggregateFunctionAnyLast, properties });
|
||||
factory.registerFunction("anyHeavy", { createAggregateFunctionAnyHeavy, properties });
|
||||
factory.registerFunction("argMin", { createAggregateFunctionArgMin, properties });
|
||||
factory.registerFunction("argMax", { createAggregateFunctionArgMax, properties });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -117,8 +117,10 @@ AggregateFunctionPtr createAggregateFunctionTopK(const std::string & name, const
|
||||
|
||||
void registerAggregateFunctionTopK(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("topK", createAggregateFunctionTopK<false>);
|
||||
factory.registerFunction("topKWeighted", createAggregateFunctionTopK<true>);
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("topK", { createAggregateFunctionTopK<false>, properties });
|
||||
factory.registerFunction("topKWeighted", { createAggregateFunctionTopK<true>, properties });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -122,14 +122,16 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
|
||||
void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
|
||||
{
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false };
|
||||
|
||||
factory.registerFunction("uniq",
|
||||
{createAggregateFunctionUniq<AggregateFunctionUniqUniquesHashSetData, AggregateFunctionUniqUniquesHashSetDataForVariadic>, {true}});
|
||||
{createAggregateFunctionUniq<AggregateFunctionUniqUniquesHashSetData, AggregateFunctionUniqUniquesHashSetDataForVariadic>, properties});
|
||||
|
||||
factory.registerFunction("uniqHLL12",
|
||||
{createAggregateFunctionUniq<false, AggregateFunctionUniqHLL12Data, AggregateFunctionUniqHLL12DataForVariadic>, {true}});
|
||||
{createAggregateFunctionUniq<false, AggregateFunctionUniqHLL12Data, AggregateFunctionUniqHLL12DataForVariadic>, properties});
|
||||
|
||||
factory.registerFunction("uniqExact",
|
||||
{createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>, {true}});
|
||||
{createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>, properties});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -289,6 +289,11 @@ struct AggregateFunctionProperties
|
||||
* or we should return non-Nullable type with default value (example: count, countDistinct).
|
||||
*/
|
||||
bool returns_default_when_only_null = false;
|
||||
|
||||
/** Result varies depending on the data order (example: groupArray).
|
||||
* Some may also name this property as "non-commutative".
|
||||
*/
|
||||
bool is_order_dependent = false;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -73,6 +73,10 @@ if(USE_RDKAFKA)
|
||||
add_headers_and_sources(dbms Storages/Kafka)
|
||||
endif()
|
||||
|
||||
if (USE_AMQPCPP)
|
||||
add_headers_and_sources(dbms Storages/RabbitMQ)
|
||||
endif()
|
||||
|
||||
if (USE_AWS_S3)
|
||||
add_headers_and_sources(dbms Common/S3)
|
||||
add_headers_and_sources(dbms Disks/S3)
|
||||
@ -262,6 +266,9 @@ if (USE_RDKAFKA)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (USE_AMQPCPP)
|
||||
dbms_target_link_libraries(PUBLIC amqp-cpp)
|
||||
endif()
|
||||
|
||||
if(RE2_INCLUDE_DIR)
|
||||
target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
|
||||
|
||||
@ -496,6 +496,7 @@ namespace ErrorCodes
|
||||
extern const int NO_SUITABLE_FUNCTION_IMPLEMENTATION = 527;
|
||||
extern const int CASSANDRA_INTERNAL_ERROR = 528;
|
||||
extern const int NOT_A_LEADER = 529;
|
||||
extern const int CANNOT_CONNECT_RABBITMQ = 530;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
|
||||
@ -30,8 +30,21 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
StatusFile::StatusFile(const std::string & path_)
|
||||
: path(path_)
|
||||
StatusFile::FillFunction StatusFile::write_pid = [](WriteBuffer & out)
|
||||
{
|
||||
out << getpid();
|
||||
};
|
||||
|
||||
StatusFile::FillFunction StatusFile::write_full_info = [](WriteBuffer & out)
|
||||
{
|
||||
out << "PID: " << getpid() << "\n"
|
||||
<< "Started at: " << LocalDateTime(time(nullptr)) << "\n"
|
||||
<< "Revision: " << ClickHouseRevision::get() << "\n";
|
||||
};
|
||||
|
||||
|
||||
StatusFile::StatusFile(std::string path_, FillFunction fill_)
|
||||
: path(std::move(path_)), fill(std::move(fill_))
|
||||
{
|
||||
/// If file already exists. NOTE Minor race condition.
|
||||
if (Poco::File(path).exists())
|
||||
@ -72,13 +85,8 @@ StatusFile::StatusFile(const std::string & path_)
|
||||
throwFromErrnoWithPath("Cannot lseek " + path, path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
||||
|
||||
/// Write information about current server instance to the file.
|
||||
{
|
||||
WriteBufferFromFileDescriptor out(fd, 1024);
|
||||
out
|
||||
<< "PID: " << getpid() << "\n"
|
||||
<< "Started at: " << LocalDateTime(time(nullptr)) << "\n"
|
||||
<< "Revision: " << ClickHouseRevision::get() << "\n";
|
||||
}
|
||||
WriteBufferFromFileDescriptor out(fd, 1024);
|
||||
fill(out);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -1,23 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <functional>
|
||||
#include <boost/noncopyable.hpp>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class WriteBuffer;
|
||||
|
||||
|
||||
/** Provides that no more than one server works with one data directory.
|
||||
*/
|
||||
class StatusFile : private boost::noncopyable
|
||||
{
|
||||
public:
|
||||
explicit StatusFile(const std::string & path_);
|
||||
using FillFunction = std::function<void(WriteBuffer&)>;
|
||||
|
||||
StatusFile(std::string path_, FillFunction fill_);
|
||||
~StatusFile();
|
||||
|
||||
/// You can use one of these functions to fill the file or provide your own.
|
||||
static FillFunction write_pid;
|
||||
static FillFunction write_full_info;
|
||||
|
||||
private:
|
||||
const std::string path;
|
||||
FillFunction fill;
|
||||
int fd = -1;
|
||||
};
|
||||
|
||||
|
||||
@ -74,6 +74,7 @@ struct Settings : public SettingsCollection<Settings>
|
||||
M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when the connection pool is full.", 0) \
|
||||
M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \
|
||||
M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \
|
||||
M(SettingMilliseconds, rabbitmq_max_wait_ms, 5000, "The wait time for reading from RabbitMQ before retry.", 0) \
|
||||
M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \
|
||||
M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \
|
||||
M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.", 0) \
|
||||
@ -335,6 +336,7 @@ struct Settings : public SettingsCollection<Settings>
|
||||
M(SettingBool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \
|
||||
M(SettingBool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \
|
||||
M(SettingBool, optimize_aggregation_in_order, false, "Enable GROUP BY optimization for aggregating data in corresponding order in MergeTree tables.", 0) \
|
||||
M(SettingUInt64, read_in_order_two_level_merge_threshold, 100, "Minimal number of parts to read to run preliminary merge step during multithread reading in order of primary key.", 0) \
|
||||
M(SettingBool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \
|
||||
M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \
|
||||
M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#cmakedefine01 USE_ICU
|
||||
#cmakedefine01 USE_MYSQL
|
||||
#cmakedefine01 USE_RDKAFKA
|
||||
#cmakedefine01 USE_AMQPCPP
|
||||
#cmakedefine01 USE_EMBEDDED_COMPILER
|
||||
#cmakedefine01 USE_INTERNAL_LLVM_LIBRARY
|
||||
#cmakedefine01 USE_SSL
|
||||
|
||||
@ -31,8 +31,8 @@ struct BlockIO
|
||||
QueryPipeline pipeline;
|
||||
|
||||
/// Callbacks for query logging could be set here.
|
||||
std::function<void(IBlockInputStream *, IBlockOutputStream *)> finish_callback;
|
||||
std::function<void()> exception_callback;
|
||||
std::function<void(IBlockInputStream *, IBlockOutputStream *, QueryPipeline *)> finish_callback;
|
||||
std::function<void()> exception_callback;
|
||||
|
||||
/// When it is true, don't bother sending any non-empty blocks to the out stream
|
||||
bool null_format = false;
|
||||
@ -41,7 +41,13 @@ struct BlockIO
|
||||
void onFinish()
|
||||
{
|
||||
if (finish_callback)
|
||||
finish_callback(in.get(), out.get());
|
||||
{
|
||||
QueryPipeline * pipeline_ptr = nullptr;
|
||||
if (pipeline.initialized())
|
||||
pipeline_ptr = &pipeline;
|
||||
|
||||
finish_callback(in.get(), out.get(), pipeline_ptr);
|
||||
}
|
||||
}
|
||||
|
||||
void onException()
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Interpreters/ExpressionActions.h>
|
||||
#include <Columns/ColumnsCommon.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
@ -14,6 +15,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int VIOLATED_CONSTRAINT;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
|
||||
@ -40,46 +42,72 @@ void CheckConstraintsBlockOutputStream::write(const Block & block)
|
||||
for (size_t i = 0; i < expressions.size(); ++i)
|
||||
{
|
||||
auto constraint_expr = expressions[i];
|
||||
|
||||
constraint_expr->execute(block_to_calculate);
|
||||
|
||||
auto * constraint_ptr = constraints.constraints[i]->as<ASTConstraintDeclaration>();
|
||||
|
||||
ColumnWithTypeAndName res_column = block_to_calculate.getByPosition(block_to_calculate.columns() - 1);
|
||||
const ColumnUInt8 & res_column_uint8 = assert_cast<const ColumnUInt8 &>(*res_column.column);
|
||||
|
||||
const UInt8 * data = res_column_uint8.getData().data();
|
||||
size_t size = res_column_uint8.size();
|
||||
if (!isUInt8(res_column.type))
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Constraint {} does not return a value of type UInt8",
|
||||
backQuote(constraint_ptr->name));
|
||||
|
||||
/// Is violated.
|
||||
if (!memoryIsByte(data, size, 1))
|
||||
if (const ColumnConst * res_const = typeid_cast<const ColumnConst *>(res_column.column.get()))
|
||||
{
|
||||
size_t row_idx = 0;
|
||||
for (; row_idx < size; ++row_idx)
|
||||
if (data[row_idx] != 1)
|
||||
break;
|
||||
UInt8 value = res_const->getValue<UInt64>();
|
||||
|
||||
Names related_columns = constraint_expr->getRequiredColumns();
|
||||
|
||||
std::stringstream exception_message;
|
||||
|
||||
auto * constraint_ptr = constraints.constraints[i]->as<ASTConstraintDeclaration>();
|
||||
exception_message << "Constraint " << backQuote(constraint_ptr->name)
|
||||
<< " for table " << table_id.getNameForLogs()
|
||||
<< " is violated at row " << (rows_written + row_idx + 1)
|
||||
<< ". Expression: (" << serializeAST(*(constraint_ptr->expr), true) << ")"
|
||||
<< ". Column values";
|
||||
|
||||
bool first = true;
|
||||
for (const auto & name : related_columns)
|
||||
/// Is violated.
|
||||
if (!value)
|
||||
{
|
||||
const IColumn & column = *block.getByName(name).column;
|
||||
assert(row_idx < column.size());
|
||||
std::stringstream exception_message;
|
||||
|
||||
exception_message << (first ? ": " : ", ")
|
||||
<< backQuoteIfNeed(name) << " = " << applyVisitor(FieldVisitorToString(), column[row_idx]);
|
||||
exception_message << "Constraint " << backQuote(constraint_ptr->name)
|
||||
<< " for table " << table_id.getNameForLogs()
|
||||
<< " is violated, because it is a constant expression returning 0."
|
||||
<< " It is most likely an error in table definition.";
|
||||
|
||||
first = false;
|
||||
throw Exception{exception_message.str(), ErrorCodes::VIOLATED_CONSTRAINT};
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const ColumnUInt8 & res_column_uint8 = assert_cast<const ColumnUInt8 &>(*res_column.column);
|
||||
|
||||
throw Exception{exception_message.str(), ErrorCodes::VIOLATED_CONSTRAINT};
|
||||
const UInt8 * data = res_column_uint8.getData().data();
|
||||
size_t size = res_column_uint8.size();
|
||||
|
||||
/// Is violated.
|
||||
if (!memoryIsByte(data, size, 1))
|
||||
{
|
||||
size_t row_idx = 0;
|
||||
for (; row_idx < size; ++row_idx)
|
||||
if (data[row_idx] != 1)
|
||||
break;
|
||||
|
||||
Names related_columns = constraint_expr->getRequiredColumns();
|
||||
|
||||
std::stringstream exception_message;
|
||||
|
||||
exception_message << "Constraint " << backQuote(constraint_ptr->name)
|
||||
<< " for table " << table_id.getNameForLogs()
|
||||
<< " is violated at row " << (rows_written + row_idx + 1)
|
||||
<< ". Expression: (" << serializeAST(*(constraint_ptr->expr), true) << ")"
|
||||
<< ". Column values";
|
||||
|
||||
bool first = true;
|
||||
for (const auto & name : related_columns)
|
||||
{
|
||||
const IColumn & column = *block.getByName(name).column;
|
||||
assert(row_idx < column.size());
|
||||
|
||||
exception_message << (first ? ": " : ", ")
|
||||
<< backQuoteIfNeed(name) << " = " << applyVisitor(FieldVisitorToString(), column[row_idx]);
|
||||
|
||||
first = false;
|
||||
}
|
||||
|
||||
throw Exception{exception_message.str(), ErrorCodes::VIOLATED_CONSTRAINT};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -90,7 +90,7 @@ public:
|
||||
bool canBeComparedWithCollation() const override { return nested_data_type->canBeComparedWithCollation(); }
|
||||
bool canBeUsedAsVersion() const override { return false; }
|
||||
bool isSummable() const override { return nested_data_type->isSummable(); }
|
||||
bool canBeUsedInBooleanContext() const override { return nested_data_type->canBeUsedInBooleanContext(); }
|
||||
bool canBeUsedInBooleanContext() const override { return nested_data_type->canBeUsedInBooleanContext() || onlyNull(); }
|
||||
bool haveMaximumSizeOfValue() const override { return nested_data_type->haveMaximumSizeOfValue(); }
|
||||
size_t getMaximumSizeOfValueInMemory() const override { return 1 + nested_data_type->getMaximumSizeOfValueInMemory(); }
|
||||
bool isNullable() const override { return true; }
|
||||
|
||||
@ -28,7 +28,11 @@ namespace
|
||||
if (!load_result.config)
|
||||
return nullptr;
|
||||
DictionaryStructure dictionary_structure = ExternalDictionariesLoader::getDictionaryStructure(*load_result.config);
|
||||
return StorageDictionary::create(StorageID(database_name, load_result.name), load_result.name, dictionary_structure);
|
||||
return StorageDictionary::create(
|
||||
StorageID(database_name, load_result.name),
|
||||
load_result.name,
|
||||
dictionary_structure,
|
||||
StorageDictionary::Location::DictionaryDatabase);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
|
||||
@ -49,7 +49,8 @@ void DatabaseWithDictionaries::attachDictionary(const String & dictionary_name,
|
||||
StorageDictionary::create(
|
||||
StorageID(getDatabaseName(), dictionary_name),
|
||||
full_name,
|
||||
ExternalDictionariesLoader::getDictionaryStructure(*attach_info.config)),
|
||||
ExternalDictionariesLoader::getDictionaryStructure(*attach_info.config),
|
||||
StorageDictionary::Location::SameDatabaseAndNameAsDictionary),
|
||||
lock);
|
||||
}
|
||||
catch (...)
|
||||
|
||||
@ -99,6 +99,8 @@ public:
|
||||
|
||||
void createHardLink(const String & src_path, const String & dst_path) override;
|
||||
|
||||
const String getType() const override { return "local"; }
|
||||
|
||||
private:
|
||||
bool tryReserve(UInt64 bytes);
|
||||
|
||||
|
||||
@ -90,6 +90,8 @@ public:
|
||||
|
||||
void createHardLink(const String & src_path, const String & dst_path) override;
|
||||
|
||||
const String getType() const override { return "memory"; }
|
||||
|
||||
private:
|
||||
void createDirectoriesImpl(const String & path);
|
||||
void replaceFileImpl(const String & from_path, const String & to_path);
|
||||
|
||||
@ -171,6 +171,9 @@ public:
|
||||
|
||||
/// Create hardlink from `src_path` to `dst_path`.
|
||||
virtual void createHardLink(const String & src_path, const String & dst_path) = 0;
|
||||
|
||||
/// Return disk type - "local", "s3", etc.
|
||||
virtual const String getType() const = 0;
|
||||
};
|
||||
|
||||
using DiskPtr = std::shared_ptr<IDisk>;
|
||||
|
||||
@ -96,6 +96,8 @@ public:
|
||||
|
||||
void setReadOnly(const String & path) override;
|
||||
|
||||
const String getType() const override { return "s3"; }
|
||||
|
||||
private:
|
||||
bool tryReserve(UInt64 bytes);
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic> // for std::atomic<size_t>
|
||||
|
||||
#include "ProxyConfiguration.h"
|
||||
|
||||
namespace DB::S3
|
||||
|
||||
@ -46,7 +46,8 @@ namespace
|
||||
throw Exception("Only HTTP/HTTPS schemas allowed in proxy resolver config: " + proxy_scheme, ErrorCodes::BAD_ARGUMENTS);
|
||||
auto proxy_port = proxy_resolver_config.getUInt(prefix + ".proxy_port");
|
||||
|
||||
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Configured proxy resolver: {}, Scheme: {}, Port: {}", endpoint.toString(), proxy_scheme, proxy_port);
|
||||
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Configured proxy resolver: {}, Scheme: {}, Port: {}",
|
||||
endpoint.toString(), proxy_scheme, proxy_port);
|
||||
|
||||
return std::make_shared<S3::ProxyResolverConfiguration>(endpoint, proxy_scheme, proxy_port);
|
||||
}
|
||||
|
||||
@ -78,6 +78,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionToDecimal128OrNull>();
|
||||
|
||||
factory.registerFunction<FunctionParseDateTimeBestEffort>();
|
||||
factory.registerFunction<FunctionParseDateTimeBestEffortUS>();
|
||||
factory.registerFunction<FunctionParseDateTimeBestEffortOrZero>();
|
||||
factory.registerFunction<FunctionParseDateTimeBestEffortOrNull>();
|
||||
factory.registerFunction<FunctionParseDateTime64BestEffort>();
|
||||
|
||||
@ -534,7 +534,8 @@ enum class ConvertFromStringExceptionMode
|
||||
enum class ConvertFromStringParsingMode
|
||||
{
|
||||
Normal,
|
||||
BestEffort /// Only applicable for DateTime. Will use sophisticated method, that is slower.
|
||||
BestEffort, /// Only applicable for DateTime. Will use sophisticated method, that is slower.
|
||||
BestEffortUS
|
||||
};
|
||||
|
||||
template <typename FromDataType, typename ToDataType, typename Name,
|
||||
@ -586,7 +587,7 @@ struct ConvertThroughParsing
|
||||
local_time_zone = &extractTimeZoneFromFunctionArguments(block, arguments, 1, 0);
|
||||
}
|
||||
|
||||
if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffort)
|
||||
if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffort || parsing_mode == ConvertFromStringParsingMode::BestEffortUS)
|
||||
utc_time_zone = &DateLUT::instance("UTC");
|
||||
}
|
||||
|
||||
@ -674,6 +675,12 @@ struct ConvertThroughParsing
|
||||
vec_to[i] = res;
|
||||
}
|
||||
}
|
||||
else if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffortUS)
|
||||
{
|
||||
time_t res;
|
||||
parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone);
|
||||
vec_to[i] = res;
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (to_datetime64)
|
||||
@ -1578,6 +1585,7 @@ using FunctionToDecimal64OrNull = FunctionConvertFromString<DataTypeDecimal<Deci
|
||||
using FunctionToDecimal128OrNull = FunctionConvertFromString<DataTypeDecimal<Decimal128>, NameToDecimal128OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
|
||||
struct NameParseDateTimeBestEffort { static constexpr auto name = "parseDateTimeBestEffort"; };
|
||||
struct NameParseDateTimeBestEffortUS { static constexpr auto name = "parseDateTimeBestEffortUS"; };
|
||||
struct NameParseDateTimeBestEffortOrZero { static constexpr auto name = "parseDateTimeBestEffortOrZero"; };
|
||||
struct NameParseDateTimeBestEffortOrNull { static constexpr auto name = "parseDateTimeBestEffortOrNull"; };
|
||||
struct NameParseDateTime64BestEffort { static constexpr auto name = "parseDateTime64BestEffort"; };
|
||||
@ -1587,6 +1595,8 @@ struct NameParseDateTime64BestEffortOrNull { static constexpr auto name = "parse
|
||||
|
||||
using FunctionParseDateTimeBestEffort = FunctionConvertFromString<
|
||||
DataTypeDateTime, NameParseDateTimeBestEffort, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::BestEffort>;
|
||||
using FunctionParseDateTimeBestEffortUS = FunctionConvertFromString<
|
||||
DataTypeDateTime, NameParseDateTimeBestEffortUS, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::BestEffortUS>;
|
||||
using FunctionParseDateTimeBestEffortOrZero = FunctionConvertFromString<
|
||||
DataTypeDateTime, NameParseDateTimeBestEffortOrZero, ConvertFromStringExceptionMode::Zero, ConvertFromStringParsingMode::BestEffort>;
|
||||
using FunctionParseDateTimeBestEffortOrNull = FunctionConvertFromString<
|
||||
|
||||
@ -115,6 +115,7 @@ inline Encoded merge(const Encoded & encodedLon, const Encoded & encodedLat, uin
|
||||
result.fill(0);
|
||||
|
||||
const auto bits = (precision * BITS_PER_SYMBOL) / 2;
|
||||
assert(bits <= 255);
|
||||
uint8_t i = 0;
|
||||
for (; i < bits; ++i)
|
||||
{
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
#include <common/types.h>
|
||||
#include <Common/Volnitsky.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
@ -27,7 +28,7 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
/// Is the LIKE expression reduced to finding a substring in a string?
|
||||
/// Is the [I]LIKE expression reduced to finding a substring in a string?
|
||||
static inline bool likePatternIsStrstr(const String & pattern, String & res)
|
||||
{
|
||||
res = "";
|
||||
@ -67,17 +68,21 @@ static inline bool likePatternIsStrstr(const String & pattern, String & res)
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 'like' - if true, treat pattern as SQL LIKE; if false - treat pattern as re2 regexp.
|
||||
/** 'like' - if true, treat pattern as SQL LIKE or ILIKE; if false - treat pattern as re2 regexp.
|
||||
* NOTE: We want to run regexp search for whole block by one call (as implemented in function 'position')
|
||||
* but for that, regexp engine must support \0 bytes and their interpretation as string boundaries.
|
||||
*/
|
||||
template <bool like, bool revert = false>
|
||||
template <bool like, bool revert = false, bool case_insensitive = false>
|
||||
struct MatchImpl
|
||||
{
|
||||
static constexpr bool use_default_implementation_for_constants = true;
|
||||
|
||||
using ResultType = UInt8;
|
||||
|
||||
using Searcher = std::conditional_t<case_insensitive,
|
||||
VolnitskyCaseInsensitiveUTF8,
|
||||
VolnitskyUTF8>;
|
||||
|
||||
static void vectorConstant(
|
||||
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & pattern, PaddedPODArray<UInt8> & res)
|
||||
{
|
||||
@ -85,7 +90,8 @@ struct MatchImpl
|
||||
return;
|
||||
|
||||
String strstr_pattern;
|
||||
/// A simple case where the LIKE expression reduces to finding a substring in a string
|
||||
|
||||
/// A simple case where the [I]LIKE expression reduces to finding a substring in a string
|
||||
if (like && likePatternIsStrstr(pattern, strstr_pattern))
|
||||
{
|
||||
const UInt8 * begin = data.data();
|
||||
@ -96,7 +102,7 @@ struct MatchImpl
|
||||
size_t i = 0;
|
||||
|
||||
/// TODO You need to make that `searcher` is common to all the calls of the function.
|
||||
Volnitsky searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos);
|
||||
Searcher searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos);
|
||||
|
||||
/// We will search for the next occurrence in all rows at once.
|
||||
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
||||
@ -126,7 +132,10 @@ struct MatchImpl
|
||||
{
|
||||
size_t size = offsets.size();
|
||||
|
||||
auto regexp = Regexps::get<like, true>(pattern);
|
||||
constexpr int flags = case_insensitive ?
|
||||
Regexps::Regexp::RE_CASELESS : 0;
|
||||
|
||||
auto regexp = Regexps::get<like, true>(pattern, flags);
|
||||
|
||||
std::string required_substring;
|
||||
bool is_trivial;
|
||||
@ -170,7 +179,7 @@ struct MatchImpl
|
||||
/// The current index in the array of strings.
|
||||
size_t i = 0;
|
||||
|
||||
Volnitsky searcher(required_substring.data(), required_substring.size(), end - pos);
|
||||
Searcher searcher(required_substring.data(), required_substring.size(), end - pos);
|
||||
|
||||
/// We will search for the next occurrence in all rows at once.
|
||||
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
||||
@ -248,7 +257,7 @@ struct MatchImpl
|
||||
/// If pattern is larger than string size - it cannot be found.
|
||||
if (strstr_pattern.size() <= n)
|
||||
{
|
||||
Volnitsky searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos);
|
||||
Searcher searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos);
|
||||
|
||||
/// We will search for the next occurrence in all rows at once.
|
||||
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
||||
@ -328,7 +337,7 @@ struct MatchImpl
|
||||
/// If required substring is larger than string size - it cannot be found.
|
||||
if (strstr_pattern.size() <= n)
|
||||
{
|
||||
Volnitsky searcher(required_substring.data(), required_substring.size(), end - pos);
|
||||
Searcher searcher(required_substring.data(), required_substring.size(), end - pos);
|
||||
|
||||
/// We will search for the next occurrence in all rows at once.
|
||||
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
||||
|
||||
@ -315,7 +315,7 @@ void PointInPolygonWithGrid<CoordinateType>::buildGrid()
|
||||
if (has_empty_bound)
|
||||
return;
|
||||
|
||||
cells.assign(grid_size * grid_size, {});
|
||||
cells.assign(size_t(grid_size) * grid_size, {});
|
||||
|
||||
const Point & min_corner = box.min_corner();
|
||||
|
||||
|
||||
@ -59,19 +59,20 @@ namespace Regexps
|
||||
* In destructor, it returns the object back to the Pool for further reuse.
|
||||
*/
|
||||
template <bool like, bool no_capture>
|
||||
inline Pool::Pointer get(const std::string & pattern)
|
||||
inline Pool::Pointer get(const std::string & pattern, int flags = 0)
|
||||
{
|
||||
/// C++11 has thread-safe function-local statics on most modern compilers.
|
||||
static Pool known_regexps; /// Different variables for different pattern parameters.
|
||||
|
||||
return known_regexps.get(pattern, [&pattern]
|
||||
return known_regexps.get(pattern, [flags, &pattern]
|
||||
{
|
||||
int flags = OptimizedRegularExpression::RE_DOT_NL;
|
||||
int flags_final = flags | OptimizedRegularExpression::RE_DOT_NL;
|
||||
|
||||
if (no_capture)
|
||||
flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
|
||||
flags_final |= OptimizedRegularExpression::RE_NO_CAPTURE;
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::RegexpCreated);
|
||||
return new Regexp{createRegexp<like>(pattern, flags)};
|
||||
return new Regexp{createRegexp<like>(pattern, flags_final)};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -51,6 +51,7 @@ struct ArraySumImpl
|
||||
|
||||
const ColVecType * column = checkAndGetColumn<ColVecType>(&*mapped);
|
||||
|
||||
/// Constant case.
|
||||
if (!column)
|
||||
{
|
||||
const ColumnConst * column_const = checkAndGetColumnConst<ColVecType>(&*mapped);
|
||||
@ -58,7 +59,7 @@ struct ArraySumImpl
|
||||
if (!column_const)
|
||||
return false;
|
||||
|
||||
const Element x = column_const->template getValue<Element>();
|
||||
const Result x = column_const->template getValue<Element>(); // NOLINT
|
||||
|
||||
typename ColVecResult::MutablePtr res_column;
|
||||
if constexpr (IsDecimalNumber<Element>)
|
||||
@ -75,6 +76,7 @@ struct ArraySumImpl
|
||||
size_t pos = 0;
|
||||
for (size_t i = 0; i < offsets.size(); ++i)
|
||||
{
|
||||
/// Just multiply the value by array size.
|
||||
res[i] = x * (offsets[i] - pos);
|
||||
pos = offsets[i];
|
||||
}
|
||||
|
||||
@ -188,6 +188,16 @@ private:
|
||||
writeNumber2(target, ToISOWeekImpl::execute(source, timezone));
|
||||
}
|
||||
|
||||
static void ISO8601Year2(char * target, Time source, const DateLUTImpl & timezone) // NOLINT
|
||||
{
|
||||
writeNumber2(target, ToISOYearImpl::execute(source, timezone) % 100);
|
||||
}
|
||||
|
||||
static void ISO8601Year4(char * target, Time source, const DateLUTImpl & timezone) // NOLINT
|
||||
{
|
||||
writeNumber4(target, ToISOYearImpl::execute(source, timezone));
|
||||
}
|
||||
|
||||
static void year2(char * target, Time source, const DateLUTImpl & timezone)
|
||||
{
|
||||
writeNumber2(target, ToYearImpl::execute(source, timezone) % 100);
|
||||
@ -459,6 +469,18 @@ public:
|
||||
result.append("0000-00-00");
|
||||
break;
|
||||
|
||||
// Last two digits of year of ISO 8601 week number (see %G)
|
||||
case 'g':
|
||||
instructions.emplace_back(&Action<T>::ISO8601Year2, 2);
|
||||
result.append("00");
|
||||
break;
|
||||
|
||||
// Year of ISO 8601 week number (see %V)
|
||||
case 'G':
|
||||
instructions.emplace_back(&Action<T>::ISO8601Year4, 4);
|
||||
result.append("0000");
|
||||
break;
|
||||
|
||||
// Day of the year (001-366) 235
|
||||
case 'j':
|
||||
instructions.emplace_back(&Action<T>::dayOfYear, 3);
|
||||
|
||||
@ -102,8 +102,6 @@ void geodistInit()
|
||||
inline float geodistDegDiff(float f)
|
||||
{
|
||||
f = fabsf(f);
|
||||
while (f > 360)
|
||||
f -= 360;
|
||||
if (f > 180)
|
||||
f = 360 - f;
|
||||
return f;
|
||||
|
||||
@ -432,8 +432,7 @@ private:
|
||||
const PaddedPODArray<UInt8> & cond_data = cond_col->getData();
|
||||
size_t rows = cond_data.size();
|
||||
|
||||
if ((col_then_fixed || col_then_const_fixed)
|
||||
&& (col_else_fixed || col_else_const_fixed))
|
||||
if (isFixedString(block.getByPosition(result).type))
|
||||
{
|
||||
/// The result is FixedString.
|
||||
|
||||
@ -448,16 +447,19 @@ private:
|
||||
else if (col_then_const_fixed && col_else_fixed)
|
||||
conditional(ConstSource<FixedStringSource>(*col_then_const_fixed), FixedStringSource(*col_else_fixed), sink, cond_data);
|
||||
else if (col_then_const_fixed && col_else_const_fixed)
|
||||
conditional(ConstSource<FixedStringSource>(*col_then_const_fixed), ConstSource<FixedStringSource>(*col_else_const_fixed), sink, cond_data);
|
||||
conditional(ConstSource<FixedStringSource>(*col_then_const_fixed),
|
||||
ConstSource<FixedStringSource>(*col_else_const_fixed), sink, cond_data);
|
||||
else
|
||||
return false;
|
||||
|
||||
block.getByPosition(result).column = std::move(col_res_untyped);
|
||||
return true;
|
||||
}
|
||||
|
||||
if ((col_then || col_then_const || col_then_fixed || col_then_const_fixed)
|
||||
&& (col_else || col_else_const || col_else_fixed || col_else_const_fixed))
|
||||
if (isString(block.getByPosition(result).type))
|
||||
{
|
||||
/// The result is String.
|
||||
|
||||
auto col_res = ColumnString::create();
|
||||
auto sink = StringSink(*col_res, rows);
|
||||
|
||||
@ -485,6 +487,17 @@ private:
|
||||
conditional(ConstSource<StringSource>(*col_then_const), ConstSource<FixedStringSource>(*col_else_const_fixed), sink, cond_data);
|
||||
else if (col_then_const_fixed && col_else_const)
|
||||
conditional(ConstSource<FixedStringSource>(*col_then_const_fixed), ConstSource<StringSource>(*col_else_const), sink, cond_data);
|
||||
else if (col_then_fixed && col_else_fixed)
|
||||
conditional(FixedStringSource(*col_then_fixed), FixedStringSource(*col_else_fixed), sink, cond_data);
|
||||
else if (col_then_fixed && col_else_const_fixed)
|
||||
conditional(FixedStringSource(*col_then_fixed), ConstSource<FixedStringSource>(*col_else_const_fixed), sink, cond_data);
|
||||
else if (col_then_const_fixed && col_else_fixed)
|
||||
conditional(ConstSource<FixedStringSource>(*col_then_const_fixed), FixedStringSource(*col_else_fixed), sink, cond_data);
|
||||
else if (col_then_const_fixed && col_else_const_fixed)
|
||||
conditional(ConstSource<FixedStringSource>(*col_then_const_fixed),
|
||||
ConstSource<FixedStringSource>(*col_else_const_fixed), sink, cond_data);
|
||||
else
|
||||
return false;
|
||||
|
||||
block.getByPosition(result).column = std::move(col_res);
|
||||
return true;
|
||||
@ -590,7 +603,8 @@ private:
|
||||
return true;
|
||||
}
|
||||
|
||||
static void executeGeneric(const ColumnUInt8 * cond_col, Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count)
|
||||
static void executeGeneric(
|
||||
const ColumnUInt8 * cond_col, Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count)
|
||||
{
|
||||
/// Convert both columns to the common type (if needed).
|
||||
|
||||
|
||||
24
src/Functions/ilike.cpp
Normal file
24
src/Functions/ilike.cpp
Normal file
@ -0,0 +1,24 @@
|
||||
#include "FunctionsStringSearch.h"
|
||||
#include "FunctionFactory.h"
|
||||
#include "MatchImpl.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
struct NameILike
|
||||
{
|
||||
static constexpr auto name = "ilike";
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
using ILikeImpl = MatchImpl<true, false, /*case-insensitive*/true>;
|
||||
}
|
||||
|
||||
using FunctionILike = FunctionsStringSearch<ILikeImpl, NameILike>;
|
||||
|
||||
void registerFunctionILike(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionILike>();
|
||||
}
|
||||
}
|
||||
@ -26,12 +26,11 @@ struct DivideIntegralByConstantImpl
|
||||
|
||||
static NO_INLINE void vectorConstant(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size)
|
||||
{
|
||||
if (unlikely(b == 0))
|
||||
throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION);
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
|
||||
/// Division by -1. By the way, we avoid FPE by division of the largest negative number by -1.
|
||||
/// And signed integer overflow is well defined in C++20.
|
||||
if (unlikely(is_signed_v<B> && b == -1))
|
||||
{
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
@ -39,8 +38,20 @@ struct DivideIntegralByConstantImpl
|
||||
return;
|
||||
}
|
||||
|
||||
/// Division with too large divisor.
|
||||
if (unlikely(b > std::numeric_limits<A>::max()
|
||||
|| (std::is_signed_v<A> && std::is_signed_v<B> && b < std::numeric_limits<A>::lowest())))
|
||||
{
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c_pos[i] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
if (unlikely(static_cast<A>(b) == 0))
|
||||
throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION);
|
||||
|
||||
libdivide::divider<A> divider(b);
|
||||
|
||||
const A * a_end = a_pos + size;
|
||||
|
||||
@ -11,11 +11,15 @@ struct NameLike
|
||||
static constexpr auto name = "like";
|
||||
};
|
||||
|
||||
using FunctionLike = FunctionsStringSearch<MatchImpl<true>, NameLike>;
|
||||
namespace
|
||||
{
|
||||
using LikeImpl = MatchImpl</*SQL LIKE */ true, /*revert*/false>;
|
||||
}
|
||||
|
||||
using FunctionLike = FunctionsStringSearch<LikeImpl, NameLike>;
|
||||
|
||||
void registerFunctionLike(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionLike>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -4,7 +4,8 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/// Transforms the LIKE expression into regexp re2. For example, abc%def -> ^abc.*def$
|
||||
|
||||
/// Transforms the [I]LIKE expression into regexp re2. For example, abc%def -> ^abc.*def$
|
||||
inline String likePatternToRegexp(const String & pattern)
|
||||
{
|
||||
String res;
|
||||
|
||||
@ -27,12 +27,10 @@ struct ModuloByConstantImpl
|
||||
|
||||
static NO_INLINE void vectorConstant(const A * __restrict src, B b, ResultType * __restrict dst, size_t size)
|
||||
{
|
||||
if (unlikely(b == 0))
|
||||
throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION);
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
|
||||
/// Modulo with too small divisor.
|
||||
if (unlikely((std::is_signed_v<B> && b == -1) || b == 1))
|
||||
{
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
@ -40,8 +38,20 @@ struct ModuloByConstantImpl
|
||||
return;
|
||||
}
|
||||
|
||||
/// Modulo with too large divisor.
|
||||
if (unlikely(b > std::numeric_limits<A>::max()
|
||||
|| (std::is_signed_v<A> && std::is_signed_v<B> && b < std::numeric_limits<A>::lowest())))
|
||||
{
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
dst[i] = src[i];
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
if (unlikely(static_cast<A>(b) == 0))
|
||||
throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION);
|
||||
|
||||
libdivide::divider<A> divider(b);
|
||||
|
||||
/// Here we failed to make the SSE variant from libdivide give an advantage.
|
||||
|
||||
@ -39,6 +39,7 @@ public:
|
||||
bool isVariadic() const override { return true; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool useDefaultImplementationForNulls() const override { return false; }
|
||||
|
||||
ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t number_of_arguments) const override
|
||||
{
|
||||
ColumnNumbers args;
|
||||
@ -70,7 +71,6 @@ public:
|
||||
throw Exception{"Invalid number of arguments for function " + getName(),
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
|
||||
|
||||
|
||||
for_conditions([&](const DataTypePtr & arg)
|
||||
{
|
||||
const IDataType * nested_type;
|
||||
|
||||
24
src/Functions/notILike.cpp
Normal file
24
src/Functions/notILike.cpp
Normal file
@ -0,0 +1,24 @@
|
||||
#include "FunctionsStringSearch.h"
|
||||
#include "FunctionFactory.h"
|
||||
#include "MatchImpl.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
struct NameNotILike
|
||||
{
|
||||
static constexpr auto name = "notILike";
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
using NotILikeImpl = MatchImpl<true, true, /*case-insensitive*/true>;
|
||||
}
|
||||
|
||||
using FunctionNotILike = FunctionsStringSearch<NotILikeImpl, NameNotILike>;
|
||||
|
||||
void registerFunctionNotILike(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionNotILike>();
|
||||
}
|
||||
}
|
||||
@ -4,7 +4,9 @@ namespace DB
|
||||
class FunctionFactory;
|
||||
|
||||
void registerFunctionLike(FunctionFactory &);
|
||||
void registerFunctionILike(FunctionFactory &);
|
||||
void registerFunctionNotLike(FunctionFactory &);
|
||||
void registerFunctionNotILike(FunctionFactory &);
|
||||
void registerFunctionMatch(FunctionFactory &);
|
||||
void registerFunctionExtract(FunctionFactory &);
|
||||
void registerFunctionReplaceOne(FunctionFactory &);
|
||||
@ -24,7 +26,9 @@ void registerFunctionExtractAllGroupsHorizontal(FunctionFactory &);
|
||||
void registerFunctionsStringRegexp(FunctionFactory & factory)
|
||||
{
|
||||
registerFunctionLike(factory);
|
||||
registerFunctionILike(factory);
|
||||
registerFunctionNotLike(factory);
|
||||
registerFunctionNotILike(factory);
|
||||
registerFunctionMatch(factory);
|
||||
registerFunctionExtract(factory);
|
||||
registerFunctionReplaceOne(factory);
|
||||
@ -41,6 +45,4 @@ void registerFunctionsStringRegexp(FunctionFactory & factory)
|
||||
registerFunctionExtractAllGroupsVertical(factory);
|
||||
registerFunctionExtractAllGroupsHorizontal(factory);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -92,8 +92,8 @@ SRCS(
|
||||
array/emptyArrayToSingle.cpp
|
||||
array/hasAll.cpp
|
||||
array/hasAny.cpp
|
||||
array/hasSubstr.cpp
|
||||
array/has.cpp
|
||||
array/hasSubstr.cpp
|
||||
array/indexOf.cpp
|
||||
array/length.cpp
|
||||
array/range.cpp
|
||||
@ -228,6 +228,7 @@ SRCS(
|
||||
ifNull.cpp
|
||||
IFunction.cpp
|
||||
ignore.cpp
|
||||
ilike.cpp
|
||||
in.cpp
|
||||
intDiv.cpp
|
||||
intDivOrZero.cpp
|
||||
@ -288,6 +289,7 @@ SRCS(
|
||||
neighbor.cpp
|
||||
notEmpty.cpp
|
||||
notEquals.cpp
|
||||
notILike.cpp
|
||||
notLike.cpp
|
||||
now64.cpp
|
||||
now.cpp
|
||||
|
||||
@ -88,7 +88,7 @@ struct DateTimeSubsecondPart
|
||||
UInt8 digits;
|
||||
};
|
||||
|
||||
template <typename ReturnType>
|
||||
template <typename ReturnType, bool is_us_style>
|
||||
ReturnType parseDateTimeBestEffortImpl(
|
||||
time_t & res,
|
||||
ReadBuffer & in,
|
||||
@ -264,11 +264,11 @@ ReturnType parseDateTimeBestEffortImpl(
|
||||
/// DD-MM-YY
|
||||
/// DD
|
||||
|
||||
UInt8 hour_or_day_of_month = 0;
|
||||
UInt8 hour_or_day_of_month_or_month = 0;
|
||||
if (num_digits == 2)
|
||||
readDecimalNumber<2>(hour_or_day_of_month, digits);
|
||||
readDecimalNumber<2>(hour_or_day_of_month_or_month, digits);
|
||||
else if (num_digits == 1) //-V547
|
||||
readDecimalNumber<1>(hour_or_day_of_month, digits);
|
||||
readDecimalNumber<1>(hour_or_day_of_month_or_month, digits);
|
||||
else
|
||||
return on_error("Cannot read DateTime: logical error, unexpected branch in code", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
@ -277,7 +277,7 @@ ReturnType parseDateTimeBestEffortImpl(
|
||||
if (has_time)
|
||||
return on_error("Cannot read DateTime: time component is duplicated", ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
|
||||
hour = hour_or_day_of_month;
|
||||
hour = hour_or_day_of_month_or_month;
|
||||
has_time = true;
|
||||
|
||||
num_digits = readDigits(digits, sizeof(digits), in);
|
||||
@ -309,29 +309,48 @@ ReturnType parseDateTimeBestEffortImpl(
|
||||
if (month)
|
||||
return on_error("Cannot read DateTime: month is duplicated", ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
|
||||
day_of_month = hour_or_day_of_month;
|
||||
|
||||
num_digits = readDigits(digits, sizeof(digits), in);
|
||||
|
||||
if (num_digits == 2)
|
||||
readDecimalNumber<2>(month, digits);
|
||||
else if (num_digits == 1)
|
||||
readDecimalNumber<1>(month, digits);
|
||||
else if (num_digits == 0)
|
||||
if constexpr (is_us_style)
|
||||
{
|
||||
/// Month in alphabetical form
|
||||
|
||||
char alpha[9]; /// The longest month name: September
|
||||
size_t num_alpha = readAlpha(alpha, sizeof(alpha), in);
|
||||
|
||||
if (num_alpha < 3)
|
||||
return on_error("Cannot read DateTime: unexpected number of alphabetical characters after day of month: " + toString(num_alpha), ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
|
||||
if (!read_alpha_month(alpha))
|
||||
return on_error("Cannot read DateTime: alphabetical characters after day of month don't look like month: " + std::string(alpha, 3), ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
month = hour_or_day_of_month_or_month;
|
||||
num_digits = readDigits(digits, sizeof(digits), in);
|
||||
if (num_digits == 2)
|
||||
readDecimalNumber<2>(day_of_month, digits);
|
||||
else if (num_digits == 1)
|
||||
readDecimalNumber<1>(day_of_month, digits);
|
||||
else
|
||||
return on_error("Cannot read DateTime: unexpected number of decimal digits after month: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
}
|
||||
else
|
||||
return on_error("Cannot read DateTime: unexpected number of decimal digits after day of month: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
{
|
||||
day_of_month = hour_or_day_of_month_or_month;
|
||||
|
||||
num_digits = readDigits(digits, sizeof(digits), in);
|
||||
|
||||
if (num_digits == 2)
|
||||
readDecimalNumber<2>(month, digits);
|
||||
else if (num_digits == 1)
|
||||
readDecimalNumber<1>(month, digits);
|
||||
else if (num_digits == 0)
|
||||
{
|
||||
/// Month in alphabetical form
|
||||
|
||||
char alpha[9]; /// The longest month name: September
|
||||
size_t num_alpha = readAlpha(alpha, sizeof(alpha), in);
|
||||
|
||||
if (num_alpha < 3)
|
||||
return on_error("Cannot read DateTime: unexpected number of alphabetical characters after day of month: " + toString(num_alpha), ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
|
||||
if (!read_alpha_month(alpha))
|
||||
return on_error("Cannot read DateTime: alphabetical characters after day of month don't look like month: " + std::string(alpha, 3), ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
}
|
||||
else
|
||||
return on_error("Cannot read DateTime: unexpected number of decimal digits after day of month: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
|
||||
|
||||
}
|
||||
|
||||
if (month > 12)
|
||||
std::swap(month, day_of_month);
|
||||
|
||||
if (checkChar('/', in) || checkChar('.', in) || checkChar('-', in))
|
||||
{
|
||||
@ -358,9 +377,9 @@ ReturnType parseDateTimeBestEffortImpl(
|
||||
else
|
||||
{
|
||||
if (day_of_month)
|
||||
hour = hour_or_day_of_month;
|
||||
hour = hour_or_day_of_month_or_month;
|
||||
else
|
||||
day_of_month = hour_or_day_of_month;
|
||||
day_of_month = hour_or_day_of_month_or_month;
|
||||
}
|
||||
}
|
||||
else if (num_digits != 0)
|
||||
@ -522,6 +541,22 @@ ReturnType parseDateTimeBestEffortImpl(
|
||||
if (!day_of_month)
|
||||
day_of_month = 1;
|
||||
|
||||
auto is_leap_year = (year % 400 == 0) || (year % 100 != 0 && year % 4 == 0);
|
||||
|
||||
auto check_date = [](const auto & is_leap_year_, const auto & month_, const auto & day_)
|
||||
{
|
||||
if ((month_ == 1 || month_ == 3 || month_ == 5 || month_ == 7 || month_ == 8 || month_ == 10 || month_ == 12) && day_ >= 1 && day_ <= 31)
|
||||
return true;
|
||||
else if (month_ == 2 && ((is_leap_year_ && day_ >= 1 && day_ <= 29) || (!is_leap_year_ && day_ >= 1 && day_ <= 28)))
|
||||
return true;
|
||||
else if ((month_ == 4 || month_ == 6 || month_ == 9 || month_ == 11) && day_ >= 1 && day_ <= 30)
|
||||
return true;
|
||||
return false;
|
||||
};
|
||||
|
||||
if (!check_date(is_leap_year, month, day_of_month))
|
||||
return on_error("Cannot read DateTime: logical error, unexpected date: " + std::to_string(year) + "-" + std::to_string(month) + "-" + std::to_string(day_of_month), ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (is_pm && hour < 12)
|
||||
hour += 12;
|
||||
|
||||
@ -565,12 +600,12 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf
|
||||
|
||||
if constexpr (std::is_same_v<ReturnType, bool>)
|
||||
{
|
||||
if (!parseDateTimeBestEffortImpl<bool>(whole, in, local_time_zone, utc_time_zone, &subsecond))
|
||||
if (!parseDateTimeBestEffortImpl<bool, false>(whole, in, local_time_zone, utc_time_zone, &subsecond))
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
parseDateTimeBestEffortImpl<ReturnType>(whole, in, local_time_zone, utc_time_zone, &subsecond);
|
||||
parseDateTimeBestEffortImpl<ReturnType, false>(whole, in, local_time_zone, utc_time_zone, &subsecond);
|
||||
}
|
||||
|
||||
|
||||
@ -598,12 +633,17 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf
|
||||
|
||||
void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
parseDateTimeBestEffortImpl<void>(res, in, local_time_zone, utc_time_zone, nullptr);
|
||||
parseDateTimeBestEffortImpl<void, false>(res, in, local_time_zone, utc_time_zone, nullptr);
|
||||
}
|
||||
|
||||
void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
parseDateTimeBestEffortImpl<void, true>(res, in, local_time_zone, utc_time_zone, nullptr);
|
||||
}
|
||||
|
||||
bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
return parseDateTimeBestEffortImpl<bool>(res, in, local_time_zone, utc_time_zone, nullptr);
|
||||
return parseDateTimeBestEffortImpl<bool, false>(res, in, local_time_zone, utc_time_zone, nullptr);
|
||||
}
|
||||
|
||||
void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user