thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-15 20:24:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into poc_orc_lc_read

Browse Source
This commit is contained in:
李扬 2024-10-15 16:11:08 +08:00 committed by GitHub
commit 8cbaec96db
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
335 changed files with 5532 additions and 1123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/ISSUE_TEMPLATE/20_feature-request.md vendored
View File

@ -15,7 +15,7 @@ assignees: ''
**Use case**
> A clear and concise description of what is the intended usage scenario is.
> A clear and concise description of what the intended usage scenario is.
**Describe the solution you'd like**

3
.github/workflows/pull_request.yml vendored
View File

@ -33,9 +33,6 @@ jobs:
filter: tree:0
- name: Debug Info
uses: ./.github/actions/debug
- name: Cancel previous Sync PR workflow
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
- name: Set pending Sync status
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status

32
cmake/cpu_features.cmake
View File

@ -11,6 +11,38 @@ option (ARCH_NATIVE "Add -march=native compiler flag. This makes your binaries n
if (ARCH_NATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
# Populate the ENABLE_ option flags. This is required for the build of some third-party dependencies, specifically snappy, which
# (somewhat weirdly) expects the relative SNAPPY_HAVE_ preprocessor variables to be populated, in addition to the microarchitecture
# feature flags being enabled in the compiler. This fixes the ARCH_NATIVE flag by automatically populating the ENABLE_ option flags
# according to the current CPU's capabilities, detected using clang.
if (ARCH_AMD64)
execute_process(
COMMAND sh -c "clang -E - -march=native -###"
INPUT_FILE /dev/null
OUTPUT_QUIET
ERROR_VARIABLE TEST_FEATURE_RESULT)
macro(TEST_AMD64_FEATURE TEST_FEATURE_RESULT feat flag)
if (${TEST_FEATURE_RESULT} MATCHES "\"\\+${feat}\"")
set(${flag} ON)
else ()
set(${flag} OFF)
endif ()
endmacro()
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} ssse3 ENABLE_SSSE3)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} sse4.1 ENABLE_SSE41)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} sse4.2 ENABLE_SSE42)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} vpclmulqdq ENABLE_PCLMULQDQ)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} popcnt ENABLE_POPCNT)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx ENABLE_AVX)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx2 ENABLE_AVX2)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx512f ENABLE_AVX512)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx512vbmi ENABLE_AVX512_VBMI)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} bmi ENABLE_BMI)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} bmi2 ENABLE_BMI2)
endif ()
elseif (ARCH_AARCH64)
# ARM publishes almost every year a new revision of it's ISA [1]. Each version comes with new mandatory and optional features from
# which CPU vendors can pick and choose. This creates a lot of variability ... We provide two build "profiles", one for maximum

49
docs/changelogs/v24.3.12.75-lts.md Normal file
View File

@ -0,0 +1,49 @@
---
sidebar_position: 1
sidebar_label: 2024
---
# 2024 Changelog
### ClickHouse release v24.3.12.75-lts (7cb5dff8019) FIXME as compared to v24.3.11.7-lts (28795d0a47e)
#### Improvement
* Backported in [#69607](https://github.com/ClickHouse/ClickHouse/issues/69607): Improved memory accounting for cgroups v2 to exclude the amount occupied by the page cache. [#65470](https://github.com/ClickHouse/ClickHouse/pull/65470) ([Nikita Taranov](https://github.com/nickitat)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Backported in [#69785](https://github.com/ClickHouse/ClickHouse/issues/69785): Fix attaching table when pg dbname contains "-" in MaterializedPostgreSQL. [#62730](https://github.com/ClickHouse/ClickHouse/pull/62730) ([takakawa](https://github.com/takakawa)).
* Backported in [#69461](https://github.com/ClickHouse/ClickHouse/issues/69461): Fix expiration in `RoleCache`. [#67748](https://github.com/ClickHouse/ClickHouse/pull/67748) ([Vitaly Baranov](https://github.com/vitlibar)).
* Backported in [#68217](https://github.com/ClickHouse/ClickHouse/issues/68217): Fixed a NULL pointer dereference, triggered by a specially crafted query, that crashed the server via hopEnd, hopStart, tumbleEnd, and tumbleStart. [#68098](https://github.com/ClickHouse/ClickHouse/pull/68098) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
* Backported in [#69437](https://github.com/ClickHouse/ClickHouse/issues/69437): After unexpected restart, fail to start replication of ReplicatedMergeTree due to abnormal handling of covered-by-broken part. [#68584](https://github.com/ClickHouse/ClickHouse/pull/68584) ([baolin](https://github.com/baolinhuang)).
* Backported in [#69827](https://github.com/ClickHouse/ClickHouse/issues/69827): Make `ColumnsDescription::toString` format each column using the same `IAST::FormatState object`. This results in uniform columns metadata being written to disk and ZooKeeper. [#68733](https://github.com/ClickHouse/ClickHouse/pull/68733) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
* Backported in [#69294](https://github.com/ClickHouse/ClickHouse/issues/69294): Fix merging of aggregated data for grouping sets. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Backported in [#70470](https://github.com/ClickHouse/ClickHouse/issues/70470): Fix inf loop after `restore replica` in the replicated merge tree with zero copy. [#69293](https://github.com/ClickHouse/ClickHouse/pull/69293) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
* Backported in [#69456](https://github.com/ClickHouse/ClickHouse/issues/69456): Fix undefined behavior when all connection attempts fail getting a connection for insertions. [#69390](https://github.com/ClickHouse/ClickHouse/pull/69390) ([Pablo Marcos](https://github.com/pamarcos)).
* Backported in [#69497](https://github.com/ClickHouse/ClickHouse/issues/69497): Fixed a `LOGICAL_ERROR` with function `sqidDecode` ([#69450](https://github.com/ClickHouse/ClickHouse/issues/69450)). [#69451](https://github.com/ClickHouse/ClickHouse/pull/69451) ([Robert Schulze](https://github.com/rschu1ze)).
* Backported in [#69724](https://github.com/ClickHouse/ClickHouse/issues/69724): Keep original order of conditions during move to prewhere. Previously the order could change and it could lead to failing queries when the order is important. [#69560](https://github.com/ClickHouse/ClickHouse/pull/69560) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#69668](https://github.com/ClickHouse/ClickHouse/issues/69668): Fix Keeper multi-request preprocessing after ZNOAUTH error. [#69627](https://github.com/ClickHouse/ClickHouse/pull/69627) ([Antonio Andelic](https://github.com/antonio2368)).
* Backported in [#69792](https://github.com/ClickHouse/ClickHouse/issues/69792): Make getHyperrectangleForRowGroup not throw an exception when the data type in parquet file is not convertable into the requested data type. Solved the user's problem when the Parquet file had Decimal64 data type and the column data type was DateTime. [#69745](https://github.com/ClickHouse/ClickHouse/pull/69745) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
* Backported in [#70089](https://github.com/ClickHouse/ClickHouse/issues/70089): Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
* Backported in [#70077](https://github.com/ClickHouse/ClickHouse/issues/70077): Closes [#69752](https://github.com/ClickHouse/ClickHouse/issues/69752). [#69985](https://github.com/ClickHouse/ClickHouse/pull/69985) ([pufit](https://github.com/pufit)).
* Backported in [#70162](https://github.com/ClickHouse/ClickHouse/issues/70162): Fix wrong LOGICAL_ERROR when replacing literals in ranges. [#70122](https://github.com/ClickHouse/ClickHouse/pull/70122) ([Pablo Marcos](https://github.com/pamarcos)).
* Backported in [#70232](https://github.com/ClickHouse/ClickHouse/issues/70232): Check for Nullable(Nothing) type during ALTER TABLE MODIFY COLUMN/QUERY to prevent tables with such data type. [#70123](https://github.com/ClickHouse/ClickHouse/pull/70123) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70179](https://github.com/ClickHouse/ClickHouse/issues/70179): Fix data race in ColumnObject/ColumnTuple decompress method that could lead to heap use after free. [#70137](https://github.com/ClickHouse/ClickHouse/pull/70137) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70241](https://github.com/ClickHouse/ClickHouse/issues/70241): Fix the password being displayed in `system.query_log` for users with bcrypt password authentication method. [#70148](https://github.com/ClickHouse/ClickHouse/pull/70148) ([Nikolay Degterinsky](https://github.com/evillique)).
* Backported in [#70397](https://github.com/ClickHouse/ClickHouse/issues/70397): Fix crash when using WITH FILL incorrectly. [#70338](https://github.com/ClickHouse/ClickHouse/pull/70338) ([Raúl Marín](https://github.com/Algunenano)).
#### NO CL CATEGORY
* Backported in [#69526](https://github.com/ClickHouse/ClickHouse/issues/69526):. [#67029](https://github.com/ClickHouse/ClickHouse/pull/67029) ([Alexander Tokmakov](https://github.com/tavplubix)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Backported in [#69506](https://github.com/ClickHouse/ClickHouse/issues/69506): Better handling of errors from azure storage. [#62306](https://github.com/ClickHouse/ClickHouse/pull/62306) ([Anton Popov](https://github.com/CurtizJ)).
* Backported in [#69955](https://github.com/ClickHouse/ClickHouse/issues/69955): Output an operation error for ZK Multi request failed operation into log. [#68127](https://github.com/ClickHouse/ClickHouse/pull/68127) ([Aleksei Filatov](https://github.com/aalexfvk)).
* Backported in [#69485](https://github.com/ClickHouse/ClickHouse/issues/69485): Fix test_role & test_keeper_s3_snapshot integration tests. [#69013](https://github.com/ClickHouse/ClickHouse/pull/69013) ([Shankar](https://github.com/shiyer7474)).
* Backported in [#70028](https://github.com/ClickHouse/ClickHouse/issues/70028): Remove stale moving parts without zookeeper. [#69075](https://github.com/ClickHouse/ClickHouse/pull/69075) ([Kirill](https://github.com/kirillgarbar)).
* Backported in [#69421](https://github.com/ClickHouse/ClickHouse/issues/69421): Fix: Not-ready Set with parallel replicas. [#69264](https://github.com/ClickHouse/ClickHouse/pull/69264) ([Igor Nikonov](https://github.com/devcrafter)).
* Backported in [#69747](https://github.com/ClickHouse/ClickHouse/issues/69747): Add function `kill_ci_runner`. Kill runner when pre-pull failed. [#69557](https://github.com/ClickHouse/ClickHouse/pull/69557) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#69636](https://github.com/ClickHouse/ClickHouse/issues/69636): Add more contexts to the debug action and use it broadly. [#69599](https://github.com/ClickHouse/ClickHouse/pull/69599) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* The test is flaky and the feature experimental. [#70269](https://github.com/ClickHouse/ClickHouse/pull/70269) ([Raúl Marín](https://github.com/Algunenano)).
* Fix test distributed inter server secret in 24.3. [#70325](https://github.com/ClickHouse/ClickHouse/pull/70325) ([Raúl Marín](https://github.com/Algunenano)).

73
docs/changelogs/v24.8.5.115-lts.md Normal file
View File

@ -0,0 +1,73 @@
---
sidebar_position: 1
sidebar_label: 2024
---
# 2024 Changelog
### ClickHouse release v24.8.5.115-lts (8c4cb00a384) FIXME as compared to v24.8.4.13-lts (53195bc189b)
#### Improvement
* Backported in [#70046](https://github.com/ClickHouse/ClickHouse/issues/70046): Add new column readonly_duration to the system.replicas table. Needed to be able to distinguish actual readonly replicas from sentinel ones in alerts. [#69871](https://github.com/ClickHouse/ClickHouse/pull/69871) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Backported in [#69786](https://github.com/ClickHouse/ClickHouse/issues/69786): Fix attaching table when pg dbname contains "-" in MaterializedPostgreSQL. [#62730](https://github.com/ClickHouse/ClickHouse/pull/62730) ([takakawa](https://github.com/takakawa)).
* Backported in [#70318](https://github.com/ClickHouse/ClickHouse/issues/70318): Fixed error on generated columns in MaterializedPostgreSQL when adnum ordering is broken [#63161](https://github.com/ClickHouse/ClickHouse/issues/63161). Fixed error on id column with nextval expression as default MaterializedPostgreSQL when there are generated columns in table. Fixed error on dropping publication with symbols except [a-z1-9-]. [#67664](https://github.com/ClickHouse/ClickHouse/pull/67664) ([Kruglov Kirill](https://github.com/1on)).
* Backported in [#69467](https://github.com/ClickHouse/ClickHouse/issues/69467): Fix expiration in `RoleCache`. [#67748](https://github.com/ClickHouse/ClickHouse/pull/67748) ([Vitaly Baranov](https://github.com/vitlibar)).
* Backported in [#69735](https://github.com/ClickHouse/ClickHouse/issues/69735): Fix crash in `lag`/`lead` which is introduced in [#67091](https://github.com/ClickHouse/ClickHouse/issues/67091). [#68262](https://github.com/ClickHouse/ClickHouse/pull/68262) ([lgbo](https://github.com/lgbo-ustc)).
* Backported in [#69444](https://github.com/ClickHouse/ClickHouse/issues/69444): After unexpected restart, fail to start replication of ReplicatedMergeTree due to abnormal handling of covered-by-broken part. [#68584](https://github.com/ClickHouse/ClickHouse/pull/68584) ([baolin](https://github.com/baolinhuang)).
* Backported in [#69810](https://github.com/ClickHouse/ClickHouse/issues/69810): Make `ColumnsDescription::toString` format each column using the same `IAST::FormatState object`. This results in uniform columns metadata being written to disk and ZooKeeper. [#68733](https://github.com/ClickHouse/ClickHouse/pull/68733) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
* Backported in [#69757](https://github.com/ClickHouse/ClickHouse/issues/69757): Fix incorrect results of Fix uniq and GROUP BY for JSON/Dynamic types. [#69203](https://github.com/ClickHouse/ClickHouse/pull/69203) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70195](https://github.com/ClickHouse/ClickHouse/issues/70195): Fix insertion of incomplete type into Dynamic during deserialization. It could lead to `Parameter out of bound` errors. [#69291](https://github.com/ClickHouse/ClickHouse/pull/69291) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#69398](https://github.com/ClickHouse/ClickHouse/issues/69398): Mark Dynamic type as not safe primary key type to avoid issues with Fields. [#69311](https://github.com/ClickHouse/ClickHouse/pull/69311) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#69704](https://github.com/ClickHouse/ClickHouse/issues/69704): Improve restoring of access entities' dependencies. [#69346](https://github.com/ClickHouse/ClickHouse/pull/69346) ([Vitaly Baranov](https://github.com/vitlibar)).
* Backported in [#69459](https://github.com/ClickHouse/ClickHouse/issues/69459): Fix undefined behavior when all connection attempts fail getting a connection for insertions. [#69390](https://github.com/ClickHouse/ClickHouse/pull/69390) ([Pablo Marcos](https://github.com/pamarcos)).
* Backported in [#69503](https://github.com/ClickHouse/ClickHouse/issues/69503): Fixed a `LOGICAL_ERROR` with function `sqidDecode` ([#69450](https://github.com/ClickHouse/ClickHouse/issues/69450)). [#69451](https://github.com/ClickHouse/ClickHouse/pull/69451) ([Robert Schulze](https://github.com/rschu1ze)).
* Backported in [#69480](https://github.com/ClickHouse/ClickHouse/issues/69480): Quick fix for s3queue problem on 24.6 or create query with database replicated. [#69454](https://github.com/ClickHouse/ClickHouse/pull/69454) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Backported in [#69535](https://github.com/ClickHouse/ClickHouse/issues/69535): Fixed case when memory consumption was too high because of the squashing in `INSERT INTO ... SELECT` or `CREATE TABLE AS SELECT` queries. [#69469](https://github.com/ClickHouse/ClickHouse/pull/69469) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
* Backported in [#69696](https://github.com/ClickHouse/ClickHouse/issues/69696): Keep original order of conditions during move to prewhere. Previously the order could change and it could lead to failing queries when the order is important. [#69560](https://github.com/ClickHouse/ClickHouse/pull/69560) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70439](https://github.com/ClickHouse/ClickHouse/issues/70439): Fix vrash during insertion into FixedString column in PostgreSQL engine. [#69584](https://github.com/ClickHouse/ClickHouse/pull/69584) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#69666](https://github.com/ClickHouse/ClickHouse/issues/69666): Fix Keeper multi-request preprocessing after ZNOAUTH error. [#69627](https://github.com/ClickHouse/ClickHouse/pull/69627) ([Antonio Andelic](https://github.com/antonio2368)).
* Backported in [#70191](https://github.com/ClickHouse/ClickHouse/issues/70191): Fix crash when executing `create view t as (with recursive 42 as ttt select ttt);`. [#69676](https://github.com/ClickHouse/ClickHouse/pull/69676) ([Han Fei](https://github.com/hanfei1991)).
* Backported in [#69798](https://github.com/ClickHouse/ClickHouse/issues/69798): Make getHyperrectangleForRowGroup not throw an exception when the data type in parquet file is not convertable into the requested data type. Solved the user's problem when the Parquet file had Decimal64 data type and the column data type was DateTime. [#69745](https://github.com/ClickHouse/ClickHouse/pull/69745) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
* Backported in [#70410](https://github.com/ClickHouse/ClickHouse/issues/70410): Fixed `maxMapState` throwing 'Bad get' if value type is DateTime64. [#69787](https://github.com/ClickHouse/ClickHouse/pull/69787) ([Michael Kolupaev](https://github.com/al13n321)).
* Backported in [#70019](https://github.com/ClickHouse/ClickHouse/issues/70019): Fix analyzer default with old compatibility value. [#69895](https://github.com/ClickHouse/ClickHouse/pull/69895) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#69941](https://github.com/ClickHouse/ClickHouse/issues/69941): Don't check dependencies during CREATE OR REPLACE VIEW during DROP of old table. Previously CREATE OR REPLACE query failed when there are dependent tables of the recreated view. [#69907](https://github.com/ClickHouse/ClickHouse/pull/69907) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70001](https://github.com/ClickHouse/ClickHouse/issues/70001): Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
* Backported in [#70081](https://github.com/ClickHouse/ClickHouse/issues/70081): Closes [#69752](https://github.com/ClickHouse/ClickHouse/issues/69752). [#69985](https://github.com/ClickHouse/ClickHouse/pull/69985) ([pufit](https://github.com/pufit)).
* Backported in [#70068](https://github.com/ClickHouse/ClickHouse/issues/70068): Fixes `Block structure mismatch` for queries with nested views and `WHERE` condition. Fixes [#66209](https://github.com/ClickHouse/ClickHouse/issues/66209). [#70054](https://github.com/ClickHouse/ClickHouse/pull/70054) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Backported in [#70166](https://github.com/ClickHouse/ClickHouse/issues/70166): Fix wrong LOGICAL_ERROR when replacing literals in ranges. [#70122](https://github.com/ClickHouse/ClickHouse/pull/70122) ([Pablo Marcos](https://github.com/pamarcos)).
* Backported in [#70236](https://github.com/ClickHouse/ClickHouse/issues/70236): Check for Nullable(Nothing) type during ALTER TABLE MODIFY COLUMN/QUERY to prevent tables with such data type. [#70123](https://github.com/ClickHouse/ClickHouse/pull/70123) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70203](https://github.com/ClickHouse/ClickHouse/issues/70203): Fix wrong result with skipping index. [#70127](https://github.com/ClickHouse/ClickHouse/pull/70127) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#70183](https://github.com/ClickHouse/ClickHouse/issues/70183): Fix data race in ColumnObject/ColumnTuple decompress method that could lead to heap use after free. [#70137](https://github.com/ClickHouse/ClickHouse/pull/70137) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70251](https://github.com/ClickHouse/ClickHouse/issues/70251): Fix possible hung in ALTER COLUMN with Dynamic type. [#70144](https://github.com/ClickHouse/ClickHouse/pull/70144) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70228](https://github.com/ClickHouse/ClickHouse/issues/70228): Use correct `max_types` parameter during Dynamic type creation for JSON subcolumn. [#70147](https://github.com/ClickHouse/ClickHouse/pull/70147) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70243](https://github.com/ClickHouse/ClickHouse/issues/70243): Fix the password being displayed in `system.query_log` for users with bcrypt password authentication method. [#70148](https://github.com/ClickHouse/ClickHouse/pull/70148) ([Nikolay Degterinsky](https://github.com/evillique)).
* Backported in [#70432](https://github.com/ClickHouse/ClickHouse/issues/70432): Fix possible crash in JSON column. [#70172](https://github.com/ClickHouse/ClickHouse/pull/70172) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70307](https://github.com/ClickHouse/ClickHouse/issues/70307): Fix multiple issues with arrayMin and arrayMax. [#70207](https://github.com/ClickHouse/ClickHouse/pull/70207) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#70274](https://github.com/ClickHouse/ClickHouse/issues/70274): Respect setting allow_simdjson in JSON type parser. [#70218](https://github.com/ClickHouse/ClickHouse/pull/70218) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70345](https://github.com/ClickHouse/ClickHouse/issues/70345): Don't modify global settings with startup scripts. Previously, changing a setting in a startup script would change it globally. [#70310](https://github.com/ClickHouse/ClickHouse/pull/70310) ([Antonio Andelic](https://github.com/antonio2368)).
* Backported in [#70426](https://github.com/ClickHouse/ClickHouse/issues/70426): Fix ALTER of Dynamic type with reducing max_types parameter that could lead to server crash. [#70328](https://github.com/ClickHouse/ClickHouse/pull/70328) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#70371](https://github.com/ClickHouse/ClickHouse/issues/70371): Fix crash when using WITH FILL incorrectly. [#70338](https://github.com/ClickHouse/ClickHouse/pull/70338) ([Raúl Marín](https://github.com/Algunenano)).
#### NO CL ENTRY
* NO CL ENTRY: 'Revert "Backport [#70146](https://github.com/ClickHouse/ClickHouse/issues/70146) to 24.8: Upgrade integration-runner image"'. [#70324](https://github.com/ClickHouse/ClickHouse/pull/70324) ([Max K.](https://github.com/maxknv)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Backported in [#69961](https://github.com/ClickHouse/ClickHouse/issues/69961): Output an operation error for ZK Multi request failed operation into log. [#68127](https://github.com/ClickHouse/ClickHouse/pull/68127) ([Aleksei Filatov](https://github.com/aalexfvk)).
* Backported in [#69491](https://github.com/ClickHouse/ClickHouse/issues/69491): Fix test_role & test_keeper_s3_snapshot integration tests. [#69013](https://github.com/ClickHouse/ClickHouse/pull/69013) ([Shankar](https://github.com/shiyer7474)).
* Backported in [#69953](https://github.com/ClickHouse/ClickHouse/issues/69953): Remove stale moving parts without zookeeper. [#69075](https://github.com/ClickHouse/ClickHouse/pull/69075) ([Kirill](https://github.com/kirillgarbar)).
* Backported in [#69353](https://github.com/ClickHouse/ClickHouse/issues/69353): Fix: Not-ready Set with parallel replicas. [#69264](https://github.com/ClickHouse/ClickHouse/pull/69264) ([Igor Nikonov](https://github.com/devcrafter)).
* Backported in [#69427](https://github.com/ClickHouse/ClickHouse/issues/69427): Fix 24.8 setting compatibility `rows_before_aggregation`. [#69394](https://github.com/ClickHouse/ClickHouse/pull/69394) ([Nikita Fomichev](https://github.com/fm4v)).
* Backported in [#69689](https://github.com/ClickHouse/ClickHouse/issues/69689): Add function `kill_ci_runner`. Kill runner when pre-pull failed. [#69557](https://github.com/ClickHouse/ClickHouse/pull/69557) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#69639](https://github.com/ClickHouse/ClickHouse/issues/69639): Add more contexts to the debug action and use it broadly. [#69599](https://github.com/ClickHouse/ClickHouse/pull/69599) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#69721](https://github.com/ClickHouse/ClickHouse/issues/69721): Prohibit `ALTER TABLE ... ADD INDEX ... TYPE` inverted if setting = 0. [#69684](https://github.com/ClickHouse/ClickHouse/pull/69684) ([Robert Schulze](https://github.com/rschu1ze)).
* Backported in [#69972](https://github.com/ClickHouse/ClickHouse/issues/69972): S3Queue: support having deprecated settings to not fail server startup. [#69769](https://github.com/ClickHouse/ClickHouse/pull/69769) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Backported in [#70283](https://github.com/ClickHouse/ClickHouse/issues/70283): Improve pipdeptree generator for docker images. - Update requirements.txt for the integration tests runner container - Remove some small dependencies, improve `helpers/retry_decorator.py` - Upgrade docker-compose from EOL version 1 to version 2. [#70146](https://github.com/ClickHouse/ClickHouse/pull/70146) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#70260](https://github.com/ClickHouse/ClickHouse/issues/70260): Update test_storage_s3_queue/test.py. [#70159](https://github.com/ClickHouse/ClickHouse/pull/70159) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Backported in [#70314](https://github.com/ClickHouse/ClickHouse/issues/70314): CI: Remove await feature from release branches. [#70294](https://github.com/ClickHouse/ClickHouse/pull/70294) ([Max K.](https://github.com/maxknv)).
* Backported in [#70380](https://github.com/ClickHouse/ClickHouse/issues/70380): Fix tiny mistake, responsible for some of kafka test flaps. Example [report](https://s3.amazonaws.com/clickhouse-test-reports/0/3198aafac59c368993e7b5f49d95674cc1b1be18/integration_tests__release__[2_4].html). [#70352](https://github.com/ClickHouse/ClickHouse/pull/70352) ([filimonov](https://github.com/filimonov)).
* Backported in [#70405](https://github.com/ClickHouse/ClickHouse/issues/70405): Closes [#69634](https://github.com/ClickHouse/ClickHouse/issues/69634). [#70354](https://github.com/ClickHouse/ClickHouse/pull/70354) ([pufit](https://github.com/pufit)).

2
docs/en/development/continuous-integration.md
View File

@ -82,7 +82,7 @@ cd ./utils/check-style
# Check duplicate includes
./check-duplicate-includes.sh
# Check c++ formatiing
# Check c++ formatting
./check-style
# Check python formatting with black

27
docs/en/engines/table-engines/integrations/azureBlobStorage.md
View File

@ -63,7 +63,34 @@ Currently there are 3 ways to authenticate:
- `SAS Token` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. It is identified by presence of '?' in the url.
- `Workload Identity` - Can be used by providing an `endpoint` or `storage_account_url`. If `use_workload_identity` parameter is set in config, ([workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications)) is used for authentication.
### Data cache {#data-cache}
`Azure` table engine supports data caching on local disk.
See filesystem cache configuration options and usage in this [section](/docs/en/operations/storing-data.md/#using-local-cache).
Caching is made depending on the path and ETag of the storage object, so clickhouse will not read a stale cache version.
To enable caching use a setting `filesystem_cache_name = '<name>'` and `enable_filesystem_cache = 1`.
```sql
SELECT *
FROM azureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;', 'test_container', 'test_table', 'CSV')
SETTINGS filesystem_cache_name = 'cache_for_azure', enable_filesystem_cache = 1;
```
1. add the following section to clickhouse configuration file:
``` xml
<clickhouse>
<filesystem_caches>
<cache_for_azure>
<path>path to cache directory</path>
<max_size>10Gi</max_size>
</cache_for_azure>
</filesystem_caches>
</clickhouse>
```
2. reuse cache configuration (and therefore cache storage) from clickhouse `storage_configuration` section, [described here](/docs/en/operations/storing-data.md/#using-local-cache)
## See also

4
docs/en/engines/table-engines/integrations/deltalake.md
View File

@ -48,6 +48,10 @@ Using named collections:
CREATE TABLE deltalake ENGINE=DeltaLake(deltalake_conf, filename = 'test_table')
```
### Data cache {#data-cache}
`Iceberg` table engine and table function support data caching same as `S3`, `AzureBlobStorage`, `HDFS` storages. See [here](../../../engines/table-engines/integrations/s3.md#data-cache).
## See also
- [deltaLake table function](../../../sql-reference/table-functions/deltalake.md)

11
docs/en/engines/table-engines/integrations/iceberg.md
View File

@ -6,7 +6,7 @@ sidebar_label: Iceberg
# Iceberg Table Engine
This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure and locally stored tables.
This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS and locally stored tables.
## Create Table
@ -19,13 +19,16 @@ CREATE TABLE iceberg_table_s3
CREATE TABLE iceberg_table_azure
ENGINE = IcebergAzure(connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression])
CREATE TABLE iceberg_table_hdfs
ENGINE = IcebergHDFS(path_to_table, [,format] [,compression_method])
CREATE TABLE iceberg_table_local
ENGINE = IcebergLocal(path_to_table, [,format] [,compression_method])
```
**Engine arguments**
Description of the arguments coincides with description of arguments in engines `S3`, `AzureBlobStorage` and `File` correspondingly.
Description of the arguments coincides with description of arguments in engines `S3`, `AzureBlobStorage`, `HDFS` and `File` correspondingly.
`format` stands for the format of data files in the Iceberg table.
Engine parameters can be specified using [Named Collections](../../../operations/named-collections.md)
@ -60,6 +63,10 @@ CREATE TABLE iceberg_table ENGINE=IcebergS3(iceberg_conf, filename = 'test_table
Table engine `Iceberg` is an alias to `IcebergS3` now.
### Data cache {#data-cache}
`Iceberg` table engine and table function support data caching same as `S3`, `AzureBlobStorage`, `HDFS` storages. See [here](../../../engines/table-engines/integrations/s3.md#data-cache).
## See also
- [iceberg table function](/docs/en/sql-reference/table-functions/iceberg.md)

4
docs/en/engines/table-engines/integrations/mysql.md
View File

@ -4,12 +4,8 @@ sidebar_position: 138
sidebar_label: MySQL
---
import CloudAvailableBadge from '@theme/badges/CloudAvailableBadge';
# MySQL Table Engine
<CloudAvailableBadge />
The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server.
## Creating a Table {#creating-a-table}

32
docs/en/engines/table-engines/integrations/s3.md
View File

@ -26,6 +26,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
│ two │ 2 │
└──────┴───────┘
```
## Create Table {#creating-a-table}
``` sql
@ -43,6 +44,37 @@ CREATE TABLE s3_engine_table (name String, value UInt32)
- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3).
- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension.
### Data cache {#data-cache}
`S3` table engine supports data caching on local disk.
See filesystem cache configuration options and usage in this [section](/docs/en/operations/storing-data.md/#using-local-cache).
Caching is made depending on the path and ETag of the storage object, so clickhouse will not read a stale cache version.
To enable caching use a setting `filesystem_cache_name = '<name>'` and `enable_filesystem_cache = 1`.
```sql
SELECT *
FROM s3('http://minio:10000/clickhouse//test_3.csv', 'minioadmin', 'minioadminpassword', 'CSV')
SETTINGS filesystem_cache_name = 'cache_for_s3', enable_filesystem_cache = 1;
```
There are two ways to define cache in configuration file.
1. add the following section to clickhouse configuration file:
``` xml
<clickhouse>
<filesystem_caches>
<cache_for_s3>
<path>path to cache directory</path>
<max_size>10Gi</max_size>
</cache_for_s3>
</filesystem_caches>
</clickhouse>
```
2. reuse cache configuration (and therefore cache storage) from clickhouse `storage_configuration` section, [described here](/docs/en/operations/storing-data.md/#using-local-cache)
### PARTITION BY
`PARTITION BY` — Optional. In most cases you don't need a partition key, and if it is needed you generally don't need a partition key more granular than by month. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead, make client identifier or name the first column in the ORDER BY expression).

6
docs/en/engines/table-engines/mergetree-family/mergetree.md
View File

@ -374,15 +374,15 @@ Users can create [UDF](/docs/en/sql-reference/statements/create/function.md) to
```sql
CREATE FUNCTION bfEstimateFunctions [ON CLUSTER cluster]
AS
(total_nubmer_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_nubmer_of_all_grams) * log(2));
(total_number_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_number_of_all_grams) * log(2));
CREATE FUNCTION bfEstimateBmSize [ON CLUSTER cluster]
AS
(total_nubmer_of_all_grams, probability_of_false_positives) -> ceil((total_nubmer_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2))));
(total_number_of_all_grams, probability_of_false_positives) -> ceil((total_number_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2))));
CREATE FUNCTION bfEstimateFalsePositive [ON CLUSTER cluster]
AS
(total_nubmer_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_nubmer_of_all_grams)), number_of_hash_functions);
(total_number_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_number_of_all_grams)), number_of_hash_functions);
CREATE FUNCTION bfEstimateGramNumber [ON CLUSTER cluster]
AS

2
docs/en/engines/table-engines/special/keepermap.md
View File

@ -35,7 +35,7 @@ Engine parameters:
- `root_path` - ZooKeeper path where the `table_name` will be stored.
This path should not contain the prefix defined by `<keeper_map_path_prefix>` config because the prefix will be automatically appended to the `root_path`.
Additionally, format of `auxiliary_zookeper_cluster_name:/some/path` is also supported where `auxiliary_zookeper_cluster` is a ZooKeeper cluster defined inside `<auxiliary_zookeepers>` config.
Additionally, format of `auxiliary_zookeeper_cluster_name:/some/path` is also supported where `auxiliary_zookeeper_cluster` is a ZooKeeper cluster defined inside `<auxiliary_zookeepers>` config.
By default, ZooKeeper cluster defined inside `<zookeeper>` config is used.
- `keys_limit` - number of keys allowed inside the table.
This limit is a soft limit and it can be possible that more keys will end up in the table for some edge cases.

10
docs/en/interfaces/formats.md
View File

@ -877,7 +877,7 @@ INSERT INTO json_as_object (json) FORMAT JSONAsObject {"any json stucture":1}
SELECT time, json FROM json_as_object FORMAT JSONEachRow
```
```resonse
```response
{"time":"2024-09-16 12:18:10","json":{}}
{"time":"2024-09-16 12:18:13","json":{"any json stucture":"1"}}
{"time":"2024-09-16 12:18:08","json":{"foo":{"bar":{"x":"y"},"baz":"1"}}}
@ -1598,10 +1598,6 @@ the columns from input data will be mapped to the columns from the table by thei
Otherwise, the first row will be skipped.
If setting [input_format_with_types_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_types_use_header) is set to 1,
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
If setting [output_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#output_format_binary_encode_types_in_binary_format) is set to 1,
the types in header will be written using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes output format.
If setting [input_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#input_format_binary_encode_types_in_binary_format) is set to 1,
the types in header will be read using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes input format.
:::
## RowBinaryWithDefaults {#rowbinarywithdefaults}
@ -1624,6 +1620,10 @@ For column `y` data starts with byte `00` that indicates that column has actual
## RowBinary format settings {#row-binary-format-settings}
- [format_binary_max_string_size](/docs/en/operations/settings/settings-formats.md/#format_binary_max_string_size) - The maximum allowed size for String in RowBinary format. Default value - `1GiB`.
- [output_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#output_format_binary_encode_types_in_binary_format) - Allows to write types in header using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes output format. Default value - `false`.
- [input_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#input_format_binary_encode_types_in_binary_format) - Allows to read types in header using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes input format. Default value - `false`.
- [output_format_binary_write_json_as_string](/docs/en/operations/settings/settings-formats.md/#output_format_binary_write_json_as_string) - Allows to write values of [JSON](/docs/en/sql-reference/data-types/newjson.md) data type as JSON [String](/docs/en/sql-reference/data-types/string.md) values in RowBinary output format. Default value - `false`.
- [input_format_binary_read_json_as_string](/docs/en/operations/settings/settings-formats.md/#input_format_binary_read_json_as_string) - Allows to read values of [JSON](/docs/en/sql-reference/data-types/newjson.md) data type as JSON [String](/docs/en/sql-reference/data-types/string.md) values in RowBinary input format. Default value - `false`.
## Values {#data-format-values}

6
docs/en/interfaces/schema-inference.md
View File

@ -509,7 +509,7 @@ DESC format(JSONEachRow, $$
{"value" : "424242424242"}
$$)
```
```reponse
```response
┌─name──┬─type────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ value │ Nullable(Int64) │ │ │ │ │ │
└───────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
@ -910,9 +910,9 @@ This setting is disabled by default.
```sql
SET input_format_json_try_infer_numbers_from_strings = 1;
DESC format(CSV, '"42","42.42"');
DESC format(CSV, '42,42.42');
```
```reponse
```response
┌─name─┬─type──────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ c1 │ Nullable(Int64) │ │ │ │ │ │
│ c2 │ Nullable(Float64) │ │ │ │ │ │

13
docs/en/interfaces/third-party/gui.md vendored
View File

@ -19,6 +19,19 @@ Features:
- Performance Optimizations: Utilizes Indexed DB for efficient caching and state management.
- Local Data Storage: All data is stored locally in the browser, ensuring no data is sent anywhere else.
### ChartDB {#chartdb}
[ChartDB](https://chartdb.io) is a free and open-source tool for visualizing and designing database schemas, including ClickHouse, with a single query. Built with React, it provides a seamless and user-friendly experience, requiring no database credentials or signup to get started.
Features:
- Schema Visualization: Instantly import and visualize your ClickHouse schema, including ER diagrams with materialized views and standard views, showing references to tables.
- AI-Powered DDL Export: Generate DDL scripts effortlessly for better schema management and documentation.
- Multi-SQL Dialect Support: Compatible with a range of SQL dialects, making it versatile for various database environments.
- No Signup or Credentials Needed: All functionality is accessible directly in the browser, keeping it frictionless and secure.
[ChartDB Source Code](https://github.com/chartdb/chartdb).
### Tabix {#tabix}
Web interface for ClickHouse in the [Tabix](https://github.com/tabixio/tabix) project.

12
docs/en/operations/settings/query-complexity.md
View File

@ -49,6 +49,18 @@ Default values are defined in [Settings.h](https://github.com/ClickHouse/ClickHo
See also the description of [max_memory_usage](#settings_max_memory_usage).
For example if you want to set `max_memory_usage_for_user` to 1000 bytes for a user named `clickhouse_read`, you can use the statement
``` sql
ALTER USER clickhouse_read SETTINGS max_memory_usage_for_user = 1000;
```
You can verify it worked by logging out of your client, logging back in, then use the `getSetting` function:
```sql
SELECT getSetting('max_memory_usage_for_user');
```
## max_rows_to_read {#max-rows-to-read}
The following restrictions can be checked on each block (instead of on each row). That is, the restrictions can be broken a little.

26
docs/en/operations/settings/settings-formats.md
View File

@ -493,6 +493,14 @@ Default value: 0
Read data types in binary format instead of type names in RowBinaryWithNamesAndTypes input format
### input_format_binary_read_json_as_string {#input_format_binary_read_json_as_string}
Type: Bool
Default value: 0
Read values of [JSON](../../sql-reference/data-types/newjson.md) data type as JSON [String](../../sql-reference/data-types/string.md) values in RowBinary input format.
## input_format_bson_skip_fields_with_unsupported_types_in_schema_inference {#input_format_bson_skip_fields_with_unsupported_types_in_schema_inference}
Type: Bool
@ -1569,6 +1577,14 @@ Default value: 0
Write data types in binary format instead of type names in RowBinaryWithNamesAndTypes output format
### output_format_binary_write_json_as_string {#output_format_binary_write_json_as_string}
Type: Bool
Default value: 0
Write values of [JSON](../../sql-reference/data-types/newjson.md) data type as JSON [String](../../sql-reference/data-types/string.md) values in RowBinary output format.
## output_format_bson_string_as_string {#output_format_bson_string_as_string}
Type: Bool
@ -1859,6 +1875,14 @@ Default value: 0
Write data types in binary format instead of type names in Native output format
### output_format_native_write_json_as_string {#output_format_native_write_json_as_string}
Type: Bool
Default value: 0
Write data of [JSON](../../sql-reference/data-types/newjson.md) column as [String](../../sql-reference/data-types/string.md) column containing JSON strings instead of default native JSON serialization.
## output_format_orc_compression_method {#output_format_orc_compression_method}
Type: ORCCompression
@ -2297,5 +2321,3 @@ Type: Bool
Default value: 1
Validate usage of experimental and suspicious types inside nested types like Array/Map/Tuple

19
docs/en/operations/settings/settings.md
View File

@ -413,14 +413,6 @@ Default value: 0
Experimental data deduplication for SELECT queries based on part UUIDs
## allow_experimental_refreshable_materialized_view {#allow_experimental_refreshable_materialized_view}
Type: Bool
Default value: 0
Allow refreshable materialized views (CREATE MATERIALIZED VIEW \\<name\\> REFRESH ...).
## allow_experimental_shared_set_join {#allow_experimental_shared_set_join}
Type: Bool
@ -5533,6 +5525,12 @@ Default value: -1
Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.
## max_parts_to_move {#max_parts_to_move}
Limit the number of parts that can be moved in one query. Zero means unlimited.
Default value: `1000`.
## max_query_size {#max_query_size}
Type: UInt64
@ -9723,6 +9721,10 @@ Default value: 15
The heartbeat interval in seconds to indicate watch query is alive.
## enforce_strict_identifier_format
If enabled, only allow identifiers containing alphanumeric characters and underscores.
## workload {#workload}
Type: String
@ -9747,4 +9749,5 @@ Default value: 0
Allows you to select the max window log of ZSTD (it will not be used for MergeTree family)
Default value: `true`.

2
docs/en/sql-reference/aggregate-functions/combinators.md
View File

@ -124,7 +124,7 @@ Converts an aggregate function for tables into an aggregate function for arrays
## -Distinct
Every unique combination of arguments will be aggregated only once. Repeating values are ignored.
Examples: `sum(DISTINCT x)`, `groupArray(DISTINCT x)`, `corrStableDistinct(DISTINCT x, y)` and so on.
Examples: `sum(DISTINCT x)` (or `sumDistinct(x)`), `groupArray(DISTINCT x)` (or `groupArrayDistinct(x)`), `corrStable(DISTINCT x, y)` (or `corrStableDistinct(x, y)`) and so on.
## -OrDefault

4
docs/en/sql-reference/data-types/data-types-binary-encoding.md
View File

@ -86,7 +86,7 @@ The table below describes how different interval kinds of `Interval` data type a
### Aggregate function parameter binary encoding
The table below describes how parameters of `AggragateFunction` and `SimpleAggregateFunction` are encoded.
The table below describes how parameters of `AggregateFunction` and `SimpleAggregateFunction` are encoded.
The encoding of a parameter consists of 1 byte indicating the type of the parameter and the value itself.
| Parameter type | Binary encoding |
@ -106,7 +106,7 @@ The encoding of a parameter consists of 1 byte indicating the type of the parame
| `String` | `0x0C<var_uint_size><data>` |
| `Array` | `0x0D<var_uint_size><value_encoding_1>...<value_encoding_N>` |
| `Tuple` | `0x0E<var_uint_size><value_encoding_1>...<value_encoding_N>` |
| `Map` | `0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_endoding_N><value_encoding_N>` |
| `Map` | `0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_encoding_N><value_encoding_N>` |
| `IPv4` | `0x10<uint32_little_endian_value>` |
| `IPv6` | `0x11<uint128_little_endian_value>` |
| `UUID` | `0x12<uuid_value>` |

321
docs/en/sql-reference/data-types/dynamic.md
View File

@ -297,99 +297,257 @@ $$)
└───────────────┴────────────────┴───────────────┴──────┴───────┴────────────┴─────────┘
```
## Comparing values of Dynamic type
## Using Dynamic type in functions
Values of `Dynamic` types are compared similar to values of `Variant` type:
Most of the functions support arguments with type `Dynamic`. In this case the function is executed separately on each internal data type stored inside `Dynamic` column.
When the result type of the function depends on the arguments types, the result of such function executed with `Dynamic` arguments will be `Dynamic`. When the result type of the function doesn't depend on the arguments types - the result will be `Nullable(T)` where `T` the usual result type of this function.
Examples:
```sql
CREATE TABLE test (d Dynamic) ENGINE=Memory;
INSERT INTO test VALUES (NULL), (1::Int8), (2::Int16), (3::Int32), (4::Int64);
```
```sql
SELECT d, dynamicType(d) FROM test;
```
```text
┌─d────┬─dynamicType(d)─┐
│ ᴺᵁᴸᴸ │ None │
│ 1 │ Int8 │
│ 2 │ Int16 │
│ 3 │ Int32 │
│ 4 │ Int64 │
└──────┴────────────────┘
```
```sql
SELECT d, d + 1 AS res, toTypeName(res), dynamicType(res) FROM test;
```
```text
┌─d────┬─res──┬─toTypeName(res)─┬─dynamicType(res)─┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dynamic │ None │
│ 1 │ 2 │ Dynamic │ Int16 │
│ 2 │ 3 │ Dynamic │ Int32 │
│ 3 │ 4 │ Dynamic │ Int64 │
│ 4 │ 5 │ Dynamic │ Int64 │
└──────┴──────┴─────────────────┴──────────────────┘
```
```sql
SELECT d, d + d AS res, toTypeName(res), dynamicType(res) FROM test;
```
```text
┌─d────┬─res──┬─toTypeName(res)─┬─dynamicType(res)─┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dynamic │ None │
│ 1 │ 2 │ Dynamic │ Int16 │
│ 2 │ 4 │ Dynamic │ Int32 │
│ 3 │ 6 │ Dynamic │ Int64 │
│ 4 │ 8 │ Dynamic │ Int64 │
└──────┴──────┴─────────────────┴──────────────────┘
```
```sql
SELECT d, d < 3 AS res, toTypeName(res) FROM test;
```
```text
┌─d────┬──res─┬─toTypeName(res)─┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(UInt8) │
│ 1 │ 1 │ Nullable(UInt8) │
│ 2 │ 1 │ Nullable(UInt8) │
│ 3 │ 0 │ Nullable(UInt8) │
│ 4 │ 0 │ Nullable(UInt8) │
└──────┴──────┴─────────────────┘
```
```sql
SELECT d, exp2(d) AS res, toTypeName(res) FROM test;
```
```sql
┌─d────┬──res─┬─toTypeName(res)───┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(Float64) │
│ 1 │ 2 │ Nullable(Float64) │
│ 2 │ 4 │ Nullable(Float64) │
│ 3 │ 8 │ Nullable(Float64) │
│ 4 │ 16 │ Nullable(Float64) │
└──────┴──────┴───────────────────┘
```
```sql
TRUNCATE TABLE test;
INSERT INTO test VALUES (NULL), ('str_1'), ('str_2');
SELECT d, dynamicType(d) FROM test;
```
```text
┌─d─────┬─dynamicType(d)─┐
│ ᴺᵁᴸᴸ │ None │
│ str_1 │ String │
│ str_2 │ String │
└───────┴────────────────┘
```
```sql
SELECT d, upper(d) AS res, toTypeName(res) FROM test;
```
```text
┌─d─────┬─res───┬─toTypeName(res)──┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(String) │
│ str_1 │ STR_1 │ Nullable(String) │
│ str_2 │ STR_2 │ Nullable(String) │
└───────┴───────┴──────────────────┘
```
```sql
SELECT d, extract(d, '([0-3])') AS res, toTypeName(res) FROM test;
```
```text
┌─d─────┬─res──┬─toTypeName(res)──┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(String) │
│ str_1 │ 1 │ Nullable(String) │
│ str_2 │ 2 │ Nullable(String) │
└───────┴──────┴──────────────────┘
```
```sql
TRUNCATE TABLE test;
INSERT INTO test VALUES (NULL), ([1, 2]), ([3, 4]);
SELECT d, dynamicType(d) FROM test;
```
```text
┌─d─────┬─dynamicType(d)─┐
│ ᴺᵁᴸᴸ │ None │
│ [1,2] │ Array(Int64) │
│ [3,4] │ Array(Int64) │
└───────┴────────────────┘
```
```sql
SELECT d, d[1] AS res, toTypeName(res), dynamicType(res) FROM test;
```
```text
┌─d─────┬─res──┬─toTypeName(res)─┬─dynamicType(res)─┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dynamic │ None │
│ [1,2] │ 1 │ Dynamic │ Int64 │
│ [3,4] │ 3 │ Dynamic │ Int64 │
└───────┴──────┴─────────────────┴──────────────────┘
```
If function cannot be executed on some type inside `Dynamic` column, the exception will be thrown:
```sql
INSERT INTO test VALUES (42), (43), ('str_1');
SELECT d, dynamicType(d) FROM test;
```
```text
┌─d─────┬─dynamicType(d)─┐
│ 42 │ Int64 │
│ 43 │ Int64 │
│ str_1 │ String │
└───────┴────────────────┘
┌─d─────┬─dynamicType(d)─┐
│ ᴺᵁᴸᴸ │ None │
│ [1,2] │ Array(Int64) │
│ [3,4] │ Array(Int64) │
└───────┴────────────────┘
```
```sql
SELECT d, d + 1 AS res, toTypeName(res), dynamicType(d) FROM test;
```
```text
Received exception:
Code: 43. DB::Exception: Illegal types Array(Int64) and UInt8 of arguments of function plus: while executing 'FUNCTION plus(__table1.d : 3, 1_UInt8 :: 1) -> plus(__table1.d, 1_UInt8) Dynamic : 0'. (ILLEGAL_TYPE_OF_ARGUMENT)
```
We can filter out unneeded types:
```sql
SELECT d, d + 1 AS res, toTypeName(res), dynamicType(res) FROM test WHERE dynamicType(d) NOT IN ('String', 'Array(Int64)', 'None')
```
```text
┌─d──┬─res─┬─toTypeName(res)─┬─dynamicType(res)─┐
│ 42 │ 43 │ Dynamic │ Int64 │
│ 43 │ 44 │ Dynamic │ Int64 │
└────┴─────┴─────────────────┴──────────────────┘
```
Or extract required type as subcolumn:
```sql
SELECT d, d.Int64 + 1 AS res, toTypeName(res) FROM test;
```
```text
┌─d─────┬──res─┬─toTypeName(res)─┐
│ 42 │ 43 │ Nullable(Int64) │
│ 43 │ 44 │ Nullable(Int64) │
│ str_1 │ ᴺᵁᴸᴸ │ Nullable(Int64) │
└───────┴──────┴─────────────────┘
┌─d─────┬──res─┬─toTypeName(res)─┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(Int64) │
│ [1,2] │ ᴺᵁᴸᴸ │ Nullable(Int64) │
│ [3,4] │ ᴺᵁᴸᴸ │ Nullable(Int64) │
└───────┴──────┴─────────────────┘
```
## Using Dynamic type in ORDER BY and GROUP BY
During `ORDER BY` and `GROUP BY` values of `Dynamic` types are compared similar to values of `Variant` type:
The result of operator `<` for values `d1` with underlying type `T1` and `d2` with underlying type `T2` of a type `Dynamic` is defined as follows:
- If `T1 = T2 = T`, the result will be `d1.T < d2.T` (underlying values will be compared).
- If `T1 != T2`, the result will be `T1 < T2` (type names will be compared).
Examples:
```sql
CREATE TABLE test (d1 Dynamic, d2 Dynamic) ENGINE=Memory;
INSERT INTO test VALUES (42, 42), (42, 43), (42, 'abc'), (42, [1, 2, 3]), (42, []), (42, NULL);
CREATE TABLE test (d Dynamic) ENGINE=Memory;
INSERT INTO test VALUES (42), (43), ('abc'), ('abd'), ([1, 2, 3]), ([]), (NULL);
```
```sql
SELECT d2, dynamicType(d2) as d2_type from test order by d2;
SELECT d, dynamicType(d) FROM test;
```
```text
┌─d2──────┬─d2_type──────┐
┌─d───────┬─dynamicType(d)─┐
│ 42 │ Int64 │
│ 43 │ Int64 │
│ abc │ String │
│ abd │ String │
│ [1,2,3] │ Array(Int64) │
│ [] │ Array(Int64) │
│ ᴺᵁᴸᴸ │ None │
└─────────┴────────────────┘
```
```sql
SELECT d, dynamicType(d) FROM test ORDER BY d;
```
```sql
┌─d───────┬─dynamicType(d)─┐
│ [] │ Array(Int64) │
│ [1,2,3] │ Array(Int64) │
│ 42 │ Int64 │
│ 43 │ Int64 │
│ abc │ String │
│ abd │ String │
│ ᴺᵁᴸᴸ │ None │
└─────────┴──────────────┘
```
```sql
SELECT d1, dynamicType(d1) as d1_type, d2, dynamicType(d2) as d2_type, d1 = d2, d1 < d2, d1 > d2 from test;
```
```text
┌─d1─┬─d1_type─┬─d2──────┬─d2_type──────┬─equals(d1, d2)─┬─less(d1, d2)─┬─greater(d1, d2)─┐
│ 42 │ Int64 │ 42 │ Int64 │ 1 │ 0 │ 0 │
│ 42 │ Int64 │ 43 │ Int64 │ 0 │ 1 │ 0 │
│ 42 │ Int64 │ abc │ String │ 0 │ 1 │ 0 │
│ 42 │ Int64 │ [1,2,3] │ Array(Int64) │ 0 │ 0 │ 1 │
│ 42 │ Int64 │ [] │ Array(Int64) │ 0 │ 0 │ 1 │
│ 42 │ Int64 │ ᴺᵁᴸᴸ │ None │ 0 │ 1 │ 0 │
└────┴─────────┴─────────┴──────────────┴────────────────┴──────────────┴─────────────────┘
```
If you need to find the row with specific `Dynamic` value, you can do one of the following:
- Cast value to the `Dynamic` type:
```sql
SELECT * FROM test WHERE d2 == [1,2,3]::Array(UInt32)::Dynamic;
```
```text
┌─d1─┬─d2──────┐
│ 42 │ [1,2,3] │
└────┴─────────┘
```
- Compare `Dynamic` subcolumn with required type:
```sql
SELECT * FROM test WHERE d2.`Array(Int65)` == [1,2,3] -- or using variantElement(d2, 'Array(UInt32)')
```
```text
┌─d1─┬─d2──────┐
│ 42 │ [1,2,3] │
└────┴─────────┘
```
Sometimes it can be useful to make additional check on dynamic type as subcolumns with complex types like `Array/Map/Tuple` cannot be inside `Nullable` and will have default values instead of `NULL` on rows with different types:
```sql
SELECT d2, d2.`Array(Int64)`, dynamicType(d2) FROM test WHERE d2.`Array(Int64)` == [];
```
```text
┌─d2───┬─d2.Array(UInt32)─┬─dynamicType(d2)─┐
│ 42 │ [] │ Int64 │
│ 43 │ [] │ Int64 │
│ abc │ [] │ String │
│ [] │ [] │ Array(Int32) │
│ ᴺᵁᴸᴸ │ [] │ None │
└──────┴──────────────────┴─────────────────┘
```
```sql
SELECT d2, d2.`Array(Int64)`, dynamicType(d2) FROM test WHERE dynamicType(d2) == 'Array(Int64)' AND d2.`Array(Int64)` == [];
```
```text
┌─d2─┬─d2.Array(UInt32)─┬─dynamicType(d2)─┐
│ [] │ [] │ Array(Int64) │
└────┴──────────────────┴─────────────────┘
└─────────┴────────────────┘
```
**Note:** values of dynamic types with different numeric types are considered as different values and not compared between each other, their type names are compared instead.
@ -411,6 +569,21 @@ SELECT d, dynamicType(d) FROM test ORDER by d;
└─────┴────────────────┘
```
```sql
SELECT d, dynamicType(d) FROM test GROUP by d;
```
```text
┌─d───┬─dynamicType(d)─┐
│ 1 │ Int64 │
│ 100 │ UInt32 │
│ 1 │ UInt32 │
│ 100 │ Int64 │
└─────┴────────────────┘
```
**Note**: the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
## Reaching the limit in number of different data types stored inside Dynamic
`Dynamic` data type can store only limited number of different data types as separate subcolumns. By default, this limit is 32, but you can change it in type declaration using syntax `Dynamic(max_types=N)` where N is between 0 and 254 (due to implementation details, it's impossible to have more than 254 different data types that can be stored as separate subcolumns inside Dynamic).

40
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -1972,7 +1972,7 @@ Result:
## toISOYear
Converts a date, or date with time, to a UInt16 number containing the ISO Year number.
Converts a date, or date with time, to the ISO year as a UInt16 number.
**Syntax**
@ -1982,11 +1982,11 @@ toISOYear(value)
**Arguments**
- `value` — The value with date or date with time.
- `value` — The value with date or date with time. [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
**Returned value**
- `value` converted to the current ISO year number. [UInt16](../data-types/int-uint.md).
- The input value converted to a ISO year number. [UInt16](../data-types/int-uint.md).
**Example**
@ -1995,7 +1995,7 @@ Query:
```sql
SELECT
toISOYear(toDate('2024/10/02')) as year1,
toISOYear(toDateTime('2024/10/02 01:30:00')) as year2
toISOYear(toDateTime('2024-10-02 01:30:00')) as year2
```
Result:
@ -2010,6 +2010,38 @@ Result:
Converts a date, or date with time, to a UInt8 number containing the ISO Week number.
**Syntax**
```sql
toISOWeek(value)
```
**Arguments**
- `value` — The value with date or date with time.
**Returned value**
- `value` converted to the current ISO week number. [UInt8](../data-types/int-uint.md).
**Example**
Query:
```sql
SELECT
toISOWeek(toDate('2024/10/02')) AS week1,
toISOWeek(toDateTime('2024/10/02 01:30:00')) AS week2
```
Response:
```response
┌─week1─┬─week2─┐
│ 40 │ 40 │
└───────┴───────┘
```
## toWeek
This function returns the week number for date or datetime. The two-argument form of `toWeek()` enables you to specify whether the week starts on Sunday or Monday and whether the return value should be in the range from 0 to 53 or from 1 to 53. If the mode argument is omitted, the default mode is 0.

8
docs/en/sql-reference/functions/ip-address-functions.md
View File

@ -288,11 +288,11 @@ toIPv4OrDefault(value)
**Arguments**
- `value` — The value with IPv4 address.
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
**Returned value**
- `value` converted to the current IPv4 address. [String](../data-types/string.md).
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
**Example**
@ -324,11 +324,11 @@ toIPv4OrNull(value)
**Arguments**
- `value` — The value with IPv4 address.
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
**Returned value**
- `value` converted to the current IPv4 address. [String](../data-types/string.md).
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
**Example**

118
docs/en/sql-reference/functions/other-functions.md
View File

@ -202,7 +202,7 @@ Result:
Returns the type name of the passed argument.
If `NULL` is passed, then the function returns type `Nullable(Nothing)`, which corresponds to ClickHouse's internal `NULL` representation.
If `NULL` is passed, the function returns type `Nullable(Nothing)`, which corresponds to ClickHouse's internal `NULL` representation.
**Syntax**
@ -212,11 +212,11 @@ toTypeName(value)
**Arguments**
- `value` — The value with any arbitrary.
- `value` — A value of arbitrary type.
**Returned value**
- `value` converted to the current data type name. [String](../data-types/string.md).
- The data type name of the input value. [String](../data-types/string.md).
**Example**
@ -410,13 +410,37 @@ Code: 44. DB::Exception: Received from localhost:9000. DB::Exception: Illegal ty
## ignore
Accepts any arguments, including `NULL` and does nothing. Always returns 0.
The argument is internally still evaluated. Useful e.g. for benchmarks.
Accepts arbitrary arguments and unconditionally returns `0`.
The argument is still evaluated internally, making it useful for eg. benchmarking.
**Syntax**
```sql
ignore(x)
ignore([arg1[, arg2[, ...]])
```
**Arguments**
- Accepts arbitrarily many arguments of arbitrary type, including `NULL`.
**Returned value**
- Returns `0`.
**Example**
Query:
```sql
SELECT ignore(0, 'ClickHouse', NULL);
```
Result:
```response
┌─ignore(0, 'ClickHouse', NULL)─┐
│ 0 │
└───────────────────────────────┘
```
## sleep
@ -524,13 +548,9 @@ Useful in table engine parameters of `CREATE TABLE` queries where you need to sp
currentDatabase()
```
**Arguments**
None.
**Returned value**
- `value` returns the current database name. [String](../data-types/string.md).
- Returns the current database name. [String](../data-types/string.md).
**Example**
@ -579,6 +599,42 @@ Result:
└───────────────┘
```
## currentSchemas
Returns a single-element array with the name of the current database schema.
**Syntax**
```sql
currentSchemas(bool)
```
Alias: `current_schemas`.
**Arguments**
- `bool`: A boolean value. [Bool](../data-types/boolean.md).
:::note
The boolean argument is ignored. It only exists for the sake of compatibility with the [implementation](https://www.postgresql.org/docs/7.3/functions-misc.html) of this function in PostgreSQL.
:::
**Returned values**
- Returns a single-element array with the name of the current database
**Example**
```sql
SELECT currentSchemas(true);
```
Result:
```response
['default']
```
## isConstant
Returns whether the argument is a constant expression.
@ -1821,7 +1877,7 @@ toColumnTypeName(value)
**Example**
Difference between `toTypeName ' and ' toColumnTypeName`:
Difference between `toTypeName` and `toColumnTypeName`:
```sql
SELECT toTypeName(CAST('2018-01-01 01:02:03' AS DateTime))
@ -3897,13 +3953,15 @@ Retrieves the connection ID of the client that submitted the current query and r
connectionId()
```
Alias: `connection_id`.
**Parameters**
None.
**Returned value**
Returns an integer of type UInt64.
The current connection ID. [UInt64](../data-types/int-uint.md).
**Implementation details**
@ -3921,40 +3979,6 @@ SELECT connectionId();
0
```
## connection_id
An alias of `connectionId`. Retrieves the connection ID of the client that submitted the current query and returns it as a UInt64 integer.
**Syntax**
```sql
connection_id()
```
**Parameters**
None.
**Returned value**
Returns an integer of type UInt64.
**Implementation details**
This function is most useful in debugging scenarios or for internal purposes within the MySQL handler. It was created for compatibility with [MySQL's `CONNECTION_ID` function](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_connection-id) It is not typically used in production queries.
**Example**
Query:
```sql
SELECT connection_id();
```
```response
0
```
## getClientHTTPHeader
Get the value of an HTTP header.

104
docs/en/sql-reference/functions/string-search-functions.md
View File

@ -755,7 +755,7 @@ Result:
## match {#match}
Returns whether string `haystack` matches the regular expression `pattern` in [re2 regular syntax](https://github.com/google/re2/wiki/Syntax).
Returns whether string `haystack` matches the regular expression `pattern` in [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax).
Matching is based on UTF-8, e.g. `.` matches the Unicode code point `¥` which is represented in UTF-8 using two bytes. The regular
expression must not contain null bytes. If the haystack or the pattern are not valid UTF-8, then the behavior is undefined.
@ -852,9 +852,10 @@ multiFuzzyMatchAllIndices(haystack, distance, \[pattern<sub>1</sub>, pattern<sub
## extract
Extracts a fragment of a string using a regular expression. If `haystack` does not match the `pattern` regex, an empty string is returned.
Returns the first match of a regular expression in a string.
If `haystack` does not match the `pattern` regex, an empty string is returned.
For regex without subpatterns, the function uses the fragment that matches the entire regex. Otherwise, it uses the fragment that matches the first subpattern.
If the regular expression has capturing groups, the function matches the input string against the first capturing group.
**Syntax**
@ -862,13 +863,36 @@ For regex without subpatterns, the function uses the fragment that matches the e
extract(haystack, pattern)
```
*Arguments**
- `haystack` — Input string. [String](../data-types/string.md).
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax).
**Returned value**
- The first match of the regular expression in the haystack string. [String](../data-types/string.md).
**Example**
Query:
```sql
SELECT extract('number: 1, number: 2, number: 3', '\\d+') AS result;
```
Result:
```response
┌─result─┐
│ 1 │
└────────┘
```
## extractAll
Extracts all fragments of a string using a regular expression. If `haystack` does not match the `pattern` regex, an empty string is returned.
Returns an array of all matches of a regular expression in a string. If `haystack` does not match the `pattern` regex, an empty string is returned.
Returns an array of strings consisting of all matches of the regex.
The behavior with respect to subpatterns is the same as in function `extract`.
The behavior with respect to sub-patterns is the same as in function [`extract`](#extract).
**Syntax**
@ -876,6 +900,31 @@ The behavior with respect to subpatterns is the same as in function `extract`.
extractAll(haystack, pattern)
```
*Arguments**
- `haystack` — Input string. [String](../data-types/string.md).
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax).
**Returned value**
- Array of matches of the regular expression in the haystack string. [Array](../data-types/array.md)([String](../data-types/string.md)).
**Example**
Query:
```sql
SELECT extractAll('number: 1, number: 2, number: 3', '\\d+') AS result;
```
Result:
```response
┌─result────────┐
│ ['1','2','3'] │
└───────────────┘
```
## extractAllGroupsHorizontal
Matches all groups of the `haystack` string using the `pattern` regular expression. Returns an array of arrays, where the first array includes all fragments matching the first group, the second array - matching the second group, etc.
@ -891,7 +940,7 @@ extractAllGroupsHorizontal(haystack, pattern)
**Arguments**
- `haystack` — Input string. [String](../data-types/string.md).
- `pattern` — Regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
**Returned value**
@ -915,6 +964,39 @@ Result:
└──────────────────────────────────────────────────────────────────────────────────────────┘
```
## extractGroups
Match all groups of given input string with a given regular expression, returns an array of arrays of matches.
**Syntax**
``` sql
extractGroups(haystack, pattern)
```
**Arguments**
- `haystack` — Input string. [String](../data-types/string.md).
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
**Returned value**
- Array of arrays of matches. [Array](../data-types/array.md).
**Example**
``` sql
SELECT extractGroups('hello abc=111 world', '("[^"]+"|\\w+)=("[^"]+"|\\w+)') AS result;
```
Result:
``` text
┌─result────────┐
│ ['abc','111'] │
└───────────────┘
```
## extractAllGroupsVertical
Matches all groups of the `haystack` string using the `pattern` regular expression. Returns an array of arrays, where each array includes matching fragments from every group. Fragments are grouped in order of appearance in the `haystack`.
@ -928,7 +1010,7 @@ extractAllGroupsVertical(haystack, pattern)
**Arguments**
- `haystack` — Input string. [String](../data-types/string.md).
- `pattern` — Regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
**Returned value**
@ -1484,7 +1566,7 @@ countMatches(haystack, pattern)
**Arguments**
- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
- `pattern` — The regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
**Returned value**
@ -1529,7 +1611,7 @@ countMatchesCaseInsensitive(haystack, pattern)
**Arguments**
- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
- `pattern` — The regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
**Returned value**

41
docs/en/sql-reference/functions/type-conversion-functions.md
View File

@ -5230,15 +5230,52 @@ Result:
Also see the `toUnixTimestamp` function.
## toFixedString(s, N)
## toFixedString
Converts a [String](../data-types/string.md) type argument to a [FixedString(N)](../data-types/fixedstring.md) type (a string of fixed length N).
If the string has fewer bytes than N, it is padded with null bytes to the right. If the string has more bytes than N, an exception is thrown.
## toStringCutToZero(s)
**Syntax**
```sql
toFixedString(s, N)
```
**Arguments**
- `s` — A String to convert to a fixed string. [String](../data-types/string.md).
- `N` — Length N. [UInt8](../data-types/int-uint.md)
**Returned value**
- An N length fixed string of `s`. [FixedString](../data-types/fixedstring.md).
**Example**
Query:
``` sql
SELECT toFixedString('foo', 8) AS s;
```
Result:
```response
┌─s─────────────┐
│ foo\0\0\0\0\0 │
└───────────────┘
```
## toStringCutToZero
Accepts a String or FixedString argument. Returns the String with the content truncated at the first zero byte found.
**Syntax**
```sql
toStringCutToZero(s)
```
**Example**
Query:

2
docs/en/sql-reference/statements/alter/view.md
View File

@ -41,7 +41,7 @@ ORDER BY ts, event_type;
│ 2020-01-02 00:00:00 │ imp │ 2 │
└─────────────────────┴────────────┴─────────────────┘
-- Let's add the new measurment `cost`
-- Let's add the new measurement `cost`
-- and the new dimension `browser`.
ALTER TABLE events

2
docs/en/sql-reference/statements/check-table.md
View File

@ -46,7 +46,7 @@ The `CHECK TABLE` query supports the following table engines:
- [StripeLog](../../engines/table-engines/log-family/stripelog.md)
- [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md)
Performed over the tables with another table engines causes an `NOT_IMPLEMETED` exception.
Performed over the tables with another table engines causes an `NOT_IMPLEMENTED` exception.
Engines from the `*Log` family do not provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner.

2
docs/en/sql-reference/statements/create/table.md
View File

@ -442,7 +442,7 @@ DEFLATE_QPL is not available in ClickHouse Cloud.
### Specialized Codecs
These codecs are designed to make compression more effective by exploiting specific features of the data. Some of these codecs do not compress data themself, they instead preprocess the data such that a second compression stage using a general-purpose codec can achieve a higher data compression rate.
These codecs are designed to make compression more effective by exploiting specific features of the data. Some of these codecs do not compress data themselves, they instead preprocess the data such that a second compression stage using a general-purpose codec can achieve a higher data compression rate.
#### Delta

2
docs/en/sql-reference/statements/create/view.md
View File

@ -194,7 +194,7 @@ REFRESH EVERY 1 MONTH OFFSET 5 DAY 2 HOUR -- on 6th day of every month, at 2:00
REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm
REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc
REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day
-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax errror, OFFSET is not allowed with AFTER
-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax error, OFFSET is not allowed with AFTER
REFRESH EVERY 1 WEEK 2 DAYS -- every 9 days, not on any particular day of the week or month;
-- specifically, when day number (since 1969-12-29) is divisible by 9
REFRESH EVERY 5 MONTHS -- every 5 months, different months each year (as 12 is not divisible by 5);

6
docs/en/sql-reference/statements/show.md
View File

@ -351,11 +351,15 @@ Shows privileges for a user.
**Syntax**
``` sql
SHOW GRANTS [FOR user1 [, user2 ...]]
SHOW GRANTS [FOR user1 [, user2 ...]] [WITH IMPLICIT] [FINAL]
```
If user is not specified, the query returns privileges for the current user.
The `WITH IMPLICIT` modifier allows to show the implicit grants (e.g., `GRANT SELECT ON system.one`)
The `FINAL` modifier merges all grants from the user and its granted roles (with inheritance)
## SHOW CREATE USER
Shows parameters that were used at a [user creation](../../sql-reference/statements/create/user.md).

9
docs/en/sql-reference/table-functions/iceberg.md
View File

@ -6,7 +6,7 @@ sidebar_label: iceberg
# iceberg Table Function
Provides a read-only table-like interface to Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure or locally stored.
Provides a read-only table-like interface to Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS or locally stored.
## Syntax
@ -17,13 +17,16 @@ icebergS3(named_collection[, option=value [,..]])
icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method])
icebergAzure(named_collection[, option=value [,..]])
icebergHDFS(path_to_table, [,format] [,compression_method])
icebergHDFS(named_collection[, option=value [,..]])
icebergLocal(path_to_table, [,format] [,compression_method])
icebergLocal(named_collection[, option=value [,..]])
```
## Arguments
Description of the arguments coincides with description of arguments in table functions `s3`, `azureBlobStorage` and `file` correspondingly.
Description of the arguments coincides with description of arguments in table functions `s3`, `azureBlobStorage`, `HDFS` and `file` correspondingly.
`format` stands for the format of data files in the Iceberg table.
**Returned value**
@ -36,7 +39,7 @@ SELECT * FROM icebergS3('http://test.s3.amazonaws.com/clickhouse-bucket/test_tab
```
:::important
ClickHouse currently supports reading v1 and v2 of the Iceberg format via the `icebergS3`, `icebergAzure` and `icebergLocal` table functions and `IcebergS3`, `icebergAzure` ans `icebergLocal` table engines.
ClickHouse currently supports reading v1 and v2 of the Iceberg format via the `icebergS3`, `icebergAzure`, `icebergHDFS` and `icebergLocal` table functions and `IcebergS3`, `icebergAzure`, `IcebergHDFS` and `IcebergLocal` table engines.
:::
## Defining a named collection

13
docs/ru/interfaces/third-party/gui.md vendored
View File

@ -9,6 +9,19 @@ sidebar_label: "Визуальные интерфейсы от сторонни
## С открытым исходным кодом {#s-otkrytym-iskhodnym-kodom}
### ChartDB {#chartdb}
[ChartDB](https://chartdb.io) — бесплатный и открытый инструмент для визуализации и проектирования схем баз данных, включая ClickHouse, с помощью одного запроса. Разработан на базе React, обеспечивает удобный и простой интерфейс, не требует ввода учетных данных или регистрации.
Основные возможности:
- Визуализация схем: мгновенно импортируйте и визуализируйте схему ClickHouse, включая ER-диаграммы с материализованными представлениями и стандартными представлениями, показывающими ссылки на таблицы;
- Экспорт DDL с поддержкой ИИ: легко генерируйте DDL-скрипты для лучшего управления и документирования схем;
- Поддержка различных SQL-диалектов: совместим с широким спектром SQL-диалектов, что делает его универсальным для разных сред баз данных;
- Без регистрации и учетных данных: весь функционал доступен прямо в браузере, обеспечивая бесшовное и безопасное использование.
[Исходный код ChartDB](https://github.com/chartdb/chartdb).
### Tabix {#tabix}
Веб-интерфейс для ClickHouse в проекте [Tabix](https://github.com/tabixio/tabix).

2
docs/ru/sql-reference/aggregate-functions/combinators.md
View File

@ -93,7 +93,7 @@ WITH anySimpleState(number) AS c SELECT toTypeName(c), c FROM numbers(1);
## -Distinct {#agg-functions-combinator-distinct}
При наличии комбинатора Distinct, каждое уникальное значение аргументов, будет учитано в агрегатной функции только один раз.
Примеры: `sum(DISTINCT x)`, `groupArray(DISTINCT x)`, `corrStableDistinct(DISTINCT x, y)` и т.п.
Примеры: `sum(DISTINCT x)` (или `sumDistinct(x)`), `groupArray(DISTINCT x)` (или `groupArrayDistinct(x)`), `corrStable(DISTINCT x, y)` (или `corrStableDistinct(x, y)`) и т.п.
## -OrDefault {#agg-functions-combinator-ordefault}

2
docs/ru/sql-reference/statements/grant.md
View File

@ -464,7 +464,7 @@ GRANT INSERT(x,y) ON db.table TO john
- `FILE`. Уровень: `GLOBAL`
- `URL`. Уровень: `GLOBAL`
- `REMOTE`. Уровень: `GLOBAL`
- `YSQL`. Уровень: `GLOBAL`
- `MYSQL`. Уровень: `GLOBAL`
- `ODBC`. Уровень: `GLOBAL`
- `JDBC`. Уровень: `GLOBAL`
- `HDFS`. Уровень: `GLOBAL`

5
docs/ru/sql-reference/statements/show.md
View File

@ -234,11 +234,14 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2
### Синтаксис {#show-grants-syntax}
``` sql
SHOW GRANTS [FOR user]
SHOW GRANTS [FOR user1 [, user2 ...]] [WITH IMPLICIT] [FINAL]
```
Если пользователь не задан, запрос возвращает привилегии текущего пользователя.
`WITH IMPLICIT` добавляет неявные привилегии (например `GRANT SELECT ON system.one`).
`FINAL` объединяет все текущие привилегии с привилегиями всех ролей пользователя (с наследованием).
## SHOW CREATE USER {#show-create-user-statement}

13
docs/zh/interfaces/third-party/gui.md vendored
View File

@ -5,6 +5,19 @@ slug: /zh/interfaces/third-party/gui
## 开源 {#kai-yuan}
### ChartDB {#chartdb}
ClickHouse 的数据库模式可视化工具 [ChartDB](https://chartdb.io)。
主要功能：
- 一键导入并可视化 ClickHouse 模式，包括带有物化视图和标准视图的 ER 图表，并显示表之间的引用关系。
- 支持 AI 驱动的 DDL 导出功能，便于数据库模式管理和文档生成。
- 支持多种 SQL 方言，适用于各种数据库环境。
- 直接在浏览器中使用，无需数据库凭证或注册，使用简单且安全。
[ChartDB 源代码](https://github.com/chartdb/chartdb).
### Tabix {#tabix}
ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix).

2
docs/zh/sql-reference/statements/show.md
View File

@ -110,7 +110,7 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2
### 语法 {#show-grants-syntax}
``` sql
SHOW GRANTS [FOR user]
SHOW GRANTS [FOR user1 [, user2 ...]] [WITH IMPLICIT] [FINAL]
```
如果未指定用户，输出当前用户的权限

2
programs/bash-completion/completions/clickhouse-bootstrap
View File

@ -155,7 +155,7 @@ function _clickhouse_quote()
function _clickhouse_get_options()
{
# By default --help will not print all settings, this is done only under --verbose
"$@" --help --verbose 2>&1 | awk -F '[ ,=<>.]' '{ for (i=1; i <= NF; ++i) { if (substr($i, 1, 1) == "-" && length($i) > 1) print $i; } }' | sort -u
"$@" --help --verbose 2>&1 | LANG=c awk -F '[ ,=<>.]' '{ for (i=1; i <= NF; ++i) { if (substr($i, 1, 1) == "-" && length($i) > 1) print $i; } }' | sort -u
}
function _complete_for_clickhouse_generic_bin_impl()

11
programs/server/Server.cpp
View File

@ -69,6 +69,7 @@
#include <Interpreters/registerInterpreters.h>
#include <Interpreters/JIT/CompiledExpressionCache.h>
#include <Access/AccessControl.h>
#include <Storages/MaterializedView/RefreshSet.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/System/attachSystemTables.h>
@ -1495,6 +1496,8 @@ try
NamedCollectionFactory::instance().loadIfNot();
FileCacheFactory::instance().loadDefaultCaches(config());
/// Initialize main config reloader.
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
@ -2083,6 +2086,12 @@ try
try
{
/// Don't run background queries until we loaded tables.
/// (In particular things would break if a background drop query happens before the
/// loadMarkedAsDroppedTables() call below - it'll see dropped table metadata and try to
/// drop the table a second time and throw an exception.)
global_context->getRefreshSet().setRefreshesStopped(true);
auto & database_catalog = DatabaseCatalog::instance();
/// We load temporary database first, because projections need it.
database_catalog.initializeAndLoadTemporaryDatabase();
@ -2122,6 +2131,8 @@ try
database_catalog.assertDatabaseExists(default_database);
/// Load user-defined SQL functions.
global_context->getUserDefinedSQLObjectsStorage().loadObjects();
global_context->getRefreshSet().setRefreshesStopped(false);
}
catch (...)
{

3
src/Access/AccessRights.cpp
View File

@ -1155,9 +1155,6 @@ private:
calculateMinMaxFlags();
if (!isLeaf())
return;
auto new_flags = function(flags, min_flags_with_children, max_flags_with_children, level, grant_option);
if (new_flags != flags)

39
src/Access/AuthenticationData.cpp
View File

@ -117,20 +117,20 @@ bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs)
}
void AuthenticationData::setPassword(const String & password_)
void AuthenticationData::setPassword(const String & password_, bool validate)
{
switch (type)
{
case AuthenticationType::PLAINTEXT_PASSWORD:
setPasswordHashBinary(Util::stringToDigest(password_));
setPasswordHashBinary(Util::stringToDigest(password_), validate);
return;
case AuthenticationType::SHA256_PASSWORD:
setPasswordHashBinary(Util::encodeSHA256(password_));
setPasswordHashBinary(Util::encodeSHA256(password_), validate);
return;
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
setPasswordHashBinary(Util::encodeDoubleSHA1(password_));
setPasswordHashBinary(Util::encodeDoubleSHA1(password_), validate);
return;
case AuthenticationType::BCRYPT_PASSWORD:
@ -149,12 +149,12 @@ void AuthenticationData::setPassword(const String & password_)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "setPassword(): authentication type {} not supported", toString(type));
}
void AuthenticationData::setPasswordBcrypt(const String & password_, int workfactor_)
void AuthenticationData::setPasswordBcrypt(const String & password_, int workfactor_, bool validate)
{
if (type != AuthenticationType::BCRYPT_PASSWORD)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot specify bcrypt password for authentication type {}", toString(type));
setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_));
setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_), validate);
}
String AuthenticationData::getPassword() const
@ -165,7 +165,7 @@ String AuthenticationData::getPassword() const
}
void AuthenticationData::setPasswordHashHex(const String & hash)
void AuthenticationData::setPasswordHashHex(const String & hash, bool validate)
{
Digest digest;
digest.resize(hash.size() / 2);
@ -179,7 +179,7 @@ void AuthenticationData::setPasswordHashHex(const String & hash)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read password hash in hex, check for valid characters [0-9a-fA-F] and length");
}
setPasswordHashBinary(digest);
setPasswordHashBinary(digest, validate);
}
@ -195,7 +195,7 @@ String AuthenticationData::getPasswordHashHex() const
}
void AuthenticationData::setPasswordHashBinary(const Digest & hash)
void AuthenticationData::setPasswordHashBinary(const Digest & hash, bool validate)
{
switch (type)
{
@ -217,7 +217,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
{
if (hash.size() != 20)
if (validate && hash.size() != 20)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Password hash for the 'DOUBLE_SHA1_PASSWORD' authentication type has length {} "
"but must be exactly 20 bytes.", hash.size());
@ -231,7 +231,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
/// However the library we use to encode it requires hash string to be 64 characters long,
/// so we also allow the hash of this length.
if (hash.size() != 59 && hash.size() != 60 && hash.size() != 64)
if (validate && hash.size() != 59 && hash.size() != 60 && hash.size() != 64)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Password hash for the 'BCRYPT_PASSWORD' authentication type has length {} "
"but must be 59 or 60 bytes.", hash.size());
@ -240,10 +240,13 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
resized.resize(64);
#if USE_BCRYPT
if (validate)
{
/// Verify that it is a valid hash
int ret = bcrypt_checkpw("", reinterpret_cast<const char *>(resized.data()));
if (ret == -1)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not decode the provided hash with 'bcrypt_hash'");
}
#endif
password_hash = hash;
@ -385,7 +388,7 @@ std::shared_ptr<ASTAuthenticationData> AuthenticationData::toAST() const
}
AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & query, ContextPtr context, bool check_password_rules)
AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & query, ContextPtr context, bool validate)
{
if (query.type && query.type == AuthenticationType::NO_PASSWORD)
return AuthenticationData();
@ -431,7 +434,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
if (!query.type && !context)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get default password type without context");
if (check_password_rules && !context)
if (validate && !context)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot check password complexity rules without context");
if (query.type == AuthenticationType::BCRYPT_PASSWORD && !context)
@ -448,13 +451,13 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
AuthenticationData auth_data(current_type);
if (check_password_rules)
if (validate)
context->getAccessControl().checkPasswordComplexityRules(value);
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
{
int workfactor = context->getAccessControl().getBcryptWorkfactor();
auth_data.setPasswordBcrypt(value, workfactor);
auth_data.setPasswordBcrypt(value, workfactor, validate);
return auth_data;
}
@ -486,7 +489,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
#endif
}
auth_data.setPassword(value);
auth_data.setPassword(value, validate);
return auth_data;
}
@ -498,11 +501,11 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
{
auth_data.setPasswordHashBinary(AuthenticationData::Util::stringToDigest(value));
auth_data.setPasswordHashBinary(AuthenticationData::Util::stringToDigest(value), validate);
return auth_data;
}
auth_data.setPasswordHashHex(value);
auth_data.setPasswordHashHex(value, validate);
if (query.type == AuthenticationType::SHA256_PASSWORD && args_size == 2)

10
src/Access/AuthenticationData.h
View File

@ -31,17 +31,17 @@ public:
AuthenticationType getType() const { return type; }
/// Sets the password and encrypt it using the authentication type set in the constructor.
void setPassword(const String & password_);
void setPassword(const String & password_, bool validate);
/// Returns the password. Allowed to use only for Type::PLAINTEXT_PASSWORD.
String getPassword() const;
/// Sets the password as a string of hexadecimal digits.
void setPasswordHashHex(const String & hash);
void setPasswordHashHex(const String & hash, bool validate);
String getPasswordHashHex() const;
/// Sets the password in binary form.
void setPasswordHashBinary(const Digest & hash);
void setPasswordHashBinary(const Digest & hash, bool validate);
const Digest & getPasswordHashBinary() const { return password_hash; }
/// Sets the salt in String form.
@ -49,7 +49,7 @@ public:
String getSalt() const;
/// Sets the password using bcrypt hash with specified workfactor
void setPasswordBcrypt(const String & password_, int workfactor_);
void setPasswordBcrypt(const String & password_, int workfactor_, bool validate);
/// Sets the server name for authentication type LDAP.
const String & getLDAPServerName() const { return ldap_server_name; }
@ -77,7 +77,7 @@ public:
friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs);
friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); }
static AuthenticationData fromAST(const ASTAuthenticationData & query, ContextPtr context, bool check_password_rules);
static AuthenticationData fromAST(const ASTAuthenticationData & query, ContextPtr context, bool validate);
std::shared_ptr<ASTAuthenticationData> toAST() const;
struct Util

42
src/Access/ContextAccess.cpp
View File

@ -64,7 +64,27 @@ namespace
}
AccessRights addImplicitAccessRights(const AccessRights & access, const AccessControl & access_control)
std::array<UUID, 1> to_array(const UUID & id)
{
std::array<UUID, 1> ids;
ids[0] = id;
return ids;
}
/// Helper for using in templates.
std::string_view getDatabase() { return {}; }
template <typename... OtherArgs>
std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
std::string_view getTableEngine() { return {}; }
template <typename... OtherArgs>
std::string_view getTableEngine(std::string_view arg1, const OtherArgs &...) { return arg1; }
}
AccessRights ContextAccess::addImplicitAccessRights(const AccessRights & access, const AccessControl & access_control)
{
AccessFlags max_flags;
@ -254,26 +274,6 @@ namespace
}
std::array<UUID, 1> to_array(const UUID & id)
{
std::array<UUID, 1> ids;
ids[0] = id;
return ids;
}
/// Helper for using in templates.
std::string_view getDatabase() { return {}; }
template <typename... OtherArgs>
std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
std::string_view getTableEngine() { return {}; }
template <typename... OtherArgs>
std::string_view getTableEngine(std::string_view arg1, const OtherArgs &...) { return arg1; }
}
std::shared_ptr<const ContextAccess> ContextAccess::fromContext(const ContextPtr & context)
{
return ContextAccessWrapper::fromContext(context)->getAccess();

2
src/Access/ContextAccess.h
View File

@ -132,6 +132,8 @@ public:
/// Checks if grantees are allowed for the current user, throws an exception if not.
void checkGranteesAreAllowed(const std::vector<UUID> & grantee_ids) const;
static AccessRights addImplicitAccessRights(const AccessRights & access, const AccessControl & access_control);
ContextAccess(const AccessControl & access_control_, const Params & params_);
~ContextAccess();

7
src/Access/UsersConfigAccessStorage.cpp
View File

@ -121,6 +121,7 @@ namespace
bool allow_no_password,
bool allow_plaintext_password)
{
const bool validate = true;
auto user = std::make_shared<User>();
user->setName(user_name);
String user_config = "users." + user_name;
@ -157,17 +158,17 @@ namespace
if (has_password_plaintext)
{
user->authentication_methods.emplace_back(AuthenticationType::PLAINTEXT_PASSWORD);
user->authentication_methods.back().setPassword(config.getString(user_config + ".password"));
user->authentication_methods.back().setPassword(config.getString(user_config + ".password"), validate);
}
else if (has_password_sha256_hex)
{
user->authentication_methods.emplace_back(AuthenticationType::SHA256_PASSWORD);
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"));
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"), validate);
}
else if (has_password_double_sha1_hex)
{
user->authentication_methods.emplace_back(AuthenticationType::DOUBLE_SHA1_PASSWORD);
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"));
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"), validate);
}
else if (has_ldap)
{

23
src/AggregateFunctions/WindowFunction.h
View File

@ -78,11 +78,6 @@ struct WindowFunction : public IAggregateFunctionHelper<WindowFunction>, public
}
String getName() const override { return name; }
void create(AggregateDataPtr __restrict) const override { }
void destroy(AggregateDataPtr __restrict) const noexcept override { }
bool hasTrivialDestructor() const override { return true; }
size_t sizeOfData() const override { return 0; }
size_t alignOfData() const override { return 1; }
void add(AggregateDataPtr __restrict, const IColumn **, size_t, Arena *) const override { fail(); }
void merge(AggregateDataPtr __restrict, ConstAggregateDataPtr, Arena *) const override { fail(); }
void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional<size_t>) const override { fail(); }
@ -90,6 +85,22 @@ struct WindowFunction : public IAggregateFunctionHelper<WindowFunction>, public
void insertResultInto(AggregateDataPtr __restrict, IColumn &, Arena *) const override { fail(); }
};
struct StatelessWindowFunction : public WindowFunction
{
StatelessWindowFunction(
const std::string & name_, const DataTypes & argument_types_, const Array & parameters_, const DataTypePtr & result_type_)
: WindowFunction(name_, argument_types_, parameters_, result_type_)
{
}
size_t sizeOfData() const override { return 0; }
size_t alignOfData() const override { return 1; }
void create(AggregateDataPtr __restrict) const override { }
void destroy(AggregateDataPtr __restrict) const noexcept override { }
bool hasTrivialDestructor() const override { return true; }
};
template <typename State>
struct StatefulWindowFunction : public WindowFunction
{
@ -100,7 +111,7 @@ struct StatefulWindowFunction : public WindowFunction
}
size_t sizeOfData() const override { return sizeof(State); }
size_t alignOfData() const override { return 1; }
size_t alignOfData() const override { return alignof(State); }
void create(AggregateDataPtr __restrict place) const override { new (place) State(); }

6
src/Analyzer/Passes/CrossToInnerJoinPass.cpp
View File

@ -34,7 +34,7 @@ namespace ErrorCodes
namespace
{
void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other)
void extractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other)
{
auto * func = node->as<FunctionNode>();
if (!func)
@ -52,7 +52,7 @@ void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi
else if (func->getFunctionName() == "and")
{
for (const auto & arg : args)
exctractJoinConditions(arg, equi_conditions, other);
extractJoinConditions(arg, equi_conditions, other);
}
else
{
@ -118,7 +118,7 @@ public:
QueryTreeNodes equi_conditions;
QueryTreeNodes other_conditions;
exctractJoinConditions(where_condition, equi_conditions, other_conditions);
extractJoinConditions(where_condition, equi_conditions, other_conditions);
bool can_convert_cross_to_inner = false;
for (auto & condition : equi_conditions)
{

2
src/Analyzer/Resolve/QueryAnalyzer.cpp
View File

@ -1950,7 +1950,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher(
{
bool table_expression_in_resolve_process = nearest_query_scope->table_expressions_in_resolve_process.contains(table_expression.get());
if (auto * /*array_join_node*/ _ = table_expression->as<ArrayJoinNode>())
if (table_expression->as<ArrayJoinNode>())
{
if (table_expressions_column_nodes_with_names_stack.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR,

23
src/Analyzer/ValidationUtils.cpp
View File

@ -19,6 +19,8 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER;
extern const int ILLEGAL_PREWHERE;
extern const int UNSUPPORTED_METHOD;
extern const int UNEXPECTED_EXPRESSION;
}
namespace
@ -26,11 +28,24 @@ namespace
void validateFilter(const QueryTreeNodePtr & filter_node, std::string_view exception_place_message, const QueryTreeNodePtr & query_node)
{
if (filter_node->getNodeType() == QueryTreeNodeType::LIST)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Unsupported expression '{}' in filter", filter_node->formatASTForErrorMessage());
DataTypePtr filter_node_result_type;
try
{
filter_node_result_type = filter_node->getResultType();
}
catch (const DB::Exception &e)
{
if (e.code() != ErrorCodes::UNSUPPORTED_METHOD)
e.rethrow();
}
if (!filter_node_result_type)
throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION,
"Unexpected expression '{}' in filter in {}. In query {}",
filter_node->formatASTForErrorMessage(),
exception_place_message,
query_node->formatASTForErrorMessage());
auto filter_node_result_type = filter_node->getResultType();
if (!filter_node_result_type->canBeUsedInBooleanContext())
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER,
"Invalid type for filter in {}: {}. In query {}",

6
src/Client/ConnectionParameters.h
View File

@ -15,7 +15,7 @@ namespace DB
{
struct ConnectionParameters
{
std::string host;
String host;
UInt16 port{};
std::string default_database;
std::string user;
@ -30,8 +30,8 @@ struct ConnectionParameters
ConnectionTimeouts timeouts;
ConnectionParameters() = default;
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host);
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host, std::optional<UInt16> port);
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, String host);
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, String host, std::optional<UInt16> port);
static UInt16 getPortFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & connection_host);

9
src/Client/TerminalKeystrokeInterceptor.cpp
View File

@ -9,6 +9,7 @@
#include <termios.h>
#include <unistd.h>
#include <base/defines.h>
#include <sys/ioctl.h>
namespace DB::ErrorCodes
{
@ -112,6 +113,14 @@ void TerminalKeystrokeInterceptor::run(TerminalKeystrokeInterceptor::CallbackMap
void TerminalKeystrokeInterceptor::runImpl(const DB::TerminalKeystrokeInterceptor::CallbackMap & map) const
{
char ch;
int available = 0;
if (ioctl(fd, FIONREAD, &available) < 0)
throw DB::ErrnoException(DB::ErrorCodes::SYSTEM_ERROR, "ioctl({}, FIONREAD)", fd);
if (available <= 0)
return;
if (read(fd, &ch, 1) > 0)
{
auto it = map.find(ch);

4
src/Columns/ColumnNullable.cpp
View File

@ -8,6 +8,7 @@
#include <Columns/ColumnString.h>
#include <Columns/ColumnCompressed.h>
#include <Columns/ColumnLowCardinality.h>
#include <Columns/MaskOperations.h>
#if USE_EMBEDDED_COMPILER
#include <DataTypes/Native.h>
@ -311,7 +312,8 @@ ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint)
void ColumnNullable::expand(const IColumn::Filter & mask, bool inverted)
{
nested_column->expand(mask, inverted);
null_map->expand(mask, inverted);
/// Use 1 as default value so column will contain NULLs on rows where filter has 0.
expandDataByMask<UInt8>(getNullMapData(), mask, inverted, 1);
}
ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const

13
src/Columns/ColumnObject.cpp
View File

@ -323,6 +323,19 @@ void ColumnObject::setDynamicPaths(const std::vector<String> & paths)
}
}
void ColumnObject::setDynamicPaths(const std::vector<std::pair<String, ColumnPtr>> & paths)
{
if (paths.size() > max_dynamic_paths)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot set dynamic paths to Object column, the number of paths ({}) exceeds the limit ({})", paths.size(), max_dynamic_paths);
for (const auto & [path, column] : paths)
{
auto it = dynamic_paths.emplace(path, column).first;
dynamic_paths_ptrs[path] = assert_cast<ColumnDynamic *>(it->second.get());
sorted_dynamic_paths.insert(it->first);
}
}
void ColumnObject::insert(const Field & x)
{
const auto & object = x.safeGet<Object>();

1
src/Columns/ColumnObject.h
View File

@ -222,6 +222,7 @@ public:
void addNewDynamicPath(std::string_view path);
void setDynamicPaths(const std::vector<String> & paths);
void setDynamicPaths(const std::vector<std::pair<String, ColumnPtr>> & paths);
void setMaxDynamicPaths(size_t max_dynamic_paths_);
void setStatistics(const StatisticsPtr & statistics_) { statistics = statistics_; }

25
src/Columns/ColumnVariant.cpp
View File

@ -1451,6 +1451,31 @@ std::optional<ColumnVariant::Discriminator> ColumnVariant::getLocalDiscriminator
return std::nullopt;
}
std::optional<ColumnVariant::Discriminator> ColumnVariant::getGlobalDiscriminatorOfOneNoneEmptyVariantNoNulls() const
{
if (auto local_discr = getLocalDiscriminatorOfOneNoneEmptyVariantNoNulls())
return globalDiscriminatorByLocal(*local_discr);
return std::nullopt;
}
std::optional<ColumnVariant::Discriminator> ColumnVariant::getGlobalDiscriminatorOfOneNoneEmptyVariant() const
{
std::optional<ColumnVariant::Discriminator> discr;
for (size_t i = 0; i != variants.size(); ++i)
{
if (!variants[i]->empty())
{
/// Check if we already had non-empty variant.
if (discr)
return std::nullopt;
discr = globalDiscriminatorByLocal(i);
}
}
return discr;
}
void ColumnVariant::applyNullMap(const ColumnVector<UInt8>::Container & null_map)
{
applyNullMapImpl<false>(null_map);

6
src/Columns/ColumnVariant.h
View File

@ -314,6 +314,12 @@ public:
/// Check if we have only 1 non-empty variant and no NULL values,
/// and if so, return the discriminator of this non-empty column.
std::optional<Discriminator> getLocalDiscriminatorOfOneNoneEmptyVariantNoNulls() const;
std::optional<Discriminator> getGlobalDiscriminatorOfOneNoneEmptyVariantNoNulls() const;
/// Check if we have only 1 non-empty variant,
/// and if so, return the discriminator of this non-empty column.
std::optional<Discriminator> getGlobalDiscriminatorOfOneNoneEmptyVariant() const;
/// Apply null map to a Variant column.
/// Replace corresponding discriminators with NULL_DISCRIMINATOR

19
src/Common/CurrentMetrics.cpp
View File

@ -291,9 +291,14 @@
M(CacheWarmerBytesInProgress, "Total size of remote file segments waiting to be asynchronously loaded into filesystem cache.") \
M(DistrCacheOpenedConnections, "Number of open connections to Distributed Cache") \
M(DistrCacheUsedConnections, "Number of currently used connections to Distributed Cache") \
M(DistrCacheAllocatedConnections, "Number of currently allocated connections to Distributed Cache connection pool") \
M(DistrCacheBorrowedConnections, "Number of currently borrowed connections to Distributed Cache connection pool") \
M(DistrCacheReadRequests, "Number of executed Read requests to Distributed Cache") \
M(DistrCacheWriteRequests, "Number of executed Write requests to Distributed Cache") \
M(DistrCacheServerConnections, "Number of open connections to ClickHouse server from Distributed Cache") \
M(DistrCacheRegisteredServers, "Number of distributed cache registered servers") \
M(DistrCacheRegisteredServersCurrentAZ, "Number of distributed cache registered servers in current az") \
M(DistrCacheServerS3CachedClients, "Number of distributed cache S3 cached clients") \
\
M(SchedulerIOReadScheduled, "Number of IO reads are being scheduled currently") \
M(SchedulerIOWriteScheduled, "Number of IO writes are being scheduled currently") \
@ -314,6 +319,20 @@
M(FilteringMarksWithSecondaryKeys, "Number of threads currently doing filtering of mark ranges by secondary keys") \
\
M(DiskS3NoSuchKeyErrors, "The number of `NoSuchKey` errors that occur when reading data from S3 cloud storage through ClickHouse disks.") \
\
M(SharedCatalogStateApplicationThreads, "Number of threads in the threadpool for state application in Shared Catalog.") \
M(SharedCatalogStateApplicationThreadsActive, "Number of active threads in the threadpool for state application in Shared Catalog.") \
M(SharedCatalogStateApplicationThreadsScheduled, "Number of queued or active jobs in the threadpool for state application in Shared Catalog.") \
\
M(SharedCatalogDropLocalThreads, "Number of threads in the threadpool for drop of local tables in Shared Catalog.") \
M(SharedCatalogDropLocalThreadsActive, "Number of active threads in the threadpool for drop of local tables in Shared Catalog.") \
M(SharedCatalogDropLocalThreadsScheduled, "Number of queued or active jobs in the threadpool for drop of local tables in Shared Catalog.") \
\
M(SharedCatalogDropZooKeeperThreads, "Number of threads in the threadpool for drop of object in ZooKeeper in Shared Catalog.") \
M(SharedCatalogDropZooKeeperThreadsActive, "Number of active threads in the threadpool for drop of object in ZooKeeper in Shared Catalog.") \
M(SharedCatalogDropZooKeeperThreadsScheduled, "Number of queued or active jobs in the threadpool for drop of object in ZooKeeper in Shared Catalog.") \
\
M(SharedDatabaseCatalogTablesInLocalDropDetachQueue, "Number of tables in the queue for local drop or detach in Shared Catalog.") \
#ifdef APPLY_FOR_EXTERNAL_METRICS
#define APPLY_FOR_METRICS(M) APPLY_FOR_BUILTIN_METRICS(M) APPLY_FOR_EXTERNAL_METRICS(M)

1
src/Common/ErrorCodes.cpp
View File

@ -614,6 +614,7 @@
\
M(900, DISTRIBUTED_CACHE_ERROR) \
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
M(902, PROTOCOL_VERSION_MISMATCH) \
\
M(999, KEEPER_EXCEPTION) \
M(1000, POCO_EXCEPTION) \

10
src/Common/FailPoint.cpp
View File

@ -49,11 +49,21 @@ static struct InitFiu
ONCE(smt_commit_write_zk_fail_before_op) \
ONCE(smt_commit_merge_change_version_before_op) \
ONCE(smt_merge_mutate_intention_freeze_in_destructor) \
ONCE(smt_add_part_sleep_after_add_before_commit) \
ONCE(smt_sleep_in_constructor) \
ONCE(meta_in_keeper_create_metadata_failure) \
ONCE(smt_insert_retry_timeout) \
ONCE(smt_insert_fake_hardware_error) \
ONCE(smt_sleep_after_hardware_in_insert) \
ONCE(smt_throw_keeper_exception_after_successful_insert) \
REGULAR(smt_dont_merge_first_part) \
REGULAR(smt_sleep_in_schedule_data_processing_job) \
REGULAR(cache_warmer_stall) \
REGULAR(check_table_query_delay_for_part) \
REGULAR(dummy_failpoint) \
REGULAR(prefetched_reader_pool_failpoint) \
REGULAR(shared_set_sleep_during_update) \
REGULAR(smt_outdated_parts_exception_response) \
PAUSEABLE_ONCE(replicated_merge_tree_insert_retry_pause) \
PAUSEABLE_ONCE(finish_set_quorum_failed_parts) \
PAUSEABLE_ONCE(finish_clean_quorum_failed_parts) \

98
src/Common/ProfileEvents.cpp
View File

@ -241,6 +241,8 @@
M(MergeVerticalStageExecuteMilliseconds, "Total busy time spent for execution of vertical stage of background merges", ValueType::Milliseconds) \
M(MergeProjectionStageTotalMilliseconds, "Total time spent for projection stage of background merges", ValueType::Milliseconds) \
M(MergeProjectionStageExecuteMilliseconds, "Total busy time spent for execution of projection stage of background merges", ValueType::Milliseconds) \
M(MergePrewarmStageTotalMilliseconds, "Total time spent for prewarm stage of background merges", ValueType::Milliseconds) \
M(MergePrewarmStageExecuteMilliseconds, "Total busy time spent for execution of prewarm stage of background merges", ValueType::Milliseconds) \
\
M(MergingSortedMilliseconds, "Total time spent while merging sorted columns", ValueType::Milliseconds) \
M(AggregatingSortedMilliseconds, "Total time spent while aggregating sorted columns", ValueType::Milliseconds) \
@ -639,6 +641,8 @@ The server successfully detected this situation and will download merged part fr
M(MetadataFromKeeperBackgroundCleanupTransactions, "Number of times old transaction idempotency token was cleaned up by background task", ValueType::Number) \
M(MetadataFromKeeperBackgroundCleanupErrors, "Number of times an error was encountered in background cleanup task", ValueType::Number) \
\
M(SharedMergeTreeMetadataCacheHintLoadedFromCache, "Number of times metadata cache hint was found without going to Keeper", ValueType::Number) \
\
M(KafkaRebalanceRevocations, "Number of partition revocations (the first stage of consumer group rebalance)", ValueType::Number) \
M(KafkaRebalanceAssignments, "Number of partition assignments (the final stage of consumer group rebalance)", ValueType::Number) \
M(KafkaRebalanceErrors, "Number of failed consumer group rebalances", ValueType::Number) \
@ -742,29 +746,51 @@ The server successfully detected this situation and will download merged part fr
M(ConnectionPoolIsFullMicroseconds, "Total time spent waiting for a slot in connection pool.", ValueType::Microseconds) \
M(AsyncLoaderWaitMicroseconds, "Total time a query was waiting for async loader jobs.", ValueType::Microseconds) \
\
M(DistrCacheServerSwitches, "Number of server switches between distributed cache servers in read/write-through cache", ValueType::Number) \
M(DistrCacheReadMicroseconds, "Time spent reading from distributed cache", ValueType::Microseconds) \
M(DistrCacheFallbackReadMicroseconds, "Time spend reading from fallback buffer instead of distribted cache", ValueType::Microseconds) \
M(DistrCachePrecomputeRangesMicroseconds, "Time spent to precompute read ranges", ValueType::Microseconds) \
M(DistrCacheNextImplMicroseconds, "Time spend in ReadBufferFromDistributedCache::nextImpl", ValueType::Microseconds) \
M(DistrCacheOpenedConnections, "The number of open connections to distributed cache", ValueType::Number) \
M(DistrCacheReusedConnections, "The number of reused connections to distributed cache", ValueType::Number) \
M(DistrCacheHoldConnections, "The number of used connections to distributed cache", ValueType::Number) \
M(DistrCacheServerSwitches, "Distributed Cache read buffer event. Number of server switches between distributed cache servers in read/write-through cache", ValueType::Number) \
M(DistrCacheReadMicroseconds, "Distributed Cache read buffer event. Time spent reading from distributed cache", ValueType::Microseconds) \
M(DistrCacheFallbackReadMicroseconds, "Distributed Cache read buffer event. Time spend reading from fallback buffer instead of distributed cache", ValueType::Microseconds) \
M(DistrCachePrecomputeRangesMicroseconds, "Distributed Cache read buffer event. Time spent to precompute read ranges", ValueType::Microseconds) \
M(DistrCacheNextImplMicroseconds, "Distributed Cache read buffer event. Time spend in ReadBufferFromDistributedCache::nextImpl", ValueType::Microseconds) \
M(DistrCacheStartRangeMicroseconds, "Distributed Cache read buffer event. Time spent to start a new read range with distributed cache", ValueType::Microseconds) \
M(DistrCacheIgnoredBytesWhileWaitingProfileEvents, "Distributed Cache read buffer event. Ignored bytes while waiting for profile events in distributed cache", ValueType::Number) \
M(DistrCacheRangeChange, "Distributed Cache read buffer event. Number of times we changed read range because of seek/last_position change", ValueType::Number) \
\
M(DistrCacheGetResponseMicroseconds, "Time spend to wait for response from distributed cache", ValueType::Microseconds) \
M(DistrCacheStartRangeMicroseconds, "Time spent to start a new read range with distributed cache", ValueType::Microseconds) \
M(DistrCacheLockRegistryMicroseconds, "Time spent to take DistributedCacheRegistry lock", ValueType::Microseconds) \
M(DistrCacheUnusedPackets, "Number of skipped unused packets from distributed cache", ValueType::Number) \
M(DistrCachePackets, "Total number of packets received from distributed cache", ValueType::Number) \
M(DistrCacheUnusedPacketsBytes, "The number of bytes in Data packets which were ignored", ValueType::Bytes) \
M(DistrCacheRegistryUpdateMicroseconds, "Time spent updating distributed cache registry", ValueType::Microseconds) \
M(DistrCacheRegistryUpdates, "Number of distributed cache registry updates", ValueType::Number) \
M(DistrCacheGetResponseMicroseconds, "Distributed Cache client event. Time spend to wait for response from distributed cache", ValueType::Microseconds) \
M(DistrCacheReadErrors, "Distributed Cache client event. Number of distributed cache errors during read", ValueType::Number) \
M(DistrCacheMakeRequestErrors, "Distributed Cache client event. Number of distributed cache errors when making a request", ValueType::Number) \
M(DistrCacheReceiveResponseErrors, "Distributed Cache client event. Number of distributed cache errors when receiving response a request", ValueType::Number) \
\
M(DistrCacheConnectMicroseconds, "The time spent to connect to distributed cache", ValueType::Microseconds) \
M(DistrCacheConnectAttempts, "The number of connection attempts to distributed cache", ValueType::Number) \
M(DistrCacheGetClient, "Number of client access times", ValueType::Number) \
M(DistrCachePackets, "Distributed Cache client event. Total number of packets received from distributed cache", ValueType::Number) \
M(DistrCachePacketsBytes, "Distributed Cache client event. The number of bytes in Data packets which were not ignored", ValueType::Bytes) \
M(DistrCacheUnusedPackets, "Distributed Cache client event. Number of skipped unused packets from distributed cache", ValueType::Number) \
M(DistrCacheUnusedPacketsBytes, "Distributed Cache client event. The number of bytes in Data packets which were ignored", ValueType::Bytes) \
M(DistrCacheUnusedPacketsBufferAllocations, "Distributed Cache client event. The number of extra buffer allocations in case we could not reuse existing buffer", ValueType::Number) \
\
M(DistrCacheServerProcessRequestMicroseconds, "Time spent processing request on DistributedCache server side", ValueType::Microseconds) \
M(DistrCacheLockRegistryMicroseconds, "Distributed Cache registry event. Time spent to take DistributedCacheRegistry lock", ValueType::Microseconds) \
M(DistrCacheRegistryUpdateMicroseconds, "Distributed Cache registry event. Time spent updating distributed cache registry", ValueType::Microseconds) \
M(DistrCacheRegistryUpdates, "Distributed Cache registry event. Number of distributed cache registry updates", ValueType::Number) \
M(DistrCacheHashRingRebuilds, "Distributed Cache registry event. Number of distributed cache hash ring rebuilds", ValueType::Number) \
\
M(DistrCacheReadBytesFromCache, "Distributed Cache read buffer event. Bytes read from distributed cache", ValueType::Bytes) \
M(DistrCacheReadBytesFromFallbackBuffer, "Distributed Cache read buffer event. Bytes read from fallback buffer", ValueType::Number) \
\
M(DistrCacheRangeResetBackward, "Distributed Cache read buffer event. Number of times we reset read range because of seek/last_position change", ValueType::Number) \
M(DistrCacheRangeResetForward, "Distributed Cache read buffer event. Number of times we reset read range because of seek/last_position change", ValueType::Number) \
\
M(DistrCacheOpenedConnections, "Distributed Cache connection event. The number of open connections to distributed cache", ValueType::Number) \
M(DistrCacheReusedConnections, "Distributed Cache connection event. The number of reused connections to distributed cache", ValueType::Number) \
M(DistrCacheOpenedConnectionsBypassingPool, "Distributed Cache connection event. The number of open connections to distributed cache bypassing pool", ValueType::Number) \
M(DistrCacheConnectMicroseconds, "Distributed Cache connection event. The time spent to connect to distributed cache", ValueType::Microseconds) \
M(DistrCacheConnectAttempts, "Distributed Cache connection event. The number of connection attempts to distributed cache", ValueType::Number) \
M(DistrCacheGetClientMicroseconds, "Distributed Cache connection event. Time spent getting client for distributed cache", ValueType::Microseconds) \
\
M(DistrCacheServerProcessRequestMicroseconds, "Distributed Cache server event. Time spent processing request on DistributedCache server side", ValueType::Microseconds) \
M(DistrCacheServerStartRequestPackets, "Distributed Cache server event. Number of StartRequest packets in DistributedCacheServer", ValueType::Number) \
M(DistrCacheServerContinueRequestPackets, "Distributed Cache server event. Number of ContinueRequest packets in DistributedCacheServer", ValueType::Number) \
M(DistrCacheServerEndRequestPackets, "Distributed Cache server event. Number of EndRequest packets in DistributedCacheServer", ValueType::Number) \
M(DistrCacheServerAckRequestPackets, "Distributed Cache server event. Number of AckRequest packets in DistributedCacheServer", ValueType::Number) \
M(DistrCacheServerNewS3CachedClients, "Distributed Cache server event. The number of new cached s3 clients", ValueType::Number) \
M(DistrCacheServerReusedS3CachedClients, "Distributed Cache server event. The number of reused cached s3 clients", ValueType::Number) \
\
M(LogTest, "Number of log messages with level Test", ValueType::Number) \
M(LogTrace, "Number of log messages with level Trace", ValueType::Number) \
@ -788,15 +814,38 @@ The server successfully detected this situation and will download merged part fr
M(InterfacePostgreSQLReceiveBytes, "Number of bytes received through PostgreSQL interfaces", ValueType::Bytes) \
\
M(ParallelReplicasUsedCount, "Number of replicas used to execute a query with task-based parallel replicas", ValueType::Number) \
M(ParallelReplicasAvailableCount, "Number of replicas available to execute a query with task-based parallel replicas", ValueType::Number) \
M(ParallelReplicasUnavailableCount, "Number of replicas which was chosen, but found to be unavailable during query execution with task-based parallel replicas", ValueType::Number) \
\
M(SharedMergeTreeVirtualPartsUpdates, "Virtual parts update count", ValueType::Number) \
M(SharedMergeTreeVirtualPartsUpdatesByLeader, "Virtual parts updates by leader", ValueType::Number) \
M(SharedMergeTreeVirtualPartsUpdateMicroseconds, "Virtual parts update microseconds", ValueType::Microseconds) \
M(SharedMergeTreeVirtualPartsUpdatesFromZooKeeper, "Virtual parts updates count from ZooKeeper", ValueType::Number) \
M(SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds, "Virtual parts updates from ZooKeeper microseconds", ValueType::Microseconds) \
M(SharedMergeTreeVirtualPartsUpdatesPeerNotFound, "Virtual updates from peer failed because no one found", ValueType::Number) \
M(SharedMergeTreeVirtualPartsUpdatesFromPeer, "Virtual parts updates count from peer", ValueType::Number) \
M(SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds, "Virtual parts updates from peer microseconds", ValueType::Microseconds) \
M(SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus, "Virtual parts updates from non-default background job", ValueType::Number) \
M(SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection, "Virtual parts updates leader election failed", ValueType::Number) \
M(SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection, "Virtual parts updates leader election successful", ValueType::Number) \
M(SharedMergeTreeMergeMutationAssignmentAttempt, "How many times we tried to assign merge or mutation", ValueType::Number) \
M(SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo, "How many times we tried to assign merge or mutation and failed because nothing to merge", ValueType::Number) \
M(SharedMergeTreeMergeMutationAssignmentFailedWithConflict, "How many times we tried to assign merge or mutation and failed because of conflict in Keeper", ValueType::Number) \
M(SharedMergeTreeMergeMutationAssignmentSuccessful, "How many times we tried to assign merge or mutation", ValueType::Number) \
M(SharedMergeTreeMergePartsMovedToOudated, "How many parts moved to oudated directory", ValueType::Number) \
M(SharedMergeTreeMergePartsMovedToCondemned, "How many parts moved to condemned directory", ValueType::Number) \
M(SharedMergeTreeOutdatedPartsConfirmationRequest, "How many ZooKeeper requests were used to config outdated parts", ValueType::Number) \
M(SharedMergeTreeOutdatedPartsConfirmationInvocations, "How many invocations were made to confirm outdated parts", ValueType::Number) \
M(SharedMergeTreeOutdatedPartsHTTPRequest, "How many HTTP requests were send to confirm outdated parts", ValueType::Number) \
M(SharedMergeTreeOutdatedPartsHTTPResponse, "How many HTTP responses were send to confirm outdated parts", ValueType::Number) \
M(SharedMergeTreeCondemnedPartsKillRequest, "How many ZooKeeper requests were used to remove condemned parts", ValueType::Number) \
M(SharedMergeTreeCondemnedPartsLockConfict, "How many times we failed to acquite lock because of conflict", ValueType::Number) \
M(SharedMergeTreeCondemnedPartsRemoved, "How many condemned parts were removed", ValueType::Number) \
M(KeeperLogsEntryReadFromLatestCache, "Number of log entries in Keeper being read from latest logs cache", ValueType::Number) \
M(KeeperLogsEntryReadFromCommitCache, "Number of log entries in Keeper being read from commit logs cache", ValueType::Number) \
M(KeeperLogsEntryReadFromFile, "Number of log entries in Keeper being read directly from the changelog file", ValueType::Number) \
M(KeeperLogsPrefetchedEntries, "Number of log entries in Keeper being prefetched from the changelog file", ValueType::Number) \
\
M(ParallelReplicasAvailableCount, "Number of replicas available to execute a query with task-based parallel replicas", ValueType::Number) \
M(ParallelReplicasUnavailableCount, "Number of replicas which was chosen, but found to be unavailable during query execution with task-based parallel replicas", ValueType::Number) \
\
M(StorageConnectionsCreated, "Number of created connections for storages", ValueType::Number) \
M(StorageConnectionsReused, "Number of reused connections for storages", ValueType::Number) \
M(StorageConnectionsReset, "Number of reset connections for storages", ValueType::Number) \
@ -828,6 +877,9 @@ The server successfully detected this situation and will download merged part fr
M(ReadWriteBufferFromHTTPRequestsSent, "Number of HTTP requests sent by ReadWriteBufferFromHTTP", ValueType::Number) \
M(ReadWriteBufferFromHTTPBytes, "Total size of payload bytes received and sent by ReadWriteBufferFromHTTP. Doesn't include HTTP headers.", ValueType::Bytes) \
\
M(SharedDatabaseCatalogFailedToApplyState, "Number of failures to apply new state in SharedDatabaseCatalog", ValueType::Number) \
M(SharedDatabaseCatalogStateApplicationMicroseconds, "Total time spend on application of new state in SharedDatabaseCatalog", ValueType::Microseconds) \
\
M(GWPAsanAllocateSuccess, "Number of successful allocations done by GWPAsan", ValueType::Number) \
M(GWPAsanAllocateFailed, "Number of failed allocations done by GWPAsan (i.e. filled pool)", ValueType::Number) \
M(GWPAsanFree, "Number of free operations done by GWPAsan", ValueType::Number) \

3
src/Common/ProfileEvents.h
View File

@ -38,6 +38,9 @@ namespace ProfileEvents
};
Timer(Counters & counters_, Event timer_event_, Resolution resolution_);
Timer(Counters & counters_, Event timer_event_, Event counter_event, Resolution resolution_);
Timer(Timer && other) noexcept
: counters(other.counters), timer_event(std::move(other.timer_event)), watch(std::move(other.watch)), resolution(std::move(other.resolution))
{}
~Timer() { end(); }
void cancel() { watch.reset(); }
void restart() { watch.restart(); }

2
src/Common/ZooKeeper/ZooKeeperRetries.h
View File

@ -159,6 +159,8 @@ public:
const std::string & getLastKeeperErrorMessage() const { return keeper_error.message; }
/// action will be called only once and only after latest failed retry
/// NOTE: this one will be called only in case when retries finishes with Keeper exception
/// if it will be some other exception this function will not be called.
void actionAfterLastFailedRetry(std::function<void()> f) { action_after_last_failed_retry = std::move(f); }
const std::string & getName() const { return name; }

16
src/Core/BaseSettingsProgramOptions.h
View File

@ -11,9 +11,17 @@ template <typename T>
void addProgramOptionAsMultitoken(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field)
{
auto on_program_option = boost::function1<void, const Strings &>([&cmd_settings, name](const Strings & values) { cmd_settings.set(name, values.back()); });
if (field.getTypeName() == "Bool")
{
options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
name.data(), boost::program_options::value<Strings>()->multitoken()->composing()->implicit_value(std::vector<std::string>{"1"}, "1")->notifier(on_program_option), field.getDescription())));
}
else
{
options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
name.data(), boost::program_options::value<Strings>()->multitoken()->composing()->notifier(on_program_option), field.getDescription())));
}
}
template <typename T>
void addProgramOptionsAsMultitokens(T &cmd_settings, boost::program_options::options_description & options)
@ -36,9 +44,17 @@ template <typename T>
void addProgramOption(T & cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field)
{
auto on_program_option = boost::function1<void, const std::string &>([&cmd_settings, name](const std::string & value) { cmd_settings.set(name, value); });
if (field.getTypeName() == "Bool")
{
options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
name.data(), boost::program_options::value<std::string>()->composing()->implicit_value("1")->notifier(on_program_option), field.getDescription()))); // NOLINT
}
else
{
options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
name.data(), boost::program_options::value<std::string>()->composing()->notifier(on_program_option), field.getDescription()))); // NOLINT
}
}
template <typename T>
void addProgramOptions(T &cmd_settings, boost::program_options::options_description & options)

9
src/Core/FormatFactorySettingsDeclaration.h
View File

@ -540,6 +540,9 @@ Read data types in binary format instead of type names in Native input format
)", 0) \
M(Bool, output_format_native_encode_types_in_binary_format, false, R"(
Write data types in binary format instead of type names in Native output format
)", 0) \
M(Bool, output_format_native_write_json_as_string, false, R"(
Write data of [JSON](../../sql-reference/data-types/newjson.md) column as [String](../../sql-reference/data-types/string.md) column containing JSON strings instead of default native JSON serialization.
)", 0) \
\
M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, R"(
@ -651,6 +654,12 @@ Write data types in binary format instead of type names in RowBinaryWithNamesAnd
)", 0) \
M(URI, format_avro_schema_registry_url, "", R"(
For AvroConfluent format: Confluent Schema Registry URL.
)", 0) \
M(Bool, input_format_binary_read_json_as_string, false, R"(
Read values of [JSON](../../sql-reference/data-types/newjson.md) data type as JSON [String](../../sql-reference/data-types/string.md) values in RowBinary input format.
)", 0) \
M(Bool, output_format_binary_write_json_as_string, false, R"(
Write values of [JSON](../../sql-reference/data-types/newjson.md) data type as JSON [String](../../sql-reference/data-types/string.md) values in RowBinary output format.
)", 0) \
\
M(Bool, output_format_json_quote_64bit_integers, true, R"(

12
src/Core/Settings.cpp
View File

@ -3067,6 +3067,7 @@ Possible values:
M(Bool, allow_drop_detached, false, R"(
Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries
)", 0) \
M(UInt64, max_parts_to_move, 1000, "Limit the number of parts that can be moved in one query. Zero means unlimited.", 0) \
\
M(UInt64, max_table_size_to_drop, 50000000000lu, R"(
Restriction on deleting tables in query time. The value 0 means that you can delete all tables without any restrictions.
@ -4811,6 +4812,9 @@ Max attempts to read with backoff
)", 0) \
M(Bool, enable_filesystem_cache, true, R"(
Use cache for remote filesystem. This setting does not turn on/off cache for disks (must be done via disk config), but allows to bypass cache for some queries if intended
)", 0) \
M(String, filesystem_cache_name, "", R"(
Filesystem cache name to use for stateless table engines or data lakes
)", 0) \
M(Bool, enable_filesystem_cache_on_write_operations, false, R"(
Write into cache on write operations. To actually work this setting requires be added to disk config too
@ -5497,8 +5501,8 @@ Replace external dictionary sources to Null on restore. Useful for testing purpo
M(Bool, create_if_not_exists, false, R"(
Enable `IF NOT EXISTS` for `CREATE` statement by default. If either this setting or `IF NOT EXISTS` is specified and a table with the provided name already exists, no exception will be thrown.
)", 0) \
M(Bool, enable_secure_identifiers, false, R"(
If enabled, only allow secure identifiers which contain only underscore and alphanumeric characters
M(Bool, enforce_strict_identifier_format, false, R"(
If enabled, only allow identifiers containing alphanumeric characters and underscores.
)", 0) \
M(Bool, mongodb_throw_on_unsupported_query, true, R"(
If enabled, MongoDB tables will return an error when a MongoDB query cannot be built. Otherwise, ClickHouse reads the full table and processes it locally. This option does not apply to the legacy implementation or when 'allow_experimental_analyzer=0'.
@ -5784,9 +5788,6 @@ The heartbeat interval in seconds to indicate watch query is alive.
Timeout for waiting for window view fire signal in event time processing
)", 0) \
\
M(Bool, allow_experimental_refreshable_materialized_view, false, R"(
Allow refreshable materialized views (CREATE MATERIALIZED VIEW \\<name\\> REFRESH ...).
)", 0) \
M(Bool, stop_refreshable_materialized_views_on_startup, false, R"(
On server startup, prevent scheduling of refreshable materialized views, as if with SYSTEM STOP VIEWS. You can manually start them with SYSTEM START VIEWS or SYSTEM START VIEW \\<name\\> afterwards. Also applies to newly created views. Has no effect on non-refreshable materialized views.
)", 0) \
@ -5822,6 +5823,7 @@ Experimental data deduplication for SELECT queries based on part UUIDs
MAKE_OBSOLETE(M, Bool, allow_experimental_alter_materialized_view_structure, true) \
MAKE_OBSOLETE(M, Bool, allow_experimental_shared_merge_tree, true) \
MAKE_OBSOLETE(M, Bool, allow_experimental_database_replicated, true) \
MAKE_OBSOLETE(M, Bool, allow_experimental_refreshable_materialized_view, true) \
\
MAKE_OBSOLETE(M, Milliseconds, async_insert_stale_timeout_ms, 0) \
MAKE_OBSOLETE(M, StreamingHandleErrorMode, handle_kafka_error_mode, StreamingHandleErrorMode::DEFAULT) \

10
src/Core/SettingsChangesHistory.cpp
View File

@ -68,14 +68,18 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
},
{"24.10",
{
{"enforce_strict_identifier_format", false, false, "New setting."},
{"enable_parsing_to_custom_serialization", false, true, "New setting"},
{"mongodb_throw_on_unsupported_query", false, true, "New setting."},
{"enable_parallel_replicas", false, false, "Parallel replicas with read tasks became the Beta tier feature."},
{"parallel_replicas_mode", "read_tasks", "read_tasks", "This setting was introduced as a part of making parallel replicas feature Beta"},
{"filesystem_cache_name", "", "", "Filesystem cache name to use for stateless table engines or data lakes"},
{"restore_replace_external_dictionary_source_to_null", false, false, "New setting."},
{"show_create_query_identifier_quoting_rule", "when_necessary", "when_necessary", "New setting."},
{"show_create_query_identifier_quoting_style", "Backticks", "Backticks", "New setting."},
{"enable_secure_identifiers", false, false, "New setting."},
{"output_format_native_write_json_as_string", false, false, "Add new setting to allow write JSON column as single String column in Native format"},
{"output_format_binary_write_json_as_string", false, false, "Add new setting to write values of JSON type as JSON string in RowBinary output format"},
{"input_format_binary_read_json_as_string", false, false, "Add new setting to read values of JSON type as JSON string in RowBinary input format"},
{"min_free_disk_bytes_to_perform_insert", 0, 0, "New setting."},
{"min_free_disk_ratio_to_perform_insert", 0.0, 0.0, "New setting."},
{"cloud_mode_database_engine", 1, 1, "A setting for ClickHouse Cloud"},
@ -95,6 +99,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"distributed_cache_max_unacked_inflight_packets", 10, 10, "A setting for ClickHouse Cloud"},
{"distributed_cache_data_packet_ack_window", 5, 5, "A setting for ClickHouse Cloud"},
{"input_format_orc_dictionary_as_low_cardinality", false, true, "Treat ORC dictionary encoded columns as LowCardinality columns while reading ORC files"},
{"allow_experimental_refreshable_materialized_view", false, true, "Not experimental anymore"},
{"max_parts_to_move", 1000, 1000, "New setting"},
}
},
{"24.9",
@ -107,7 +113,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"allow_materialized_view_with_bad_select", true, true, "Support (but not enable yet) stricter validation in CREATE MATERIALIZED VIEW"},
{"parallel_replicas_mark_segment_size", 128, 0, "Value for this setting now determined automatically"},
{"database_replicated_allow_replicated_engine_arguments", 1, 0, "Don't allow explicit arguments by default"},
{"database_replicated_allow_explicit_uuid", 0, 0, "Added a new setting to disallow explicitly specifying table UUID"},
{"database_replicated_allow_explicit_uuid", 1, 0, "Added a new setting to disallow explicitly specifying table UUID"},
{"parallel_replicas_local_plan", false, false, "Use local plan for local replica in a query with parallel replicas"},
{"join_to_sort_minimum_perkey_rows", 0, 40, "The lower limit of per-key average rows in the right table to determine whether to rerange the right table by key in left or inner join. This setting ensures that the optimization is not applied for sparse table keys"},
{"join_to_sort_maximum_table_rows", 0, 10000, "The maximum number of rows in the right table to determine whether to rerange the right table by key in left or inner join"},

10
src/DataTypes/DataTypeObject.cpp
View File

@ -356,17 +356,13 @@ std::unique_ptr<ISerialization::SubstreamData> DataTypeObject::getDynamicSubcolu
result_typed_columns[getSubPath(path, prefix)] = column;
}
auto & result_dynamic_columns = result_object_column.getDynamicPaths();
auto & result_dynamic_columns_ptrs = result_object_column.getDynamicPathsPtrs();
std::vector<std::pair<String, ColumnPtr>> result_dynamic_paths;
for (const auto & [path, column] : object_column.getDynamicPaths())
{
if (path.starts_with(prefix) && path.size() != prefix.size())
{
auto sub_path = getSubPath(path, prefix);
result_dynamic_columns[sub_path] = column;
result_dynamic_columns_ptrs[sub_path] = assert_cast<ColumnDynamic *>(result_dynamic_columns[sub_path].get());
}
result_dynamic_paths.emplace_back(getSubPath(path, prefix), column);
}
result_object_column.setDynamicPaths(result_dynamic_paths);
const auto & shared_data_offsets = object_column.getSharedDataOffsets();
const auto [shared_data_paths, shared_data_values] = object_column.getSharedDataPathsAndValues();

3
src/DataTypes/Serializations/ISerialization.h
View File

@ -276,6 +276,9 @@ public:
bool use_compact_variant_discriminators_serialization = false;
/// Serialize JSON column as single String column with serialized JSON values.
bool write_json_as_string = false;
enum class ObjectAndDynamicStatisticsMode
{
NONE, /// Don't write statistics.

12
src/DataTypes/Serializations/SerializationJSON.cpp
View File

@ -291,7 +291,7 @@ void SerializationJSON<Parser>::serializeTextImpl(const IColumn & column, size_t
}
template <typename Parser>
void SerializationJSON<Parser>::deserializeTextImpl(IColumn & column, std::string_view object, const FormatSettings & settings) const
void SerializationJSON<Parser>::deserializeObject(IColumn & column, std::string_view object, const FormatSettings & settings) const
{
typename Parser::Element document;
auto parser = parsers_pool.get([] { return new Parser; });
@ -314,7 +314,7 @@ void SerializationJSON<Parser>::deserializeWholeText(IColumn & column, ReadBuffe
{
String object;
readStringUntilEOF(object, istr);
deserializeTextImpl(column, object, settings);
deserializeObject(column, object, settings);
}
template <typename Parser>
@ -330,7 +330,7 @@ void SerializationJSON<Parser>::deserializeTextEscaped(IColumn & column, ReadBuf
{
String object;
readEscapedString(object, istr);
deserializeTextImpl(column, object, settings);
deserializeObject(column, object, settings);
}
template <typename Parser>
@ -346,7 +346,7 @@ void SerializationJSON<Parser>::deserializeTextQuoted(IColumn & column, ReadBuff
{
String object;
readQuotedString(object, istr);
deserializeTextImpl(column, object, settings);
deserializeObject(column, object, settings);
}
template <typename Parser>
@ -362,7 +362,7 @@ void SerializationJSON<Parser>::deserializeTextCSV(IColumn & column, ReadBuffer
{
String object;
readCSVString(object, istr, settings.csv);
deserializeTextImpl(column, object, settings);
deserializeObject(column, object, settings);
}
template <typename Parser>
@ -390,7 +390,7 @@ void SerializationJSON<Parser>::deserializeTextJSON(IColumn & column, ReadBuffer
{
String object_buffer;
auto object_view = readJSONObjectAsViewPossiblyInvalid(istr, object_buffer);
deserializeTextImpl(column, object_view, settings);
deserializeObject(column, object_view, settings);
}
#if USE_SIMDJSON

3
src/DataTypes/Serializations/SerializationJSON.h
View File

@ -36,9 +36,10 @@ public:
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override;
void deserializeObject(IColumn & column, std::string_view object, const FormatSettings & settings) const override;
private:
void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings, bool pretty = false, size_t indent = 0) const;
void deserializeTextImpl(IColumn & column, std::string_view object, const FormatSettings & settings) const;
std::unique_ptr<JSONExtractTreeNode<Parser>> json_extract_tree;
JSONExtractInsertSettings insert_settings;

93
src/DataTypes/Serializations/SerializationObject.cpp
View File

@ -1,11 +1,13 @@
#include <DataTypes/Serializations/SerializationObject.h>
#include <DataTypes/Serializations/SerializationObjectTypedPath.h>
#include <DataTypes/Serializations/SerializationString.h>
#include <Columns/ColumnObject.h>
#include <DataTypes/DataTypeObject.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeString.h>
#include <IO/ReadBufferFromString.h>
namespace DB
{
@ -68,7 +70,7 @@ SerializationObject::ObjectSerializationVersion::ObjectSerializationVersion(UInt
void SerializationObject::ObjectSerializationVersion::checkVersion(UInt64 version)
{
if (version != BASIC)
if (version != V1 && version != STRING)
throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Object structure serialization.");
}
@ -193,10 +195,16 @@ void SerializationObject::serializeBinaryBulkStatePrefix(
throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for Object column structure during serialization of binary bulk state prefix");
/// Write serialization version.
UInt64 serialization_version = ObjectSerializationVersion::Value::BASIC;
UInt64 serialization_version = settings.write_json_as_string ? ObjectSerializationVersion::Value::STRING : ObjectSerializationVersion::Value::V1;
writeBinaryLittleEndian(serialization_version, *stream);
auto object_state = std::make_shared<SerializeBinaryBulkStateObject>(serialization_version);
if (serialization_version == ObjectSerializationVersion::Value::STRING)
{
state = std::move(object_state);
return;
}
object_state->max_dynamic_paths = column_object.getMaxDynamicPaths();
/// Write max_dynamic_paths parameter.
writeVarUInt(object_state->max_dynamic_paths, *stream);
@ -309,6 +317,13 @@ void SerializationObject::deserializeBinaryBulkStatePrefix(
auto object_state = std::make_shared<DeserializeBinaryBulkStateObject>();
object_state->structure_state = std::move(structure_state);
auto * structure_state_concrete = checkAndGetState<DeserializeBinaryBulkStateObjectStructure>(object_state->structure_state);
if (structure_state_concrete->serialization_version.value == ObjectSerializationVersion::Value::STRING)
{
state = std::move(object_state);
return;
}
settings.path.push_back(Substream::ObjectData);
for (const auto & path : sorted_typed_paths)
@ -319,8 +334,7 @@ void SerializationObject::deserializeBinaryBulkStatePrefix(
settings.path.pop_back();
}
const auto & sorted_dynamic_paths = checkAndGetState<DeserializeBinaryBulkStateObjectStructure>(object_state->structure_state)->sorted_dynamic_paths;
for (const auto & path : sorted_dynamic_paths)
for (const auto & path : structure_state_concrete->sorted_dynamic_paths)
{
settings.path.push_back(Substream::ObjectDynamicPath);
settings.path.back().object_path_name = path;
@ -353,6 +367,8 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb
UInt64 serialization_version;
readBinaryLittleEndian(serialization_version, *structure_stream);
auto structure_state = std::make_shared<DeserializeBinaryBulkStateObjectStructure>(serialization_version);
if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::V1)
{
/// Read max_dynamic_paths parameter.
readVarUInt(structure_state->max_dynamic_paths, *structure_stream);
/// Read the sorted list of dynamic paths.
@ -389,6 +405,7 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb
structure_state->statistics = std::make_shared<const ColumnObject::Statistics>(std::move(statistics));
}
}
state = std::move(structure_state);
addToSubstreamsDeserializeStatesCache(cache, settings.path, state);
@ -405,11 +422,36 @@ void SerializationObject::serializeBinaryBulkWithMultipleStreams(
SerializeBinaryBulkSettings & settings,
SerializeBinaryBulkStatePtr & state) const
{
auto * object_state = checkAndGetState<SerializeBinaryBulkStateObject>(state);
if (object_state->serialization_version.value == ObjectSerializationVersion::Value::STRING)
{
/// Serialize JSON column as single stream of JSON strings.
settings.path.push_back(Substream::ObjectData);
auto * data_stream = settings.getter(settings.path);
settings.path.pop_back();
if (!data_stream)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty stream for String data in SerializationObject::serializeBinaryBulkWithMultipleStreams");
size_t end = limit && offset + limit < column.size() ? offset + limit : column.size();
WriteBufferFromOwnString buf;
FormatSettings format_settings;
for (size_t i = offset; i != end; ++i)
{
serializeText(column, i, buf, format_settings);
const auto & data = buf.str();
writeStringBinary(data, *data_stream);
buf.restart();
}
return;
}
const auto & column_object = assert_cast<const ColumnObject &>(column);
const auto & typed_paths = column_object.getTypedPaths();
const auto & dynamic_paths = column_object.getDynamicPaths();
const auto & shared_data = column_object.getSharedDataPtr();
auto * object_state = checkAndGetState<SerializeBinaryBulkStateObject>(state);
if (column_object.getMaxDynamicPaths() != object_state->max_dynamic_paths)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of max_dynamic_paths parameter of Object. Expected: {}, Got: {}", object_state->max_dynamic_paths, column_object.getMaxDynamicPaths());
@ -474,6 +516,9 @@ void SerializationObject::serializeBinaryBulkStateSuffix(
SerializeBinaryBulkSettings & settings, SerializeBinaryBulkStatePtr & state) const
{
auto * object_state = checkAndGetState<SerializeBinaryBulkStateObject>(state);
if (object_state->serialization_version.value == ObjectSerializationVersion::Value::STRING)
return;
settings.path.push_back(Substream::ObjectStructure);
auto * stream = settings.getter(settings.path);
settings.path.pop_back();
@ -534,6 +579,27 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams(
auto * object_state = checkAndGetState<DeserializeBinaryBulkStateObject>(state);
auto * structure_state = checkAndGetState<DeserializeBinaryBulkStateObjectStructure>(object_state->structure_state);
auto mutable_column = column->assumeMutable();
if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::STRING)
{
/// Read JSON column as single stream of JSON strings.
settings.path.push_back(Substream::ObjectData);
auto * data_stream = settings.getter(settings.path);
settings.path.pop_back();
if (!data_stream)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for Object data serialization in SerializationObject::deserializeBinaryBulkWithMultipleStreams");
String data;
FormatSettings format_settings;
for (size_t i = 0; i != limit; ++i)
{
readStringBinary(data, *data_stream);
ReadBufferFromString buf(data);
deserializeObject(*mutable_column, data, format_settings);
}
return;
}
auto & column_object = assert_cast<ColumnObject &>(*mutable_column);
/// If it's a new object column, set dynamic paths and statistics.
if (column_object.empty())
@ -587,6 +653,15 @@ void SerializationObject::serializeBinary(const Field & field, WriteBuffer & ost
void SerializationObject::serializeBinary(const IColumn & col, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
if (settings.binary.write_json_as_string)
{
/// Serialize row as JSON string.
WriteBufferFromOwnString buf;
serializeText(col, row_num, buf, settings);
writeStringBinary(buf.str(), ostr);
return;
}
const auto & column_object = assert_cast<const ColumnObject &>(col);
const auto & typed_paths = column_object.getTypedPaths();
const auto & dynamic_paths = column_object.getDynamicPaths();
@ -678,6 +753,14 @@ void SerializationObject::restoreColumnObject(ColumnObject & column_object, size
void SerializationObject::deserializeBinary(IColumn & col, ReadBuffer & istr, const FormatSettings & settings) const
{
if (settings.binary.read_json_as_string)
{
String data;
readStringBinary(data, istr);
deserializeObject(col, data, settings);
return;
}
auto & column_object = assert_cast<ColumnObject &>(col);
auto & typed_paths = column_object.getTypedPaths();
auto & dynamic_paths = column_object.getDynamicPaths();

22
src/DataTypes/Serializations/SerializationObject.h
View File

@ -19,7 +19,21 @@ public:
{
enum Value
{
BASIC = 0,
/// V1 serialization:
/// - ObjectStructure stream:
/// <max_dynamic_paths parameter>
/// <actual number of dynamic paths>
/// <sorted list of dynamic paths>
/// <statistics with number of non-null values for dynamic paths> (only in MergeTree serialization)
/// <statistics with number of non-null values for some paths in shared data> (only in MergeTree serialization)
/// - ObjectData stream:
/// - ObjectTypedPath stream for each column in typed paths
/// - ObjectDynamicPath stream for each column in dynamic paths
/// - ObjectSharedData stream shared data column.
V1 = 0,
/// String serialization:
/// - ObjectData stream with single String column containing serialized JSON.
STRING = 1,
};
Value value;
@ -72,6 +86,8 @@ public:
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeBinary(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
virtual void deserializeObject(IColumn & column, std::string_view object, const FormatSettings & settings) const = 0;
static void restoreColumnObject(ColumnObject & column_object, size_t prev_size);
private:
@ -81,14 +97,14 @@ private:
/// State of an Object structure. Can be also used during deserializing of Object subcolumns.
struct DeserializeBinaryBulkStateObjectStructure : public ISerialization::DeserializeBinaryBulkState
{
ObjectSerializationVersion structure_version;
ObjectSerializationVersion serialization_version;
size_t max_dynamic_paths;
std::vector<String> sorted_dynamic_paths;
std::unordered_set<String> dynamic_paths;
/// Paths statistics. Map (dynamic path) -> (number of non-null values in this path).
ColumnObject::StatisticsPtr statistics;
explicit DeserializeBinaryBulkStateObjectStructure(UInt64 structure_version_) : structure_version(structure_version_) {}
explicit DeserializeBinaryBulkStateObjectStructure(UInt64 serialization_version_) : serialization_version(serialization_version_) {}
};
static DeserializeBinaryBulkStatePtr deserializeObjectStructureStatePrefix(

2
src/Databases/DatabaseMemory.cpp
View File

@ -150,7 +150,7 @@ void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & tabl
if (it == create_queries.end() || !it->second)
throw Exception(ErrorCodes::UNKNOWN_TABLE, "Cannot alter: There is no metadata of table {}", table_id.getNameForLogs());
applyMetadataChangesToCreateQuery(it->second, metadata);
applyMetadataChangesToCreateQuery(it->second, metadata, local_context);
/// The create query of the table has been just changed, we need to update dependencies too.
auto ref_dependencies = getDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second, local_context->getCurrentDatabase());

2
src/Databases/DatabaseOnDisk.cpp
View File

@ -123,7 +123,7 @@ std::pair<String, StoragePtr> createTableFromAST(
else
{
columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, mode);
constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints, columns, context);
}
}

2
src/Databases/DatabaseOrdinary.cpp
View File

@ -567,7 +567,7 @@ void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & ta
local_context->getSettingsRef()[Setting::max_parser_depth],
local_context->getSettingsRef()[Setting::max_parser_backtracks]);
applyMetadataChangesToCreateQuery(ast, metadata);
applyMetadataChangesToCreateQuery(ast, metadata, local_context);
statement = getObjectDefinitionFromCreateQuery(ast);
{

2
src/Databases/DatabaseReplicated.h
View File

@ -87,7 +87,7 @@ public:
void shutdown() override;
std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override;
std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr &) const override;
void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr<IRestoreCoordination> restore_coordination, UInt64 timeout_ms) override;
bool shouldReplicateQuery(const ContextPtr & query_context, const ASTPtr & query_ptr) const override;

96
src/Databases/DatabasesCommon.cpp
View File

@ -2,15 +2,22 @@
#include <Backups/BackupEntriesCollector.h>
#include <Backups/RestorerFromBackup.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
#include <Interpreters/DatabaseCatalog.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/TreeRewriter.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/KeyDescription.h>
#include <Storages/StorageDictionary.h>
#include <Storages/StorageFactory.h>
#include <Storages/TTLDescription.h>
#include <Storages/Utils.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Common/CurrentMetrics.h>
@ -18,10 +25,14 @@
#include <Common/logger_useful.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace Setting
{
extern const SettingsUInt64 max_parser_backtracks;
extern const SettingsUInt64 max_parser_depth;
}
namespace ErrorCodes
{
extern const int TABLE_ALREADY_EXISTS;
@ -31,8 +42,87 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
}
namespace
{
void validateCreateQuery(const ASTCreateQuery & query, ContextPtr context)
{
/// First validate that the query can be parsed
const auto serialized_query = serializeAST(query);
ParserCreateQuery parser;
ASTPtr new_query_raw = parseQuery(
parser,
serialized_query.data(),
serialized_query.data() + serialized_query.size(),
"after altering table ",
0,
context->getSettingsRef()[Setting::max_parser_backtracks],
context->getSettingsRef()[Setting::max_parser_depth]);
const auto & new_query = new_query_raw->as<const ASTCreateQuery &>();
/// If there are no columns, then there is nothing much we can do
if (!new_query.columns_list || !new_query.columns_list->columns)
return;
void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata)
const auto & columns = *new_query.columns_list;
/// Do some basic sanity checks. We cannot do the same strict checks as on create, because context might not have the same settings if it is not called directly from an alter query.
/// SECONDARY_CREATE should check most of the important things.
const auto columns_desc
= InterpreterCreateQuery::getColumnsDescription(*columns.columns, context, LoadingStrictnessLevel::SECONDARY_CREATE, false);
/// Default expressions are only validated in level CREATE, so let's check them now
DefaultExpressionsInfo default_expr_info{std::make_shared<ASTExpressionList>()};
for (const auto & ast : columns.columns->children)
{
const auto & col_decl = ast->as<ASTColumnDeclaration &>();
/// There might be some special columns for which `columns_desc.get` would throw, e.g. Nested column when flatten_nested is enabled.
/// At the time of writing I am not aware of anything else, but my knowledge is limited and new types might be added, so let's be safe.
if (!col_decl.default_expression)
continue;
/// If no column description for the name, let's skip the validation of default expressions, but let's log the fact that something went wrong
if (const auto * maybe_column_desc = columns_desc.tryGet(col_decl.name); maybe_column_desc)
getDefaultExpressionInfoInto(col_decl, maybe_column_desc->type, default_expr_info);
else
LOG_WARNING(getLogger("validateCreateQuery"), "Couldn't get column description for column {}", col_decl.name);
}
if (default_expr_info.expr_list)
validateColumnsDefaultsAndGetSampleBlock(default_expr_info.expr_list, columns_desc.getAll(), context);
if (columns.indices)
{
for (const auto & child : columns.indices->children)
IndexDescription::getIndexFromAST(child, columns_desc, context);
}
if (columns.constraints)
{
InterpreterCreateQuery::getConstraintsDescription(columns.constraints, columns_desc, context);
}
if (columns.projections)
{
for (const auto & child : columns.projections->children)
ProjectionDescription::getProjectionFromAST(child, columns_desc, context);
}
if (!new_query.storage)
return;
const auto & storage = *new_query.storage;
std::optional<KeyDescription> primary_key;
/// First get the key description from order by, so if there is no primary key we will use that
if (storage.order_by)
primary_key = KeyDescription::getKeyFromAST(storage.order_by->ptr(), columns_desc, context);
if (storage.primary_key)
primary_key = KeyDescription::getKeyFromAST(storage.primary_key->ptr(), columns_desc, context);
if (storage.partition_by)
KeyDescription::getKeyFromAST(storage.partition_by->ptr(), columns_desc, context);
if (storage.sample_by)
KeyDescription::getKeyFromAST(storage.sample_by->ptr(), columns_desc, context);
if (storage.ttl_table && primary_key.has_value())
TTLTableDescription::getTTLForTableFromAST(storage.ttl_table->ptr(), columns_desc, context, *primary_key, true);
}
}
void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata, ContextPtr context)
{
auto & ast_create_query = query->as<ASTCreateQuery &>();
@ -115,6 +205,8 @@ void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemo
ast_create_query.reset(ast_create_query.comment);
else
ast_create_query.set(ast_create_query.comment, std::make_shared<ASTLiteral>(metadata.comment));
validateCreateQuery(ast_create_query, context);
}

9
src/Databases/DatabasesCommon.h
View File

@ -1,10 +1,9 @@
#pragma once
#include <base/types.h>
#include <Databases/IDatabase.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/IStorage_fwd.h>
#include <Databases/IDatabase.h>
#include <mutex>
#include <base/types.h>
/// General functionality for several different database engines.
@ -12,15 +11,13 @@
namespace DB
{
void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata);
void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata, ContextPtr context);
ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_storage, bool only_ordinary,
uint32_t max_parser_depth, uint32_t max_parser_backtracks, bool throw_on_error);
/// Cleans a CREATE QUERY from temporary flags like "IF NOT EXISTS", "OR REPLACE", "AS SELECT" (for non-views), etc.
void cleanupObjectDefinitionFromTemporaryFlags(ASTCreateQuery & query);
class Context;
/// A base class for databases that manage their own list of tables.
class DatabaseWithOwnTablesBase : public IDatabase, protected WithContext
{

4
src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
View File

@ -31,7 +31,7 @@ CachedObjectStorage::CachedObjectStorage(
FileCache::Key CachedObjectStorage::getCacheKey(const std::string & path) const
{
return cache->createKeyForPath(path);
return FileCacheKey::fromPath(path);
}
ObjectStorageKey
@ -71,7 +71,7 @@ std::unique_ptr<ReadBufferFromFileBase> CachedObjectStorage::readObject( /// NOL
{
if (cache->isInitialized())
{
auto cache_key = cache->createKeyForPath(object.remote_path);
auto cache_key = FileCacheKey::fromPath(object.remote_path);
auto global_context = Context::getGlobalContextInstance();
auto modified_read_settings = read_settings.withNestedBuffer();

3
src/Formats/FormatFactory.cpp
View File

@ -305,9 +305,12 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
format_settings.binary.max_binary_array_size = settings[Setting::format_binary_max_array_size];
format_settings.binary.encode_types_in_binary_format = settings[Setting::output_format_binary_encode_types_in_binary_format];
format_settings.binary.decode_types_in_binary_format = settings[Setting::input_format_binary_decode_types_in_binary_format];
format_settings.binary.write_json_as_string = settings[Setting::output_format_binary_write_json_as_string];
format_settings.binary.read_json_as_string = settings[Setting::input_format_binary_read_json_as_string];
format_settings.native.allow_types_conversion = settings[Setting::input_format_native_allow_types_conversion];
format_settings.native.encode_types_in_binary_format = settings[Setting::output_format_native_encode_types_in_binary_format];
format_settings.native.decode_types_in_binary_format = settings[Setting::input_format_native_decode_types_in_binary_format];
format_settings.native.write_json_as_string = settings[Setting::output_format_native_write_json_as_string];
format_settings.max_parser_depth = settings[Setting::max_parser_depth];
format_settings.date_time_overflow_behavior = settings[Setting::date_time_overflow_behavior];
format_settings.try_infer_variant = settings[Setting::input_format_try_infer_variants];

3
src/Formats/FormatSettings.h
View File

@ -126,6 +126,8 @@ struct FormatSettings
UInt64 max_binary_array_size = 1_GiB;
bool encode_types_in_binary_format = false;
bool decode_types_in_binary_format = false;
bool read_json_as_string = false;
bool write_json_as_string = false;
} binary{};
struct
@ -474,6 +476,7 @@ struct FormatSettings
bool allow_types_conversion = true;
bool encode_types_in_binary_format = false;
bool decode_types_in_binary_format = false;
bool write_json_as_string = false;
} native{};
struct

1
src/Formats/NativeWriter.cpp
View File

@ -69,6 +69,7 @@ static void writeData(const ISerialization & serialization, const ColumnPtr & co
settings.position_independent_encoding = false;
settings.low_cardinality_max_dictionary_size = 0;
settings.data_types_binary_encoding = format_settings && format_settings->native.encode_types_in_binary_format;
settings.write_json_as_string = format_settings && format_settings->native.write_json_as_string;
ISerialization::SerializeBinaryBulkStatePtr state;
serialization.serializeBinaryBulkStatePrefix(*full_column, settings, state);

1
src/Functions/CMakeLists.txt
View File

@ -7,6 +7,7 @@ add_headers_and_sources(clickhouse_functions .)
# This allows less dependency and linker work (specially important when building many example executables)
set(DBMS_FUNCTIONS
IFunction.cpp # IFunctionOverloadResolver::getLambdaArgumentTypes, IExecutableFunction::execute... (Many AST visitors, analyzer passes, some storages...)
FunctionDynamicAdaptor.cpp # IFunctionOverloadResolver::getLambdaArgumentTypes, IExecutableFunction::execute... (Many AST visitors, analyzer passes, some storages...)
FunctionFactory.cpp # FunctionFactory::instance() (Many AST visitors, analyzer passes, some storages...)
FunctionHelpers.cpp # convertConstTupleToConstantElements, checkAndGetColumnConstStringOrFixedString, checkAndGetNestedArrayOffset ...)
FunctionsLogical.cpp # createInternalFunctionAndOverloadResolve / createInternalFunctionOrOverloadResolver ... (Multiple)

5
src/Functions/FunctionBase58Conversion.h
View File

@ -147,6 +147,11 @@ public:
return std::make_shared<DataTypeString>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeString>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const ColumnPtr column_string = arguments[0].column;

5
src/Functions/FunctionBase64Conversion.h
View File

@ -207,6 +207,11 @@ public:
return std::make_shared<DataTypeString>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeString>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto & input_column = arguments[0].column;

5
src/Functions/FunctionBitTestMany.h
View File

@ -57,6 +57,11 @@ public:
return std::make_shared<DataTypeUInt8>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeUInt8>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const auto * value_col = arguments.front().column.get();

5
src/Functions/FunctionChar.cpp
View File

@ -52,6 +52,11 @@ public:
return std::make_shared<DataTypeString>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeString>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override

5
src/Functions/FunctionCustomWeekToSomething.h
View File

@ -23,6 +23,11 @@ public:
return std::make_shared<ToDataType>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<ToDataType>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const IDataType * from_type = arguments[0].type.get();

8
src/Functions/FunctionDateOrDateTimeToSomething.h
View File

@ -56,6 +56,14 @@ public:
return std::make_shared<ToDataType>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
/// If result type is DateTime or DateTime64 we don't know the timezone and scale without argument types.
if constexpr (!std::is_same_v<ToDataType, DataTypeDateTime> && !std::is_same_v<ToDataType, DataTypeDateTime64>)
return std::make_shared<ToDataType>();
return nullptr;
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const IDataType * from_type = arguments[0].type.get();

476
src/Functions/FunctionDynamicAdaptor.cpp Normal file
View File

@ -0,0 +1,476 @@
#include <Functions/FunctionDynamicAdaptor.h>
#include <DataTypes/DataTypeDynamic.h>
#include <DataTypes/DataTypeVariant.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypesBinaryEncoding.h>
#include <Columns/ColumnDynamic.h>
#include <Columns/ColumnNullable.h>
#include <Interpreters/castColumn.h>
#include <IO/ReadBufferFromMemory.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
ColumnPtr ExecutableFunctionDynamicAdaptor::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t, bool dry_run) const
{
auto column = arguments[dynamic_argument_index].column->convertToFullColumnIfConst();
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(*column);
if (dynamic_column.empty())
return result_type->createColumn();
const auto & variant_info = dynamic_column.getVariantInfo();
const auto & variant_types = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariants();
const auto & variant_column = dynamic_column.getVariantColumn();
auto shared_variant_discr = dynamic_column.getSharedVariantDiscriminator();
/// We use default implementation for Dynamic type only when default implementation for NULLs is used.
/// If current column contains only NULLs, result column will also contain only NULLs.
if (variant_column.hasOnlyNulls())
{
auto result = result_type->createColumn();
result->insertManyDefaults(variant_column.size());
return result;
}
/// Check if this Dynamic column contains only values of one type and no NULLs.
/// In this case we can replace argument with this variant and execute the function without changing all other arguments.
auto non_empty_variant_discr_no_nulls = variant_column.getGlobalDiscriminatorOfOneNoneEmptyVariantNoNulls();
if (non_empty_variant_discr_no_nulls && *non_empty_variant_discr_no_nulls != shared_variant_discr)
{
/// Crete new arguments and replace our Dynamic column with variant.
auto global_discr = *non_empty_variant_discr_no_nulls;
ColumnsWithTypeAndName new_arguments;
new_arguments.reserve(arguments.size());
for (size_t i = 0; i != arguments.size(); ++i)
{
if (i == dynamic_argument_index)
{
ColumnWithTypeAndName arg
{
variant_column.getVariantPtrByGlobalDiscriminator(global_discr),
variant_types[global_discr],
arguments[i].name,
};
new_arguments.push_back(std::move(arg));
}
else
{
new_arguments.push_back(arguments[i]);
}
}
/// Execute function on new arguments.
auto func_base = function_overload_resolver->build(new_arguments);
auto nested_result_type = func_base->getResultType();
auto nested_result = func_base->execute(new_arguments, nested_result_type, dynamic_column.size(), dry_run);
/// If result is Nullable(Nothing), just return column filled with NULLs.
if (nested_result_type->onlyNull())
{
auto res = result_type->createColumn();
res->insertManyDefaults(dynamic_column.size());
return res;
}
/// If the result of the function is not Dynamic, it means that this function returns the same
/// type for all argument types (or similar types like FixedString or String).
/// In this case we return Nullable of this type (because Dynamic can contain NULLs).
if (!isDynamic(result_type))
{
/// If return types are not the same, they must be convertible to each other (like FixedString/String).
if (!removeNullable(result_type)->equals(*removeNullable(nested_result_type)))
{
try
{
return castColumn(ColumnWithTypeAndName{makeNullableSafe(nested_result), makeNullableSafe(nested_result_type), ""}, result_type);
}
catch (const Exception & e)
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot convert nested result of function {} with type {} to the expected result type {}: {}", getName(), removeNullable(result_type)->getName(), removeNullable(nested_result_type)->getName(), e.message());
}
}
return makeNullableSafe(nested_result);
}
/// Cast column to result Dynamic type.
return castColumn(ColumnWithTypeAndName{nested_result, nested_result_type, ""}, result_type);
}
/// Second, check if this Dynamic column contains only 1 variant and NULLs.
/// In this case we can create a null-mask, filter all arguments by it and execute function
/// on this variant and filtered arguments.
auto non_empty_variant_discr = variant_column.getGlobalDiscriminatorOfOneNoneEmptyVariant();
if (non_empty_variant_discr && *non_empty_variant_discr != shared_variant_discr)
{
auto global_discr = *non_empty_variant_discr;
/// Create filter for rows containing our variant.
PaddedPODArray<UInt8> filter;
filter.reserve(variant_column.size());
const auto & local_discriminators = variant_column.getLocalDiscriminators();
auto local_discr = variant_column.localDiscriminatorByGlobal(global_discr);
for (const auto & discr : local_discriminators)
filter.push_back(discr == local_discr);
/// Filter all other arguments using created filter.
ColumnsWithTypeAndName new_arguments;
new_arguments.reserve(arguments.size());
size_t result_size_hint = variant_column.getVariantPtrByGlobalDiscriminator(global_discr)->size();
for (size_t i = 0; i != arguments.size(); ++i)
{
if (i == dynamic_argument_index)
{
ColumnWithTypeAndName arg
{
variant_column.getVariantPtrByGlobalDiscriminator(global_discr),
variant_types[global_discr],
arguments[i].name,
};
new_arguments.push_back(std::move(arg));
}
else
{
ColumnWithTypeAndName arg
{
arguments[i].column->filter(filter, result_size_hint),
arguments[i].type,
arguments[i].name
};
new_arguments.push_back(std::move(arg));
}
}
/// Execute function on new arguments.
auto func_base = function_overload_resolver->build(new_arguments);
auto nested_result_type = func_base->getResultType();
auto nested_result = func_base->execute(new_arguments, nested_result_type, new_arguments[0].column->size(), dry_run)->convertToFullColumnIfConst();
/// If result is Nullable(Nothing), just return column filled with NULLs.
if (nested_result_type->onlyNull())
{
auto res = result_type->createColumn();
res->insertManyDefaults(dynamic_column.size());
return res;
}
/// If the result of the function is not Dynamic, it means that this function returns the same
/// type for all argument types (or similar types like FixedString or String).
/// In this case we return Nullable of this type (because Dynamic can contain NULLs).
if (!isDynamic(result_type))
{
/// Expand filtered result. If it's already Nullable, it will be filled with NULLs.
nested_result->assumeMutable()->expand(filter, false);
/// If result wasn't Nullable, create null-mask from filter and make it Nullable.
if (!nested_result_type->isNullable() && nested_result_type->canBeInsideNullable())
{
for (auto & byte : filter)
byte = !byte;
auto null_map_column = ColumnUInt8::create();
null_map_column->getData() = std::move(filter);
nested_result = ColumnNullable::create(nested_result, std::move(null_map_column));
nested_result_type = makeNullable(nested_result_type);
}
/// If return types are not the same, they must be convertible to each other (like FixedString/String).
if (!result_type->equals(*nested_result_type))
{
try
{
return castColumn(ColumnWithTypeAndName{nested_result, nested_result_type, ""}, result_type);
}
catch (const Exception & e)
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot convert nested result of function {} with type {} to the expected result type {}: {}", getName(), result_type->getName(), nested_result_type->getName(), e.message());
}
}
return nested_result;
}
/// If the result of nested function is Variant type, we cannot use it as single variant inside Dynamic.
/// In this case we cast the result to Dynamic result type.
if (isVariant(nested_result_type))
{
nested_result = castColumn(ColumnWithTypeAndName{nested_result, nested_result_type, ""}, result_type);
nested_result_type = result_type;
}
/// If the result of nested function is Dynamic (or we casted to it from Variant), we can just expand it
/// and cast to the result Dynamic type (it can have different max_types parameter).
if (isDynamic(nested_result_type))
{
nested_result->assumeMutable()->expand(filter, false);
return castColumn(ColumnWithTypeAndName{nested_result, nested_result_type, ""}, result_type);
}
/// If the result of nested function is not Dynamic, we create Dynamic column with variant of this type.
auto variant = nested_result;
auto variant_type = nested_result_type;
const NullMap * null_map_ptr = nullptr;
/// If the result of nested function is Nullable, we create a null-mask and use it during Dynamic column creation,
/// also the nested column inside Nullable will be filtered by this mask (inside Dynamic we don't store default values in rows with NULLs).
if (const auto & column_nullable = typeid_cast<const ColumnNullable *>(variant.get()))
{
const auto & null_map = column_nullable->getNullMapData();
/// Create filter for nested column from null-map and calculate result size hint.
PaddedPODArray<UInt8> nested_filter;
nested_filter.reserve(null_map.size());
size_t size_hint = 0;
for (auto byte : null_map)
{
if (byte)
{
nested_filter.push_back(0);
}
else
{
nested_filter.push_back(1);
++size_hint;
}
}
variant = column_nullable->getNestedColumnPtr()->filter(nested_filter, size_hint);
variant_type = removeNullable(nested_result_type);
null_map_ptr = &null_map;
}
auto result = result_type->createColumn();
auto & result_dynamic = assert_cast<ColumnDynamic &>(*result);
if (!result_dynamic.addNewVariant(variant_type))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add new variant {} to Dynamic column {} during execution of {}", variant_type->getName(), result_type->getName(), getName());
/// Now inside Dynamic we have empty Variant containing type of our column from function execution.
/// Use our result column as variant and fill discriminators and offsets columns.
auto & result_variant = result_dynamic.getVariantColumn();
auto result_global_discr = result_dynamic.getVariantInfo().variant_name_to_discriminator.at(variant_type->getName());
auto result_local_discr = result_variant.localDiscriminatorByGlobal(result_global_discr);
result_variant.getVariantPtrByLocalDiscriminator(result_local_discr) = std::move(variant);
auto & result_local_discriminators = result_variant.getLocalDiscriminators();
result_local_discriminators.reserve(filter.size());
auto & result_offsets = result_variant.getOffsets();
result_offsets.reserve(filter.size());
/// Calculate correct offset for our variant, we cannot use initial offsets from
/// argument column because we could filter result column by its null-map.
size_t offset = 0;
/// Use initial offsets from argument column to use correct values of null-map.
const auto & offsets = variant_column.getOffsets();
for (size_t i = 0; i != filter.size(); ++i)
{
if (filter[i] && (!null_map_ptr || !(*null_map_ptr)[offsets[i]]))
{
result_local_discriminators.push_back(result_local_discr);
result_offsets.push_back(offset++);
}
else
{
result_local_discriminators.push_back(ColumnVariant::NULL_DISCRIMINATOR);
result_offsets.emplace_back();
}
}
return result;
}
/// In general case with several variants we create a selector from discriminators
/// and use it to create a set of filtered arguments for each variant.
/// Then we will execute our function over all these arguments and construct the resulting column
/// from all results based on created selector.
IColumn::Selector selector;
selector.reserve(variant_column.size());
IColumn::Offsets variants_offsets;
variants_offsets.reserve(variant_column.size());
std::vector<ColumnWithTypeAndName> variants;
/// We need to determine the selector index for rows with NULL values, but we don't know how many
/// variants we have in Dynamic column (shared variant can contain unknown amount of new variant types).
/// So, we allocate 0 index for rows with NULL values.
variants.emplace_back();
/// Remember indexes in selector for each variant type.
std::unordered_map<String, size_t> variant_indexes;
const auto & local_discriminators = variant_column.getLocalDiscriminators();
const auto & offsets = variant_column.getOffsets();
auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(dynamic_column.getSharedVariantDiscriminator());
const auto & shared_variant = dynamic_column.getSharedVariant();
/// Remember created serializations for variants in shared variant to avoid recreating it every time.
std::unordered_map<String, SerializationPtr> shared_variants_serializations;
FormatSettings format_settings;
for (size_t i = 0; i != local_discriminators.size(); ++i)
{
auto local_discr = local_discriminators[i];
if (local_discr == ColumnVariant::NULL_DISCRIMINATOR)
{
selector.push_back(0);
variants_offsets.emplace_back();
}
else if (local_discr == shared_variant_local_discr)
{
/// Deserialize type and value from shared variant row.
auto value = shared_variant.getDataAt(offsets[i]);
ReadBufferFromMemory buf(value.data, value.size);
auto type = decodeDataType(buf);
auto type_name = type->getName();
/// Check if we already allocated selector index for this variant type.
/// If not, append it to list of variants and remember its index.
auto indexes_it = variant_indexes.find(type_name);
if (indexes_it == variant_indexes.end())
{
indexes_it = variant_indexes.emplace(type_name, variants.size()).first;
variants.emplace_back(type->createColumn(), type, "");
}
auto serializations_it = shared_variants_serializations.find(type_name);
if (serializations_it == shared_variants_serializations.end())
serializations_it = shared_variants_serializations.emplace(type_name, type->getDefaultSerialization()).first;
/// Deserialize value into usual column.
serializations_it->second->deserializeBinary(*variants[indexes_it->second].column->assumeMutable(), buf, format_settings);
selector.push_back(indexes_it->second);
variants_offsets.push_back(variants[indexes_it->second].column->size() - 1);
}
else
{
auto global_discr = variant_column.globalDiscriminatorByLocal(local_discr);
/// Check if we already allocated selector index for this variant type.
/// If not, append it to list of variants and remember its index.
auto it = variant_indexes.find(variant_info.variant_names[global_discr]);
if (it == variant_indexes.end())
{
it = variant_indexes.emplace(variant_info.variant_names[global_discr], variants.size()).first;
variants.emplace_back(variant_column.getVariantPtrByLocalDiscriminator(local_discr), variant_types[global_discr], "");
}
selector.push_back(it->second);
variants_offsets.push_back(offsets[i]);
}
}
/// Create set of arguments for each variant using selector.
std::vector<ColumnsWithTypeAndName> variants_arguments;
variants_arguments.resize(variants.size());
for (size_t i = 0; i != arguments.size(); ++i)
{
if (i == dynamic_argument_index)
{
for (size_t j = 1; j != variants_arguments.size(); ++j)
variants_arguments[j].push_back(variants[j]);
}
else
{
auto columns = arguments[i].column->scatter(variants.size(), selector);
for (size_t j = 0; j != variants_arguments.size(); ++j)
variants_arguments[j].emplace_back(std::move(columns[j]), arguments[i].type, arguments[i].name);
}
}
/// Execute function over all created sets of arguments and remember all results.
std::vector<ColumnPtr> variants_results;
variants_results.reserve(variants.size());
/// 0 index is allocated for rows with NULL values, it doesn't have any result,
/// we will insert NULL values in these rows.
variants_results.emplace_back();
for (size_t i = 1; i != variants_arguments.size(); ++i)
{
auto func_base = function_overload_resolver->build(variants_arguments[i]);
auto nested_result_type = func_base->getResultType();
auto nested_result = func_base->execute(variants_arguments[i], nested_result_type, variants_arguments[i][0].column->size(), dry_run)->convertToFullColumnIfConst();
/// Append nullptr in case of only NULL values, we will insert NULL for rows of this selector.
if (nested_result_type->onlyNull())
{
variants_results.emplace_back();
}
/// If the result of the function is not Dynamic, it means that this function returns the same
/// type for all argument types (or similar types like FixedString or String).
/// In this case we return Nullable of this type (because Dynamic can contain NULLs).
else if (!isDynamic(result_type))
{
/// If return types are not the same, they must be convertible to each other (like FixedString/String).
if (!removeNullable(result_type)->equals(*removeNullable(nested_result_type)))
{
try
{
variants_results.push_back(castColumn(ColumnWithTypeAndName{makeNullableSafe(nested_result), makeNullableSafe(nested_result_type), ""}, result_type));
}
catch (const Exception & e)
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot convert nested result of function {} with type {} to the expected result type {}: {}", getName(), result_type->getName(), nested_result_type->getName(), e.message());
}
}
else
{
variants_results.push_back(makeNullableSafe(nested_result));
}
}
/// Otherwise cast this result to the resulting Dynamic type.
else
{
variants_results.push_back(castColumn(ColumnWithTypeAndName{nested_result, nested_result_type, ""}, result_type));
}
}
/// Construct resulting Dynamic column from all results.
auto result = result_type->createColumn();
result->reserve(dynamic_column.size());
for (size_t i = 0; i != selector.size(); ++i)
{
if (selector[i] == 0 || !variants_results[selector[i]])
result->insertDefault();
else
result->insertFrom(*variants_results[selector[i]], variants_offsets[i]);
}
return result;
}
ColumnPtr ExecutableFunctionDynamicAdaptor::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
{
return executeImpl(arguments, result_type, input_rows_count, false);
}
ColumnPtr ExecutableFunctionDynamicAdaptor::executeDryRunImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
{
return executeImpl(arguments, result_type, input_rows_count, true);
}
FunctionBaseDynamicAdaptor::FunctionBaseDynamicAdaptor(std::shared_ptr<const IFunctionOverloadResolver> function_overload_resolver_, DataTypes arguments_) : function_overload_resolver(function_overload_resolver_), arguments(arguments_)
{
/// For resulting Dynamic type use the maximum max_dynamic_types from all Dynamic arguments.
size_t result_max_dynamic_type;
bool first = true;
for (size_t i = 0; i != arguments.size(); ++i)
{
if (const auto * dynamic_type = typeid_cast<const DataTypeDynamic *>(arguments[i].get()))
{
if (first)
{
result_max_dynamic_type = dynamic_type->getMaxDynamicTypes();
dynamic_argument_index = i;
first = false;
}
else
{
result_max_dynamic_type = std::max(result_max_dynamic_type, dynamic_type->getMaxDynamicTypes());
}
}
}
if (auto type = function_overload_resolver->getReturnTypeForDefaultImplementationForDynamic())
return_type = makeNullableSafe(type);
else
return_type = std::make_shared<DataTypeDynamic>(result_max_dynamic_type);
}
}

79
src/Functions/FunctionDynamicAdaptor.h Normal file
View File

@ -0,0 +1,79 @@
#pragma once
#include <Functions/IFunction.h>
namespace DB
{
/// Special adapter classes that implement functions execution with Dynamic arguments.
class ExecutableFunctionDynamicAdaptor final : public IExecutableFunction
{
public:
explicit ExecutableFunctionDynamicAdaptor(
std::shared_ptr<const IFunctionOverloadResolver> function_overload_resolver_, size_t dynamic_argument_index_)
: function_overload_resolver(std::move(function_overload_resolver_)), dynamic_argument_index(dynamic_argument_index_)
{
}
String getName() const override { return function_overload_resolver->getName(); }
protected:
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const final;
ColumnPtr executeDryRunImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const final;
/// Disable all default implementations, they will be used later.
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForNothing() const override { return false; }
bool useDefaultImplementationForConstants() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
bool useDefaultImplementationForSparseColumns() const override { return false; }
bool canBeExecutedOnDefaultArguments() const override { return false; }
private:
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run) const;
/// We remember the original IFunctionOverloadResolver to be able to build function for types inside Dynamic column.
std::shared_ptr<const IFunctionOverloadResolver> function_overload_resolver;
size_t dynamic_argument_index;
};
class FunctionBaseDynamicAdaptor final : public IFunctionBase
{
public:
FunctionBaseDynamicAdaptor(std::shared_ptr<const IFunctionOverloadResolver> function_overload_resolver_, DataTypes arguments_);
String getName() const override { return function_overload_resolver->getName(); }
const DataTypes & getArgumentTypes() const override { return arguments; }
const DataTypePtr & getResultType() const override { return return_type; }
ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName & /*arguments*/) const override
{
return std::make_unique<ExecutableFunctionDynamicAdaptor>(function_overload_resolver, dynamic_argument_index);
}
bool isSuitableForConstantFolding() const override { return false; }
bool isStateful() const override { return function_overload_resolver->isStateful(); }
bool isDeterministic() const override { return function_overload_resolver->isDeterministic(); }
bool isDeterministicInScopeOfQuery() const override { return function_overload_resolver->isDeterministicInScopeOfQuery(); }
bool isServerConstant() const override { return function_overload_resolver->isServerConstant(); }
bool isShortCircuit(ShortCircuitSettings & settings, size_t number_of_arguments) const override { return function_overload_resolver->isShortCircuit(settings, number_of_arguments); }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; }
private:
/// We remember the original IFunctionOverloadResolver to be able to build function for types inside Dynamic column.
std::shared_ptr<const IFunctionOverloadResolver> function_overload_resolver;
DataTypes arguments;
DataTypePtr return_type;
size_t dynamic_argument_index;
};
}

5
src/Functions/FunctionFile.cpp
View File

@ -64,6 +64,11 @@ public:
return std::make_shared<DataTypeString>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeString>();
}
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
bool useDefaultImplementationForNulls() const override { return false; }

5
src/Functions/FunctionMathBinaryFloat64.h
View File

@ -53,6 +53,11 @@ private:
return std::make_shared<DataTypeFloat64>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeFloat64>();
}
template <typename LeftType, typename RightType>
static ColumnPtr executeTyped(const ColumnConst * left_arg, const IColumn * right_arg, size_t input_rows_count)
{

5
src/Functions/FunctionMathUnary.h
View File

@ -57,6 +57,11 @@ private:
return argument;
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return Impl::always_returns_float64 ? std::make_shared<DataTypeFloat64>() : nullptr;
}
template <typename T, typename ReturnType>
static void executeInIterations(const T * src_data, ReturnType * dst_data, size_t size)
{

5
src/Functions/FunctionNumericPredicate.h
View File

@ -51,6 +51,11 @@ public:
return std::make_shared<DataTypeUInt8>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeUInt8>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override

5
src/Functions/FunctionSpaceFillingCurve.h
View File

@ -70,6 +70,11 @@ public:
}
return std::make_shared<DataTypeUInt64>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeUInt64>();
}
};
template <UInt8 max_dimensions, UInt8 min_ratio, UInt8 max_ratio>

5
src/Functions/FunctionStartsEndsWith.h
View File

@ -91,6 +91,11 @@ public:
arguments[0]->getName(), arguments[1]->getName(), getName());
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeUInt8>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
auto data_type = arguments[0].type;

5
src/Functions/FunctionStringOrArrayToT.h
View File

@ -61,6 +61,11 @@ public:
return std::make_shared<DataTypeNumber<ResultType>>();
}
DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
{
return std::make_shared<DataTypeNumber<ResultType>>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override

Some files were not shown because too many files have changed in this diff Show More